fix: prevent unhandled promise rejection for scheduler status fetch

Attach .catch at promise creation to mark it as handled by the runtime,
then check the result type after await. This prevents an
unhandledrejection event if the fetch rejects before the await.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.github/workflows/e2e.yml

@@ -135,7 +135,7 @@ jobs:
 
     - name: Archive logs
       if: always()
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       with:
         name: output-logs
         path: docker/output.log

.github/workflows/ec-integration-tests.yml

@@ -52,7 +52,7 @@ jobs:
 
     - name: Archive logs
       if: failure()
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       with:
         name: ec-integration-test-logs
         path: |

.github/workflows/ec-integration.yml

@@ -42,7 +42,7 @@ jobs:
         
     - name: Upload test logs on failure
       if: failure()
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       with:
         name: ec-test-logs
         path: test/erasure_coding/admin_dockertest/tmp/logs/

.github/workflows/fuse-integration.yml

@@ -183,7 +183,7 @@ jobs:
         
     - name: Upload Test Artifacts
       if: always()
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       with:
         name: fuse-integration-test-results
         path: |

.github/workflows/java_unit_tests.yml

@@ -49,7 +49,7 @@ jobs:
       
       - name: Upload Test Reports
         if: always()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: test-reports-java-${{ matrix.java }}
           path: |

.github/workflows/metadata-subscribe-tests.yml

@@ -70,7 +70,7 @@ jobs:
         
     - name: Archive logs on failure
       if: failure()
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       with:
         name: metadata-subscribe-test-logs
         path: |

.github/workflows/postgres-tests.yml

@@ -62,7 +62,7 @@ jobs:
 
     - name: Archive logs
       if: always()
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       with:
         name: postgres-logs
         path: test/postgres/postgres-output.log

.github/workflows/s3-example-integration-tests.yml

@@ -57,7 +57,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: integration-test-logs
           path: test/s3/normal/*.log

.github/workflows/s3-filer-group-tests.yml

@@ -77,7 +77,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-filer-group-test-logs
           path: test/s3/filer_group/weed-test*.log

.github/workflows/s3-go-tests.yml

@@ -76,7 +76,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-versioning-test-logs-${{ matrix.test-type }}
           path: test/s3/versioning/weed-test*.log
@@ -124,7 +124,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-versioning-compatibility-logs
           path: test/s3/versioning/weed-test*.log
@@ -172,7 +172,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-cors-compatibility-logs
           path: test/s3/cors/weed-test*.log
@@ -239,7 +239,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-retention-test-logs-${{ matrix.test-type }}
           path: test/s3/retention/weed-test*.log
@@ -306,7 +306,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-cors-test-logs-${{ matrix.test-type }}
           path: test/s3/cors/weed-test*.log
@@ -355,7 +355,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-retention-worm-logs
           path: test/s3/retention/weed-test*.log
@@ -422,7 +422,7 @@ jobs:
 
       - name: Upload stress test logs
         if: always()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-versioning-stress-logs
           path: test/s3/versioning/weed-test*.log
@@ -478,7 +478,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-tagging-test-logs
           path: test/s3/tagging/weed-test*.log
@@ -531,7 +531,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-remote-cache-test-logs
           path: |

.github/workflows/s3-iam-tests.yml

@@ -65,7 +65,7 @@ jobs:
 
       - name: Upload test results on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: iam-unit-test-results
           path: |
@@ -162,7 +162,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-iam-integration-logs-${{ matrix.test-type }}
           path: test/s3/iam/weed-*.log
@@ -222,7 +222,7 @@ jobs:
 
       - name: Upload distributed test logs
         if: always()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-iam-distributed-logs
           path: test/s3/iam/weed-*.log
@@ -274,7 +274,7 @@ jobs:
 
       - name: Upload performance test results
         if: always()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-iam-performance-results
           path: |

.github/workflows/s3-keycloak-tests.yml

@@ -152,7 +152,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-keycloak-test-logs
           path: |

.github/workflows/s3-parquet-tests.yml

@@ -121,7 +121,7 @@ jobs:
       
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: test-logs-python-${{ matrix.python-version }}
           path: |

.github/workflows/s3-policy-tests.yml

@@ -70,7 +70,7 @@ jobs:
 
       - name: Upload test results on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: policy-unit-test-results
           path: |
@@ -178,7 +178,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-policy-variables-test-logs
           path: /tmp/weed_policy_test_server.log
@@ -299,7 +299,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-policy-enforcement-logs-${{ matrix.test-case }}
           path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
@@ -386,7 +386,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: trusted-proxy-test-logs
           path: /tmp/weed_proxy_test.log

.github/workflows/s3-spark-tests.yml

@@ -73,7 +73,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-spark-test-logs
           path: test/s3/spark/test-output.log

.github/workflows/s3-sse-tests.yml

@@ -95,7 +95,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-sse-test-logs-${{ matrix.test-type }}
           path: test/s3/sse/weed-test*.log
@@ -143,7 +143,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-sse-compatibility-logs
           path: test/s3/sse/weed-test*.log
@@ -192,7 +192,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-sse-metadata-persistence-logs
           path: test/s3/sse/weed-test*.log
@@ -241,7 +241,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-sse-copy-operations-logs
           path: test/s3/sse/weed-test*.log
@@ -290,7 +290,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-sse-multipart-logs
           path: test/s3/sse/weed-test*.log
@@ -340,7 +340,7 @@ jobs:
 
       - name: Upload performance test logs
         if: always()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-sse-performance-logs
           path: test/s3/sse/weed-test*.log
@@ -389,7 +389,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-volume-encryption-logs
           path: /tmp/seaweedfs-sse-*.log

.github/workflows/s3-tables-tests.yml

@@ -66,7 +66,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: s3-tables-test-logs
           path: test/s3tables/table-buckets/test-output.log
@@ -122,7 +122,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: iceberg-catalog-test-logs
           path: test/s3tables/catalog/test-output.log
@@ -188,12 +188,73 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: trino-iceberg-catalog-test-logs
           path: test/s3tables/catalog_trino/test-output.log
           retention-days: 3
 
+  polaris-integration-tests:
+    name: Polaris Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Run go mod tidy
+        run: go mod tidy
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false ./weed
+
+      - name: Pre-pull Polaris image
+        run: docker pull apache/polaris:latest
+
+      - name: Run Polaris Integration Tests
+        timeout-minutes: 25
+        run: |
+          set -x
+          set -o pipefail
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          df -h
+          echo "=== Starting Polaris Tests ==="
+
+          go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
+            echo "Polaris integration tests failed"
+            exit 1
+          }
+
+      - name: Show test output on failure
+        if: failure()
+        working-directory: test/s3tables/polaris
+        run: |
+          echo "=== Test Output ==="
+          if [ -f test-output.log ]; then
+            tail -200 test-output.log
+          fi
+
+          echo "=== Process information ==="
+          ps aux | grep -E "(weed|test|docker)" || true
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: polaris-test-logs
+          path: test/s3tables/polaris/test-output.log
+          retention-days: 3
+
   spark-iceberg-catalog-tests:
     name: Spark Iceberg Catalog Integration Tests
     runs-on: ubuntu-22.04
@@ -254,7 +315,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: spark-iceberg-catalog-test-logs
           path: test/s3tables/catalog_spark/test-output.log
@@ -322,7 +383,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: risingwave-catalog-test-logs
           path: test/s3tables/catalog_risingwave/test-output.log
@@ -388,7 +449,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: sts-integration-test-logs
           path: test/s3tables/sts_integration/test-output.log
@@ -457,7 +518,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: lakekeeper-integration-test-logs
           path: test/s3tables/lakekeeper/test-output.log

.github/workflows/spark-integration-tests.yml

@@ -125,7 +125,7 @@ jobs:
 
     - name: Upload test results
       if: always()
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       with:
         name: spark-test-results
         path: test/java/spark/target/surefire-reports/

.github/workflows/test-s3-over-https-using-awscli.yml

@@ -103,7 +103,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: seaweedfs-logs
           # Note: actions don't use defaults.run.working-directory, so path is relative to workspace root

.github/workflows/tus-tests.yml

@@ -106,7 +106,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: tus-test-logs
           path: |

.github/workflows/volume-server-integration-tests.yml

@@ -90,7 +90,7 @@ jobs:
 
       - name: Archive logs on failure
         if: failure()
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: volume-server-integration-test-logs
           path: /tmp/volume-server-it-logs/

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
-	github.com/go-redsync/redsync/v4 v4.15.0
+	github.com/go-redsync/redsync/v4 v4.16.0
 	github.com/go-sql-driver/mysql v1.9.3
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/golang/protobuf v1.5.4
@@ -89,20 +89,20 @@ require (
 	go.etcd.io/etcd/client/v3 v3.6.7
 	go.mongodb.org/mongo-driver v1.17.6
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.44.0
-	gocloud.dev/pubsub/natspubsub v0.44.0
+	gocloud.dev v0.45.0
+	gocloud.dev/pubsub/natspubsub v0.45.0
 	gocloud.dev/pubsub/rabbitpubsub v0.44.0
 	golang.org/x/crypto v0.48.0
 	golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
 	golang.org/x/image v0.36.0
 	golang.org/x/net v0.49.0
-	golang.org/x/oauth2 v0.34.0
+	golang.org/x/oauth2 v0.35.0
 	golang.org/x/sys v0.41.0
 	golang.org/x/text v0.34.0 // indirect
 	golang.org/x/tools v0.41.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.258.0
-	google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect
+	google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 // indirect
 	google.golang.org/grpc v1.78.0
 	google.golang.org/protobuf v1.36.11
 	gopkg.in/inf.v0 v0.9.1 // indirect
@@ -138,8 +138,7 @@ require (
 	github.com/hashicorp/raft-boltdb/v2 v2.3.1
 	github.com/hashicorp/vault/api v1.22.0
 	github.com/jhump/protoreflect v1.18.0
-	github.com/lib/pq v1.11.1
-	github.com/linkedin/goavro/v2 v2.14.1
+	github.com/linkedin/goavro/v2 v2.15.0
 	github.com/mattn/go-sqlite3 v1.14.34
 	github.com/minio/crc64nvme v1.1.1
 	github.com/orcaman/concurrent-map/v2 v2.0.1
@@ -151,7 +150,7 @@ require (
 	github.com/redis/go-redis/v9 v9.18.0
 	github.com/schollz/progressbar/v3 v3.19.0
 	github.com/seaweedfs/go-fuse/v2 v2.9.1
-	github.com/shirou/gopsutil/v4 v4.26.1
+	github.com/shirou/gopsutil/v4 v4.26.2
 	github.com/tarantool/go-tarantool/v2 v2.4.1
 	github.com/testcontainers/testcontainers-go v0.39.0
 	github.com/tikv/client-go/v2 v2.0.7
@@ -172,7 +171,7 @@ require (
 	atomicgo.dev/keyboard v0.2.9 // indirect
 	atomicgo.dev/schedule v0.1.0 // indirect
 	cloud.google.com/go/longrunning v0.7.0 // indirect
-	cloud.google.com/go/pubsub/v2 v2.2.1 // indirect
+	cloud.google.com/go/pubsub/v2 v2.3.0 // indirect
 	dario.cat/mergo v1.0.2 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
@@ -237,6 +236,7 @@ require (
 	github.com/klauspost/asmfmt v1.3.2 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
+	github.com/lib/pq v1.11.1 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
 	github.com/magiconair/properties v1.8.10 // indirect
@@ -279,10 +279,10 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
 	go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.7.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.uber.org/mock v0.5.2 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
@@ -292,12 +292,12 @@ require (
 )
 
 require (
-	cel.dev/expr v0.24.0 // indirect
+	cel.dev/expr v0.25.1 // indirect
 	cloud.google.com/go/auth v0.17.0 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	cloud.google.com/go/iam v1.5.3 // indirect
-	cloud.google.com/go/monitoring v1.24.2 // indirect
+	cloud.google.com/go/monitoring v1.24.3 // indirect
 	filippo.io/edwards25519 v1.1.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
@@ -327,7 +327,7 @@ require (
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
@@ -336,11 +336,11 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.6
 	github.com/aws/smithy-go v1.24.0
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
@@ -352,7 +352,7 @@ require (
 	github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
 	github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
 	github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
-	github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
+	github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e // indirect
 	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
 	github.com/creasty/defaults v1.8.0 // indirect
 	github.com/cronokirby/saferith v0.33.0 // indirect
@@ -360,11 +360,11 @@ require (
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
-	github.com/ebitengine/purego v0.9.1 // indirect
+	github.com/ebitengine/purego v0.10.0 // indirect
 	github.com/elastic/gosigar v0.14.3 // indirect
 	github.com/emersion/go-message v0.18.2 // indirect
 	github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
-	github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
+	github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/fatih/color v1.18.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -428,8 +428,8 @@ require (
 	github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
 	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/nats-io/nats.go v1.43.0 // indirect
-	github.com/nats-io/nkeys v0.4.11 // indirect
+	github.com/nats-io/nats.go v1.48.0 // indirect
+	github.com/nats-io/nkeys v0.4.12 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/ncruces/go-strftime v1.0.0 // indirect
 	github.com/ncw/swift/v2 v2.0.5 // indirect
@@ -493,11 +493,11 @@ require (
 	go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
-	go.opentelemetry.io/otel v1.38.0 // indirect
-	go.opentelemetry.io/otel/metric v1.38.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.38.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
-	go.opentelemetry.io/otel/trace v1.38.0 // indirect
+	go.opentelemetry.io/otel v1.40.0 // indirect
+	go.opentelemetry.io/otel/metric v1.40.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.40.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
+	go.opentelemetry.io/otel/trace v1.40.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
 	golang.org/x/term v0.40.0 // indirect

go.sum

@@ -6,8 +6,8 @@ atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8=
 atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ=
 atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs=
 atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU=
-cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
-cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
+cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
+cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -310,8 +310,8 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6
 cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
 cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
 cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
-cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc=
-cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA=
+cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
+cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
 cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
 cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
 cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
@@ -338,8 +338,8 @@ cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJP
 cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk=
 cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4=
 cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w=
-cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM=
-cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U=
+cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE=
+cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI=
 cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA=
 cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o=
 cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM=
@@ -393,8 +393,8 @@ cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhz
 cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
 cloud.google.com/go/pubsub v1.50.1 h1:fzbXpPyJnSGvWXF1jabhQeXyxdbCIkXTpjXHy7xviBM=
 cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk=
-cloud.google.com/go/pubsub/v2 v2.2.1 h1:3brZcshL3fIiD1qOxAE2QW9wxsfjioy014x4yC9XuYI=
-cloud.google.com/go/pubsub/v2 v2.2.1/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
+cloud.google.com/go/pubsub/v2 v2.3.0 h1:DgAN907x+sP0nScYfBzneRiIhWoXcpCD8ZAut8WX9vs=
+cloud.google.com/go/pubsub/v2 v2.3.0/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
 cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg=
 cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k=
 cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
@@ -504,8 +504,8 @@ cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV6
 cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28=
 cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y=
 cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA=
-cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4=
-cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI=
+cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U=
+cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
 cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs=
 cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg=
 cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos=
@@ -687,6 +687,8 @@ github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUS
 github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
 github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op h1:Ucf+QxEKMbPogRO5guBNe5cgd9uZgfoJLOYs8WWhtjM=
+github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
 github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
 github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
 github.com/apache/arrow-go/v18 v18.4.1 h1:q/jVkBWCJOB9reDgaIZIdruLQUb1kbkvOnOFezVH1C4=
@@ -724,8 +726,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUT
 github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 h1:2fjfz3/G9BRvIKuNZ655GwzpklC2kEH0cowZQGO7uBg=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4/go.mod h1:Ymws824lvMypLFPwyyUXM52SXuGgxpu0+DISLfKvB+c=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
@@ -746,10 +748,10 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZi
 github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
-github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE=
-github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
-github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
-github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
+github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
+github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
+github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
+github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
 github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
 github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds=
@@ -855,8 +857,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
-github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
+github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e h1:gt7U1Igw0xbJdyaCM5H2CnlAlPSkzrhsebQB6WQWjLA=
+github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI=
 github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=
 github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc=
 github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
@@ -972,8 +974,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A
 github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
-github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
+github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 h1:XBBHcIb256gUJtLmY22n99HaZTz+r2Z51xUPi01m3wg=
 github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203/go.mod h1:E1jcSv8FaEny+OP/5k9UxZVw9YFWGj7eI4KR/iOBqCg=
 github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
@@ -1000,8 +1002,8 @@ github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJ
 github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI=
 github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
 github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
-github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
-github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
+github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g=
+github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98=
 github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
 github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
@@ -1124,8 +1126,8 @@ github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOr
 github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
-github.com/go-redsync/redsync/v4 v4.15.0 h1:KH/XymuxSV7vyKs6z1Cxxj+N+N18JlPxgXeP6x4JY54=
-github.com/go-redsync/redsync/v4 v4.15.0/go.mod h1:qNp+lLs3vkfZbtA/aM/OjlZHfEr5YTAYhRktFPKHC7s=
+github.com/go-redsync/redsync/v4 v4.16.0 h1:bNcOzeHH9d3s6pghU9NJFMPrQa41f5Nx3L4YKr3BdEU=
+github.com/go-redsync/redsync/v4 v4.16.0/go.mod h1:V4gagqgyASWBZuwx4xGzu72aZNb/6Mo05byUa3mVmKQ=
 github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
 github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
 github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
@@ -1237,6 +1239,8 @@ github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV
 github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI=
 github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk=
 github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
+github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -1514,8 +1518,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI=
 github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
-github.com/linkedin/goavro/v2 v2.14.1 h1:/8VjDpd38PRsy02JS0jflAu7JZPfJcGTwqWgMkFS2iI=
-github.com/linkedin/goavro/v2 v2.14.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
+github.com/linkedin/goavro/v2 v2.15.0 h1:pDj1UrjUOO62iXhgBiE7jQkpNIc5/tA5eZsgolMjgVI=
+github.com/linkedin/goavro/v2 v2.15.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
 github.com/linxGnu/grocksdb v1.10.7 h1:fCi4qvZWo04VgFwGWmO8HQJgUVounJBy+C2TMVPU/ho=
 github.com/linxGnu/grocksdb v1.10.7/go.mod h1:OLQKZwiKwaJiAVCsOzWKvwiLwfZ5Vz8Md5TYR7t7pM8=
 github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=
@@ -1563,8 +1567,8 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
 github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
 github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
 github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
-github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
-github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
+github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk=
+github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -1622,14 +1626,14 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak=
-github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI=
-github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU=
-github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0=
-github.com/nats-io/nats.go v1.43.0 h1:uRFZ2FEoRvP64+UUhaTokyS18XBCR/xM2vQZKO4i8ug=
-github.com/nats-io/nats.go v1.43.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
-github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
-github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
+github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g=
+github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
+github.com/nats-io/nats-server/v2 v2.11.12 h1:jGDXTkcjqQ5fCRstwIxvv1K0RHfftFUoSCT/iIZcqOc=
+github.com/nats-io/nats-server/v2 v2.11.12/go.mod h1:5MCp/pqm5SEfsvVZ31ll1088ZTwEUdvRX1Hmh/mTTDg=
+github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U=
+github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
+github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc=
+github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
@@ -1800,10 +1804,10 @@ github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6ha
 github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
 github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
 github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
-github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
-github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
-github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
-github.com/redis/rueidis/rueidiscompat v1.0.69/go.mod h1:iC4Y8DoN0Uth0Uezg9e2trvNRC7QAgGeuP2OPLb5ccI=
+github.com/redis/rueidis v1.0.71 h1:pODtnAR5GAB7j4ekhldZ29HKOxe4Hph0GTDGk1ayEQY=
+github.com/redis/rueidis v1.0.71/go.mod h1:lfdcZzJ1oKGKL37vh9fO3ymwt+0TdjkkUCJxbgpmcgQ=
+github.com/redis/rueidis/rueidiscompat v1.0.71 h1:wNZ//kEjMZgBM0KCk7ncOX8KmAgROU2kDdDNpwheG4w=
+github.com/redis/rueidis/rueidiscompat v1.0.71/go.mod h1:esmCLJvaRzZoKlgB82G1bY7Iky5TnO9Rz+NlhbEccFI=
 github.com/rekby/fixenv v0.3.2/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
 github.com/rekby/fixenv v0.6.1 h1:jUFiSPpajT4WY2cYuc++7Y1zWrnCxnovGCIX72PZniM=
 github.com/rekby/fixenv v0.6.1/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
@@ -1857,8 +1861,8 @@ github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQ
 github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
 github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=
 github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=
-github.com/shirou/gopsutil/v4 v4.26.1 h1:TOkEyriIXk2HX9d4isZJtbjXbEjf5qyKPAzbzY0JWSo=
-github.com/shirou/gopsutil/v4 v4.26.1/go.mod h1:medLI9/UNAb0dOI9Q3/7yWSqKkj00u+1tgY8nvv41pc=
+github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI=
+github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
@@ -2130,35 +2134,35 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
-go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
-go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0 h1:zG8GlgXCJQd5BU98C0hZnBbElszTmUgCNCfYneaDL0A=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0/go.mod h1:hOfBCz8kv/wuq73Mx2H2QnWokh/kHZxkh6SNF2bdKtw=
+go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
+go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0 h1:ZsXq73BERAiNuuFXYqP4MR5hBrjXfMGSO+Cx7qoOZiM=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0/go.mod h1:hg1zaDMpyZJuUzjFxFsRYBoccE86tM9Uf4IqNMUxvrY=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2TYsQw2r1IASwoROaCnjdj2cvC2+Jbxvk6nHnWU=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83wzcKzGGqAnnfLFyFe7RslekZuv+VI=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w=
 go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
 go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
-go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
-go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
-go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
-go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
-go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
-go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
-go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
-go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
+go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
+go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
+go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
+go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
+go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
+go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
+go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
+go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
-go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
-go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
+go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
+go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@@ -2182,10 +2186,10 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
 go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-gocloud.dev v0.44.0 h1:iVyMAqFl2r6xUy7M4mfqwlN+21UpJoEtgHEcfiLMUXs=
-gocloud.dev v0.44.0/go.mod h1:ZmjROXGdC/eKZLF1N+RujDlFRx3D+4Av2thREKDMVxY=
-gocloud.dev/pubsub/natspubsub v0.44.0 h1:1Us76ckkdgtiE1p1rJZ+38b9TQP051bmjAiQlFQzYrM=
-gocloud.dev/pubsub/natspubsub v0.44.0/go.mod h1:PvVAGIhL14PWGwWIXX/zAK42ixr2/PKP4Q4yMiAUraQ=
+gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY=
+gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI=
+gocloud.dev/pubsub/natspubsub v0.45.0 h1:kfCupVejeynIQcS1GaIvkY3Nj/acLlM5yPWyxZN8wJ8=
+gocloud.dev/pubsub/natspubsub v0.45.0/go.mod h1:WU5SMDWuF7CCr2U8UpbrEONYYFOmvigCAvTpOOpUvYE=
 gocloud.dev/pubsub/rabbitpubsub v0.44.0 h1:MpRIO6XJ/JTqrlUWt3CxwDe1LvaiXUVu4sS5cv4f/AM=
 gocloud.dev/pubsub/rabbitpubsub v0.44.0/go.mod h1:BB9+qT3r6g4M5+4asiXaEeqw4QAOzsWusO5krYaqkdA=
 golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
@@ -2376,8 +2380,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
 golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
 golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
-golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
-golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
+golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -2840,8 +2844,8 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
 google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
 google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
 google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
-google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc=
-google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc=
+google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 h1:dDbsTLIK7EzwUq36kCSAsk0slouq/S0tWHeeGi97cD8=
+google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846/go.mod h1:PP0g88Dz3C7hRAfbQCQggeWAXjuqGsNPLE4s7jh0RGU=
 google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846 h1:ZdyUkS9po3H7G0tuh955QVyyotWvOD4W0aEapeGeUYk=
 google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846/go.mod h1:Fk4kyraUvqD7i5H6S43sj2W98fbZa75lpZz/eUyhfO0=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=

k8s/charts/seaweedfs/openshift-values.yaml

@@ -100,12 +100,19 @@ filer:
 
 # S3 gateway (if enabled)
 s3:
+  enabled: true
+  replicas: 1
+  port: 8333
+  enableAuth: true
   podSecurityContext:
     enabled: true
     # On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
     # controller assign them automatically based on the namespace's SCC.
     runAsNonRoot: true
 
+  logs:
+    type: "emptyDir"
+
   containerSecurityContext:
     enabled: true
     allowPrivilegeEscalation: false

k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml

@@ -118,8 +118,10 @@ spec:
                   fieldPath: metadata.namespace
             - name: SEAWEEDFS_FULLNAME
               value: "{{ include "seaweedfs.fullname" . }}"
-            {{- if .Values.admin.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.admin.extraEnvironmentVars }}
+            {{- $mergedExtraEnvironmentVars := dict }}
+            {{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.admin "target" $mergedExtraEnvironmentVars) }}
+            {{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
+            {{- $value := index $mergedExtraEnvironmentVars $key }}
             - name: {{ $key }}
             {{- if kindIs "string" $value }}
               value: {{ tpl $value $ | quote }}
@@ -128,18 +130,6 @@ spec:
                 {{ toYaml $value | nindent 16 | trim }}
             {{- end -}}
             {{- end }}
-            {{- end }}
-            {{- if .Values.global.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
-            - name: {{ $key }}
-            {{- if kindIs "string" $value }}
-              value: {{ tpl $value $ | quote }}
-            {{- else }}
-              valueFrom:
-                {{ toYaml $value | nindent 16 | trim }}
-            {{- end -}}
-            {{- end }}
-            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"

k8s/charts/seaweedfs/templates/cosi/cosi-deployment.yaml

@@ -96,8 +96,10 @@ spec:
             - name: WEED_GRPC_CA
               value: /usr/local/share/ca-certificates/client/ca.crt
             {{- end }}
-            {{- if .Values.cosi.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.cosi.extraEnvironmentVars }}
+            {{- $mergedExtraEnvironmentVars := dict }}
+            {{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.cosi "target" $mergedExtraEnvironmentVars) }}
+            {{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
+            {{- $value := index $mergedExtraEnvironmentVars $key }}
             - name: {{ $key }}
             {{- if kindIs "string" $value }}
               value: {{ tpl $value $ | quote }}
@@ -106,18 +108,6 @@ spec:
                 {{ toYaml $value | nindent 16 | trim }}
             {{- end -}}
             {{- end }}
-            {{- end }}
-            {{- if .Values.global.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
-            - name: {{ $key }}
-            {{- if kindIs "string" $value }}
-              value: {{ tpl $value $ | quote }}
-            {{- else }}
-              valueFrom:
-                {{ toYaml $value | nindent 16 | trim }}
-            {{- end -}}
-            {{- end }}
-            {{- end }}
           volumeMounts:
             - mountPath: /var/lib/cosi
               name: socket

k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml

@@ -114,8 +114,10 @@ spec:
                   optional: true
             - name: SEAWEEDFS_FULLNAME
               value: "{{ include "seaweedfs.fullname" . }}"
-            {{- if .Values.filer.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.filer.extraEnvironmentVars }}
+            {{- $mergedExtraEnvironmentVars := dict }}
+            {{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.filer "target" $mergedExtraEnvironmentVars) }}
+            {{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
+            {{- $value := index $mergedExtraEnvironmentVars $key }}
             - name: {{ $key }}
             {{- if kindIs "string" $value }}
               value: {{ tpl $value $ | quote }}
@@ -124,18 +126,6 @@ spec:
                 {{ toYaml $value | nindent 16 | trim }}
             {{- end -}}
             {{- end }}
-            {{- end }}
-            {{- if .Values.global.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
-            - name: {{ $key }}
-            {{- if kindIs "string" $value }}
-              value: {{ tpl $value $ | quote }}
-            {{- else }}
-              valueFrom:
-                {{ toYaml $value | nindent 16 | trim }}
-            {{- end -}}
-            {{- end }}
-            {{- end }}
             {{- if .Values.filer.secretExtraEnvironmentVars }}
             {{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
             - name: {{ $key }}

k8s/charts/seaweedfs/templates/master/master-statefulset.yaml

@@ -69,9 +69,7 @@ spec:
       priorityClassName: {{ .Values.master.priorityClassName | quote }}
       {{- end }}
       enableServiceLinks: false
-      {{- if .Values.global.createClusterRole }}
       serviceAccountName: {{ .Values.master.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration
-      {{- end }}
       {{- if .Values.master.initContainers }}
       initContainers:
         {{ tpl .Values.master.initContainers . | nindent 8 | trim }}
@@ -98,8 +96,10 @@ spec:
                   fieldPath: metadata.namespace
             - name: SEAWEEDFS_FULLNAME
               value: "{{ include "seaweedfs.fullname" . }}"
-            {{- if .Values.master.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.master.extraEnvironmentVars }}
+            {{- $mergedExtraEnvironmentVars := dict }}
+            {{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.master "target" $mergedExtraEnvironmentVars) }}
+            {{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
+            {{- $value := index $mergedExtraEnvironmentVars $key }}
             - name: {{ $key }}
             {{- if kindIs "string" $value }}
               value: {{ tpl $value $ | quote }}
@@ -108,18 +108,6 @@ spec:
                 {{ toYaml $value | nindent 16 | trim }}
             {{- end -}}
             {{- end }}
-            {{- end }}
-            {{- if .Values.global.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
-            - name: {{ $key }}
-            {{- if kindIs "string" $value }}
-              value: {{ tpl $value $ | quote }}
-            {{- else }}
-              valueFrom:
-                {{ toYaml $value | nindent 16 | trim }}
-            {{- end -}}
-            {{- end }}
-            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"

k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml

@@ -90,8 +90,10 @@ spec:
                   fieldPath: metadata.namespace
             - name: SEAWEEDFS_FULLNAME
               value: "{{ include "seaweedfs.fullname" . }}"
-            {{- if .Values.s3.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.s3.extraEnvironmentVars }}
+            {{- $mergedExtraEnvironmentVars := dict }}
+            {{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.s3 "target" $mergedExtraEnvironmentVars) }}
+            {{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
+            {{- $value := index $mergedExtraEnvironmentVars $key }}
             - name: {{ $key }}
             {{- if kindIs "string" $value }}
               value: {{ tpl $value $ | quote }}
@@ -100,18 +102,6 @@ spec:
                 {{ toYaml $value | nindent 16 | trim }}
             {{- end -}}
             {{- end }}
-            {{- end }}
-            {{- if .Values.global.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
-            - name: {{ $key }}
-            {{- if kindIs "string" $value }}
-              value: {{ tpl $value $ | quote }}
-            {{- else }}
-              valueFrom:
-                {{ toYaml $value | nindent 16 | trim }}
-            {{- end -}}
-            {{- end }}
-            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"

k8s/charts/seaweedfs/templates/sftp/sftp-deployment.yaml

@@ -90,8 +90,10 @@ spec:
                   fieldPath: metadata.namespace
             - name: SEAWEEDFS_FULLNAME
               value: "{{ include "seaweedfs.fullname" . }}"
-            {{- if .Values.sftp.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.sftp.extraEnvironmentVars }}
+            {{- $mergedExtraEnvironmentVars := dict }}
+            {{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.sftp "target" $mergedExtraEnvironmentVars) }}
+            {{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
+            {{- $value := index $mergedExtraEnvironmentVars $key }}
             - name: {{ $key }}
             {{- if kindIs "string" $value }}
               value: {{ tpl $value $ | quote }}
@@ -100,18 +102,6 @@ spec:
                 {{ toYaml $value | nindent 16 | trim }}
             {{- end -}}
             {{- end }}
-            {{- end }}
-            {{- if .Values.global.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
-            - name: {{ $key }}
-            {{- if kindIs "string" $value }}
-              value: {{ tpl $value $ | quote }}
-            {{- else }}
-              valueFrom:
-                {{ toYaml $value | nindent 16 | trim }}
-            {{- end -}}
-            {{- end }}
-            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"

k8s/charts/seaweedfs/templates/shared/_helpers.tpl

@@ -59,6 +59,18 @@ Inject extra environment vars in the format key:value, if populated
 {{- end -}}
 {{- end -}}
 
+{{- define "seaweedfs.mergeExtraEnvironmentVars" -}}
+{{- $global := ((.global | default dict).extraEnvironmentVars | default dict) -}}
+{{- $component := ((.component | default dict).extraEnvironmentVars | default dict) -}}
+{{- $target := .target -}}
+{{- range $key, $value := $global }}
+{{- $_ := set $target $key $value }}
+{{- end }}
+{{- range $key, $value := $component }}
+{{- $_ := set $target $key $value }}
+{{- end }}
+{{- end -}}
+
 {{/* Return the proper filer image */}}
 {{- define "filer.image" -}}
 {{- if .Values.filer.imageOverride -}}

k8s/charts/seaweedfs/templates/volume/volume-statefulset.yaml

@@ -69,9 +69,7 @@ spec:
       priorityClassName: {{ $volume.priorityClassName | quote }}
       {{- end }}
       enableServiceLinks: false
-      {{- if $.Values.global.createClusterRole }}
       serviceAccountName: {{ $volume.serviceAccountName | default (include "seaweedfs.serviceAccountName" $) | quote }} # for deleting statefulset pods after migration
-      {{- end }}
       {{- $initContainers_exists := include "volume.initContainers_exists" $ -}}
       {{- if $initContainers_exists }}
       initContainers:
@@ -118,8 +116,10 @@ spec:
                   fieldPath: status.hostIP
             - name: SEAWEEDFS_FULLNAME
               value: "{{ include "seaweedfs.fullname" $ }}"
-            {{- if $volume.extraEnvironmentVars }}
-            {{- range $key, $value := $volume.extraEnvironmentVars }}
+            {{- $mergedExtraEnvironmentVars := dict }}
+            {{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" $.Values.global "component" $volume "target" $mergedExtraEnvironmentVars) }}
+            {{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
+            {{- $value := index $mergedExtraEnvironmentVars $key }}
             - name: {{ $key }}
             {{- if kindIs "string" $value }}
               value: {{ tpl $value $ | quote }}
@@ -128,18 +128,6 @@ spec:
                 {{ toYaml $value | nindent 16 | trim }}
             {{- end -}}
             {{- end }}
-            {{- end }}
-            {{- if $.Values.global.extraEnvironmentVars }}
-            {{- range $key, $value := $.Values.global.extraEnvironmentVars }}
-            - name: {{ $key }}
-            {{- if kindIs "string" $value }}
-              value: {{ tpl $value $ | quote }}
-            {{- else }}
-              valueFrom:
-                {{ toYaml $value | nindent 16 | trim }}
-            {{- end -}}
-            {{- end }}
-            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"

k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml

@@ -93,8 +93,10 @@ spec:
                   fieldPath: metadata.namespace
             - name: SEAWEEDFS_FULLNAME
               value: "{{ include "seaweedfs.fullname" . }}"
-            {{- if .Values.worker.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.worker.extraEnvironmentVars }}
+            {{- $mergedExtraEnvironmentVars := dict }}
+            {{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.worker "target" $mergedExtraEnvironmentVars) }}
+            {{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
+            {{- $value := index $mergedExtraEnvironmentVars $key }}
             - name: {{ $key }}
             {{- if kindIs "string" $value }}
               value: {{ tpl $value $ | quote }}
@@ -103,18 +105,6 @@ spec:
                 {{ toYaml $value | nindent 16 | trim }}
             {{- end -}}
             {{- end }}
-            {{- end }}
-            {{- if .Values.global.extraEnvironmentVars }}
-            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
-            - name: {{ $key }}
-            {{- if kindIs "string" $value }}
-              value: {{ tpl $value $ | quote }}
-            {{- else }}
-              valueFrom:
-                {{ toYaml $value | nindent 16 | trim }}
-            {{- end -}}
-            {{- end }}
-            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"

test/erasure_coding/admin_dockertest/ec_integration_test.go

@@ -135,7 +135,7 @@ func ensureEnvironment(t *testing.T) {
 	// 6. Start Admin
 	os.RemoveAll(filepath.Join("tmp", "admin"))
 	os.MkdirAll(filepath.Join("tmp", "admin"), 0755)
-	startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin")
+	startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin", "-scheduler.idleSleep=2")
 	waitForUrl(t, AdminUrl+"/health", 60)
 
 	t.Log("Environment started successfully")

test/java/spark/pom.xml

@@ -21,7 +21,7 @@
         <scala.binary.version>2.12</scala.binary.version>
         <junit.version>4.13.2</junit.version>
         <seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
-        <jackson.version>2.18.2</jackson.version> <!-- Upgraded from 2.15.3 -->
+        <jackson.version>2.18.6</jackson.version> <!-- Upgraded from 2.15.3 -->
         <netty.version>4.1.129.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
         <parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
         <parquet.format.version>2.12.0</parquet.format.version>

test/s3tables/lakekeeper/lakekeeper_test.go

@@ -1,15 +1,32 @@
 package lakekeeper
 
 import (
+	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/aws-sdk-go-v2/service/sts"
+
 	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
 )
 
 type TestEnvironment struct {
@@ -31,15 +48,19 @@ type TestEnvironment struct {
 	secretKey      string
 }
 
+const (
+	lakekeeperRegion      = "us-east-1"
+	lakekeeperRoleArn     = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
+	lakekeeperSessionName = "lakekeeper-session"
+)
+
+type lakekeeperSession struct {
+	endpoint string
+	region   string
+	creds    aws.Credentials
+}
+
 func TestLakekeeperIntegration(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	if !testutil.HasDocker() {
-		t.Skip("Docker not available, skipping Lakekeeper integration test")
-	}
-
 	env := NewTestEnvironment(t)
 	defer env.Cleanup(t)
 
@@ -51,6 +72,18 @@ func TestLakekeeperIntegration(t *testing.T) {
 	runLakekeeperRepro(t, env)
 }
 
+func TestLakekeeperTableBucketIntegration(t *testing.T) {
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	fmt.Printf(">>> Starting SeaweedFS with Lakekeeper configuration...\n")
+	env.StartSeaweedFS(t)
+	fmt.Printf(">>> SeaweedFS started.\n")
+
+	// Run python script in docker to test STS and S3 Tables operations
+	runLakekeeperTableBucketRepro(t, env)
+}
+
 func NewTestEnvironment(t *testing.T) *TestEnvironment {
 	t.Helper()
 
@@ -218,124 +251,439 @@ func (env *TestEnvironment) Cleanup(t *testing.T) {
 func runLakekeeperRepro(t *testing.T, env *TestEnvironment) {
 	t.Helper()
 
-	scriptContent := fmt.Sprintf(`
-import boto3
-import botocore.config
-import botocore
-from botocore.exceptions import ClientError
-import os
-import sys
-import time
-import logging
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
 
-# Enable botocore debug logging to see signature calculation
-logging.basicConfig(level=logging.DEBUG)
-botocore.session.get_session().set_debug_logger()
-
-print("Starting Lakekeeper repro test...")
-
-endpoint_url = "http://host.docker.internal:%d"
-access_key = "%s"
-secret_key = "%s"
-region = "us-east-1"
-
-print(f"Connecting to {endpoint_url}")
-
-try:
-    config = botocore.config.Config(
-        retries={'max_attempts': 3}
-    )
-    sts = boto3.client(
-        'sts',
-        endpoint_url=endpoint_url,
-        aws_access_key_id=access_key,
-        aws_secret_access_key=secret_key,
-        region_name=region,
-        config=config
-    )
-
-    role_arn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
-    session_name = "lakekeeper-session"
-
-    print(f"Calling AssumeRole on {role_arn} with POST body...")
-    
-    # Standard boto3 call sends parameters in POST body
-    response = sts.assume_role(
-        RoleArn=role_arn,
-        RoleSessionName=session_name
-    )
-
-    creds = response['Credentials']
-    access_key_id = creds['AccessKeyId']
-    secret_access_key = creds['SecretAccessKey']
-    session_token = creds['SessionToken']
-
-    print(f"Success! Got credentials with prefix: {access_key_id[:4]}")
-    
-    if not access_key_id.startswith("ASIA"):
-        print(f"FAILED: Expected ASIA prefix, got {access_key_id}")
-        sys.exit(1)
-
-    print("Verifying S3 operations with vended credentials...")
-    s3 = boto3.client(
-        's3',
-        endpoint_url=endpoint_url,
-        aws_access_key_id=access_key_id,
-        aws_secret_access_key=secret_access_key,
-        aws_session_token=session_token,
-        region_name=region,
-        config=config
-    )
-
-    bucket = "lakekeeper-vended-bucket"
-    print(f"Creating bucket {bucket}...")
-    s3.create_bucket(Bucket=bucket)
-
-    print("Listing buckets...")
-    response = s3.list_buckets()
-    buckets = [b['Name'] for b in response['Buckets']]
-    print(f"Found buckets: {buckets}")
-
-    if bucket not in buckets:
-        print(f"FAILED: Bucket {bucket} not found in list")
-        sys.exit(1)
-
-    print("SUCCESS: Lakekeeper flow verified!")
-    sys.exit(0)
-    
-except Exception as e:
-    print(f"FAILED: {e}")
-    # Print more details if it is a ClientError
-    if hasattr(e, 'response'):
-        print(f"Response: {e.response}")
-    sys.exit(1)
-`, env.s3Port, env.accessKey, env.secretKey)
-
-	scriptPath := filepath.Join(env.dataDir, "lakekeeper_repro.py")
-	if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil {
-		t.Fatalf("Failed to write python script: %v", err)
-	}
-
-	containerName := "seaweed-lakekeeper-client-" + fmt.Sprintf("%d", time.Now().UnixNano())
-
-	// Create a context with timeout for the docker run command
-	dockerCtx, dockerCancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	defer dockerCancel()
-
-	cmd := exec.CommandContext(dockerCtx, "docker", "run", "--rm",
-		"--name", containerName,
-		"--add-host", "host.docker.internal:host-gateway",
-		"-v", fmt.Sprintf("%s:/work", env.dataDir),
-		"python:3",
-		"/bin/bash", "-c", "pip install boto3 && python /work/lakekeeper_repro.py",
-	)
-
-	output, err := cmd.CombinedOutput()
+	session, err := newLakekeeperSession(ctx, env)
 	if err != nil {
-		if dockerCtx.Err() == context.DeadlineExceeded {
-			t.Fatalf("Lakekeeper repro client timed out after 5 minutes\nOutput:\n%s", string(output))
-		}
-		t.Fatalf("Lakekeeper repro client failed: %v\nOutput:\n%s", err, string(output))
+		t.Fatalf("AssumeRole failed: %v", err)
 	}
-	t.Logf("Lakekeeper repro client output:\n%s", string(output))
+
+	s3Client, err := newS3Client(ctx, session.endpoint, session.region, session.creds)
+	if err != nil {
+		t.Fatalf("Create S3 client failed: %v", err)
+	}
+
+	bucketName := fmt.Sprintf("lakekeeper-vended-bucket-%d", time.Now().UnixNano())
+	if _, err := s3Client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	}); err != nil {
+		t.Fatalf("CreateBucket failed: %v", err)
+	}
+
+	bucketCreated := true
+	defer func() {
+		if !bucketCreated {
+			return
+		}
+		_, _ = s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+			Bucket: aws.String(bucketName),
+		})
+	}()
+
+	listResp, err := s3Client.ListBuckets(ctx, &s3.ListBucketsInput{})
+	if err != nil {
+		t.Fatalf("ListBuckets failed: %v", err)
+	}
+
+	found := false
+	for _, bucket := range listResp.Buckets {
+		if aws.ToString(bucket.Name) == bucketName {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatalf("Bucket %s not found in list", bucketName)
+	}
+
+	if _, err := s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	}); err != nil {
+		t.Fatalf("DeleteBucket failed: %v", err)
+	}
+	bucketCreated = false
+}
+
+func runLakekeeperTableBucketRepro(t *testing.T, env *TestEnvironment) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	session, err := newLakekeeperSession(ctx, env)
+	if err != nil {
+		t.Fatalf("AssumeRole failed: %v", err)
+	}
+
+	client := newS3TablesClient(session.endpoint, session.region, session.creds)
+	bucketName := fmt.Sprintf("lakekeeper-table-bucket-%d", time.Now().UnixNano())
+	bucketARN, err := client.CreateTableBucket(ctx, bucketName)
+	if err != nil {
+		t.Fatalf("CreateTableBucket failed: %v", err)
+	}
+
+	bucketDeleted := false
+	namespaceCreated := false
+	tableCreated := false
+	namespaceName := fmt.Sprintf("lakekeeper_ns_%d", time.Now().UnixNano())
+	tableName := fmt.Sprintf("lakekeeper_table_%d", time.Now().UnixNano())
+	defer func() {
+		if tableCreated {
+			if err := client.DeleteTable(ctx, bucketARN, namespaceName, tableName); err != nil {
+				t.Logf("Failed to delete table: %v", err)
+			}
+		}
+		if namespaceCreated {
+			if err := client.DeleteNamespace(ctx, bucketARN, namespaceName); err != nil {
+				t.Logf("Failed to delete namespace: %v", err)
+			}
+		}
+		if bucketDeleted {
+			return
+		}
+		if err := client.DeleteTableBucket(ctx, bucketARN); err != nil {
+			t.Logf("Failed to delete table bucket: %v", err)
+		}
+	}()
+
+	buckets, err := client.ListTableBuckets(ctx)
+	if err != nil {
+		t.Fatalf("ListTableBuckets failed: %v", err)
+	}
+	found := false
+	for _, b := range buckets {
+		if b.Name == bucketName {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatalf("Created table bucket %s not found in list", bucketName)
+	}
+
+	if _, err := client.GetTableBucket(ctx, bucketARN); err != nil {
+		t.Fatalf("GetTableBucket failed: %v", err)
+	}
+
+	if err := client.CreateNamespace(ctx, bucketARN, namespaceName); err != nil {
+		t.Fatalf("CreateNamespace failed: %v", err)
+	}
+	namespaceCreated = true
+
+	if err := client.CreateTable(ctx, bucketARN, namespaceName, tableName); err != nil {
+		t.Fatalf("CreateTable failed: %v", err)
+	}
+	tableCreated = true
+
+	s3Client, err := newS3Client(ctx, session.endpoint, session.region, session.creds)
+	if err != nil {
+		t.Fatalf("Create S3 client failed: %v", err)
+	}
+
+	objectKey := fmt.Sprintf("%s/%s/data/part-%d.parquet", namespaceName, tableName, time.Now().UnixNano())
+	createResp, err := s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	if err != nil {
+		t.Fatalf("CreateMultipartUpload failed: %v", err)
+	}
+	uploadID := aws.ToString(createResp.UploadId)
+	multipartCompleted := false
+	defer func() {
+		if uploadID == "" || multipartCompleted {
+			return
+		}
+		_, _ = s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
+			Bucket:   aws.String(bucketName),
+			Key:      aws.String(objectKey),
+			UploadId: aws.String(uploadID),
+		})
+	}()
+
+	partSize := 5 * 1024 * 1024
+	part1 := bytes.Repeat([]byte("a"), partSize)
+	part2 := bytes.Repeat([]byte("b"), 1024*1024)
+
+	part1Resp, err := s3Client.UploadPart(ctx, &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   aws.String(uploadID),
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(part1),
+	})
+	if err != nil {
+		t.Fatalf("UploadPart 1 failed: %v", err)
+	}
+
+	part2Resp, err := s3Client.UploadPart(ctx, &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   aws.String(uploadID),
+		PartNumber: aws.Int32(2),
+		Body:       bytes.NewReader(part2),
+	})
+	if err != nil {
+		t.Fatalf("UploadPart 2 failed: %v", err)
+	}
+
+	_, err = s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: aws.String(uploadID),
+		MultipartUpload: &s3types.CompletedMultipartUpload{
+			Parts: []s3types.CompletedPart{
+				{
+					ETag:       part1Resp.ETag,
+					PartNumber: aws.Int32(1),
+				},
+				{
+					ETag:       part2Resp.ETag,
+					PartNumber: aws.Int32(2),
+				},
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("CompleteMultipartUpload failed: %v", err)
+	}
+	multipartCompleted = true
+
+	headResp, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	if err != nil {
+		t.Fatalf("HeadObject after multipart upload failed: %v", err)
+	}
+	expectedSize := int64(len(part1) + len(part2))
+	if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
+		t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
+	}
+
+	if err := client.DeleteTable(ctx, bucketARN, namespaceName, tableName); err != nil {
+		t.Fatalf("DeleteTable failed: %v", err)
+	}
+	tableCreated = false
+
+	if err := client.DeleteNamespace(ctx, bucketARN, namespaceName); err != nil {
+		t.Fatalf("DeleteNamespace failed: %v", err)
+	}
+	namespaceCreated = false
+
+	if err := client.DeleteTableBucket(ctx, bucketARN); err != nil {
+		t.Fatalf("DeleteTableBucket failed: %v", err)
+	}
+	bucketDeleted = true
+
+	if _, err := client.GetTableBucket(ctx, bucketARN); err == nil {
+		t.Fatalf("expected GetTableBucket to fail after deletion")
+	}
+}
+
+type s3TablesClient struct {
+	endpoint   string
+	region     string
+	creds      aws.Credentials
+	httpClient *http.Client
+}
+
+func newS3TablesClient(endpoint, region string, creds aws.Credentials) *s3TablesClient {
+	return &s3TablesClient{
+		endpoint: endpoint,
+		region:   region,
+		creds:    creds,
+		httpClient: &http.Client{
+			Timeout: 30 * time.Second,
+		},
+	}
+}
+
+func (c *s3TablesClient) CreateTableBucket(ctx context.Context, name string) (string, error) {
+	req := &s3tables.CreateTableBucketRequest{Name: name}
+	var resp s3tables.CreateTableBucketResponse
+	if err := c.doRequest(ctx, "CreateTableBucket", http.MethodPut, "/buckets", req, &resp); err != nil {
+		return "", err
+	}
+	return resp.ARN, nil
+}
+
+func (c *s3TablesClient) GetTableBucket(ctx context.Context, arn string) (*s3tables.GetTableBucketResponse, error) {
+	path := "/buckets/" + url.PathEscape(arn)
+	var resp s3tables.GetTableBucketResponse
+	if err := c.doRequest(ctx, "GetTableBucket", http.MethodGet, path, nil, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (c *s3TablesClient) ListTableBuckets(ctx context.Context) ([]s3tables.TableBucketSummary, error) {
+	var resp s3tables.ListTableBucketsResponse
+	if err := c.doRequest(ctx, "ListTableBuckets", http.MethodGet, "/buckets", nil, &resp); err != nil {
+		return nil, err
+	}
+	return resp.TableBuckets, nil
+}
+
+func (c *s3TablesClient) DeleteTableBucket(ctx context.Context, arn string) error {
+	path := "/buckets/" + url.PathEscape(arn)
+	return c.doRequest(ctx, "DeleteTableBucket", http.MethodDelete, path, nil, nil)
+}
+
+func (c *s3TablesClient) CreateNamespace(ctx context.Context, bucketARN, namespace string) error {
+	req := &s3tables.CreateNamespaceRequest{
+		Namespace: []string{namespace},
+	}
+	path := "/namespaces/" + url.PathEscape(bucketARN)
+	return c.doRequest(ctx, "CreateNamespace", http.MethodPut, path, req, nil)
+}
+
+func (c *s3TablesClient) DeleteNamespace(ctx context.Context, bucketARN, namespace string) error {
+	path := "/namespaces/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace)
+	return c.doRequest(ctx, "DeleteNamespace", http.MethodDelete, path, nil, nil)
+}
+
+func (c *s3TablesClient) CreateTable(ctx context.Context, bucketARN, namespace, name string) error {
+	req := &s3tables.CreateTableRequest{
+		Name:   name,
+		Format: "ICEBERG",
+	}
+	path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace)
+	return c.doRequest(ctx, "CreateTable", http.MethodPut, path, req, nil)
+}
+
+func (c *s3TablesClient) DeleteTable(ctx context.Context, bucketARN, namespace, name string) error {
+	path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace) + "/" + url.PathEscape(name)
+	return c.doRequest(ctx, "DeleteTable", http.MethodDelete, path, nil, nil)
+}
+
+func (c *s3TablesClient) doRequest(ctx context.Context, operation, method, path string, body interface{}, out interface{}) error {
+	var bodyBytes []byte
+	if body != nil {
+		encoded, err := json.Marshal(body)
+		if err != nil {
+			return fmt.Errorf("%s: marshal request: %w", operation, err)
+		}
+		bodyBytes = encoded
+	}
+	req, err := http.NewRequestWithContext(ctx, method, c.endpoint+path, bytes.NewReader(bodyBytes))
+	if err != nil {
+		return fmt.Errorf("%s: create request: %w", operation, err)
+	}
+	if body != nil {
+		req.Header.Set("Content-Type", "application/x-amz-json-1.1")
+	}
+	req.Host = req.URL.Host
+	req.Header.Set("Host", req.URL.Host)
+
+	payloadHash := sha256.Sum256(bodyBytes)
+	if err := v4.NewSigner().SignHTTP(ctx, c.creds, req, hex.EncodeToString(payloadHash[:]), "s3tables", c.region, time.Now()); err != nil {
+		return fmt.Errorf("%s: sign request: %w", operation, err)
+	}
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("%s: request failed: %w", operation, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		bodyBytes, readErr := io.ReadAll(resp.Body)
+		if readErr != nil {
+			return fmt.Errorf("%s failed with status %d and could not read response body: %v", operation, resp.StatusCode, readErr)
+		}
+		var errResp s3tables.S3TablesError
+		if jsonErr := json.Unmarshal(bodyBytes, &errResp); jsonErr == nil && (errResp.Type != "" || errResp.Message != "") {
+			return fmt.Errorf("%s failed: %s - %s", operation, errResp.Type, errResp.Message)
+		}
+		return fmt.Errorf("%s failed with status %d: %s", operation, resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
+	}
+
+	if out == nil {
+		return nil
+	}
+	if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
+		return fmt.Errorf("%s: decode response: %w", operation, err)
+	}
+	return nil
+}
+
+func assumeRole(ctx context.Context, endpoint, region, accessKey, secretKey, roleArn, sessionName string) (aws.Credentials, error) {
+	resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+		if service == sts.ServiceID {
+			return aws.Endpoint{
+				URL:               endpoint,
+				SigningRegion:     region,
+				HostnameImmutable: true,
+			}, nil
+		}
+		return aws.Endpoint{}, &aws.EndpointNotFoundError{}
+	})
+
+	cfg, err := config.LoadDefaultConfig(ctx,
+		config.WithRegion(region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
+		config.WithEndpointResolverWithOptions(resolver),
+	)
+	if err != nil {
+		return aws.Credentials{}, err
+	}
+
+	client := sts.NewFromConfig(cfg)
+	resp, err := client.AssumeRole(ctx, &sts.AssumeRoleInput{
+		RoleArn:         aws.String(roleArn),
+		RoleSessionName: aws.String(sessionName),
+	})
+	if err != nil {
+		return aws.Credentials{}, err
+	}
+	if resp.Credentials == nil {
+		return aws.Credentials{}, fmt.Errorf("missing credentials in AssumeRole response")
+	}
+	return aws.Credentials{
+		AccessKeyID:     aws.ToString(resp.Credentials.AccessKeyId),
+		SecretAccessKey: aws.ToString(resp.Credentials.SecretAccessKey),
+		SessionToken:    aws.ToString(resp.Credentials.SessionToken),
+		Source:          "lakekeeper-sts",
+	}, nil
+}
+
+func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials) (*s3.Client, error) {
+	resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+		if service == s3.ServiceID {
+			return aws.Endpoint{
+				URL:               endpoint,
+				SigningRegion:     region,
+				HostnameImmutable: true,
+			}, nil
+		}
+		return aws.Endpoint{}, &aws.EndpointNotFoundError{}
+	})
+
+	cfg, err := config.LoadDefaultConfig(ctx,
+		config.WithRegion(region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
+		config.WithEndpointResolverWithOptions(resolver),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	}), nil
+}
+
+func newLakekeeperSession(ctx context.Context, env *TestEnvironment) (lakekeeperSession, error) {
+	endpoint := fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
+	creds, err := assumeRole(ctx, endpoint, lakekeeperRegion, env.accessKey, env.secretKey, lakekeeperRoleArn, lakekeeperSessionName)
+	if err != nil {
+		return lakekeeperSession{}, err
+	}
+	return lakekeeperSession{
+		endpoint: endpoint,
+		region:   lakekeeperRegion,
+		creds:    creds,
+	}, nil
 }

test/s3tables/polaris/polaris_env_test.go

@@ -0,0 +1,377 @@
+package polaris
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+
+	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+)
+
+const (
+	polarisImage            = "apache/polaris:latest"
+	polarisRealm            = "POLARIS"
+	polarisRootClientID     = "root"
+	polarisRootClientSecret = "s3cr3t"
+	polarisRegion           = "us-east-1"
+	polarisRoleArn          = "arn:aws:iam::000000000000:role/PolarisVendedRole"
+	polarisSigningKey       = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz" // gitleaks:allow - test signing key
+)
+
+type TestEnvironment struct {
+	seaweedDir       string
+	weedBinary       string
+	dataDir          string
+	bindIP           string
+	s3Port           int
+	s3GrpcPort       int
+	masterPort       int
+	masterGrpcPort   int
+	filerPort        int
+	filerGrpcPort    int
+	volumePort       int
+	volumeGrpcPort   int
+	polarisPort      int
+	polarisAdminPort int
+	weedProcess      *exec.Cmd
+	weedCancel       context.CancelFunc
+	polarisContainer string
+	accessKey        string
+	secretKey        string
+}
+
+func NewTestEnvironment(t *testing.T) *TestEnvironment {
+	t.Helper()
+
+	if !testutil.HasDocker() {
+		t.Skip("Docker is required for Polaris integration tests")
+	}
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("Failed to get working directory: %v", err)
+	}
+
+	seaweedDir := wd
+	for i := 0; i < 6; i++ {
+		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
+			break
+		}
+		seaweedDir = filepath.Dir(seaweedDir)
+	}
+
+	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
+	if _, err := os.Stat(weedBinary); err != nil {
+		weedBinary = "weed"
+		if _, err := exec.LookPath(weedBinary); err != nil {
+			t.Skip("weed binary not found, skipping integration test")
+		}
+	}
+
+	dataDir, err := os.MkdirTemp("", "seaweed-polaris-test-*")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %v", err)
+	}
+
+	bindIP := testutil.FindBindIP()
+
+	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
+	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
+	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
+	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
+	polarisPort, polarisAdminPort := testutil.MustFreePortPair(t, "Polaris")
+
+	return &TestEnvironment{
+		seaweedDir:       seaweedDir,
+		weedBinary:       weedBinary,
+		dataDir:          dataDir,
+		bindIP:           bindIP,
+		s3Port:           s3Port,
+		s3GrpcPort:       s3GrpcPort,
+		masterPort:       masterPort,
+		masterGrpcPort:   masterGrpcPort,
+		filerPort:        filerPort,
+		filerGrpcPort:    filerGrpcPort,
+		volumePort:       volumePort,
+		volumeGrpcPort:   volumeGrpcPort,
+		polarisPort:      polarisPort,
+		polarisAdminPort: polarisAdminPort,
+		accessKey:        "admin",
+		secretKey:        "admin",
+	}
+}
+
+func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
+	t.Helper()
+
+	iamConfigPath := filepath.Join(env.dataDir, "iam.json")
+	iamConfig := fmt.Sprintf(`{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "%s",
+          "secretKey": "%s"
+        }
+      ],
+      "actions": ["Admin", "Read", "List", "Tagging", "Write"]
+    }
+  ],
+  "sts": {
+    "tokenDuration": "12h",
+    "maxSessionLength": "24h",
+    "issuer": "seaweedfs-sts",
+    "signingKey": "%s"
+  },
+  "roles": [
+    {
+      "roleName": "PolarisVendedRole",
+      "roleArn": "%s",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": "*",
+            "Action": "sts:AssumeRole"
+          }
+        ]
+      },
+      "attachedPolicies": ["FullAccess"]
+    }
+  ],
+  "policies": [
+    {
+      "name": "FullAccess",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": "*",
+            "Resource": "*"
+          }
+        ]
+      }
+    }
+  ]
+}`, env.accessKey, env.secretKey, polarisSigningKey, polarisRoleArn)
+
+	if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
+		t.Fatalf("Failed to create IAM config: %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	env.weedCancel = cancel
+
+	cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini",
+		"-master.port", fmt.Sprintf("%d", env.masterPort),
+		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
+		"-volume.port", fmt.Sprintf("%d", env.volumePort),
+		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
+		"-filer.port", fmt.Sprintf("%d", env.filerPort),
+		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
+		"-s3.port", fmt.Sprintf("%d", env.s3Port),
+		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
+		"-s3.config", iamConfigPath,
+		"-s3.iam.config", iamConfigPath,
+		"-s3.iam.readOnly=false",
+		"-ip", env.bindIP,
+		"-ip.bind", "0.0.0.0",
+		"-dir", env.dataDir,
+	)
+	cmd.Dir = env.dataDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("Failed to start SeaweedFS: %v", err)
+	}
+	env.weedProcess = cmd
+
+	if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) {
+		t.Fatalf("S3 API failed to become ready")
+	}
+}
+
+func (env *TestEnvironment) StartPolaris(t *testing.T) {
+	t.Helper()
+
+	containerName := fmt.Sprintf("seaweed-polaris-%d", time.Now().UnixNano())
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, "docker", "run", "-d", "--rm",
+		"--name", containerName,
+		"--add-host", "host.docker.internal:host-gateway",
+		"-p", fmt.Sprintf("%d:8181", env.polarisPort),
+		"-p", fmt.Sprintf("%d:8182", env.polarisAdminPort),
+		"-e", fmt.Sprintf("AWS_REGION=%s", polarisRegion),
+		"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", env.accessKey),
+		"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", env.secretKey),
+		"-e", fmt.Sprintf("POLARIS_BOOTSTRAP_CREDENTIALS=%s,%s,%s", polarisRealm, polarisRootClientID, polarisRootClientSecret),
+		"-e", fmt.Sprintf("polaris.realm-context.realms=%s", polarisRealm),
+		"-e", "quarkus.otel.sdk.disabled=true",
+		polarisImage,
+	)
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		if ctx.Err() == context.DeadlineExceeded {
+			t.Fatalf("Timed out waiting for Polaris container: %v\nOutput:\n%s", ctx.Err(), string(output))
+		}
+		t.Fatalf("Failed to start Polaris: %v\nOutput:\n%s", err, string(output))
+	}
+	env.polarisContainer = containerName
+
+	if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/q/health", env.polarisAdminPort), 60*time.Second) {
+		logCtx, logCancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer logCancel()
+		logs, _ := exec.CommandContext(logCtx, "docker", "logs", env.polarisContainer).CombinedOutput()
+		t.Fatalf("Polaris failed to become ready\nLogs:\n%s", string(logs))
+	}
+}
+
+func (env *TestEnvironment) Cleanup(t *testing.T) {
+	t.Helper()
+	if env.weedCancel != nil {
+		env.weedCancel()
+	}
+	if env.weedProcess != nil {
+		time.Sleep(1 * time.Second)
+		_ = env.weedProcess.Wait()
+	}
+	if env.polarisContainer != "" {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		_ = exec.CommandContext(ctx, "docker", "rm", "-f", env.polarisContainer).Run()
+	}
+	if env.dataDir != "" {
+		_ = os.RemoveAll(env.dataDir)
+	}
+}
+
+func (env *TestEnvironment) polarisEndpoint() string {
+	return fmt.Sprintf("http://127.0.0.1:%d", env.polarisPort)
+}
+
+func (env *TestEnvironment) s3Endpoint() string {
+	return fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
+}
+
+func (env *TestEnvironment) s3InternalEndpoint() string {
+	return fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
+}
+
+type polarisHTTPClient struct {
+	baseURL    string
+	realm      string
+	token      string
+	httpClient *http.Client
+}
+
+func newPolarisHTTPClient(baseURL, realm, token string) *polarisHTTPClient {
+	return &polarisHTTPClient{
+		baseURL: baseURL,
+		realm:   realm,
+		token:   token,
+		httpClient: &http.Client{
+			Timeout: 30 * time.Second,
+		},
+	}
+}
+
+func (c *polarisHTTPClient) doJSON(ctx context.Context, method, path string, body interface{}, out interface{}) error {
+	return c.doJSONWithHeaders(ctx, method, path, body, out, nil)
+}
+
+func (c *polarisHTTPClient) doJSONWithHeaders(ctx context.Context, method, path string, body interface{}, out interface{}, headers map[string]string) error {
+	var reader io.Reader
+	if body != nil {
+		encoded, err := json.Marshal(body)
+		if err != nil {
+			return fmt.Errorf("encode request body: %w", err)
+		}
+		reader = bytes.NewReader(encoded)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader)
+	if err != nil {
+		return fmt.Errorf("create request: %w", err)
+	}
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	req.Header.Set("Accept", "application/json")
+	if c.realm != "" {
+		req.Header.Set("Polaris-Realm", c.realm)
+	}
+	if c.token != "" {
+		req.Header.Set("Authorization", "Bearer "+c.token)
+	}
+	for key, value := range headers {
+		req.Header.Set(key, value)
+	}
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		bodyBytes, readErr := io.ReadAll(resp.Body)
+		if readErr != nil {
+			return fmt.Errorf("request failed with status %d and could not read response body: %w", resp.StatusCode, readErr)
+		}
+		return fmt.Errorf("request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
+	}
+	if out == nil {
+		return nil
+	}
+	if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
+		return fmt.Errorf("decode response: %w", err)
+	}
+	return nil
+}
+
+func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials, pathStyle bool) (*s3.Client, error) {
+	resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+		if service == s3.ServiceID {
+			return aws.Endpoint{
+				URL:               endpoint,
+				SigningRegion:     region,
+				HostnameImmutable: true,
+			}, nil
+		}
+		return aws.Endpoint{}, &aws.EndpointNotFoundError{}
+	})
+
+	cfg, err := config.LoadDefaultConfig(ctx,
+		config.WithRegion(region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
+		config.WithEndpointResolverWithOptions(resolver),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = pathStyle
+	}), nil
+}

test/s3tables/polaris/polaris_test.go

@@ -0,0 +1,764 @@
+package polaris
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+type polarisSession struct {
+	catalogName  string
+	bucketName   string
+	token        string
+	baseLocation string
+}
+
+type polarisTableSetup struct {
+	namespace     string
+	table         string
+	dataKeyPrefix string
+	s3Client      *s3.Client
+}
+
+type polarisCatalogClient struct {
+	http    *polarisHTTPClient
+	catalog string
+}
+
+type polarisCredentials struct {
+	ClientID     string `json:"clientId"`
+	ClientSecret string `json:"clientSecret"`
+}
+
+type createPrincipalResponse struct {
+	Credentials polarisCredentials `json:"credentials"`
+}
+
+type createCatalogRequest struct {
+	Catalog polarisCatalog `json:"catalog"`
+}
+
+type polarisCatalog struct {
+	Name              string               `json:"name"`
+	Type              string               `json:"type"`
+	ReadOnly          bool                 `json:"readOnly"`
+	Properties        map[string]string    `json:"properties"`
+	StorageConfigInfo polarisStorageConfig `json:"storageConfigInfo"`
+}
+
+type polarisStorageConfig struct {
+	StorageType      string   `json:"storageType"`
+	AllowedLocations []string `json:"allowedLocations"`
+	Endpoint         string   `json:"endpoint"`
+	EndpointInternal string   `json:"endpointInternal"`
+	StsEndpoint      string   `json:"stsEndpoint"`
+	PathStyleAccess  bool     `json:"pathStyleAccess"`
+	RoleArn          string   `json:"roleArn"`
+	Region           string   `json:"region"`
+}
+
+type createNamespaceRequest struct {
+	Namespace []string `json:"namespace"`
+}
+
+type createTableRequest struct {
+	Name          string            `json:"name"`
+	Location      string            `json:"location"`
+	Schema        icebergSchema     `json:"schema"`
+	PartitionSpec icebergPartition  `json:"partition-spec"`
+	SortOrder     icebergSortOrder  `json:"sort-order"`
+	Properties    map[string]string `json:"properties"`
+}
+
+type icebergSchema struct {
+	Type     string               `json:"type"`
+	SchemaID int                  `json:"schema-id"`
+	Fields   []icebergSchemaField `json:"fields"`
+}
+
+type icebergSchemaField struct {
+	ID       int    `json:"id"`
+	Name     string `json:"name"`
+	Type     string `json:"type"`
+	Required bool   `json:"required"`
+}
+
+type icebergPartition struct {
+	SpecID int                     `json:"spec-id"`
+	Fields []icebergPartitionField `json:"fields"`
+}
+
+type icebergPartitionField struct {
+	SourceID  int    `json:"source-id"`
+	FieldID   int    `json:"field-id"`
+	Name      string `json:"name"`
+	Transform string `json:"transform"`
+}
+
+type icebergSortOrder struct {
+	OrderID int                `json:"order-id"`
+	Fields  []icebergSortField `json:"fields"`
+}
+
+type icebergSortField struct {
+	SourceID  int    `json:"source-id"`
+	Direction string `json:"direction"`
+	NullOrder string `json:"null-order"`
+}
+
+type loadTableResponse struct {
+	Config             map[string]string   `json:"config"`
+	StorageCredentials []storageCredential `json:"storage-credentials"`
+}
+
+type loadCredentialsResponse struct {
+	StorageCredentials []storageCredential `json:"storage-credentials"`
+}
+
+type storageCredential struct {
+	Prefix string            `json:"prefix"`
+	Config map[string]string `json:"config"`
+}
+
+func bootstrapPolarisTest(t *testing.T, env *TestEnvironment) (context.Context, context.CancelFunc, polarisSession, *polarisTableSetup, func()) {
+	t.Helper()
+
+	t.Logf(">>> Starting SeaweedFS with Polaris configuration...")
+	env.StartSeaweedFS(t)
+	t.Logf(">>> SeaweedFS started.")
+
+	t.Logf(">>> Starting Polaris...")
+	env.StartPolaris(t)
+	t.Logf(">>> Polaris started.")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	session := newPolarisSession(t, ctx, env)
+	setup, cleanup := setupPolarisTable(t, ctx, env, session)
+
+	return ctx, cancel, session, setup, cleanup
+}
+
+func TestPolarisIntegration(t *testing.T) {
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
+	defer cancel()
+	defer cleanup()
+
+	objectKey := fmt.Sprintf("%s/hello-%d.txt", setup.dataKeyPrefix, time.Now().UnixNano())
+	payload := []byte("polaris")
+
+	if _, err := setup.s3Client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(session.bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(payload),
+	}); err != nil {
+		t.Fatalf("PutObject failed: %v", err)
+	}
+
+	listObjects, err := setup.s3Client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
+		Bucket: aws.String(session.bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	if err != nil {
+		t.Fatalf("ListObjectsV2 failed: %v", err)
+	}
+
+	found := false
+	for _, obj := range listObjects.Contents {
+		if aws.ToString(obj.Key) == objectKey {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatalf("Object %s not found in list", objectKey)
+	}
+
+	getResp, err := setup.s3Client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(session.bucketName),
+		Key:    aws.String(objectKey),
+	})
+	if err != nil {
+		t.Fatalf("GetObject failed: %v", err)
+	}
+	body, err := io.ReadAll(getResp.Body)
+	_ = getResp.Body.Close()
+	if err != nil {
+		t.Fatalf("Read object body failed: %v", err)
+	}
+	if !bytes.Equal(body, payload) {
+		t.Fatalf("Unexpected object payload: got %q want %q", string(body), string(payload))
+	}
+
+	if _, err := setup.s3Client.DeleteObject(ctx, &s3.DeleteObjectInput{
+		Bucket: aws.String(session.bucketName),
+		Key:    aws.String(objectKey),
+	}); err != nil {
+		t.Fatalf("DeleteObject failed: %v", err)
+	}
+}
+
+func TestPolarisTableIntegration(t *testing.T) {
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
+	defer cancel()
+	defer cleanup()
+
+	objectKey := fmt.Sprintf("%s/part-%d.parquet", setup.dataKeyPrefix, time.Now().UnixNano())
+	createResp, err := setup.s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(session.bucketName),
+		Key:    aws.String(objectKey),
+	})
+	if err != nil {
+		t.Fatalf("CreateMultipartUpload failed: %v", err)
+	}
+
+	uploadID := aws.ToString(createResp.UploadId)
+	multipartCompleted := false
+	defer func() {
+		if uploadID == "" || multipartCompleted {
+			return
+		}
+		_, _ = setup.s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
+			Bucket:   aws.String(session.bucketName),
+			Key:      aws.String(objectKey),
+			UploadId: aws.String(uploadID),
+		})
+	}()
+
+	partSize := 5 * 1024 * 1024
+	part1 := bytes.Repeat([]byte("a"), partSize)
+	part2 := bytes.Repeat([]byte("b"), 1024*1024)
+
+	part1Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
+		Bucket:     aws.String(session.bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   aws.String(uploadID),
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(part1),
+	})
+	if err != nil {
+		t.Fatalf("UploadPart 1 failed: %v", err)
+	}
+
+	part2Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
+		Bucket:     aws.String(session.bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   aws.String(uploadID),
+		PartNumber: aws.Int32(2),
+		Body:       bytes.NewReader(part2),
+	})
+	if err != nil {
+		t.Fatalf("UploadPart 2 failed: %v", err)
+	}
+
+	_, err = setup.s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(session.bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: aws.String(uploadID),
+		MultipartUpload: &s3types.CompletedMultipartUpload{
+			Parts: []s3types.CompletedPart{
+				{
+					ETag:       part1Resp.ETag,
+					PartNumber: aws.Int32(1),
+				},
+				{
+					ETag:       part2Resp.ETag,
+					PartNumber: aws.Int32(2),
+				},
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("CompleteMultipartUpload failed: %v", err)
+	}
+	multipartCompleted = true
+
+	headResp, err := setup.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(session.bucketName),
+		Key:    aws.String(objectKey),
+	})
+	if err != nil {
+		t.Fatalf("HeadObject after multipart upload failed: %v", err)
+	}
+	expectedSize := int64(len(part1) + len(part2))
+	if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
+		t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
+	}
+}
+
+func newPolarisSession(t *testing.T, ctx context.Context, env *TestEnvironment) polarisSession {
+	t.Helper()
+
+	adminCreds := aws.Credentials{
+		AccessKeyID:     env.accessKey,
+		SecretAccessKey: env.secretKey,
+		Source:          "polaris-admin",
+	}
+	adminS3, err := newS3Client(ctx, env.s3Endpoint(), polarisRegion, adminCreds, true)
+	if err != nil {
+		t.Fatalf("Create admin S3 client failed: %v", err)
+	}
+
+	bucketName := fmt.Sprintf("polaris-bucket-%d", time.Now().UnixNano())
+	if _, err := adminS3.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	}); err != nil {
+		t.Fatalf("CreateBucket failed: %v", err)
+	}
+	policyDoc := map[string]interface{}{
+		"Version": "2012-10-17",
+		"Statement": []map[string]interface{}{
+			{
+				"Sid":       "AllowPolarisVendedAccess",
+				"Effect":    "Allow",
+				"Principal": "*",
+				"Action":    "s3:*",
+				"Resource": []string{
+					fmt.Sprintf("arn:aws:s3:::%s", bucketName),
+					fmt.Sprintf("arn:aws:s3:::%s/polaris/*", bucketName),
+				},
+			},
+		},
+	}
+	policyBytes, err := json.Marshal(policyDoc)
+	if err != nil {
+		t.Fatalf("Failed to marshal bucket policy: %v", err)
+	}
+	if _, err := adminS3.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+		Policy: aws.String(string(policyBytes)),
+	}); err != nil {
+		t.Fatalf("PutBucketPolicy failed: %v", err)
+	}
+
+	rootToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), polarisRootClientID, polarisRootClientSecret)
+	if err != nil {
+		t.Fatalf("Polaris root token request failed: %v", err)
+	}
+
+	managementClient := newPolarisHTTPClient(env.polarisEndpoint(), polarisRealm, rootToken)
+	catalogName := fmt.Sprintf("polaris_catalog_%d", time.Now().UnixNano())
+	baseLocation := fmt.Sprintf("s3://%s/polaris", bucketName)
+
+	catalogRequest := createCatalogRequest{
+		Catalog: polarisCatalog{
+			Name:     catalogName,
+			Type:     "INTERNAL",
+			ReadOnly: false,
+			Properties: map[string]string{
+				"default-base-location": baseLocation,
+			},
+			StorageConfigInfo: polarisStorageConfig{
+				StorageType:      "S3",
+				AllowedLocations: []string{baseLocation},
+				Endpoint:         env.s3Endpoint(),
+				EndpointInternal: env.s3InternalEndpoint(),
+				StsEndpoint:      env.s3InternalEndpoint(),
+				PathStyleAccess:  true,
+				RoleArn:          polarisRoleArn,
+				Region:           polarisRegion,
+			},
+		},
+	}
+
+	if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/catalogs", catalogRequest, nil); err != nil {
+		t.Fatalf("Create catalog failed: %v", err)
+	}
+
+	principalName := fmt.Sprintf("polaris_user_%d", time.Now().UnixNano())
+	principalRoleName := fmt.Sprintf("polaris_principal_role_%d", time.Now().UnixNano())
+	catalogRoleName := fmt.Sprintf("polaris_catalog_role_%d", time.Now().UnixNano())
+
+	var principalResp createPrincipalResponse
+	if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principals", map[string]interface{}{
+		"principal": map[string]interface{}{
+			"name":       principalName,
+			"properties": map[string]string{},
+		},
+	}, &principalResp); err != nil {
+		t.Fatalf("Create principal failed: %v", err)
+	}
+	if principalResp.Credentials.ClientID == "" || principalResp.Credentials.ClientSecret == "" {
+		t.Fatalf("Missing principal credentials in response")
+	}
+
+	if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principal-roles", map[string]interface{}{
+		"principalRole": map[string]interface{}{
+			"name":       principalRoleName,
+			"properties": map[string]string{},
+		},
+	}, nil); err != nil {
+		t.Fatalf("Create principal role failed: %v", err)
+	}
+
+	if err := managementClient.doJSON(ctx, http.MethodPost, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles", url.PathEscape(catalogName)), map[string]interface{}{
+		"catalogRole": map[string]interface{}{
+			"name":       catalogRoleName,
+			"properties": map[string]string{},
+		},
+	}, nil); err != nil {
+		t.Fatalf("Create catalog role failed: %v", err)
+	}
+
+	if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principals/%s/principal-roles", url.PathEscape(principalName)), map[string]interface{}{
+		"principalRole": map[string]interface{}{
+			"name": principalRoleName,
+		},
+	}, nil); err != nil {
+		t.Fatalf("Assign principal role failed: %v", err)
+	}
+
+	if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principal-roles/%s/catalog-roles/%s", url.PathEscape(principalRoleName), url.PathEscape(catalogName)), map[string]interface{}{
+		"catalogRole": map[string]interface{}{
+			"name": catalogRoleName,
+		},
+	}, nil); err != nil {
+		t.Fatalf("Assign catalog role failed: %v", err)
+	}
+
+	if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles/%s/grants", url.PathEscape(catalogName), url.PathEscape(catalogRoleName)), map[string]interface{}{
+		"type":      "catalog",
+		"privilege": "CATALOG_MANAGE_CONTENT",
+	}, nil); err != nil {
+		t.Fatalf("Grant catalog privilege failed: %v", err)
+	}
+
+	userToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), principalResp.Credentials.ClientID, principalResp.Credentials.ClientSecret)
+	if err != nil {
+		t.Fatalf("Polaris user token request failed: %v", err)
+	}
+
+	return polarisSession{
+		catalogName:  catalogName,
+		bucketName:   bucketName,
+		token:        userToken,
+		baseLocation: baseLocation,
+	}
+}
+
+func setupPolarisTable(t *testing.T, ctx context.Context, env *TestEnvironment, session polarisSession) (*polarisTableSetup, func()) {
+	t.Helper()
+
+	catalogClient := newPolarisCatalogClient(env.polarisEndpoint(), polarisRealm, session.token, session.catalogName)
+	namespace := fmt.Sprintf("polaris_ns_%d", time.Now().UnixNano())
+	table := fmt.Sprintf("polaris_table_%d", time.Now().UnixNano())
+
+	if err := catalogClient.CreateNamespace(ctx, namespace); err != nil {
+		t.Fatalf("CreateNamespace failed: %v", err)
+	}
+
+	location := fmt.Sprintf("%s/%s/%s", session.baseLocation, namespace, table)
+	if err := catalogClient.CreateTable(ctx, namespace, table, location); err != nil {
+		t.Fatalf("CreateTable failed: %v", err)
+	}
+
+	loadResp, err := catalogClient.LoadTable(ctx, namespace, table)
+	if err != nil {
+		t.Fatalf("LoadTable failed: %v", err)
+	}
+
+	credsResp, err := catalogClient.LoadCredentials(ctx, namespace, table)
+	if err != nil {
+		t.Fatalf("LoadCredentials failed: %v", err)
+	}
+	if len(credsResp.StorageCredentials) == 0 {
+		t.Fatalf("LoadCredentials returned no storage credentials")
+	}
+
+	credentialSource := &loadTableResponse{
+		Config:             loadResp.Config,
+		StorageCredentials: credsResp.StorageCredentials,
+	}
+
+	dataPrefix := location + "/data"
+	creds, endpoint, region, pathStyle, err := extractS3Credentials(credentialSource, dataPrefix, env.s3Endpoint(), polarisRegion)
+	if err != nil {
+		t.Fatalf("Extract vended credentials failed: %v", err)
+	}
+	dataKeyPrefix, err := s3URIToKeyPrefix(dataPrefix, session.bucketName)
+	if err != nil {
+		t.Fatalf("Invalid data prefix %s: %v", dataPrefix, err)
+	}
+
+	s3Client, err := newS3Client(ctx, endpoint, region, creds, pathStyle)
+	if err != nil {
+		t.Fatalf("Create vended S3 client failed: %v", err)
+	}
+
+	cleanup := func() {
+		cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cleanupCancel()
+		if err := catalogClient.DeleteTable(cleanupCtx, namespace, table); err != nil {
+			t.Logf("DeleteTable failed: %v", err)
+		}
+		if err := catalogClient.DeleteNamespace(cleanupCtx, namespace); err != nil {
+			t.Logf("DeleteNamespace failed: %v", err)
+		}
+	}
+
+	return &polarisTableSetup{
+		namespace:     namespace,
+		table:         table,
+		dataKeyPrefix: dataKeyPrefix,
+		s3Client:      s3Client,
+	}, cleanup
+}
+
+func fetchPolarisToken(ctx context.Context, baseURL, clientID, clientSecret string) (string, error) {
+	form := url.Values{}
+	form.Set("grant_type", "client_credentials")
+	form.Set("client_id", clientID)
+	form.Set("client_secret", clientSecret)
+	form.Set("scope", "PRINCIPAL_ROLE:ALL")
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/api/catalog/v1/oauth/tokens", strings.NewReader(form.Encode()))
+	if err != nil {
+		return "", fmt.Errorf("create token request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("Accept", "application/json")
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("token request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, readErr := io.ReadAll(resp.Body)
+		if readErr != nil {
+			return "", fmt.Errorf("token request failed with status %d and reading body: %w", resp.StatusCode, readErr)
+		}
+		return "", fmt.Errorf("token request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+
+	var tokenResp struct {
+		AccessToken string `json:"access_token"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
+		return "", fmt.Errorf("decode token response: %w", err)
+	}
+	if tokenResp.AccessToken == "" {
+		return "", fmt.Errorf("missing access token in response")
+	}
+
+	return tokenResp.AccessToken, nil
+}
+
+func newPolarisCatalogClient(baseURL, realm, token, catalog string) *polarisCatalogClient {
+	return &polarisCatalogClient{
+		http:    newPolarisHTTPClient(baseURL, realm, token),
+		catalog: catalog,
+	}
+}
+
+func (c *polarisCatalogClient) CreateNamespace(ctx context.Context, namespace string) error {
+	path := fmt.Sprintf("/api/catalog/v1/%s/namespaces", url.PathEscape(c.catalog))
+	req := createNamespaceRequest{Namespace: []string{namespace}}
+	return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
+}
+
+func (c *polarisCatalogClient) DeleteNamespace(ctx context.Context, namespace string) error {
+	path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s", url.PathEscape(c.catalog), url.PathEscape(namespace))
+	return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
+}
+
+func (c *polarisCatalogClient) CreateTable(ctx context.Context, namespace, table, location string) error {
+	path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables", url.PathEscape(c.catalog), url.PathEscape(namespace))
+
+	req := createTableRequest{
+		Name:     table,
+		Location: location,
+		Schema: icebergSchema{
+			Type:     "struct",
+			SchemaID: 0,
+			Fields: []icebergSchemaField{
+				{
+					ID:       1,
+					Name:     "id",
+					Type:     "long",
+					Required: false,
+				},
+			},
+		},
+		PartitionSpec: icebergPartition{
+			SpecID: 0,
+			Fields: []icebergPartitionField{},
+		},
+		SortOrder: icebergSortOrder{
+			OrderID: 0,
+			Fields:  []icebergSortField{},
+		},
+		Properties: map[string]string{
+			"format-version": "2",
+		},
+	}
+
+	return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
+}
+
+func (c *polarisCatalogClient) DeleteTable(ctx context.Context, namespace, table string) error {
+	path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
+	return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
+}
+
+func (c *polarisCatalogClient) LoadTable(ctx context.Context, namespace, table string) (*loadTableResponse, error) {
+	path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
+	var resp loadTableResponse
+	headers := map[string]string{
+		"X-Iceberg-Access-Delegation": "vended-credentials",
+	}
+	if err := c.http.doJSONWithHeaders(ctx, http.MethodGet, path, nil, &resp, headers); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (c *polarisCatalogClient) LoadCredentials(ctx context.Context, namespace, table string) (*loadCredentialsResponse, error) {
+	path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s/credentials", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
+	var resp loadCredentialsResponse
+	if err := c.http.doJSON(ctx, http.MethodGet, path, nil, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoint, fallbackRegion string) (aws.Credentials, string, string, bool, error) {
+	credentialConfig, err := selectStorageConfig(load, targetPrefix)
+	if err != nil {
+		return aws.Credentials{}, "", "", false, err
+	}
+
+	lookupConfig := func(key string) string {
+		if load != nil && load.Config != nil {
+			if val, ok := load.Config[key]; ok && strings.TrimSpace(val) != "" {
+				return strings.TrimSpace(val)
+			}
+		}
+		if val, ok := credentialConfig[key]; ok && strings.TrimSpace(val) != "" {
+			return strings.TrimSpace(val)
+		}
+		return ""
+	}
+
+	accessKey := strings.TrimSpace(credentialConfig["s3.access-key-id"])
+	secretKey := strings.TrimSpace(credentialConfig["s3.secret-access-key"])
+	sessionToken := strings.TrimSpace(credentialConfig["s3.session-token"])
+	if accessKey == "" || secretKey == "" {
+		return aws.Credentials{}, "", "", false, fmt.Errorf("missing s3.access-key-id or s3.secret-access-key in selected storage credential")
+	}
+
+	endpoint := lookupConfig("s3.endpoint")
+	if endpoint == "" {
+		endpoint = fallbackEndpoint
+	}
+	if endpoint != "" && !strings.HasPrefix(endpoint, "http") {
+		endpoint = "http://" + endpoint
+	}
+
+	region := lookupConfig("client.region")
+	if region == "" {
+		region = fallbackRegion
+	}
+
+	pathStyle := true
+	if value := lookupConfig("s3.path-style-access"); value != "" {
+		pathStyle = strings.EqualFold(value, "true")
+	}
+
+	return aws.Credentials{
+		AccessKeyID:     accessKey,
+		SecretAccessKey: secretKey,
+		SessionToken:    sessionToken,
+		Source:          "polaris-vended",
+	}, endpoint, region, pathStyle, nil
+}
+
+func selectStorageConfig(load *loadTableResponse, targetPrefix string) (map[string]string, error) {
+	if load == nil {
+		return nil, fmt.Errorf("load table response is nil")
+	}
+
+	switch len(load.StorageCredentials) {
+	case 0:
+		if load.Config == nil {
+			return nil, fmt.Errorf("polaris returned no storage credentials or config")
+		}
+		return load.Config, nil
+	case 1:
+		cred := load.StorageCredentials[0]
+		if cred.Config == nil {
+			return nil, fmt.Errorf("storage credential for prefix %s returned nil config", cred.Prefix)
+		}
+		return cred.Config, nil
+	default:
+		if targetPrefix == "" {
+			return nil, fmt.Errorf("multiple storage credentials (%d) returned but no target prefix provided", len(load.StorageCredentials))
+		}
+		normalizedTarget := normalizePrefix(targetPrefix)
+		if normalizedTarget == "" {
+			return nil, fmt.Errorf("target prefix %q normalized to empty string", targetPrefix)
+		}
+		var bestConfig map[string]string
+		bestLen := -1
+		for _, cred := range load.StorageCredentials {
+			if cred.Config == nil {
+				continue
+			}
+			prefix := normalizePrefix(cred.Prefix)
+			if prefix == "" {
+				if bestLen < 0 {
+					bestLen = 0
+					bestConfig = cred.Config
+				}
+				continue
+			}
+			if normalizedTarget == prefix || strings.HasPrefix(normalizedTarget, prefix+"/") {
+				if len(prefix) > bestLen {
+					bestLen = len(prefix)
+					bestConfig = cred.Config
+				}
+			}
+		}
+		if bestConfig != nil {
+			return bestConfig, nil
+		}
+		return nil, fmt.Errorf("none of the %d storage credentials matched prefix %s", len(load.StorageCredentials), targetPrefix)
+	}
+}
+
+func normalizePrefix(prefix string) string {
+	p := strings.TrimSpace(prefix)
+	p = strings.TrimSuffix(p, "/")
+	return p
+}
+
+func s3URIToKeyPrefix(uri, bucket string) (string, error) {
+	prefix := "s3://" + bucket + "/"
+	if !strings.HasPrefix(uri, prefix) {
+		return "", fmt.Errorf("uri %q does not match bucket %q", uri, bucket)
+	}
+	keyPrefix := strings.TrimPrefix(uri, prefix)
+	keyPrefix = strings.TrimPrefix(keyPrefix, "/")
+	if keyPrefix == "" {
+		return "", fmt.Errorf("empty key prefix in uri %q", uri)
+	}
+	return keyPrefix, nil
+}

weed/admin/dash/admin_lock_manager.go

@@ -0,0 +1,148 @@
+package dash
+
+import (
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
+)
+
+const (
+	adminLockName       = "shell"
+	adminLockClientName = "admin-plugin"
+)
+
+// AdminLockManager coordinates exclusive admin locks with reference counting.
+// It is safe for concurrent use.
+type AdminLockManager struct {
+	locker     *exclusive_locks.ExclusiveLocker
+	clientName string
+
+	mu        sync.Mutex
+	cond      *sync.Cond
+	acquiring bool
+	holdCount int
+
+	lastAcquiredAt time.Time
+	lastReleasedAt time.Time
+	waitingSince   time.Time
+	waitingReason  string
+	currentReason  string
+}
+
+func NewAdminLockManager(masterClient *wdclient.MasterClient, clientName string) *AdminLockManager {
+	if masterClient == nil {
+		return nil
+	}
+	if clientName == "" {
+		clientName = adminLockClientName
+	}
+	manager := &AdminLockManager{
+		locker:     exclusive_locks.NewExclusiveLocker(masterClient, adminLockName),
+		clientName: clientName,
+	}
+	manager.cond = sync.NewCond(&manager.mu)
+	return manager
+}
+
+func (m *AdminLockManager) Acquire(reason string) (func(), error) {
+	if m == nil || m.locker == nil {
+		return func() {}, nil
+	}
+
+	m.mu.Lock()
+	if reason != "" {
+		m.locker.SetMessage(reason)
+		m.currentReason = reason
+	}
+	for m.acquiring {
+		m.cond.Wait()
+	}
+	if m.holdCount == 0 {
+		m.acquiring = true
+		m.waitingSince = time.Now().UTC()
+		m.waitingReason = reason
+		m.mu.Unlock()
+		m.locker.RequestLock(m.clientName)
+		m.mu.Lock()
+		m.acquiring = false
+		m.holdCount = 1
+		m.lastAcquiredAt = time.Now().UTC()
+		m.waitingSince = time.Time{}
+		m.waitingReason = ""
+		m.cond.Broadcast()
+		m.mu.Unlock()
+		return m.Release, nil
+	}
+	m.holdCount++
+	if reason != "" {
+		m.currentReason = reason
+	}
+	m.mu.Unlock()
+	return m.Release, nil
+}
+
+func (m *AdminLockManager) Release() {
+	if m == nil || m.locker == nil {
+		return
+	}
+
+	m.mu.Lock()
+	if m.holdCount <= 0 {
+		m.mu.Unlock()
+		return
+	}
+	m.holdCount--
+	shouldRelease := m.holdCount == 0
+	m.mu.Unlock()
+
+	if shouldRelease {
+		m.mu.Lock()
+		m.lastReleasedAt = time.Now().UTC()
+		m.currentReason = ""
+		m.mu.Unlock()
+		m.locker.ReleaseLock()
+	}
+}
+
+type LockStatus struct {
+	Held           bool       `json:"held"`
+	HoldCount      int        `json:"hold_count"`
+	Acquiring      bool       `json:"acquiring"`
+	Message        string     `json:"message,omitempty"`
+	WaitingReason  string     `json:"waiting_reason,omitempty"`
+	LastAcquiredAt *time.Time `json:"last_acquired_at,omitempty"`
+	LastReleasedAt *time.Time `json:"last_released_at,omitempty"`
+	WaitingSince   *time.Time `json:"waiting_since,omitempty"`
+}
+
+func (m *AdminLockManager) Status() LockStatus {
+	if m == nil {
+		return LockStatus{}
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	status := LockStatus{
+		Held:          m.holdCount > 0,
+		HoldCount:     m.holdCount,
+		Acquiring:     m.acquiring,
+		Message:       m.currentReason,
+		WaitingReason: m.waitingReason,
+	}
+	if !m.lastAcquiredAt.IsZero() {
+		at := m.lastAcquiredAt
+		status.LastAcquiredAt = &at
+	}
+	if !m.lastReleasedAt.IsZero() {
+		at := m.lastReleasedAt
+		status.LastReleasedAt = &at
+	}
+	if !m.waitingSince.IsZero() {
+		at := m.waitingSince
+		status.WaitingSince = &at
+	}
+	return status
+}

weed/admin/dash/admin_server.go

@@ -98,6 +98,8 @@ type AdminServer struct {
 	// Maintenance system
 	maintenanceManager *maintenance.MaintenanceManager
 	plugin             *adminplugin.Plugin
+	pluginLock         *AdminLockManager
+	expireJobHandler   func(jobID string, reason string) (*adminplugin.TrackedJob, bool, error)
 
 	// Topic retention purger
 	topicRetentionPurger *TopicRetentionPurger
@@ -116,7 +118,7 @@ type AdminServer struct {
 
 // Type definitions moved to types.go
 
-func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string, icebergPort int) *AdminServer {
+func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string, icebergPort int, idleSleepDuration time.Duration) *AdminServer {
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.admin")
 
 	// Create master client with multiple master support
@@ -134,6 +136,8 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 	ctx := context.Background()
 	go masterClient.KeepConnectedToMaster(ctx)
 
+	lockManager := NewAdminLockManager(masterClient, adminLockClientName)
+
 	server := &AdminServer{
 		masterClient:                  masterClient,
 		templateFS:                    templateFS,
@@ -145,6 +149,7 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 		collectionStatsCacheThreshold: defaultStatsCacheTimeout,
 		s3TablesManager:               newS3TablesManager(),
 		icebergPort:                   icebergPort,
+		pluginLock:                    lockManager,
 	}
 
 	// Initialize topic retention purger
@@ -224,18 +229,22 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 	}
 
 	plugin, err := adminplugin.New(adminplugin.Options{
-		DataDir: dataDir,
+		DataDir:           dataDir,
+		IdleSleepDuration: idleSleepDuration,
 		ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
 			return server.buildDefaultPluginClusterContext(), nil
 		},
+		LockManager: lockManager,
 	})
 	if err != nil && dataDir != "" {
 		glog.Warningf("Failed to initialize plugin with dataDir=%q: %v. Falling back to in-memory plugin state.", dataDir, err)
 		plugin, err = adminplugin.New(adminplugin.Options{
-			DataDir: "",
+			DataDir:           "",
+			IdleSleepDuration: idleSleepDuration,
 			ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
 				return server.buildDefaultPluginClusterContext(), nil
 			},
+			LockManager: lockManager,
 		})
 	}
 	if err != nil {
@@ -889,6 +898,13 @@ func (s *AdminServer) GetPlugin() *adminplugin.Plugin {
 	return s.plugin
 }
 
+func (s *AdminServer) acquirePluginLock(reason string) (func(), error) {
+	if s == nil || s.pluginLock == nil {
+		return func() {}, nil
+	}
+	return s.pluginLock.Acquire(reason)
+}
+
 // RequestPluginJobTypeDescriptor asks one worker for job type schema and returns the descriptor.
 func (s *AdminServer) RequestPluginJobTypeDescriptor(ctx context.Context, jobType string, forceRefresh bool) (*plugin_pb.JobTypeDescriptor, error) {
 	if s.plugin == nil {
@@ -931,6 +947,13 @@ func (s *AdminServer) RunPluginDetection(
 	if s.plugin == nil {
 		return nil, fmt.Errorf("plugin is not enabled")
 	}
+	releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detection %s", jobType))
+	if err != nil {
+		return nil, err
+	}
+	if releaseLock != nil {
+		defer releaseLock()
+	}
 	return s.plugin.RunDetection(ctx, jobType, clusterContext, maxResults)
 }
 
@@ -956,6 +979,13 @@ func (s *AdminServer) RunPluginDetectionWithReport(
 	if s.plugin == nil {
 		return nil, fmt.Errorf("plugin is not enabled")
 	}
+	releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detection %s", jobType))
+	if err != nil {
+		return nil, err
+	}
+	if releaseLock != nil {
+		defer releaseLock()
+	}
 	return s.plugin.RunDetectionWithReport(ctx, jobType, clusterContext, maxResults)
 }
 
@@ -969,6 +999,17 @@ func (s *AdminServer) ExecutePluginJob(
 	if s.plugin == nil {
 		return nil, fmt.Errorf("plugin is not enabled")
 	}
+	jobType := ""
+	if job != nil {
+		jobType = strings.TrimSpace(job.JobType)
+	}
+	releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin execution %s", jobType))
+	if err != nil {
+		return nil, err
+	}
+	if releaseLock != nil {
+		defer releaseLock()
+	}
 	return s.plugin.ExecuteJob(ctx, job, clusterContext, attempt)
 }
 
@@ -1020,6 +1061,17 @@ func (s *AdminServer) GetPluginJobDetail(jobID string, activityLimit, relatedLim
 	return s.plugin.BuildJobDetail(jobID, activityLimit, relatedLimit)
 }
 
+// ExpirePluginJob marks an active plugin job as failed so it no longer blocks scheduling.
+func (s *AdminServer) ExpirePluginJob(jobID, reason string) (*adminplugin.TrackedJob, bool, error) {
+	if handler := s.expireJobHandler; handler != nil {
+		return handler(jobID, reason)
+	}
+	if s.plugin == nil {
+		return nil, false, fmt.Errorf("plugin is not enabled")
+	}
+	return s.plugin.ExpireJob(jobID, reason)
+}
+
 // ListPluginActivities returns plugin job activities for monitoring.
 func (s *AdminServer) ListPluginActivities(jobType string, limit int) []adminplugin.JobActivity {
 	if s.plugin == nil {

weed/admin/dash/plugin_api.go

@@ -5,6 +5,7 @@ import (
 	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -130,6 +131,47 @@ func (s *AdminServer) GetPluginJobDetailAPI(w http.ResponseWriter, r *http.Reque
 	writeJSON(w, http.StatusOK, detail)
 }
 
+// ExpirePluginJobAPI marks a job as failed so it no longer blocks scheduling.
+func (s *AdminServer) ExpirePluginJobAPI(w http.ResponseWriter, r *http.Request) {
+	jobID := strings.TrimSpace(mux.Vars(r)["jobId"])
+	if jobID == "" {
+		writeJSONError(w, http.StatusBadRequest, "jobId is required")
+		return
+	}
+
+	var req struct {
+		Reason string `json:"reason"`
+	}
+
+	if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil && err != io.EOF {
+		writeJSONError(w, http.StatusBadRequest, "invalid request body: "+err.Error())
+		return
+	}
+
+	job, expired, err := s.ExpirePluginJob(jobID, req.Reason)
+	if err != nil {
+		if errors.Is(err, plugin.ErrJobNotFound) {
+			writeJSONError(w, http.StatusNotFound, err.Error())
+			return
+		}
+		writeJSONError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	response := map[string]interface{}{
+		"job_id":  jobID,
+		"expired": expired,
+	}
+	if job != nil {
+		response["job"] = job
+	}
+	if !expired {
+		response["message"] = "job is not active"
+	}
+
+	writeJSON(w, http.StatusOK, response)
+}
+
 // GetPluginActivitiesAPI returns recent plugin activities.
 func (s *AdminServer) GetPluginActivitiesAPI(w http.ResponseWriter, r *http.Request) {
 	query := r.URL.Query()
@@ -172,6 +214,27 @@ func (s *AdminServer) GetPluginSchedulerStatesAPI(w http.ResponseWriter, r *http
 	writeJSON(w, http.StatusOK, states)
 }
 
+// GetPluginSchedulerStatusAPI returns scheduler status including in-process jobs and lock state.
+func (s *AdminServer) GetPluginSchedulerStatusAPI(w http.ResponseWriter, r *http.Request) {
+	pluginSvc := s.GetPlugin()
+	if pluginSvc == nil {
+		writeJSON(w, http.StatusOK, map[string]interface{}{
+			"enabled": false,
+		})
+		return
+	}
+
+	response := map[string]interface{}{
+		"enabled":   true,
+		"scheduler": pluginSvc.GetSchedulerStatus(),
+	}
+	if s.pluginLock != nil {
+		response["lock"] = s.pluginLock.Status()
+	}
+
+	writeJSON(w, http.StatusOK, response)
+}
+
 // RequestPluginJobTypeSchemaAPI asks a worker for one job type schema.
 func (s *AdminServer) RequestPluginJobTypeSchemaAPI(w http.ResponseWriter, r *http.Request) {
 	jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
@@ -235,6 +298,9 @@ func (s *AdminServer) GetPluginJobTypeConfigAPI(w http.ResponseWriter, r *http.R
 			AdminRuntime:       &plugin_pb.AdminRuntimeConfig{},
 		}
 	}
+	if descriptor, err := s.LoadPluginJobTypeDescriptor(jobType); err == nil && descriptor != nil {
+		applyDescriptorDefaultsToPersistedConfig(config, descriptor)
+	}
 
 	renderProtoJSON(w, http.StatusOK, config)
 }
@@ -413,6 +479,14 @@ func (s *AdminServer) RunPluginJobTypeAPI(w http.ResponseWriter, r *http.Request
 		writeJSONError(w, http.StatusBadRequest, "jobType is required")
 		return
 	}
+	releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detect+execute %s", jobType))
+	if err != nil {
+		writeJSONError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if releaseLock != nil {
+		defer releaseLock()
+	}
 
 	var req struct {
 		ClusterContext json.RawMessage `json:"cluster_context"`
@@ -729,6 +803,90 @@ func buildJobSpecFromProposal(jobType string, proposal *plugin_pb.JobProposal, i
 	return jobSpec
 }
 
+func applyDescriptorDefaultsToPersistedConfig(
+	config *plugin_pb.PersistedJobTypeConfig,
+	descriptor *plugin_pb.JobTypeDescriptor,
+) {
+	if config == nil || descriptor == nil {
+		return
+	}
+
+	if config.AdminConfigValues == nil {
+		config.AdminConfigValues = map[string]*plugin_pb.ConfigValue{}
+	}
+	if config.WorkerConfigValues == nil {
+		config.WorkerConfigValues = map[string]*plugin_pb.ConfigValue{}
+	}
+	if config.AdminRuntime == nil {
+		config.AdminRuntime = &plugin_pb.AdminRuntimeConfig{}
+	}
+
+	if descriptor.AdminConfigForm != nil {
+		for key, value := range descriptor.AdminConfigForm.DefaultValues {
+			if value == nil {
+				continue
+			}
+			current := config.AdminConfigValues[key]
+			if current == nil {
+				config.AdminConfigValues[key] = proto.Clone(value).(*plugin_pb.ConfigValue)
+				continue
+			}
+			if strings.EqualFold(descriptor.JobType, "admin_script") &&
+				key == "script" &&
+				isBlankStringConfigValue(current) {
+				config.AdminConfigValues[key] = proto.Clone(value).(*plugin_pb.ConfigValue)
+			}
+		}
+	}
+	if descriptor.WorkerConfigForm != nil {
+		for key, value := range descriptor.WorkerConfigForm.DefaultValues {
+			if value == nil {
+				continue
+			}
+			if config.WorkerConfigValues[key] != nil {
+				continue
+			}
+			config.WorkerConfigValues[key] = proto.Clone(value).(*plugin_pb.ConfigValue)
+		}
+	}
+	if descriptor.AdminRuntimeDefaults != nil {
+		runtime := config.AdminRuntime
+		defaults := descriptor.AdminRuntimeDefaults
+		if runtime.DetectionIntervalSeconds <= 0 {
+			runtime.DetectionIntervalSeconds = defaults.DetectionIntervalSeconds
+		}
+		if runtime.DetectionTimeoutSeconds <= 0 {
+			runtime.DetectionTimeoutSeconds = defaults.DetectionTimeoutSeconds
+		}
+		if runtime.MaxJobsPerDetection <= 0 {
+			runtime.MaxJobsPerDetection = defaults.MaxJobsPerDetection
+		}
+		if runtime.GlobalExecutionConcurrency <= 0 {
+			runtime.GlobalExecutionConcurrency = defaults.GlobalExecutionConcurrency
+		}
+		if runtime.PerWorkerExecutionConcurrency <= 0 {
+			runtime.PerWorkerExecutionConcurrency = defaults.PerWorkerExecutionConcurrency
+		}
+		if runtime.RetryBackoffSeconds <= 0 {
+			runtime.RetryBackoffSeconds = defaults.RetryBackoffSeconds
+		}
+		if runtime.RetryLimit < 0 {
+			runtime.RetryLimit = defaults.RetryLimit
+		}
+	}
+}
+
+func isBlankStringConfigValue(value *plugin_pb.ConfigValue) bool {
+	if value == nil {
+		return true
+	}
+	kind, ok := value.Kind.(*plugin_pb.ConfigValue_StringValue)
+	if !ok {
+		return false
+	}
+	return strings.TrimSpace(kind.StringValue) == ""
+}
+
 func parsePositiveInt(raw string, defaultValue int) int {
 	value, err := strconv.Atoi(strings.TrimSpace(raw))
 	if err != nil || value <= 0 {

weed/admin/dash/plugin_api_test.go

@@ -1,11 +1,120 @@
 package dash
 
 import (
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
 	"testing"
 
+	"github.com/gorilla/mux"
+	"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
 	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
 )
 
+func TestExpirePluginJobAPI(t *testing.T) {
+	makeRequest := func(adminServer *AdminServer, jobID string, body io.Reader) *httptest.ResponseRecorder {
+		req := httptest.NewRequest(http.MethodPost, "/api/plugin/jobs/"+jobID+"/expire", body)
+		req = mux.SetURLVars(req, map[string]string{"jobId": jobID})
+		recorder := httptest.NewRecorder()
+		adminServer.ExpirePluginJobAPI(recorder, req)
+		return recorder
+	}
+
+	t.Run("empty job id", func(t *testing.T) {
+		recorder := makeRequest(&AdminServer{}, "", nil)
+		if recorder.Code != http.StatusBadRequest {
+			t.Fatalf("expected 400, got %d", recorder.Code)
+		}
+	})
+
+	t.Run("invalid json", func(t *testing.T) {
+		recorder := makeRequest(&AdminServer{}, "job-id", strings.NewReader("{"))
+		if recorder.Code != http.StatusBadRequest {
+			t.Fatalf("expected 400, got %d", recorder.Code)
+		}
+	})
+
+	t.Run("job not found", func(t *testing.T) {
+		adminServer := &AdminServer{
+			expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
+				return nil, false, plugin.ErrJobNotFound
+			},
+		}
+		recorder := makeRequest(adminServer, "missing", strings.NewReader(`{"reason":"nope"}`))
+		if recorder.Code != http.StatusNotFound {
+			t.Fatalf("expected 404, got %d", recorder.Code)
+		}
+		var payload map[string]any
+		if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
+			t.Fatalf("failed to unmarshal body: %v", err)
+		}
+		if payload["error"] == nil {
+			t.Fatalf("expected error payload, got %v", payload)
+		}
+	})
+
+	t.Run("successful expire", func(t *testing.T) {
+		expected := &plugin.TrackedJob{JobID: "foo", State: "assigned"}
+		adminServer := &AdminServer{
+			expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
+				if jobID != "foo" {
+					return nil, false, errors.New("unexpected")
+				}
+				return expected, true, nil
+			},
+		}
+		recorder := makeRequest(adminServer, "foo", strings.NewReader(`{"reason":"cleanup"}`))
+		if recorder.Code != http.StatusOK {
+			t.Fatalf("expected 200, got %d", recorder.Code)
+		}
+		var payload map[string]any
+		if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
+			t.Fatalf("failed to decode payload: %v", err)
+		}
+		if payload["job_id"] != "foo" {
+			t.Fatalf("expected job_id foo, got %v", payload["job_id"])
+		}
+		if expired, ok := payload["expired"].(bool); !ok || !expired {
+			t.Fatalf("expected expired=true, got %v", payload["expired"])
+		}
+		jobData, ok := payload["job"].(map[string]any)
+		if !ok || jobData["job_id"] != "foo" {
+			t.Fatalf("expected job info with job_id, got %v", payload["job"])
+		}
+	})
+
+	t.Run("non-active job", func(t *testing.T) {
+		adminServer := &AdminServer{
+			expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
+				return nil, false, nil
+			},
+		}
+		recorder := makeRequest(adminServer, "bar", strings.NewReader(`{"reason":"ignore"}`))
+		if recorder.Code != http.StatusOK {
+			t.Fatalf("expected 200, got %d", recorder.Code)
+		}
+		var payload map[string]any
+		if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
+			t.Fatalf("failed to decode payload: %v", err)
+		}
+		if payload["job_id"] != "bar" {
+			t.Fatalf("expected job_id bar, got %v", payload["job_id"])
+		}
+		if expired, ok := payload["expired"].(bool); !ok || expired {
+			t.Fatalf("expected expired=false, got %v", payload["expired"])
+		}
+		if payload["message"] != "job is not active" {
+			t.Fatalf("expected message job is not active, got %v", payload["message"])
+		}
+		if _, exists := payload["job"]; exists {
+			t.Fatalf("expected no job payload for non-active job, got %v", payload["job"])
+		}
+	})
+}
+
 func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
 	t.Parallel()
 
@@ -31,3 +140,83 @@ func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
 		t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey)
 	}
 }
+
+func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testing.T) {
+	t.Parallel()
+
+	config := &plugin_pb.PersistedJobTypeConfig{
+		JobType:            "admin_script",
+		AdminConfigValues:  map[string]*plugin_pb.ConfigValue{},
+		WorkerConfigValues: map[string]*plugin_pb.ConfigValue{},
+		AdminRuntime:       &plugin_pb.AdminRuntimeConfig{},
+	}
+	descriptor := &plugin_pb.JobTypeDescriptor{
+		JobType: "admin_script",
+		AdminConfigForm: &plugin_pb.ConfigForm{
+			DefaultValues: map[string]*plugin_pb.ConfigValue{
+				"script": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "volume.balance -apply"},
+				},
+				"run_interval_minutes": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 17},
+				},
+			},
+		},
+		AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
+			DetectionIntervalSeconds: 60,
+			DetectionTimeoutSeconds:  300,
+		},
+	}
+
+	applyDescriptorDefaultsToPersistedConfig(config, descriptor)
+
+	script := config.AdminConfigValues["script"]
+	if script == nil {
+		t.Fatalf("expected script default to be backfilled")
+	}
+	scriptKind, ok := script.Kind.(*plugin_pb.ConfigValue_StringValue)
+	if !ok || scriptKind.StringValue == "" {
+		t.Fatalf("expected non-empty script default, got=%+v", script)
+	}
+	if config.AdminRuntime.DetectionIntervalSeconds != 60 {
+		t.Fatalf("expected runtime detection interval default to be backfilled")
+	}
+}
+
+func TestApplyDescriptorDefaultsToPersistedConfigReplacesBlankAdminScript(t *testing.T) {
+	t.Parallel()
+
+	config := &plugin_pb.PersistedJobTypeConfig{
+		JobType: "admin_script",
+		AdminConfigValues: map[string]*plugin_pb.ConfigValue{
+			"script": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "   "},
+			},
+		},
+		AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
+	}
+	descriptor := &plugin_pb.JobTypeDescriptor{
+		JobType: "admin_script",
+		AdminConfigForm: &plugin_pb.ConfigForm{
+			DefaultValues: map[string]*plugin_pb.ConfigValue{
+				"script": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "volume.fix.replication -apply"},
+				},
+			},
+		},
+	}
+
+	applyDescriptorDefaultsToPersistedConfig(config, descriptor)
+
+	script := config.AdminConfigValues["script"]
+	if script == nil {
+		t.Fatalf("expected script config value")
+	}
+	scriptKind, ok := script.Kind.(*plugin_pb.ConfigValue_StringValue)
+	if !ok {
+		t.Fatalf("expected string script config value, got=%T", script.Kind)
+	}
+	if scriptKind.StringValue != "volume.fix.replication -apply" {
+		t.Fatalf("expected blank script to be replaced by default, got=%q", scriptKind.StringValue)
+	}
+}

weed/admin/handlers/admin_handlers.go

@@ -234,6 +234,7 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
 	pluginApi.HandleFunc("/jobs/{jobId}/detail", h.adminServer.GetPluginJobDetailAPI).Methods(http.MethodGet)
 	pluginApi.HandleFunc("/activities", h.adminServer.GetPluginActivitiesAPI).Methods(http.MethodGet)
 	pluginApi.HandleFunc("/scheduler-states", h.adminServer.GetPluginSchedulerStatesAPI).Methods(http.MethodGet)
+	pluginApi.HandleFunc("/scheduler-status", h.adminServer.GetPluginSchedulerStatusAPI).Methods(http.MethodGet)
 	pluginApi.HandleFunc("/job-types/{jobType}/descriptor", h.adminServer.GetPluginJobTypeDescriptorAPI).Methods(http.MethodGet)
 	pluginApi.HandleFunc("/job-types/{jobType}/schema", h.adminServer.RequestPluginJobTypeSchemaAPI).Methods(http.MethodPost)
 	pluginApi.HandleFunc("/job-types/{jobType}/config", h.adminServer.GetPluginJobTypeConfigAPI).Methods(http.MethodGet)
@@ -242,6 +243,7 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
 	pluginApi.Handle("/job-types/{jobType}/detect", wrapWrite(h.adminServer.TriggerPluginDetectionAPI)).Methods(http.MethodPost)
 	pluginApi.Handle("/job-types/{jobType}/run", wrapWrite(h.adminServer.RunPluginJobTypeAPI)).Methods(http.MethodPost)
 	pluginApi.Handle("/jobs/execute", wrapWrite(h.adminServer.ExecutePluginJobAPI)).Methods(http.MethodPost)
+	pluginApi.Handle("/jobs/{jobId}/expire", wrapWrite(h.adminServer.ExpirePluginJobAPI)).Methods(http.MethodPost)
 
 	mqApi := api.PathPrefix("/mq").Subrouter()
 	mqApi.HandleFunc("/topics/{namespace}/{topic}", h.mqHandlers.GetTopicDetailsAPI).Methods(http.MethodGet)

weed/admin/handlers/admin_handlers_routes_test.go

@@ -21,6 +21,9 @@ func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_NoAuth(t *testing.T) {
 	if !hasRoute(router, http.MethodGet, "/api/plugin/jobs/example/detail") {
 		t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in no-auth mode")
 	}
+	if !hasRoute(router, http.MethodPost, "/api/plugin/jobs/example/expire") {
+		t.Fatalf("expected POST /api/plugin/jobs/:jobId/expire to be registered in no-auth mode")
+	}
 }
 
 func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_WithAuth(t *testing.T) {
@@ -34,6 +37,9 @@ func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_WithAuth(t *testing.T) {
 	if !hasRoute(router, http.MethodGet, "/api/plugin/jobs/example/detail") {
 		t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in auth mode")
 	}
+	if !hasRoute(router, http.MethodPost, "/api/plugin/jobs/example/expire") {
+		t.Fatalf("expected POST /api/plugin/jobs/:jobId/expire to be registered in auth mode")
+	}
 }
 
 func TestSetupRoutes_RegistersPluginPages_NoAuth(t *testing.T) {

weed/admin/plugin/lock_manager.go

@@ -0,0 +1,7 @@
+package plugin
+
+// LockManager provides a shared exclusive lock for admin-managed detection/execution.
+// Acquire returns a release function that must be called when the protected work finishes.
+type LockManager interface {
+	Acquire(reason string) (release func(), err error)
+}

weed/admin/plugin/plugin.go

@@ -24,6 +24,7 @@ const (
 	defaultHeartbeatInterval   = 30
 	defaultReconnectDelay      = 5
 	defaultPendingSchemaBuffer = 1
+	adminScriptJobType         = "admin_script"
 )
 
 type Options struct {
@@ -31,7 +32,9 @@ type Options struct {
 	OutgoingBufferSize     int
 	SendTimeout            time.Duration
 	SchedulerTick          time.Duration
+	IdleSleepDuration      time.Duration
 	ClusterContextProvider func(context.Context) (*plugin_pb.ClusterContext, error)
+	LockManager            LockManager
 }
 
 // JobTypeInfo contains metadata about a plugin job type.
@@ -51,17 +54,25 @@ type Plugin struct {
 	sendTimeout    time.Duration
 
 	schedulerTick          time.Duration
+	idleSleepDuration      time.Duration
 	clusterContextProvider func(context.Context) (*plugin_pb.ClusterContext, error)
+	lockManager            LockManager
 
-	schedulerMu       sync.Mutex
-	nextDetectionAt   map[string]time.Time
-	detectionInFlight map[string]bool
+	schedulerMu                sync.Mutex
+	schedulerPhase             string
+	currentJobType             string
+	iterationStartedAt         time.Time
+	lastIterationEndedAt       time.Time
+	lastIterationWorkDetected  bool
 
 	detectorLeaseMu sync.Mutex
 	detectorLeases  map[string]string
 
 	schedulerExecMu           sync.Mutex
 	schedulerExecReservations map[string]int
+	adminScriptRunMu          sync.RWMutex
+	schedulerDetectionMu      sync.Mutex
+	schedulerDetection        map[string]*schedulerDetectionInfo
 
 	dedupeMu           sync.Mutex
 	recentDedupeByType map[string]map[string]time.Time
@@ -80,6 +91,8 @@ type Plugin struct {
 
 	jobsMu sync.RWMutex
 	jobs   map[string]*TrackedJob
+	// serialize stale job cleanup to avoid duplicate expirations
+	staleJobsMu sync.Mutex
 
 	jobDetailsMu sync.Mutex
 
@@ -138,6 +151,10 @@ func New(options Options) (*Plugin, error) {
 	if schedulerTick <= 0 {
 		schedulerTick = defaultSchedulerTick
 	}
+	idleSleepDuration := options.IdleSleepDuration
+	if idleSleepDuration <= 0 {
+		idleSleepDuration = defaultIdleSleepDuration
+	}
 
 	plugin := &Plugin{
 		store:                     store,
@@ -145,15 +162,17 @@ func New(options Options) (*Plugin, error) {
 		outgoingBuffer:            bufferSize,
 		sendTimeout:               sendTimeout,
 		schedulerTick:             schedulerTick,
+		idleSleepDuration:         idleSleepDuration,
 		clusterContextProvider:    options.ClusterContextProvider,
+		lockManager:               options.LockManager,
+		schedulerPhase:            "idle",
 		sessions:                  make(map[string]*streamSession),
 		pendingSchema:             make(map[string]chan *plugin_pb.ConfigSchemaResponse),
 		pendingDetection:          make(map[string]*pendingDetectionState),
 		pendingExecution:          make(map[string]chan *plugin_pb.JobCompleted),
-		nextDetectionAt:           make(map[string]time.Time),
-		detectionInFlight:         make(map[string]bool),
 		detectorLeases:            make(map[string]string),
 		schedulerExecReservations: make(map[string]int),
+		schedulerDetection:        make(map[string]*schedulerDetectionInfo),
 		recentDedupeByType:        make(map[string]map[string]time.Time),
 		jobs:                      make(map[string]*TrackedJob),
 		activities:                make([]JobActivity, 0, 256),
@@ -380,6 +399,13 @@ func (r *Plugin) BaseDir() string {
 	return r.store.BaseDir()
 }
 
+func (r *Plugin) acquireAdminLock(reason string) (func(), error) {
+	if r == nil || r.lockManager == nil {
+		return func() {}, nil
+	}
+	return r.lockManager.Acquire(reason)
+}
+
 // RunDetectionWithReport requests one detector worker and returns proposals with request metadata.
 func (r *Plugin) RunDetectionWithReport(
 	ctx context.Context,
@@ -387,6 +413,9 @@ func (r *Plugin) RunDetectionWithReport(
 	clusterContext *plugin_pb.ClusterContext,
 	maxResults int32,
 ) (*DetectionReport, error) {
+	releaseGate := r.acquireDetectionExecutionGate(jobType, false)
+	defer releaseGate()
+
 	detector, err := r.pickDetector(jobType)
 	if err != nil {
 		return nil, err
@@ -401,7 +430,10 @@ func (r *Plugin) RunDetectionWithReport(
 	if err != nil {
 		return nil, err
 	}
-	lastSuccessfulRun := r.loadLastSuccessfulRun(jobType)
+	lastCompletedRun := r.loadLastSuccessfulRun(jobType)
+	if strings.EqualFold(strings.TrimSpace(jobType), adminScriptJobType) {
+		lastCompletedRun = r.loadLastCompletedRun(jobType)
+	}
 
 	state := &pendingDetectionState{
 		complete: make(chan *plugin_pb.DetectionComplete, 1),
@@ -442,7 +474,7 @@ func (r *Plugin) RunDetectionWithReport(
 				AdminConfigValues:  adminConfigValues,
 				WorkerConfigValues: workerConfigValues,
 				ClusterContext:     clusterContext,
-				LastSuccessfulRun:  lastSuccessfulRun,
+				LastSuccessfulRun:  lastCompletedRun,
 				MaxResults:         maxResults,
 			},
 		},
@@ -529,11 +561,14 @@ func (r *Plugin) ExecuteJob(
 	if job == nil {
 		return nil, fmt.Errorf("job is nil")
 	}
-	if strings.TrimSpace(job.JobType) == "" {
+	jobType := strings.TrimSpace(job.JobType)
+	if jobType == "" {
 		return nil, fmt.Errorf("job_type is required")
 	}
+	releaseGate := r.acquireDetectionExecutionGate(jobType, true)
+	defer releaseGate()
 
-	executor, err := r.registry.PickExecutor(job.JobType)
+	executor, err := r.registry.PickExecutor(jobType)
 	if err != nil {
 		return nil, err
 	}
@@ -541,6 +576,23 @@ func (r *Plugin) ExecuteJob(
 	return r.executeJobWithExecutor(ctx, executor, job, clusterContext, attempt)
 }
 
+func (r *Plugin) acquireDetectionExecutionGate(jobType string, execution bool) func() {
+	normalizedJobType := strings.ToLower(strings.TrimSpace(jobType))
+	if execution && normalizedJobType == adminScriptJobType {
+		r.adminScriptRunMu.Lock()
+		return func() {
+			r.adminScriptRunMu.Unlock()
+		}
+	}
+	if normalizedJobType != adminScriptJobType {
+		r.adminScriptRunMu.RLock()
+		return func() {
+			r.adminScriptRunMu.RUnlock()
+		}
+	}
+	return func() {}
+}
+
 func (r *Plugin) executeJobWithExecutor(
 	ctx context.Context,
 	executor *WorkerSession,
@@ -1289,6 +1341,41 @@ func (r *Plugin) loadLastSuccessfulRun(jobType string) *timestamppb.Timestamp {
 	return timestamppb.New(latest.UTC())
 }
 
+func (r *Plugin) loadLastCompletedRun(jobType string) *timestamppb.Timestamp {
+	history, err := r.store.LoadRunHistory(jobType)
+	if err != nil {
+		glog.Warningf("Plugin failed to load run history for %s: %v", jobType, err)
+		return nil
+	}
+	if history == nil {
+		return nil
+	}
+
+	var latest time.Time
+	for i := range history.SuccessfulRuns {
+		completedAt := history.SuccessfulRuns[i].CompletedAt
+		if completedAt == nil || completedAt.IsZero() {
+			continue
+		}
+		if latest.IsZero() || completedAt.After(latest) {
+			latest = *completedAt
+		}
+	}
+	for i := range history.ErrorRuns {
+		completedAt := history.ErrorRuns[i].CompletedAt
+		if completedAt == nil || completedAt.IsZero() {
+			continue
+		}
+		if latest.IsZero() || completedAt.After(latest) {
+			latest = *completedAt
+		}
+	}
+	if latest.IsZero() {
+		return nil
+	}
+	return timestamppb.New(latest.UTC())
+}
+
 func CloneConfigValueMap(in map[string]*plugin_pb.ConfigValue) map[string]*plugin_pb.ConfigValue {
 	if len(in) == 0 {
 		return map[string]*plugin_pb.ConfigValue{}

weed/admin/plugin/plugin_cancel_test.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 func TestRunDetectionSendsCancelOnContextDone(t *testing.T) {
@@ -110,3 +112,165 @@ func TestExecuteJobSendsCancelOnContextDone(t *testing.T) {
 		t.Fatalf("expected context canceled error, got %v", runErr)
 	}
 }
+
+func TestAdminScriptExecutionBlocksOtherDetection(t *testing.T) {
+	pluginSvc, err := New(Options{})
+	if err != nil {
+		t.Fatalf("New plugin error: %v", err)
+	}
+	defer pluginSvc.Shutdown()
+
+	const adminWorkerID = "worker-admin-script"
+	const otherWorkerID = "worker-vacuum"
+
+	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
+		WorkerId: adminWorkerID,
+		Capabilities: []*plugin_pb.JobTypeCapability{
+			{JobType: "admin_script", CanExecute: true, MaxExecutionConcurrency: 1},
+		},
+	})
+	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
+		WorkerId: otherWorkerID,
+		Capabilities: []*plugin_pb.JobTypeCapability{
+			{JobType: "vacuum", CanDetect: true, MaxDetectionConcurrency: 1},
+		},
+	})
+	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
+	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
+	pluginSvc.putSession(adminSession)
+	pluginSvc.putSession(otherSession)
+
+	adminErrCh := make(chan error, 1)
+	go func() {
+		_, runErr := pluginSvc.ExecuteJob(context.Background(), &plugin_pb.JobSpec{
+			JobId:   "job-admin-script-1",
+			JobType: "admin_script",
+		}, &plugin_pb.ClusterContext{}, 1)
+		adminErrCh <- runErr
+	}()
+
+	adminExecMessage := <-adminSession.outgoing
+	if adminExecMessage.GetExecuteJobRequest() == nil {
+		t.Fatalf("expected admin_script execute request")
+	}
+
+	detectErrCh := make(chan error, 1)
+	go func() {
+		_, runErr := pluginSvc.RunDetection(context.Background(), "vacuum", &plugin_pb.ClusterContext{}, 10)
+		detectErrCh <- runErr
+	}()
+
+	select {
+	case unexpected := <-otherSession.outgoing:
+		t.Fatalf("expected vacuum detection to wait while admin_script runs, got message: %+v", unexpected)
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{
+		RequestId:   adminExecMessage.RequestId,
+		JobId:       "job-admin-script-1",
+		JobType:     "admin_script",
+		Success:     true,
+		CompletedAt: timestamppb.Now(),
+	})
+	if runErr := <-adminErrCh; runErr != nil {
+		t.Fatalf("admin_script ExecuteJob error: %v", runErr)
+	}
+
+	detectMessage := <-otherSession.outgoing
+	detectRequest := detectMessage.GetRunDetectionRequest()
+	if detectRequest == nil {
+		t.Fatalf("expected vacuum detection request after admin_script completion")
+	}
+	pluginSvc.handleDetectionComplete(otherWorkerID, &plugin_pb.DetectionComplete{
+		RequestId: detectMessage.RequestId,
+		JobType:   "vacuum",
+		Success:   true,
+	})
+	if runErr := <-detectErrCh; runErr != nil {
+		t.Fatalf("vacuum RunDetection error: %v", runErr)
+	}
+}
+
+func TestAdminScriptExecutionBlocksOtherExecution(t *testing.T) {
+	pluginSvc, err := New(Options{})
+	if err != nil {
+		t.Fatalf("New plugin error: %v", err)
+	}
+	defer pluginSvc.Shutdown()
+
+	const adminWorkerID = "worker-admin-script"
+	const otherWorkerID = "worker-vacuum"
+
+	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
+		WorkerId: adminWorkerID,
+		Capabilities: []*plugin_pb.JobTypeCapability{
+			{JobType: "admin_script", CanExecute: true, MaxExecutionConcurrency: 1},
+		},
+	})
+	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
+		WorkerId: otherWorkerID,
+		Capabilities: []*plugin_pb.JobTypeCapability{
+			{JobType: "vacuum", CanExecute: true, MaxExecutionConcurrency: 1},
+		},
+	})
+	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
+	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
+	pluginSvc.putSession(adminSession)
+	pluginSvc.putSession(otherSession)
+
+	adminErrCh := make(chan error, 1)
+	go func() {
+		_, runErr := pluginSvc.ExecuteJob(context.Background(), &plugin_pb.JobSpec{
+			JobId:   "job-admin-script-2",
+			JobType: "admin_script",
+		}, &plugin_pb.ClusterContext{}, 1)
+		adminErrCh <- runErr
+	}()
+
+	adminExecMessage := <-adminSession.outgoing
+	if adminExecMessage.GetExecuteJobRequest() == nil {
+		t.Fatalf("expected admin_script execute request")
+	}
+
+	otherErrCh := make(chan error, 1)
+	go func() {
+		_, runErr := pluginSvc.ExecuteJob(context.Background(), &plugin_pb.JobSpec{
+			JobId:   "job-vacuum-1",
+			JobType: "vacuum",
+		}, &plugin_pb.ClusterContext{}, 1)
+		otherErrCh <- runErr
+	}()
+
+	select {
+	case unexpected := <-otherSession.outgoing:
+		t.Fatalf("expected vacuum execute to wait while admin_script runs, got message: %+v", unexpected)
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{
+		RequestId:   adminExecMessage.RequestId,
+		JobId:       "job-admin-script-2",
+		JobType:     "admin_script",
+		Success:     true,
+		CompletedAt: timestamppb.Now(),
+	})
+	if runErr := <-adminErrCh; runErr != nil {
+		t.Fatalf("admin_script ExecuteJob error: %v", runErr)
+	}
+
+	otherExecMessage := <-otherSession.outgoing
+	if otherExecMessage.GetExecuteJobRequest() == nil {
+		t.Fatalf("expected vacuum execute request after admin_script completion")
+	}
+	pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{
+		RequestId:   otherExecMessage.RequestId,
+		JobId:       "job-vacuum-1",
+		JobType:     "vacuum",
+		Success:     true,
+		CompletedAt: timestamppb.Now(),
+	})
+	if runErr := <-otherErrCh; runErr != nil {
+		t.Fatalf("vacuum ExecuteJob error: %v", runErr)
+	}
+}

weed/admin/plugin/plugin_detection_test.go

@@ -195,3 +195,64 @@ func TestRunDetectionWithReportCapturesDetectionActivities(t *testing.T) {
 		t.Fatalf("expected requested/proposal/completed activities, got stages=%v", stages)
 	}
 }
+
+func TestRunDetectionAdminScriptUsesLastCompletedRun(t *testing.T) {
+	pluginSvc, err := New(Options{})
+	if err != nil {
+		t.Fatalf("New plugin error: %v", err)
+	}
+	defer pluginSvc.Shutdown()
+
+	jobType := "admin_script"
+	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
+		WorkerId: "worker-admin-script",
+		Capabilities: []*plugin_pb.JobTypeCapability{
+			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
+		},
+	})
+	session := &streamSession{workerID: "worker-admin-script", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
+	pluginSvc.putSession(session)
+
+	successCompleted := time.Date(2026, 2, 1, 10, 0, 0, 0, time.UTC)
+	errorCompleted := successCompleted.Add(45 * time.Minute)
+	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
+		Outcome:     RunOutcomeSuccess,
+		CompletedAt: timeToPtr(successCompleted),
+	}); err != nil {
+		t.Fatalf("AppendRunRecord success run: %v", err)
+	}
+	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
+		Outcome:     RunOutcomeError,
+		CompletedAt: timeToPtr(errorCompleted),
+	}); err != nil {
+		t.Fatalf("AppendRunRecord error run: %v", err)
+	}
+
+	errCh := make(chan error, 1)
+	go func() {
+		_, runErr := pluginSvc.RunDetection(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10)
+		errCh <- runErr
+	}()
+
+	message := <-session.outgoing
+	detectRequest := message.GetRunDetectionRequest()
+	if detectRequest == nil {
+		t.Fatalf("expected run detection request message")
+	}
+	if detectRequest.LastSuccessfulRun == nil {
+		t.Fatalf("expected last_successful_run to be set")
+	}
+	if got := detectRequest.LastSuccessfulRun.AsTime().UTC(); !got.Equal(errorCompleted) {
+		t.Fatalf("unexpected last_successful_run, got=%s want=%s", got, errorCompleted)
+	}
+
+	pluginSvc.handleDetectionComplete("worker-admin-script", &plugin_pb.DetectionComplete{
+		RequestId: message.RequestId,
+		JobType:   jobType,
+		Success:   true,
+	})
+
+	if runErr := <-errCh; runErr != nil {
+		t.Fatalf("RunDetection error: %v", runErr)
+	}
+}

weed/admin/plugin/plugin_monitor.go

@@ -2,6 +2,7 @@ package plugin
 
 import (
 	"encoding/json"
+	"fmt"
 	"sort"
 	"strings"
 	"time"
@@ -9,12 +10,18 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
 	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 const (
 	maxTrackedJobsTotal = 1000
 	maxActivityRecords  = 4000
 	maxRelatedJobs      = 100
+
+	// stale active jobs block dedupe and scheduling; use generous defaults to
+	// avoid expiring legitimate long-running tasks.
+	defaultStaleActiveJobTimeout    = 24 * time.Hour
+	defaultOrphanedActiveJobTimeout = 15 * time.Minute
 )
 
 var (
@@ -23,6 +30,14 @@ var (
 	StateCanceled  = strings.ToLower(plugin_pb.JobState_JOB_STATE_CANCELED.String())
 )
 
+type activeJobSnapshot struct {
+	jobID      string
+	jobType    string
+	workerID   string
+	requestID  string
+	lastUpdate time.Time
+}
+
 // activityLess reports whether activity a occurred after activity b (newest-first order).
 // A nil OccurredAt is treated as the zero time.
 func activityLess(a, b JobActivity) bool {
@@ -54,6 +69,13 @@ func (r *Plugin) loadPersistedMonitorState() error {
 			if strings.TrimSpace(job.JobID) == "" {
 				continue
 			}
+			if isActiveTrackedJobState(job.State) {
+				if detail, detailErr := r.store.LoadJobDetail(job.JobID); detailErr != nil {
+					glog.Warningf("Plugin failed to load detail snapshot for job %s: %v", job.JobID, detailErr)
+				} else if detail != nil {
+					mergeTerminalDetailIntoTracked(&job, detail)
+				}
+			}
 			// Backward compatibility: migrate older inline detail payloads
 			// out of tracked_jobs.json into dedicated per-job detail files.
 			if hasTrackedJobRichDetails(job) {
@@ -81,6 +103,265 @@ func (r *Plugin) loadPersistedMonitorState() error {
 	return nil
 }
 
+// ExpireJob marks an active job as failed so it no longer blocks scheduling.
+func (r *Plugin) ExpireJob(jobID, reason string) (*TrackedJob, bool, error) {
+	normalizedJobID := strings.TrimSpace(jobID)
+	if normalizedJobID == "" {
+		return nil, false, ErrJobNotFound
+	}
+
+	reason = strings.TrimSpace(reason)
+	if reason == "" {
+		reason = "job expired by admin request"
+	}
+
+	var jobType string
+	var requestID string
+	active := false
+
+	r.jobsMu.RLock()
+	if tracked := r.jobs[normalizedJobID]; tracked != nil {
+		jobType = tracked.JobType
+		requestID = tracked.RequestID
+		active = isActiveTrackedJobState(tracked.State)
+	}
+	r.jobsMu.RUnlock()
+
+	if jobType == "" || requestID == "" || !active {
+		if detail, err := r.store.LoadJobDetail(normalizedJobID); err != nil {
+			return nil, false, err
+		} else if detail != nil {
+			if jobType == "" {
+				jobType = detail.JobType
+			}
+			if requestID == "" {
+				requestID = detail.RequestID
+			}
+			if !active && isActiveTrackedJobState(detail.State) {
+				active = true
+			}
+		}
+	}
+
+	if jobType == "" {
+		return nil, false, ErrJobNotFound
+	}
+
+	if !active {
+		current, _ := r.GetTrackedJob(normalizedJobID)
+		if current == nil {
+			if detail, err := r.store.LoadJobDetail(normalizedJobID); err == nil && detail != nil {
+				clone := cloneTrackedJob(*detail)
+				current = &clone
+			}
+		}
+		return current, false, nil
+	}
+
+	now := time.Now().UTC()
+	r.handleJobCompleted(&plugin_pb.JobCompleted{
+		JobId:        normalizedJobID,
+		JobType:      jobType,
+		RequestId:    requestID,
+		Success:      false,
+		ErrorMessage: reason,
+		CompletedAt:  timestamppb.New(now),
+	})
+	r.appendActivity(JobActivity{
+		JobID:      normalizedJobID,
+		JobType:    jobType,
+		RequestID:  requestID,
+		Source:     "admin_expire",
+		Message:    reason,
+		Stage:      "expired",
+		OccurredAt: timeToPtr(now),
+	})
+
+	updated, _ := r.GetTrackedJob(normalizedJobID)
+	return updated, true, nil
+}
+
+// expireStaleJobs marks stale active jobs as failed so they stop blocking new work.
+func (r *Plugin) expireStaleJobs(now time.Time) int {
+	if now.IsZero() {
+		now = time.Now().UTC()
+	}
+
+	r.staleJobsMu.Lock()
+	defer r.staleJobsMu.Unlock()
+
+	snapshots := r.snapshotActiveJobs()
+	if len(snapshots) == 0 {
+		return 0
+	}
+
+	expired := 0
+	for _, snap := range snapshots {
+		if snap.lastUpdate.IsZero() {
+			continue
+		}
+		if stale, _, _ := r.evaluateStaleJob(now, snap.workerID, snap.lastUpdate); !stale {
+			continue
+		}
+
+		reason := r.confirmStaleReason(now, snap.jobID)
+		if reason == "" {
+			continue
+		}
+
+		r.handleJobCompleted(&plugin_pb.JobCompleted{
+			JobId:        snap.jobID,
+			JobType:      snap.jobType,
+			RequestId:    snap.requestID,
+			Success:      false,
+			ErrorMessage: reason,
+			CompletedAt:  timestamppb.New(now),
+		})
+		expired++
+	}
+
+	return expired
+}
+
+func (r *Plugin) snapshotActiveJobs() []activeJobSnapshot {
+	r.jobsMu.RLock()
+	defer r.jobsMu.RUnlock()
+
+	if len(r.jobs) == 0 {
+		return nil
+	}
+
+	out := make([]activeJobSnapshot, 0, len(r.jobs))
+	for _, job := range r.jobs {
+		if job == nil {
+			continue
+		}
+		if !isActiveTrackedJobState(job.State) {
+			continue
+		}
+		out = append(out, activeJobSnapshot{
+			jobID:      job.JobID,
+			jobType:    job.JobType,
+			workerID:   job.WorkerID,
+			requestID:  job.RequestID,
+			lastUpdate: jobLastUpdated(job),
+		})
+	}
+	return out
+}
+
+func jobLastUpdated(job *TrackedJob) time.Time {
+	if job == nil {
+		return time.Time{}
+	}
+	if job.UpdatedAt != nil && !job.UpdatedAt.IsZero() {
+		return *job.UpdatedAt
+	}
+	if job.CreatedAt != nil && !job.CreatedAt.IsZero() {
+		return *job.CreatedAt
+	}
+	return time.Time{}
+}
+
+func (r *Plugin) evaluateStaleJob(now time.Time, workerID string, lastUpdate time.Time) (bool, time.Duration, string) {
+	if lastUpdate.IsZero() {
+		return false, 0, ""
+	}
+
+	timeout := defaultStaleActiveJobTimeout
+	reason := fmt.Sprintf("job expired after %s without progress", timeout)
+
+	workerID = strings.TrimSpace(workerID)
+	if workerID == "" {
+		reason = fmt.Sprintf("job expired after %s without executor assignment", timeout)
+	} else if !r.isWorkerAvailable(workerID) {
+		timeout = defaultOrphanedActiveJobTimeout
+		reason = fmt.Sprintf("job expired after %s without worker heartbeat (worker=%s)", timeout, workerID)
+	}
+
+	if now.Sub(lastUpdate) < timeout {
+		return false, timeout, reason
+	}
+	return true, timeout, reason
+}
+
+func (r *Plugin) confirmStaleReason(now time.Time, jobID string) string {
+	r.jobsMu.RLock()
+	job := r.jobs[jobID]
+	if job == nil || !isActiveTrackedJobState(job.State) {
+		r.jobsMu.RUnlock()
+		return ""
+	}
+	lastUpdate := jobLastUpdated(job)
+	workerID := job.WorkerID
+	r.jobsMu.RUnlock()
+
+	stale, _, reason := r.evaluateStaleJob(now, workerID, lastUpdate)
+	if !stale {
+		return ""
+	}
+	return reason
+}
+
+func (r *Plugin) isWorkerAvailable(workerID string) bool {
+	workerID = strings.TrimSpace(workerID)
+	if workerID == "" {
+		return false
+	}
+	_, ok := r.registry.Get(workerID)
+	return ok
+}
+
+func isTerminalTrackedJobState(state string) bool {
+	normalized := strings.ToLower(strings.TrimSpace(state))
+	switch normalized {
+	case StateSucceeded, StateFailed, StateCanceled:
+		return true
+	default:
+		return false
+	}
+}
+
+func mergeTerminalDetailIntoTracked(tracked *TrackedJob, detail *TrackedJob) {
+	if tracked == nil || detail == nil {
+		return
+	}
+	if !isTerminalTrackedJobState(detail.State) {
+		return
+	}
+	if !isActiveTrackedJobState(tracked.State) {
+		return
+	}
+
+	if detail.State != "" {
+		tracked.State = detail.State
+	}
+	if detail.Progress != 0 {
+		tracked.Progress = detail.Progress
+	}
+	if detail.Stage != "" {
+		tracked.Stage = detail.Stage
+	}
+	if detail.Message != "" {
+		tracked.Message = detail.Message
+	}
+	if detail.ErrorMessage != "" {
+		tracked.ErrorMessage = detail.ErrorMessage
+	}
+	if detail.ResultSummary != "" {
+		tracked.ResultSummary = detail.ResultSummary
+	}
+	if detail.CompletedAt != nil && !detail.CompletedAt.IsZero() {
+		tracked.CompletedAt = detail.CompletedAt
+	}
+	if detail.UpdatedAt != nil && !detail.UpdatedAt.IsZero() {
+		tracked.UpdatedAt = detail.UpdatedAt
+	}
+	if tracked.UpdatedAt == nil && tracked.CompletedAt != nil {
+		tracked.UpdatedAt = tracked.CompletedAt
+	}
+}
+
 func (r *Plugin) ListTrackedJobs(jobType string, state string, limit int) []TrackedJob {
 	r.jobsMu.RLock()
 	defer r.jobsMu.RUnlock()

weed/admin/plugin/plugin_scheduler.go

@@ -17,7 +17,8 @@ var errExecutorAtCapacity = errors.New("executor is at capacity")
 
 const (
 	defaultSchedulerTick                       = 5 * time.Second
-	defaultScheduledDetectionInterval          = 300 * time.Second
+	defaultIdleSleepDuration                   = 17 * time.Minute
+	defaultMaxJobTypeDuration                  = 30 * time.Minute
 	defaultScheduledDetectionTimeout           = 45 * time.Second
 	defaultScheduledExecutionTimeout           = 90 * time.Second
 	defaultScheduledMaxResults           int32 = 1000
@@ -31,9 +32,9 @@ const (
 )
 
 type schedulerPolicy struct {
-	DetectionInterval      time.Duration
 	DetectionTimeout       time.Duration
 	ExecutionTimeout       time.Duration
+	MaxJobTypeDuration     time.Duration
 	RetryBackoff           time.Duration
 	MaxResults             int32
 	ExecutionConcurrency   int
@@ -42,31 +43,63 @@ type schedulerPolicy struct {
 	ExecutorReserveBackoff time.Duration
 }
 
+func (r *Plugin) setSchedulerPhase(phase, jobType string) {
+	r.schedulerMu.Lock()
+	r.schedulerPhase = phase
+	r.currentJobType = jobType
+	r.schedulerMu.Unlock()
+}
+
 func (r *Plugin) schedulerLoop() {
 	defer r.wg.Done()
-	ticker := time.NewTicker(r.schedulerTick)
-	defer ticker.Stop()
 
 	// Try once immediately on startup.
-	r.runSchedulerTick()
+	workDetected := r.runSchedulerIteration()
 
 	for {
+		if workDetected {
+			// Immediate re-iteration when work was found.
+			select {
+			case <-r.shutdownCh:
+				return
+			default:
+			}
+		} else {
+			r.setSchedulerPhase("idle", "")
+			if !waitForShutdownOrTimer(r.shutdownCh, r.idleSleepDuration) {
+				return
+			}
+		}
+
 		select {
 		case <-r.shutdownCh:
 			return
-		case <-ticker.C:
-			r.runSchedulerTick()
+		default:
 		}
+
+		workDetected = r.runSchedulerIteration()
 	}
 }
 
-func (r *Plugin) runSchedulerTick() {
+func (r *Plugin) runSchedulerIteration() bool {
+	now := time.Now().UTC()
+
+	r.schedulerMu.Lock()
+	r.iterationStartedAt = now
+	r.schedulerMu.Unlock()
+
+	r.expireStaleJobs(now)
+
 	jobTypes := r.registry.DetectableJobTypes()
 	if len(jobTypes) == 0 {
-		return
+		r.finishIteration(false)
+		return false
 	}
 
 	active := make(map[string]struct{}, len(jobTypes))
+	enabledJobTypes := make([]string, 0, len(jobTypes))
+	policies := make(map[string]schedulerPolicy, len(jobTypes))
+
 	for _, jobType := range jobTypes {
 		active[jobType] = struct{}{}
 
@@ -80,19 +113,171 @@ func (r *Plugin) runSchedulerTick() {
 			continue
 		}
 
-		if !r.markDetectionDue(jobType, policy.DetectionInterval) {
-			continue
-		}
-
-		r.wg.Add(1)
-		go func(jt string, p schedulerPolicy) {
-			defer r.wg.Done()
-			r.runScheduledDetection(jt, p)
-		}(jobType, policy)
+		enabledJobTypes = append(enabledJobTypes, jobType)
+		policies[jobType] = policy
 	}
 
-	r.pruneSchedulerState(active)
 	r.pruneDetectorLeases(active)
+
+	if len(enabledJobTypes) == 0 {
+		r.finishIteration(false)
+		return false
+	}
+
+	// Acquire the lock ONCE for the entire iteration.
+	r.setSchedulerPhase("acquiring_lock", "")
+	releaseLock, lockErr := r.acquireAdminLock("plugin scheduler iteration")
+	if lockErr != nil {
+		glog.Warningf("Plugin scheduler failed to acquire lock: %v", lockErr)
+		r.appendActivity(JobActivity{
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduler iteration aborted: failed to acquire lock: %v", lockErr),
+			Stage:      "failed",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		r.finishIteration(false)
+		return false
+	}
+	defer releaseLock()
+
+	// Load cluster context ONCE for all job types.
+	clusterContext, err := r.loadSchedulerClusterContext()
+	if err != nil {
+		r.appendActivity(JobActivity{
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduler iteration aborted: %v", err),
+			Stage:      "failed",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		r.finishIteration(false)
+		return false
+	}
+
+	// Process each job type sequentially.
+	anyWorkDetected := false
+	for _, jobType := range enabledJobTypes {
+		select {
+		case <-r.shutdownCh:
+			r.finishIteration(anyWorkDetected)
+			return anyWorkDetected
+		default:
+		}
+
+		policy := policies[jobType]
+		r.setSchedulerPhase("processing", jobType)
+
+		if r.runJobTypeIteration(jobType, policy, clusterContext) {
+			anyWorkDetected = true
+		}
+	}
+
+	r.finishIteration(anyWorkDetected)
+	return anyWorkDetected
+}
+
+func (r *Plugin) finishIteration(workDetected bool) {
+	now := time.Now().UTC()
+	r.schedulerMu.Lock()
+	r.lastIterationEndedAt = now
+	r.lastIterationWorkDetected = workDetected
+	r.currentJobType = ""
+	r.schedulerPhase = "idle"
+	r.schedulerMu.Unlock()
+}
+
+func (r *Plugin) runJobTypeIteration(
+	jobType string,
+	policy schedulerPolicy,
+	clusterContext *plugin_pb.ClusterContext,
+) bool {
+	budget := policy.MaxJobTypeDuration
+	if budget <= 0 {
+		budget = defaultMaxJobTypeDuration
+	}
+	ctx, cancel := context.WithTimeout(r.ctx, budget)
+	defer cancel()
+
+	start := time.Now().UTC()
+	r.appendActivity(JobActivity{
+		JobType:    jobType,
+		Source:     "admin_scheduler",
+		Message:    "scheduled detection started",
+		Stage:      "detecting",
+		OccurredAt: timeToPtr(start),
+	})
+
+	if skip, waitingCount, waitingThreshold := r.shouldSkipDetectionForWaitingJobs(jobType, policy); skip {
+		r.recordSchedulerDetectionSkip(jobType, fmt.Sprintf("waiting backlog %d reached threshold %d", waitingCount, waitingThreshold))
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduled detection skipped: waiting backlog %d reached threshold %d", waitingCount, waitingThreshold),
+			Stage:      "skipped_waiting_backlog",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		return false
+	}
+
+	detectionTimeout := policy.DetectionTimeout
+	if detectionTimeout <= 0 {
+		detectionTimeout = defaultScheduledDetectionTimeout
+	}
+	detCtx, detCancel := context.WithTimeout(ctx, detectionTimeout)
+	proposals, err := r.RunDetection(detCtx, jobType, clusterContext, policy.MaxResults)
+	detCancel()
+	if err != nil {
+		r.recordSchedulerDetectionError(jobType, err)
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduled detection failed: %v", err),
+			Stage:      "failed",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		return false
+	}
+
+	r.appendActivity(JobActivity{
+		JobType:    jobType,
+		Source:     "admin_scheduler",
+		Message:    fmt.Sprintf("scheduled detection completed: %d proposal(s)", len(proposals)),
+		Stage:      "detected",
+		OccurredAt: timeToPtr(time.Now().UTC()),
+	})
+	r.recordSchedulerDetectionSuccess(jobType, len(proposals))
+
+	filteredByActive, skippedActive := r.filterProposalsWithActiveJobs(jobType, proposals)
+	if skippedActive > 0 {
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduled detection skipped %d proposal(s) due to active assigned/running jobs", skippedActive),
+			Stage:      "deduped_active_jobs",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+	}
+
+	if len(filteredByActive) == 0 {
+		return false
+	}
+
+	filtered := r.filterScheduledProposals(filteredByActive)
+	if len(filtered) != len(filteredByActive) {
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduled detection deduped %d proposal(s) within this run", len(filteredByActive)-len(filtered)),
+			Stage:      "deduped",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+	}
+
+	if len(filtered) == 0 {
+		return false
+	}
+
+	r.dispatchScheduledProposals(ctx, jobType, filtered, clusterContext, policy)
+	return true
 }
 
 func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, error) {
@@ -114,9 +299,9 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
 	}
 
 	policy := schedulerPolicy{
-		DetectionInterval:      durationFromSeconds(adminRuntime.DetectionIntervalSeconds, defaultScheduledDetectionInterval),
 		DetectionTimeout:       durationFromSeconds(adminRuntime.DetectionTimeoutSeconds, defaultScheduledDetectionTimeout),
 		ExecutionTimeout:       defaultScheduledExecutionTimeout,
+		MaxJobTypeDuration:     defaultMaxJobTypeDuration,
 		RetryBackoff:           durationFromSeconds(adminRuntime.RetryBackoffSeconds, defaultScheduledRetryBackoff),
 		MaxResults:             adminRuntime.MaxJobsPerDetection,
 		ExecutionConcurrency:   int(adminRuntime.GlobalExecutionConcurrency),
@@ -125,9 +310,6 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
 		ExecutorReserveBackoff: 200 * time.Millisecond,
 	}
 
-	if policy.DetectionInterval < r.schedulerTick {
-		policy.DetectionInterval = r.schedulerTick
-	}
 	if policy.MaxResults <= 0 {
 		policy.MaxResults = defaultScheduledMaxResults
 	}
@@ -164,28 +346,18 @@ func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
 	}
 
 	r.schedulerMu.Lock()
-	nextDetectionAt := make(map[string]time.Time, len(r.nextDetectionAt))
-	for jobType, nextRun := range r.nextDetectionAt {
-		nextDetectionAt[jobType] = nextRun
-	}
-	detectionInFlight := make(map[string]bool, len(r.detectionInFlight))
-	for jobType, inFlight := range r.detectionInFlight {
-		detectionInFlight[jobType] = inFlight
-	}
+	currentJobType := r.currentJobType
 	r.schedulerMu.Unlock()
 
 	states := make([]SchedulerJobTypeState, 0, len(jobTypes))
 	for _, jobTypeInfo := range jobTypes {
 		jobType := jobTypeInfo.JobType
 		state := SchedulerJobTypeState{
-			JobType:           jobType,
-			DetectionInFlight: detectionInFlight[jobType],
+			JobType: jobType,
 		}
 
-		if nextRun, ok := nextDetectionAt[jobType]; ok && !nextRun.IsZero() {
-			nextRunUTC := nextRun.UTC()
-			state.NextDetectionAt = &nextRunUTC
-		}
+		// Mark as in-flight if this job type is being processed right now.
+		state.DetectionInFlight = jobType == currentJobType && currentJobType != ""
 
 		policy, enabled, loadErr := r.loadSchedulerPolicy(jobType)
 
@@ -194,9 +366,9 @@ func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
 		} else {
 			state.Enabled = enabled
 			if enabled {
-				state.DetectionIntervalSeconds = secondsFromDuration(policy.DetectionInterval)
 				state.DetectionTimeoutSeconds = secondsFromDuration(policy.DetectionTimeout)
 				state.ExecutionTimeoutSeconds = secondsFromDuration(policy.ExecutionTimeout)
+				state.MaxJobTypeDurationSeconds = secondsFromDuration(policy.MaxJobTypeDuration)
 				state.MaxJobsPerDetection = policy.MaxResults
 				state.GlobalExecutionConcurrency = policy.ExecutionConcurrency
 				state.PerWorkerExecutionConcurrency = policy.PerWorkerConcurrency
@@ -259,49 +431,7 @@ func deriveSchedulerAdminRuntime(
 	}
 }
 
-func (r *Plugin) markDetectionDue(jobType string, interval time.Duration) bool {
-	now := time.Now().UTC()
-
-	r.schedulerMu.Lock()
-	defer r.schedulerMu.Unlock()
-
-	if r.detectionInFlight[jobType] {
-		return false
-	}
-
-	nextRun, exists := r.nextDetectionAt[jobType]
-	if exists && now.Before(nextRun) {
-		return false
-	}
-
-	r.nextDetectionAt[jobType] = now.Add(interval)
-	r.detectionInFlight[jobType] = true
-	return true
-}
-
-func (r *Plugin) finishDetection(jobType string) {
-	r.schedulerMu.Lock()
-	delete(r.detectionInFlight, jobType)
-	r.schedulerMu.Unlock()
-}
-
-func (r *Plugin) pruneSchedulerState(activeJobTypes map[string]struct{}) {
-	r.schedulerMu.Lock()
-	defer r.schedulerMu.Unlock()
-
-	for jobType := range r.nextDetectionAt {
-		if _, ok := activeJobTypes[jobType]; !ok {
-			delete(r.nextDetectionAt, jobType)
-			delete(r.detectionInFlight, jobType)
-		}
-	}
-}
-
 func (r *Plugin) clearSchedulerJobType(jobType string) {
-	r.schedulerMu.Lock()
-	delete(r.nextDetectionAt, jobType)
-	delete(r.detectionInFlight, jobType)
-	r.schedulerMu.Unlock()
 	r.clearDetectorLease(jobType, "")
 }
 
@@ -316,96 +446,6 @@ func (r *Plugin) pruneDetectorLeases(activeJobTypes map[string]struct{}) {
 	}
 }
 
-func (r *Plugin) runScheduledDetection(jobType string, policy schedulerPolicy) {
-	defer r.finishDetection(jobType)
-
-	start := time.Now().UTC()
-	r.appendActivity(JobActivity{
-		JobType:    jobType,
-		Source:     "admin_scheduler",
-		Message:    "scheduled detection started",
-		Stage:      "detecting",
-		OccurredAt: timeToPtr(start),
-	})
-
-	if skip, waitingCount, waitingThreshold := r.shouldSkipDetectionForWaitingJobs(jobType, policy); skip {
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection skipped: waiting backlog %d reached threshold %d", waitingCount, waitingThreshold),
-			Stage:      "skipped_waiting_backlog",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-		return
-	}
-
-	clusterContext, err := r.loadSchedulerClusterContext()
-	if err != nil {
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection aborted: %v", err),
-			Stage:      "failed",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-		return
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), policy.DetectionTimeout)
-	proposals, err := r.RunDetection(ctx, jobType, clusterContext, policy.MaxResults)
-	cancel()
-	if err != nil {
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection failed: %v", err),
-			Stage:      "failed",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-		return
-	}
-
-	r.appendActivity(JobActivity{
-		JobType:    jobType,
-		Source:     "admin_scheduler",
-		Message:    fmt.Sprintf("scheduled detection completed: %d proposal(s)", len(proposals)),
-		Stage:      "detected",
-		OccurredAt: timeToPtr(time.Now().UTC()),
-	})
-
-	filteredByActive, skippedActive := r.filterProposalsWithActiveJobs(jobType, proposals)
-	if skippedActive > 0 {
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection skipped %d proposal(s) due to active assigned/running jobs", skippedActive),
-			Stage:      "deduped_active_jobs",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-	}
-
-	if len(filteredByActive) == 0 {
-		return
-	}
-
-	filtered := r.filterScheduledProposals(filteredByActive)
-	if len(filtered) != len(filteredByActive) {
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection deduped %d proposal(s) within this run", len(filteredByActive)-len(filtered)),
-			Stage:      "deduped",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-	}
-
-	if len(filtered) == 0 {
-		return
-	}
-
-	r.dispatchScheduledProposals(jobType, filtered, clusterContext, policy)
-}
-
 func (r *Plugin) loadSchedulerClusterContext() (*plugin_pb.ClusterContext, error) {
 	if r.clusterContextProvider == nil {
 		return nil, fmt.Errorf("cluster context provider is not configured")
@@ -425,6 +465,7 @@ func (r *Plugin) loadSchedulerClusterContext() (*plugin_pb.ClusterContext, error
 }
 
 func (r *Plugin) dispatchScheduledProposals(
+	parentCtx context.Context,
 	jobType string,
 	proposals []*plugin_pb.JobProposal,
 	clusterContext *plugin_pb.ClusterContext,
@@ -438,6 +479,9 @@ func (r *Plugin) dispatchScheduledProposals(
 		case <-r.shutdownCh:
 			close(jobQueue)
 			return
+		case <-parentCtx.Done():
+			close(jobQueue)
+			return
 		default:
 			jobQueue <- job
 		}
@@ -463,6 +507,8 @@ func (r *Plugin) dispatchScheduledProposals(
 				select {
 				case <-r.shutdownCh:
 					return
+				case <-parentCtx.Done():
+					return
 				default:
 				}
 
@@ -470,10 +516,12 @@ func (r *Plugin) dispatchScheduledProposals(
 					select {
 					case <-r.shutdownCh:
 						return
+					case <-parentCtx.Done():
+						return
 					default:
 					}
 
-					executor, release, reserveErr := r.reserveScheduledExecutor(jobType, policy)
+					executor, release, reserveErr := r.reserveScheduledExecutor(parentCtx, jobType, policy)
 					if reserveErr != nil {
 						select {
 						case <-r.shutdownCh:
@@ -493,7 +541,7 @@ func (r *Plugin) dispatchScheduledProposals(
 						break
 					}
 
-					err := r.executeScheduledJobWithExecutor(executor, job, clusterContext, policy)
+					err := r.executeScheduledJobWithExecutor(parentCtx, executor, job, clusterContext, policy)
 					release()
 					if errors.Is(err, errExecutorAtCapacity) {
 						r.trackExecutionQueued(job)
@@ -538,6 +586,7 @@ func (r *Plugin) dispatchScheduledProposals(
 }
 
 func (r *Plugin) reserveScheduledExecutor(
+	ctx context.Context,
 	jobType string,
 	policy schedulerPolicy,
 ) (*WorkerSession, func(), error) {
@@ -550,6 +599,8 @@ func (r *Plugin) reserveScheduledExecutor(
 		select {
 		case <-r.shutdownCh:
 			return nil, nil, fmt.Errorf("plugin is shutting down")
+		case <-ctx.Done():
+			return nil, nil, ctx.Err()
 		default:
 		}
 
@@ -559,8 +610,8 @@ func (r *Plugin) reserveScheduledExecutor(
 
 		executors, err := r.registry.ListExecutors(jobType)
 		if err != nil {
-			if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
-				return nil, nil, fmt.Errorf("plugin is shutting down")
+			if !waitForShutdownOrCtx(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
+				return nil, nil, fmt.Errorf("plugin is shutting down or context canceled")
 			}
 			continue
 		}
@@ -573,8 +624,8 @@ func (r *Plugin) reserveScheduledExecutor(
 			return executor, release, nil
 		}
 
-		if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
-			return nil, nil, fmt.Errorf("plugin is shutting down")
+		if !waitForShutdownOrCtx(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
+			return nil, nil, fmt.Errorf("plugin is shutting down or context canceled")
 		}
 	}
 }
@@ -658,6 +709,7 @@ func schedulerWorkerExecutionLimit(executor *WorkerSession, jobType string, poli
 }
 
 func (r *Plugin) executeScheduledJobWithExecutor(
+	parentCtx context.Context,
 	executor *WorkerSession,
 	job *plugin_pb.JobSpec,
 	clusterContext *plugin_pb.ClusterContext,
@@ -673,10 +725,12 @@ func (r *Plugin) executeScheduledJobWithExecutor(
 		select {
 		case <-r.shutdownCh:
 			return fmt.Errorf("plugin is shutting down")
+		case <-parentCtx.Done():
+			return parentCtx.Err()
 		default:
 		}
 
-		execCtx, cancel := context.WithTimeout(context.Background(), policy.ExecutionTimeout)
+		execCtx, cancel := context.WithTimeout(parentCtx, policy.ExecutionTimeout)
 		_, err := r.executeJobWithExecutor(execCtx, executor, job, clusterContext, int32(attempt))
 		cancel()
 		if err == nil {
@@ -696,8 +750,8 @@ func (r *Plugin) executeScheduledJobWithExecutor(
 				Stage:      "retry",
 				OccurredAt: timeToPtr(time.Now().UTC()),
 			})
-			if !waitForShutdownOrTimer(r.shutdownCh, policy.RetryBackoff) {
-				return fmt.Errorf("plugin is shutting down")
+			if !waitForShutdownOrCtx(r.shutdownCh, parentCtx, policy.RetryBackoff) {
+				return fmt.Errorf("plugin is shutting down or context canceled")
 			}
 		}
 	}
@@ -839,11 +893,34 @@ func waitForShutdownOrTimer(shutdown <-chan struct{}, duration time.Duration) bo
 	}
 }
 
+func waitForShutdownOrCtx(shutdown <-chan struct{}, ctx context.Context, duration time.Duration) bool {
+	if duration <= 0 {
+		return true
+	}
+
+	timer := time.NewTimer(duration)
+	defer timer.Stop()
+
+	select {
+	case <-shutdown:
+		return false
+	case <-ctx.Done():
+		return false
+	case <-timer.C:
+		return true
+	}
+}
+
+// filterProposalsWithActiveJobs removes proposals whose dedupe keys already have active jobs.
+// It first expires stale tracked jobs via expireStaleJobs, which can mutate scheduler state,
+// so callers should treat this method as a stateful operation.
 func (r *Plugin) filterProposalsWithActiveJobs(jobType string, proposals []*plugin_pb.JobProposal) ([]*plugin_pb.JobProposal, int) {
 	if len(proposals) == 0 {
 		return proposals, 0
 	}
 
+	r.expireStaleJobs(time.Now().UTC())
+
 	activeKeys := make(map[string]struct{})
 	r.jobsMu.RLock()
 	for _, job := range r.jobs {

weed/admin/plugin/plugin_scheduler_test.go

@@ -1,7 +1,9 @@
 package plugin
 
 import (
+	"context"
 	"fmt"
+	"sync"
 	"testing"
 	"time"
 
@@ -53,6 +55,9 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
 	if policy.RetryLimit != 4 {
 		t.Fatalf("unexpected retry limit: got=%d", policy.RetryLimit)
 	}
+	if policy.MaxJobTypeDuration != defaultMaxJobTypeDuration {
+		t.Fatalf("unexpected max job type duration: got=%v", policy.MaxJobTypeDuration)
+	}
 }
 
 func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T) {
@@ -126,13 +131,13 @@ func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) {
 		ExecutorReserveBackoff: time.Millisecond,
 	}
 
-	executor1, release1, err := pluginSvc.reserveScheduledExecutor("balance", policy)
+	executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
 	if err != nil {
 		t.Fatalf("reserve executor 1: %v", err)
 	}
 	defer release1()
 
-	executor2, release2, err := pluginSvc.reserveScheduledExecutor("balance", policy)
+	executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
 	if err != nil {
 		t.Fatalf("reserve executor 2: %v", err)
 	}
@@ -254,7 +259,7 @@ func TestReserveScheduledExecutorTimesOutWhenNoExecutor(t *testing.T) {
 
 	start := time.Now()
 	pluginSvc.Shutdown()
-	_, _, err = pluginSvc.reserveScheduledExecutor("missing-job-type", policy)
+	_, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy)
 	if err == nil {
 		t.Fatalf("expected reservation shutdown error")
 	}
@@ -285,7 +290,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
 		ExecutorReserveBackoff: 5 * time.Millisecond,
 	}
 
-	_, release1, err := pluginSvc.reserveScheduledExecutor("balance", policy)
+	_, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
 	if err != nil {
 		t.Fatalf("reserve executor 1: %v", err)
 	}
@@ -296,7 +301,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
 	}
 	secondReserveCh := make(chan reserveResult, 1)
 	go func() {
-		_, release2, reserveErr := pluginSvc.reserveScheduledExecutor("balance", policy)
+		_, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
 		if release2 != nil {
 			release2()
 		}
@@ -407,10 +412,9 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
 		},
 	})
 
-	nextDetectionAt := time.Now().UTC().Add(2 * time.Minute).Round(time.Second)
+	// Mark this job type as currently processing to test DetectionInFlight.
 	pluginSvc.schedulerMu.Lock()
-	pluginSvc.nextDetectionAt[jobType] = nextDetectionAt
-	pluginSvc.detectionInFlight[jobType] = true
+	pluginSvc.currentJobType = jobType
 	pluginSvc.schedulerMu.Unlock()
 
 	states, err := pluginSvc.ListSchedulerStates()
@@ -429,16 +433,7 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
 		t.Fatalf("unexpected policy error: %s", state.PolicyError)
 	}
 	if !state.DetectionInFlight {
-		t.Fatalf("expected detection in flight")
-	}
-	if state.NextDetectionAt == nil {
-		t.Fatalf("expected next detection time")
-	}
-	if state.NextDetectionAt.Unix() != nextDetectionAt.Unix() {
-		t.Fatalf("unexpected next detection time: got=%v want=%v", state.NextDetectionAt, nextDetectionAt)
-	}
-	if state.DetectionIntervalSeconds != 45 {
-		t.Fatalf("unexpected detection interval: got=%d", state.DetectionIntervalSeconds)
+		t.Fatalf("expected detection in flight when current job type matches")
 	}
 	if state.DetectionTimeoutSeconds != 30 {
 		t.Fatalf("unexpected detection timeout: got=%d", state.DetectionTimeoutSeconds)
@@ -446,6 +441,9 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
 	if state.ExecutionTimeoutSeconds != 90 {
 		t.Fatalf("unexpected execution timeout: got=%d", state.ExecutionTimeoutSeconds)
 	}
+	if state.MaxJobTypeDurationSeconds != int32(defaultMaxJobTypeDuration/time.Second) {
+		t.Fatalf("unexpected max job type duration: got=%d", state.MaxJobTypeDurationSeconds)
+	}
 	if state.MaxJobsPerDetection != 80 {
 		t.Fatalf("unexpected max jobs per detection: got=%d", state.MaxJobsPerDetection)
 	}
@@ -467,6 +465,23 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
 	if state.ExecutorWorkerCount != 1 {
 		t.Fatalf("unexpected executor worker count: got=%d", state.ExecutorWorkerCount)
 	}
+
+	// Clear the current job type and verify DetectionInFlight is false.
+	pluginSvc.schedulerMu.Lock()
+	pluginSvc.currentJobType = ""
+	pluginSvc.schedulerMu.Unlock()
+
+	states2, err := pluginSvc.ListSchedulerStates()
+	if err != nil {
+		t.Fatalf("ListSchedulerStates (2): %v", err)
+	}
+	state2 := findSchedulerState(states2, jobType)
+	if state2 == nil {
+		t.Fatalf("missing scheduler state for %s (2)", jobType)
+	}
+	if state2.DetectionInFlight {
+		t.Fatalf("expected detection not in flight when current job type is empty")
+	}
 }
 
 func TestListSchedulerStatesShowsDisabledWhenNoPolicy(t *testing.T) {
@@ -581,3 +596,315 @@ func TestPickDetectorReassignsWhenLeaseIsStale(t *testing.T) {
 		t.Fatalf("expected detector lease to be updated to worker-a, got=%s", lease)
 	}
 }
+
+// mockLockManager records lock/release calls for testing.
+type mockLockManager struct {
+	mu           sync.Mutex
+	acquireCount int
+	releaseCount int
+	failAcquire  bool
+}
+
+func (m *mockLockManager) Acquire(reason string) (func(), error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.failAcquire {
+		return nil, fmt.Errorf("mock lock acquisition failed")
+	}
+	m.acquireCount++
+	return func() {
+		m.mu.Lock()
+		m.releaseCount++
+		m.mu.Unlock()
+	}, nil
+}
+
+func (m *mockLockManager) Status() interface{} {
+	return nil
+}
+
+func (m *mockLockManager) getAcquireCount() int {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.acquireCount
+}
+
+func (m *mockLockManager) getReleaseCount() int {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.releaseCount
+}
+
+func TestIterationAcquiresLockOnce(t *testing.T) {
+	t.Parallel()
+
+	lock := &mockLockManager{}
+	pluginSvc, err := New(Options{
+		LockManager: lock,
+	})
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	defer pluginSvc.Shutdown()
+
+	// Register two enabled job types.
+	for _, jt := range []string{"vacuum", "balance"} {
+		err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
+			JobType: jt,
+			AdminRuntime: &plugin_pb.AdminRuntimeConfig{
+				Enabled:                  true,
+				DetectionTimeoutSeconds:  5,
+				MaxJobsPerDetection:      10,
+				GlobalExecutionConcurrency: 1,
+				PerWorkerExecutionConcurrency: 1,
+			},
+		})
+		if err != nil {
+			t.Fatalf("SaveJobTypeConfig(%s): %v", jt, err)
+		}
+		pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
+			WorkerId: "worker-" + jt,
+			Capabilities: []*plugin_pb.JobTypeCapability{
+				{JobType: jt, CanDetect: true, CanExecute: true},
+			},
+		})
+	}
+
+	// runSchedulerIteration requires a cluster context provider.
+	pluginSvc.clusterContextProvider = func(_ context.Context) (*plugin_pb.ClusterContext, error) {
+		return &plugin_pb.ClusterContext{}, nil
+	}
+
+	pluginSvc.runSchedulerIteration()
+
+	// Lock should have been acquired exactly once (not per-job-type).
+	if lock.getAcquireCount() != 1 {
+		t.Fatalf("expected 1 lock acquisition, got %d", lock.getAcquireCount())
+	}
+	if lock.getReleaseCount() != 1 {
+		t.Fatalf("expected 1 lock release, got %d", lock.getReleaseCount())
+	}
+}
+
+func TestIterationReturnsfalseWhenLockFails(t *testing.T) {
+	t.Parallel()
+
+	lock := &mockLockManager{failAcquire: true}
+	pluginSvc, err := New(Options{
+		LockManager: lock,
+	})
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	defer pluginSvc.Shutdown()
+
+	err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
+		JobType: "vacuum",
+		AdminRuntime: &plugin_pb.AdminRuntimeConfig{
+			Enabled:                  true,
+			DetectionTimeoutSeconds:  5,
+			MaxJobsPerDetection:      10,
+			GlobalExecutionConcurrency: 1,
+			PerWorkerExecutionConcurrency: 1,
+		},
+	})
+	if err != nil {
+		t.Fatalf("SaveJobTypeConfig: %v", err)
+	}
+	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
+		WorkerId: "worker-a",
+		Capabilities: []*plugin_pb.JobTypeCapability{
+			{JobType: "vacuum", CanDetect: true, CanExecute: true},
+		},
+	})
+
+	pluginSvc.clusterContextProvider = func(_ context.Context) (*plugin_pb.ClusterContext, error) {
+		return &plugin_pb.ClusterContext{}, nil
+	}
+
+	result := pluginSvc.runSchedulerIteration()
+	if result {
+		t.Fatalf("expected false when lock acquisition fails")
+	}
+}
+
+func TestSchedulerPhaseTracking(t *testing.T) {
+	t.Parallel()
+
+	pluginSvc, err := New(Options{})
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	defer pluginSvc.Shutdown()
+
+	// Initial phase should be idle.
+	pluginSvc.schedulerMu.Lock()
+	phase := pluginSvc.schedulerPhase
+	pluginSvc.schedulerMu.Unlock()
+
+	if phase != "idle" {
+		t.Fatalf("expected initial phase to be idle, got=%s", phase)
+	}
+
+	pluginSvc.setSchedulerPhase("processing", "vacuum")
+
+	pluginSvc.schedulerMu.Lock()
+	phase = pluginSvc.schedulerPhase
+	jobType := pluginSvc.currentJobType
+	pluginSvc.schedulerMu.Unlock()
+
+	if phase != "processing" {
+		t.Fatalf("expected phase processing, got=%s", phase)
+	}
+	if jobType != "vacuum" {
+		t.Fatalf("expected current job type vacuum, got=%s", jobType)
+	}
+
+	pluginSvc.finishIteration(true)
+
+	pluginSvc.schedulerMu.Lock()
+	phase = pluginSvc.schedulerPhase
+	jobType = pluginSvc.currentJobType
+	workDetected := pluginSvc.lastIterationWorkDetected
+	lastEnded := pluginSvc.lastIterationEndedAt
+	pluginSvc.schedulerMu.Unlock()
+
+	if phase != "idle" {
+		t.Fatalf("expected phase idle after finish, got=%s", phase)
+	}
+	if jobType != "" {
+		t.Fatalf("expected empty job type after finish, got=%s", jobType)
+	}
+	if !workDetected {
+		t.Fatalf("expected last iteration work detected to be true")
+	}
+	if lastEnded.IsZero() {
+		t.Fatalf("expected last iteration ended at to be set")
+	}
+}
+
+func TestGetSchedulerStatusIncludesIterationFields(t *testing.T) {
+	t.Parallel()
+
+	pluginSvc, err := New(Options{
+		IdleSleepDuration: 10 * time.Minute,
+	})
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	defer pluginSvc.Shutdown()
+
+	now := time.Now().UTC()
+	pluginSvc.schedulerMu.Lock()
+	pluginSvc.schedulerPhase = "processing"
+	pluginSvc.currentJobType = "vacuum"
+	pluginSvc.iterationStartedAt = now.Add(-5 * time.Second)
+	pluginSvc.lastIterationEndedAt = now.Add(-20 * time.Second)
+	pluginSvc.lastIterationWorkDetected = true
+	pluginSvc.schedulerMu.Unlock()
+
+	status := pluginSvc.GetSchedulerStatus()
+
+	if status.Phase != "processing" {
+		t.Fatalf("expected phase processing, got=%s", status.Phase)
+	}
+	if status.CurrentJobType != "vacuum" {
+		t.Fatalf("expected current job type vacuum, got=%s", status.CurrentJobType)
+	}
+	if status.IdleSleepSeconds != 600 {
+		t.Fatalf("expected idle sleep 600s, got=%d", status.IdleSleepSeconds)
+	}
+	if status.IterationStartedAt == nil {
+		t.Fatalf("expected iteration started at to be set")
+	}
+	if status.LastIterationEndedAt == nil {
+		t.Fatalf("expected last iteration ended at to be set")
+	}
+	if !status.LastIterationWorkDetected {
+		t.Fatalf("expected last iteration work detected to be true")
+	}
+	// SchedulerTickSeconds should match IdleSleepSeconds for backward compat.
+	if status.SchedulerTickSeconds != 600 {
+		t.Fatalf("expected scheduler tick seconds to match idle sleep, got=%d", status.SchedulerTickSeconds)
+	}
+}
+
+func TestGracefulShutdownDuringIteration(t *testing.T) {
+	t.Parallel()
+
+	pluginSvc, err := New(Options{
+		IdleSleepDuration: time.Millisecond,
+		ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
+			return &plugin_pb.ClusterContext{}, nil
+		},
+	})
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+
+	// Register an enabled job type so the scheduler loop has work to consider.
+	err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
+		JobType: "vacuum",
+		AdminRuntime: &plugin_pb.AdminRuntimeConfig{
+			Enabled:                       true,
+			DetectionTimeoutSeconds:       5,
+			MaxJobsPerDetection:           10,
+			GlobalExecutionConcurrency:    1,
+			PerWorkerExecutionConcurrency: 1,
+		},
+	})
+	if err != nil {
+		t.Fatalf("SaveJobTypeConfig: %v", err)
+	}
+	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
+		WorkerId: "worker-a",
+		Capabilities: []*plugin_pb.JobTypeCapability{
+			{JobType: "vacuum", CanDetect: true, CanExecute: true},
+		},
+	})
+
+	// Shutdown while the scheduler loop is actively iterating.
+	done := make(chan struct{})
+	go func() {
+		pluginSvc.Shutdown()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		// Good — clean shutdown.
+	case <-time.After(5 * time.Second):
+		t.Fatalf("shutdown did not complete in time")
+	}
+}
+
+func TestIdleSleepDurationDefault(t *testing.T) {
+	t.Parallel()
+
+	pluginSvc, err := New(Options{})
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	defer pluginSvc.Shutdown()
+
+	if pluginSvc.idleSleepDuration != defaultIdleSleepDuration {
+		t.Fatalf("expected default idle sleep %v, got=%v", defaultIdleSleepDuration, pluginSvc.idleSleepDuration)
+	}
+}
+
+func TestIdleSleepDurationCustom(t *testing.T) {
+	t.Parallel()
+
+	customDuration := 5 * time.Minute
+	pluginSvc, err := New(Options{
+		IdleSleepDuration: customDuration,
+	})
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	defer pluginSvc.Shutdown()
+
+	if pluginSvc.idleSleepDuration != customDuration {
+		t.Fatalf("expected custom idle sleep %v, got=%v", customDuration, pluginSvc.idleSleepDuration)
+	}
+}

weed/admin/plugin/scheduler_status.go

@@ -0,0 +1,257 @@
+package plugin
+
+import (
+	"sort"
+	"strings"
+	"time"
+)
+
+type SchedulerStatus struct {
+	Now                       time.Time                `json:"now"`
+	SchedulerTickSeconds      int                      `json:"scheduler_tick_seconds"`
+	IdleSleepSeconds          int                      `json:"idle_sleep_seconds"`
+	Phase                     string                   `json:"phase"`
+	CurrentJobType            string                   `json:"current_job_type,omitempty"`
+	IterationStartedAt        *time.Time               `json:"iteration_started_at,omitempty"`
+	LastIterationEndedAt      *time.Time               `json:"last_iteration_ended_at,omitempty"`
+	LastIterationWorkDetected bool                     `json:"last_iteration_work_detected"`
+	Waiting                   []SchedulerWaitingStatus `json:"waiting,omitempty"`
+	InProcessJobs             []SchedulerJobStatus     `json:"in_process_jobs,omitempty"`
+	JobTypes                  []SchedulerJobTypeStatus `json:"job_types,omitempty"`
+}
+
+type SchedulerWaitingStatus struct {
+	Reason  string                 `json:"reason"`
+	JobType string                 `json:"job_type,omitempty"`
+	Since   *time.Time             `json:"since,omitempty"`
+	Until   *time.Time             `json:"until,omitempty"`
+	Details map[string]interface{} `json:"details,omitempty"`
+}
+
+type SchedulerJobStatus struct {
+	JobID           string     `json:"job_id"`
+	JobType         string     `json:"job_type"`
+	State           string     `json:"state"`
+	Stage           string     `json:"stage,omitempty"`
+	WorkerID        string     `json:"worker_id,omitempty"`
+	Message         string     `json:"message,omitempty"`
+	Progress        float64    `json:"progress,omitempty"`
+	CreatedAt       *time.Time `json:"created_at,omitempty"`
+	UpdatedAt       *time.Time `json:"updated_at,omitempty"`
+	DurationSeconds float64    `json:"duration_seconds,omitempty"`
+}
+
+type SchedulerJobTypeStatus struct {
+	JobType                      string     `json:"job_type"`
+	Enabled                      bool       `json:"enabled"`
+	DetectionInFlight            bool       `json:"detection_in_flight"`
+	MaxJobTypeDurationSeconds    int32      `json:"max_job_type_duration_seconds,omitempty"`
+	LastDetectedAt               *time.Time `json:"last_detected_at,omitempty"`
+	LastDetectedCount            int        `json:"last_detected_count,omitempty"`
+	LastDetectionError           string     `json:"last_detection_error,omitempty"`
+	LastDetectionSkipped         string     `json:"last_detection_skipped,omitempty"`
+}
+
+type schedulerDetectionInfo struct {
+	lastDetectedAt    time.Time
+	lastDetectedCount int
+	lastErrorAt       time.Time
+	lastError         string
+	lastSkippedAt     time.Time
+	lastSkippedReason string
+}
+
+func (r *Plugin) recordSchedulerDetectionSuccess(jobType string, count int) {
+	if r == nil {
+		return
+	}
+	r.schedulerDetectionMu.Lock()
+	defer r.schedulerDetectionMu.Unlock()
+	info := r.schedulerDetection[jobType]
+	if info == nil {
+		info = &schedulerDetectionInfo{}
+		r.schedulerDetection[jobType] = info
+	}
+	info.lastDetectedAt = time.Now().UTC()
+	info.lastDetectedCount = count
+	info.lastError = ""
+	info.lastSkippedReason = ""
+}
+
+func (r *Plugin) recordSchedulerDetectionError(jobType string, err error) {
+	if r == nil {
+		return
+	}
+	if err == nil {
+		return
+	}
+	r.schedulerDetectionMu.Lock()
+	defer r.schedulerDetectionMu.Unlock()
+	info := r.schedulerDetection[jobType]
+	if info == nil {
+		info = &schedulerDetectionInfo{}
+		r.schedulerDetection[jobType] = info
+	}
+	info.lastErrorAt = time.Now().UTC()
+	info.lastError = err.Error()
+}
+
+func (r *Plugin) recordSchedulerDetectionSkip(jobType string, reason string) {
+	if r == nil {
+		return
+	}
+	if strings.TrimSpace(reason) == "" {
+		return
+	}
+	r.schedulerDetectionMu.Lock()
+	defer r.schedulerDetectionMu.Unlock()
+	info := r.schedulerDetection[jobType]
+	if info == nil {
+		info = &schedulerDetectionInfo{}
+		r.schedulerDetection[jobType] = info
+	}
+	info.lastSkippedAt = time.Now().UTC()
+	info.lastSkippedReason = reason
+}
+
+func (r *Plugin) snapshotSchedulerDetection(jobType string) schedulerDetectionInfo {
+	if r == nil {
+		return schedulerDetectionInfo{}
+	}
+	r.schedulerDetectionMu.Lock()
+	defer r.schedulerDetectionMu.Unlock()
+	info := r.schedulerDetection[jobType]
+	if info == nil {
+		return schedulerDetectionInfo{}
+	}
+	return *info
+}
+
+func (r *Plugin) GetSchedulerStatus() SchedulerStatus {
+	now := time.Now().UTC()
+
+	r.schedulerMu.Lock()
+	phase := r.schedulerPhase
+	currentJobType := r.currentJobType
+	iterationStartedAt := r.iterationStartedAt
+	lastIterationEndedAt := r.lastIterationEndedAt
+	lastIterationWorkDetected := r.lastIterationWorkDetected
+	r.schedulerMu.Unlock()
+
+	status := SchedulerStatus{
+		Now:                       now,
+		SchedulerTickSeconds:      int(secondsFromDuration(r.idleSleepDuration)),
+		IdleSleepSeconds:          int(secondsFromDuration(r.idleSleepDuration)),
+		Phase:                     phase,
+		CurrentJobType:            currentJobType,
+		LastIterationWorkDetected: lastIterationWorkDetected,
+		InProcessJobs:             r.listInProcessJobs(now),
+	}
+	if !iterationStartedAt.IsZero() {
+		status.IterationStartedAt = timeToPtr(iterationStartedAt)
+	}
+	if !lastIterationEndedAt.IsZero() {
+		status.LastIterationEndedAt = timeToPtr(lastIterationEndedAt)
+	}
+
+	states, err := r.ListSchedulerStates()
+	if err != nil {
+		return status
+	}
+
+	waiting := make([]SchedulerWaitingStatus, 0)
+	jobTypes := make([]SchedulerJobTypeStatus, 0, len(states))
+
+	for _, state := range states {
+		jobType := state.JobType
+		info := r.snapshotSchedulerDetection(jobType)
+
+		jobStatus := SchedulerJobTypeStatus{
+			JobType:                   jobType,
+			Enabled:                   state.Enabled,
+			DetectionInFlight:         state.DetectionInFlight,
+			MaxJobTypeDurationSeconds: state.MaxJobTypeDurationSeconds,
+		}
+		if !info.lastDetectedAt.IsZero() {
+			jobStatus.LastDetectedAt = timeToPtr(info.lastDetectedAt)
+			jobStatus.LastDetectedCount = info.lastDetectedCount
+		}
+		if info.lastError != "" {
+			jobStatus.LastDetectionError = info.lastError
+		}
+		if info.lastSkippedReason != "" {
+			jobStatus.LastDetectionSkipped = info.lastSkippedReason
+		}
+		jobTypes = append(jobTypes, jobStatus)
+
+		if state.DetectionInFlight {
+			waiting = append(waiting, SchedulerWaitingStatus{
+				Reason:  "processing",
+				JobType: jobType,
+			})
+		}
+	}
+
+	if phase == "idle" && !lastIterationEndedAt.IsZero() {
+		waiting = append(waiting, SchedulerWaitingStatus{
+			Reason: "idle_sleep",
+		})
+	}
+
+	sort.Slice(jobTypes, func(i, j int) bool {
+		return jobTypes[i].JobType < jobTypes[j].JobType
+	})
+
+	status.Waiting = waiting
+	status.JobTypes = jobTypes
+	return status
+}
+
+func (r *Plugin) listInProcessJobs(now time.Time) []SchedulerJobStatus {
+	active := make([]SchedulerJobStatus, 0)
+	if r == nil {
+		return active
+	}
+
+	r.jobsMu.RLock()
+	for _, job := range r.jobs {
+		if job == nil {
+			continue
+		}
+		if !isActiveTrackedJobState(job.State) {
+			continue
+		}
+		start := timeToPtr(now)
+		if job.CreatedAt != nil && !job.CreatedAt.IsZero() {
+			start = job.CreatedAt
+		} else if job.UpdatedAt != nil && !job.UpdatedAt.IsZero() {
+			start = job.UpdatedAt
+		}
+		durationSeconds := 0.0
+		if start != nil {
+			durationSeconds = now.Sub(*start).Seconds()
+		}
+		active = append(active, SchedulerJobStatus{
+			JobID:           job.JobID,
+			JobType:         job.JobType,
+			State:           strings.ToLower(job.State),
+			Stage:           job.Stage,
+			WorkerID:        job.WorkerID,
+			Message:         job.Message,
+			Progress:        job.Progress,
+			CreatedAt:       job.CreatedAt,
+			UpdatedAt:       job.UpdatedAt,
+			DurationSeconds: durationSeconds,
+		})
+	}
+	r.jobsMu.RUnlock()
+
+	sort.Slice(active, func(i, j int) bool {
+		if active[i].DurationSeconds != active[j].DurationSeconds {
+			return active[i].DurationSeconds > active[j].DurationSeconds
+		}
+		return active[i].JobID < active[j].JobID
+	})
+
+	return active
+}

weed/admin/plugin/scheduler_status_test.go

@@ -0,0 +1,64 @@
+package plugin
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+)
+
+func TestGetSchedulerStatusIncludesInProcessJobs(t *testing.T) {
+	pluginSvc, err := New(Options{})
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	defer pluginSvc.Shutdown()
+
+	pluginSvc.trackExecutionStart("req-1", "worker-a", &plugin_pb.JobSpec{
+		JobId:   "job-1",
+		JobType: "vacuum",
+	}, 1)
+
+	status := pluginSvc.GetSchedulerStatus()
+	if len(status.InProcessJobs) != 1 {
+		t.Fatalf("expected one in-process job, got %d", len(status.InProcessJobs))
+	}
+	if status.InProcessJobs[0].JobID != "job-1" {
+		t.Fatalf("unexpected job id: %s", status.InProcessJobs[0].JobID)
+	}
+}
+
+func TestGetSchedulerStatusIncludesLastDetectionCount(t *testing.T) {
+	pluginSvc, err := New(Options{})
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	defer pluginSvc.Shutdown()
+
+	const jobType = "vacuum"
+	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
+		WorkerId: "worker-a",
+		Capabilities: []*plugin_pb.JobTypeCapability{
+			{JobType: jobType, CanDetect: true},
+		},
+	})
+
+	pluginSvc.recordSchedulerDetectionSuccess(jobType, 3)
+
+	status := pluginSvc.GetSchedulerStatus()
+	found := false
+	for _, jt := range status.JobTypes {
+		if jt.JobType != jobType {
+			continue
+		}
+		found = true
+		if jt.LastDetectedCount != 3 {
+			t.Fatalf("unexpected last detected count: got=%d want=3", jt.LastDetectedCount)
+		}
+		if jt.LastDetectedAt == nil {
+			t.Fatalf("expected last detected at to be set")
+		}
+	}
+	if !found {
+		t.Fatalf("expected job type status for %s", jobType)
+	}
+}

weed/admin/plugin/types.go

@@ -1,6 +1,9 @@
 package plugin
 
-import "time"
+import (
+	"errors"
+	"time"
+)
 
 const (
 	// Keep exactly the last 10 successful and last 10 error runs per job type.
@@ -8,6 +11,8 @@ const (
 	MaxErrorRunHistory      = 10
 )
 
+var ErrJobNotFound = errors.New("job not found")
+
 type RunOutcome string
 
 const (
@@ -77,22 +82,21 @@ type JobDetail struct {
 }
 
 type SchedulerJobTypeState struct {
-	JobType                       string     `json:"job_type"`
-	Enabled                       bool       `json:"enabled"`
-	PolicyError                   string     `json:"policy_error,omitempty"`
-	DetectionInFlight             bool       `json:"detection_in_flight"`
-	NextDetectionAt               *time.Time `json:"next_detection_at,omitempty"`
-	DetectionIntervalSeconds      int32      `json:"detection_interval_seconds,omitempty"`
-	DetectionTimeoutSeconds       int32      `json:"detection_timeout_seconds,omitempty"`
-	ExecutionTimeoutSeconds       int32      `json:"execution_timeout_seconds,omitempty"`
-	MaxJobsPerDetection           int32      `json:"max_jobs_per_detection,omitempty"`
-	GlobalExecutionConcurrency    int        `json:"global_execution_concurrency,omitempty"`
-	PerWorkerExecutionConcurrency int        `json:"per_worker_execution_concurrency,omitempty"`
-	RetryLimit                    int        `json:"retry_limit,omitempty"`
-	RetryBackoffSeconds           int32      `json:"retry_backoff_seconds,omitempty"`
-	DetectorAvailable             bool       `json:"detector_available"`
-	DetectorWorkerID              string     `json:"detector_worker_id,omitempty"`
-	ExecutorWorkerCount           int        `json:"executor_worker_count"`
+	JobType                       string `json:"job_type"`
+	Enabled                       bool   `json:"enabled"`
+	PolicyError                   string `json:"policy_error,omitempty"`
+	DetectionInFlight             bool   `json:"detection_in_flight"`
+	DetectionTimeoutSeconds       int32  `json:"detection_timeout_seconds,omitempty"`
+	ExecutionTimeoutSeconds       int32  `json:"execution_timeout_seconds,omitempty"`
+	MaxJobTypeDurationSeconds     int32  `json:"max_job_type_duration_seconds,omitempty"`
+	MaxJobsPerDetection           int32  `json:"max_jobs_per_detection,omitempty"`
+	GlobalExecutionConcurrency    int    `json:"global_execution_concurrency,omitempty"`
+	PerWorkerExecutionConcurrency int    `json:"per_worker_execution_concurrency,omitempty"`
+	RetryLimit                    int    `json:"retry_limit,omitempty"`
+	RetryBackoffSeconds           int32  `json:"retry_backoff_seconds,omitempty"`
+	DetectorAvailable             bool   `json:"detector_available"`
+	DetectorWorkerID              string `json:"detector_worker_id,omitempty"`
+	ExecutorWorkerCount           int    `json:"executor_worker_count"`
 }
 
 func timeToPtr(t time.Time) *time.Time {

weed/admin/topology/capacity.go

@@ -115,6 +115,7 @@ func (at *ActiveTopology) GetDisksWithEffectiveCapacity(taskType TaskType, exclu
 					RemoteVolumeCount: disk.DiskInfo.DiskInfo.RemoteVolumeCount,
 					ActiveVolumeCount: disk.DiskInfo.DiskInfo.ActiveVolumeCount,
 					FreeVolumeCount:   disk.DiskInfo.DiskInfo.FreeVolumeCount,
+					Tags:              append([]string(nil), disk.DiskInfo.DiskInfo.Tags...),
 				}
 				diskCopy.DiskInfo = diskInfoCopy
 				diskCopy.DiskInfo.MaxVolumeCount = disk.DiskInfo.DiskInfo.MaxVolumeCount // Ensure Max is set
@@ -178,6 +179,7 @@ func (at *ActiveTopology) GetDisksForPlanning(taskType TaskType, excludeNodeID s
 					RemoteVolumeCount: disk.DiskInfo.DiskInfo.RemoteVolumeCount,
 					ActiveVolumeCount: disk.DiskInfo.DiskInfo.ActiveVolumeCount,
 					FreeVolumeCount:   disk.DiskInfo.DiskInfo.FreeVolumeCount,
+					Tags:              append([]string(nil), disk.DiskInfo.DiskInfo.Tags...),
 				}
 				diskCopy.DiskInfo = diskInfoCopy
 

weed/admin/view/app/plugin.templ

@@ -115,12 +115,42 @@ templ Plugin(page string) {
                     </div>
                 </div>
             </div>
+            <div class="row mb-4">
+                <div class="col-12">
+                    <div class="card shadow-sm">
+                        <div class="card-header d-flex justify-content-between align-items-center flex-wrap gap-2">
+                            <h5 class="mb-0"><i class="fas fa-sync-alt me-2"></i>Scheduler Iteration</h5>
+                            <small class="text-muted">Current iteration phase and timing</small>
+                        </div>
+                        <div class="card-body" id="plugin-scheduler-iteration-card">
+                            <div class="row g-3">
+                                <div class="col-md-3 col-sm-6">
+                                    <div class="small text-muted">Phase</div>
+                                    <div class="fw-semibold" id="plugin-scheduler-phase">-</div>
+                                </div>
+                                <div class="col-md-3 col-sm-6">
+                                    <div class="small text-muted">Current Job Type</div>
+                                    <div class="fw-semibold" id="plugin-scheduler-current-jt">-</div>
+                                </div>
+                                <div class="col-md-3 col-sm-6">
+                                    <div class="small text-muted">Idle Sleep</div>
+                                    <div class="fw-semibold" id="plugin-scheduler-idle-sleep">-</div>
+                                </div>
+                                <div class="col-md-3 col-sm-6">
+                                    <div class="small text-muted">Last Iteration</div>
+                                    <div class="fw-semibold" id="plugin-scheduler-last-iteration">-</div>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
             <div class="row mb-4">
                 <div class="col-12">
                     <div class="card shadow-sm">
                         <div class="card-header d-flex justify-content-between align-items-center flex-wrap gap-2">
                             <h5 class="mb-0"><i class="fas fa-clock me-2"></i>Scheduler State</h5>
-                            <small class="text-muted">Per job type detection schedule and execution limits</small>
+                            <small class="text-muted">Per job type execution limits and status</small>
                         </div>
                         <div class="card-body p-0">
                             <div class="table-responsive">
@@ -130,9 +160,8 @@ templ Plugin(page string) {
                                             <th>Job Type</th>
                                             <th>Enabled</th>
                                             <th>Detector</th>
-                                            <th>In Flight</th>
-                                            <th>Next Detection</th>
-                                            <th>Interval</th>
+                                            <th>Status</th>
+                                            <th>Max Duration</th>
                                             <th>Exec Global</th>
                                             <th>Exec/Worker</th>
                                             <th>Executor Workers</th>
@@ -140,7 +169,7 @@ templ Plugin(page string) {
                                         </tr>
                                     </thead>
                                     <tbody id="plugin-scheduler-table-body">
-                                        <tr><td colspan="10" class="text-muted text-center py-3">Loading...</td></tr>
+                                        <tr><td colspan="9" class="text-muted text-center py-3">Loading...</td></tr>
                                     </tbody>
                                 </table>
                             </div>
@@ -485,6 +514,11 @@ templ Plugin(page string) {
             <div class="modal-content">
                 <div class="modal-header">
                     <h5 class="modal-title" id="plugin-job-detail-modal-label"><i class="fas fa-file-alt me-2"></i>Job Detail</h5>
+                    <div class="ms-auto me-2">
+                        <button type="button" class="btn btn-outline-danger btn-sm" id="plugin-expire-job-btn" disabled>
+                            <i class="fas fa-stop-circle me-1"></i>Expire Job
+                        </button>
+                    </div>
                     <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
                 </div>
                 <div class="modal-body" id="plugin-job-detail-content">
@@ -506,6 +540,9 @@ templ Plugin(page string) {
         .plugin-form-root .card {
             border: 1px solid #dee2e6;
         }
+        .plugin-form-root textarea {
+            min-height: 12rem;
+        }
 
         .plugin-field-hidden {
             display: none;
@@ -1073,6 +1110,66 @@ templ Plugin(page string) {
                 return html;
             }
 
+            function isActiveJobState(candidateState) {
+                var jobState = candidateState;
+                if (candidateState && typeof candidateState === 'object' && candidateState.state !== undefined) {
+                    jobState = candidateState.state;
+                }
+                var st = String(jobState || '').toLowerCase();
+                return st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' ||
+                    st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
+            }
+
+            function setExpireButtonState(job) {
+                var expireBtn = document.getElementById('plugin-expire-job-btn');
+                if (!expireBtn) {
+                    return;
+                }
+                var jobID = job && job.job_id ? String(job.job_id) : '';
+                var active = isActiveJobState(job);
+                expireBtn.setAttribute('data-job-id', jobID);
+                expireBtn.disabled = !jobID || !active;
+                if (!jobID) {
+                    expireBtn.title = 'Select a job to expire.';
+                } else if (!active) {
+                    expireBtn.title = 'Job is not active.';
+                } else {
+                    expireBtn.title = 'Expire job to unblock scheduling.';
+                }
+            }
+
+            async function expireJob(jobID) {
+                var normalizedJobID = String(jobID || '').trim();
+                if (!normalizedJobID) {
+                    return;
+                }
+
+                var reason = window.prompt('Expire job ' + normalizedJobID + '? Optional reason:', 'job expired by admin request');
+                if (reason === null) {
+                    return;
+                }
+
+                var expireBtn = document.getElementById('plugin-expire-job-btn');
+                if (expireBtn) {
+                    expireBtn.disabled = true;
+                }
+
+                try {
+                    var response = await pluginRequest('POST', '/api/plugin/jobs/' + encodePath(normalizedJobID) + '/expire', {
+                        reason: reason,
+                    });
+                    if (response && response.expired === false) {
+                        notify(response.message || 'Job is not active.', 'info');
+                    } else {
+                        notify('Job expired: ' + normalizedJobID, 'success');
+                    }
+                    await refreshJobsAndActivities();
+                    await openJobDetail(normalizedJobID);
+                } catch (e) {
+                    notify('Failed to expire job: ' + e.message, 'error');
+                }
+            }
+
             async function openJobDetail(jobID) {
                 var normalizedJobID = String(jobID || '').trim();
                 if (!normalizedJobID) {
@@ -1093,10 +1190,12 @@ templ Plugin(page string) {
                     modal.show();
                 }
 
+                setExpireButtonState(null);
                 contentRoot.innerHTML = '<div class="text-muted">Loading job detail...</div>';
                 try {
                     var detail = await pluginRequest('GET', '/api/plugin/jobs/' + encodePath(normalizedJobID) + '/detail?activity_limit=500&related_limit=20');
                     var job = (detail && detail.job) ? detail.job : {};
+                    setExpireButtonState(job);
                     var runRecord = detail && detail.run_record ? detail.run_record : null;
                     var activities = (detail && Array.isArray(detail.activities)) ? detail.activities : [];
                     var relatedJobs = (detail && Array.isArray(detail.related_jobs)) ? detail.related_jobs : [];
@@ -1197,6 +1296,7 @@ templ Plugin(page string) {
 
                     contentRoot.innerHTML = html;
                 } catch (e) {
+                    setExpireButtonState(null);
                     contentRoot.innerHTML = '<div class="alert alert-danger mb-0">Failed to load job detail: ' + escapeHtml(e.message) + '</div>';
                 }
             }
@@ -1238,8 +1338,7 @@ templ Plugin(page string) {
                 var allActivities = Array.isArray(state.allActivities) ? state.allActivities : [];
 
                 var activeCount = allJobs.filter(function(job) {
-                    var st = String(job.state || '').toLowerCase();
-                    return st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' || st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
+                    return isActiveJobState(job);
                 }).length;
 
                 document.getElementById('plugin-status-workers').textContent = String(state.workers.length);
@@ -1265,8 +1364,7 @@ templ Plugin(page string) {
                     if (!jobType) {
                         continue;
                     }
-                    var st = String(job.state || '').toLowerCase();
-                    var isActive = st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' || st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
+                    var isActive = isActiveJobState(job);
                     if (!isActive) {
                         continue;
                     }
@@ -1363,7 +1461,7 @@ templ Plugin(page string) {
 
                 var states = Array.isArray(state.schedulerStates) ? state.schedulerStates : [];
                 if (!states.length) {
-                    tbody.innerHTML = '<tr><td colspan="10" class="text-muted text-center py-3">No scheduler state available</td></tr>';
+                    tbody.innerHTML = '<tr><td colspan="9" class="text-muted text-center py-3">No scheduler state available</td></tr>';
                     return;
                 }
 
@@ -1373,8 +1471,8 @@ templ Plugin(page string) {
                     var enabled = !!item.enabled;
                     var inFlight = !!item.detection_in_flight;
                     var detector = item.detector_available ? textOrDash(item.detector_worker_id) : 'No detector';
-                    var intervalSeconds = Number(item.detection_interval_seconds || 0);
-                    var intervalText = intervalSeconds > 0 ? (String(intervalSeconds) + 's') : '-';
+                    var maxDurationSeconds = Number(item.max_job_type_duration_seconds || 0);
+                    var maxDurationText = maxDurationSeconds > 0 ? (String(Math.round(maxDurationSeconds / 60)) + 'm') : '-';
                     var globalExec = Number(item.global_execution_concurrency || 0);
                     var perWorkerExec = Number(item.per_worker_execution_concurrency || 0);
                     var executorWorkers = Number(item.executor_worker_count || 0);
@@ -1385,7 +1483,7 @@ templ Plugin(page string) {
                     var effectiveExecText = enabled ? String(effectiveExec) : '-';
 
                     var enabledBadge = enabled ? '<span class="badge bg-success">Enabled</span>' : '<span class="badge bg-secondary">Disabled</span>';
-                    var inFlightBadge = inFlight ? '<span class="badge bg-warning text-dark">Yes</span>' : '<span class="badge bg-light text-dark">No</span>';
+                    var statusBadge = inFlight ? '<span class="badge bg-warning text-dark">Processing</span>' : '<span class="badge bg-light text-dark">Idle</span>';
                     var policyErrorHtml = '';
                     if (item.policy_error) {
                         policyErrorHtml = '<div><small class="text-danger">' + escapeHtml(String(item.policy_error)) + '</small></div>';
@@ -1395,9 +1493,8 @@ templ Plugin(page string) {
                         '<td>' + escapeHtml(textOrDash(item.job_type)) + policyErrorHtml + '</td>' +
                         '<td>' + enabledBadge + '</td>' +
                         '<td><small>' + escapeHtml(detector) + '</small></td>' +
-                        '<td>' + inFlightBadge + '</td>' +
-                        '<td><small>' + escapeHtml(parseTime(item.next_detection_at) || '-') + '</small></td>' +
-                        '<td><small>' + escapeHtml(intervalText) + '</small></td>' +
+                        '<td>' + statusBadge + '</td>' +
+                        '<td><small>' + escapeHtml(maxDurationText) + '</small></td>' +
                         '<td><small>' + escapeHtml(globalExecText) + '</small></td>' +
                         '<td><small>' + escapeHtml(perWorkerExecText) + '</small></td>' +
                         '<td><small>' + escapeHtml(executorWorkersText) + '</small></td>' +
@@ -2711,6 +2808,37 @@ templ Plugin(page string) {
                 }
             }
 
+            function renderSchedulerIterationStatus(schedulerStatus) {
+                var phaseEl = document.getElementById('plugin-scheduler-phase');
+                var currentJtEl = document.getElementById('plugin-scheduler-current-jt');
+                var idleSleepEl = document.getElementById('plugin-scheduler-idle-sleep');
+                var lastIterEl = document.getElementById('plugin-scheduler-last-iteration');
+                if (!phaseEl) {
+                    return;
+                }
+
+                var sched = (schedulerStatus && schedulerStatus.scheduler) ? schedulerStatus.scheduler : {};
+                var phase = String(sched.phase || 'unknown');
+                var phaseColors = { idle: 'bg-secondary', acquiring_lock: 'bg-info', processing: 'bg-primary' };
+                var phaseColor = phaseColors[phase] || 'bg-dark';
+                phaseEl.innerHTML = '<span class="badge ' + phaseColor + '">' + escapeHtml(phase) + '</span>';
+                currentJtEl.textContent = sched.current_job_type || '-';
+
+                var idleSleep = Number(sched.idle_sleep_seconds || 0);
+                idleSleepEl.textContent = idleSleep > 0 ? (String(Math.round(idleSleep / 60)) + ' min') : '-';
+
+                var lastEnded = parseTime(sched.last_iteration_ended_at);
+                var workDetected = !!sched.last_iteration_work_detected;
+                if (lastEnded) {
+                    var workBadge = workDetected
+                        ? '<span class="badge bg-success ms-1">work found</span>'
+                        : '<span class="badge bg-light text-dark ms-1">no work</span>';
+                    lastIterEl.innerHTML = '<small>' + escapeHtml(lastEnded) + '</small>' + workBadge;
+                } else {
+                    lastIterEl.textContent = '-';
+                }
+            }
+
             async function refreshJobsAndActivities() {
                 var executionStateFilter = document.getElementById('plugin-monitor-job-state-filter');
 
@@ -2719,10 +2847,18 @@ templ Plugin(page string) {
                 var allJobsPromise = pluginRequest('GET', '/api/plugin/jobs?limit=500');
                 var allActivitiesPromise = pluginRequest('GET', '/api/plugin/activities?limit=500');
                 var schedulerPromise = pluginRequest('GET', '/api/plugin/scheduler-states');
+                var schedulerStatusPromise = pluginRequest('GET', '/api/plugin/scheduler-status').catch(function(e) { return e; });
 
                 var allJobs = await allJobsPromise;
                 var allActivities = await allActivitiesPromise;
                 var schedulerStates = await schedulerPromise;
+                var schedulerStatusResult = await schedulerStatusPromise;
+                var schedulerStatus = null;
+                if (schedulerStatusResult instanceof Error) {
+                    console.error('Failed to fetch scheduler status:', schedulerStatusResult);
+                } else {
+                    schedulerStatus = schedulerStatusResult;
+                }
 
                 state.jobs = Array.isArray(allJobs) ? allJobs : [];
                 state.activities = Array.isArray(allActivities) ? allActivities : [];
@@ -2734,6 +2870,7 @@ templ Plugin(page string) {
                 renderExecutionJobs();
                 renderExecutionActivities();
                 renderSchedulerStates();
+                renderSchedulerIterationStatus(schedulerStatus);
                 renderStatus();
                 renderJobTypeSummary();
             }
@@ -2778,6 +2915,17 @@ templ Plugin(page string) {
                     });
                 }
 
+                var expireBtn = document.getElementById('plugin-expire-job-btn');
+                if (expireBtn) {
+                    expireBtn.addEventListener('click', function() {
+                        var jobID = String(expireBtn.getAttribute('data-job-id') || '').trim();
+                        if (!jobID) {
+                            return;
+                        }
+                        expireJob(jobID);
+                    });
+                }
+
                 document.getElementById('plugin-refresh-all-btn').addEventListener('click', function() {
                     refreshAll();
                 });

weed/command/admin.go

@@ -45,6 +45,7 @@ type AdminOptions struct {
 	readOnlyPassword *string
 	dataDir          *string
 	icebergPort      *int
+	idleSleepSeconds *int
 }
 
 func init() {
@@ -60,6 +61,7 @@ func init() {
 	a.readOnlyUser = cmdAdmin.Flag.String("readOnlyUser", "", "read-only user username (optional, for view-only access)")
 	a.readOnlyPassword = cmdAdmin.Flag.String("readOnlyPassword", "", "read-only user password (optional, for view-only access; requires adminPassword to be set)")
 	a.icebergPort = cmdAdmin.Flag.Int("iceberg.port", 8181, "Iceberg REST Catalog port (0 to hide in UI)")
+	a.idleSleepSeconds = cmdAdmin.Flag.Int("scheduler.idleSleep", 0, "scheduler idle sleep in seconds between iterations when no work is found (0 = default 17 minutes)")
 }
 
 var cmdAdmin = &Command{
@@ -290,7 +292,11 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
 	}
 
 	// Create admin server (plugin is always enabled)
-	adminServer := dash.NewAdminServer(*options.master, nil, dataDir, icebergPort)
+	var idleSleep time.Duration
+	if options.idleSleepSeconds != nil && *options.idleSleepSeconds > 0 {
+		idleSleep = time.Duration(*options.idleSleepSeconds) * time.Second
+	}
+	adminServer := dash.NewAdminServer(*options.master, nil, dataDir, icebergPort, idleSleep)
 
 	// Show discovered filers
 	filers := adminServer.GetAllFilers()

weed/command/mini.go

@@ -45,7 +45,7 @@ const (
 	defaultMiniVolumeSizeMB       = 128         // Default volume size for mini mode
 	maxVolumeSizeMB               = 1024        // Maximum volume size in MB (1GB)
 	GrpcPortOffset                = 10000       // Offset used to calculate gRPC port from HTTP port
-	defaultMiniPluginJobTypes     = "vacuum,volume_balance,erasure_coding"
+	defaultMiniPluginJobTypes     = "vacuum,volume_balance,erasure_coding,admin_script"
 )
 
 var (
@@ -204,6 +204,7 @@ func initMiniVolumeFlags() {
 	miniOptions.v.publicUrl = cmdMini.Flag.String("volume.publicUrl", "", "publicly accessible address")
 	miniOptions.v.indexType = cmdMini.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	miniOptions.v.diskType = cmdMini.Flag.String("volume.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
+	miniOptions.v.tags = cmdMini.Flag.String("volume.tags", "", "comma-separated tag groups per data dir; each group uses ':' (e.g. fast:ssd,archive)")
 	miniOptions.v.fixJpgOrientation = cmdMini.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	miniOptions.v.readMode = cmdMini.Flag.String("volume.readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|read in remote node|redirect volume location'.")
 	miniOptions.v.compactionMBPerSecond = cmdMini.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")

weed/command/mini_plugin_test.go

@@ -7,7 +7,7 @@ func TestMiniDefaultPluginJobTypes(t *testing.T) {
 	if err != nil {
 		t.Fatalf("parsePluginWorkerJobTypes(mini default) err = %v", err)
 	}
-	if len(jobTypes) != 3 {
-		t.Fatalf("expected mini default job types to include 3 handlers, got %v", jobTypes)
+	if len(jobTypes) != 4 {
+		t.Fatalf("expected mini default job types to include 4 handlers, got %v", jobTypes)
 	}
 }

weed/command/plugin_worker_test.go

@@ -123,6 +123,14 @@ func TestParsePluginWorkerJobTypes(t *testing.T) {
 	if _, err = parsePluginWorkerJobTypes(" , "); err != nil {
 		t.Fatalf("expected empty list to resolve to default vacuum: %v", err)
 	}
+
+	jobTypes, err = parsePluginWorkerJobTypes("admin-script,script,admin_script")
+	if err != nil {
+		t.Fatalf("parsePluginWorkerJobTypes(admin script aliases) err = %v", err)
+	}
+	if len(jobTypes) != 1 || jobTypes[0] != "admin_script" {
+		t.Fatalf("expected admin_script alias to resolve, got %v", jobTypes)
+	}
 }
 
 func TestPluginWorkerDefaultJobTypes(t *testing.T) {
@@ -130,8 +138,8 @@ func TestPluginWorkerDefaultJobTypes(t *testing.T) {
 	if err != nil {
 		t.Fatalf("parsePluginWorkerJobTypes(default setting) err = %v", err)
 	}
-	if len(jobTypes) != 3 {
-		t.Fatalf("expected default job types to include 3 handlers, got %v", jobTypes)
+	if len(jobTypes) != 4 {
+		t.Fatalf("expected default job types to include 4 handlers, got %v", jobTypes)
 	}
 }
 

weed/command/server.go

@@ -137,6 +137,7 @@ func init() {
 	serverOptions.v.id = cmdServer.Flag.String("volume.id", "", "volume server id. If empty, default to ip:port")
 	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	serverOptions.v.diskType = cmdServer.Flag.String("volume.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
+	serverOptions.v.tags = cmdServer.Flag.String("volume.tags", "", "comma-separated tag groups per data dir; each group uses ':' (e.g. fast:ssd,archive)")
 	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	serverOptions.v.readMode = cmdServer.Flag.String("volume.readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|read in remote node|redirect volume location'.")
 	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")

weed/command/volume.go

@@ -53,6 +53,7 @@ type VolumeServerOptions struct {
 	whiteList                 []string
 	indexType                 *string
 	diskType                  *string
+	tags                      *string
 	fixJpgOrientation         *bool
 	readMode                  *string
 	cpuProfile                *string
@@ -94,6 +95,7 @@ func init() {
 	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
 	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	v.diskType = cmdVolume.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
+	v.tags = cmdVolume.Flag.String("tags", "", "comma-separated tag groups per data dir; each group uses ':' (e.g. fast:ssd,archive)")
 	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	v.readMode = cmdVolume.Flag.String("readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|proxy to remote node|redirect volume location'.")
 	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
@@ -219,6 +221,12 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		glog.Fatalf("%d directories by -dir, but only %d disk types is set by -disk", len(v.folders), len(diskTypes))
 	}
 
+	var tagsArg string
+	if v.tags != nil {
+		tagsArg = *v.tags
+	}
+	folderTags := parseVolumeTags(tagsArg, len(v.folders))
+
 	// security related white list configuration
 	v.whiteList = util.StringSplit(volumeWhiteListOption, ",")
 
@@ -269,7 +277,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 
 	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
 		*v.ip, *v.port, *v.portGrpc, *v.publicUrl, volumeServerId,
-		v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
+		v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes, folderTags,
 		*v.idxFolder,
 		volumeNeedleMapKind,
 		v.masters, constants.VolumePulsePeriod, *v.dataCenter, *v.rack,
@@ -334,6 +342,36 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 
 }
 
+func parseVolumeTags(tagsArg string, folderCount int) [][]string {
+	if folderCount <= 0 {
+		return nil
+	}
+	tagEntries := []string{}
+	if strings.TrimSpace(tagsArg) != "" {
+		tagEntries = strings.Split(tagsArg, ",")
+	}
+	folderTags := make([][]string, folderCount)
+	
+	// If exactly one tag entry provided, replicate it to all folders
+	if len(tagEntries) == 1 {
+		normalized := util.NormalizeTagList(strings.Split(tagEntries[0], ":"))
+		for i := 0; i < folderCount; i++ {
+			folderTags[i] = append([]string(nil), normalized...)
+		}
+	} else {
+		// Otherwise, assign tags to folders that have explicit entries
+		for i := 0; i < folderCount; i++ {
+			if i < len(tagEntries) {
+				folderTags[i] = util.NormalizeTagList(strings.Split(tagEntries[i], ":"))
+			} else {
+				// Initialize remaining folders with empty tag slice
+				folderTags[i] = []string{}
+			}
+		}
+	}
+	return folderTags
+}
+
 func shutdown(publicHttpDown httpdown.Server, clusterHttpServer httpdown.Server, grpcS *grpc.Server, volumeServer *weed_server.VolumeServer) {
 
 	// firstly, stop the public http service to prevent from receiving new user request

weed/command/worker.go

@@ -7,11 +7,11 @@ import (
 )
 
 var cmdWorker = &Command{
-	UsageLine: "worker -admin=<admin_server> [-id=<worker_id>] [-jobType=vacuum,volume_balance,erasure_coding] [-workingDir=<path>] [-heartbeat=15s] [-reconnect=5s] [-maxDetect=1] [-maxExecute=4] [-metricsPort=<port>] [-metricsIp=<ip>] [-debug]",
+	UsageLine: "worker -admin=<admin_server> [-id=<worker_id>] [-jobType=vacuum,volume_balance,erasure_coding,admin_script] [-workingDir=<path>] [-heartbeat=15s] [-reconnect=5s] [-maxDetect=1] [-maxExecute=4] [-metricsPort=<port>] [-metricsIp=<ip>] [-debug]",
 	Short:     "start a plugin.proto worker process",
 	Long: `Start an external plugin worker using weed/pb/plugin.proto over gRPC.
 
-This command provides vacuum, volume_balance, and erasure_coding job type
+This command provides vacuum, volume_balance, erasure_coding, and admin_script job type
 contracts with the plugin stream runtime, including descriptor delivery,
 heartbeat/load reporting, detection, and execution.
 
@@ -25,6 +25,7 @@ Examples:
   weed worker -admin=localhost:23646 -jobType=volume_balance
   weed worker -admin=localhost:23646 -jobType=vacuum,volume_balance
   weed worker -admin=localhost:23646 -jobType=erasure_coding
+  weed worker -admin=localhost:23646 -jobType=admin_script
   weed worker -admin=admin.example.com:23646 -id=plugin-vacuum-a -heartbeat=10s
   weed worker -admin=localhost:23646 -workingDir=/var/lib/seaweedfs-plugin
   weed worker -admin=localhost:23646 -metricsPort=9327 -metricsIp=0.0.0.0

weed/command/worker_runtime.go

@@ -23,7 +23,7 @@ import (
 	"google.golang.org/grpc"
 )
 
-const defaultPluginWorkerJobTypes = "vacuum,volume_balance,erasure_coding"
+const defaultPluginWorkerJobTypes = "vacuum,volume_balance,erasure_coding,admin_script"
 
 type pluginWorkerRunOptions struct {
 	AdminServer string
@@ -156,6 +156,8 @@ func buildPluginWorkerHandler(jobType string, dialOption grpc.DialOption, maxExe
 		return pluginworker.NewVolumeBalanceHandler(dialOption), nil
 	case "erasure_coding":
 		return pluginworker.NewErasureCodingHandler(dialOption, workingDir), nil
+	case "admin_script":
+		return pluginworker.NewAdminScriptHandler(dialOption), nil
 	default:
 		return nil, fmt.Errorf("unsupported plugin job type %q", canonicalJobType)
 	}
@@ -220,6 +222,8 @@ func canonicalPluginWorkerJobType(jobType string) (string, error) {
 		return "volume_balance", nil
 	case "erasure_coding", "erasure-coding", "erasure.coding", "ec":
 		return "erasure_coding", nil
+	case "admin_script", "admin-script", "admin.script", "script", "admin":
+		return "admin_script", nil
 	default:
 		return "", fmt.Errorf("unsupported plugin job type %q", jobType)
 	}

weed/command/worker_test.go

@@ -7,7 +7,7 @@ func TestWorkerDefaultJobTypes(t *testing.T) {
 	if err != nil {
 		t.Fatalf("parsePluginWorkerJobTypes(default worker flag) err = %v", err)
 	}
-	if len(jobTypes) != 3 {
-		t.Fatalf("expected default worker job types to include 3 handlers, got %v", jobTypes)
+	if len(jobTypes) != 4 {
+		t.Fatalf("expected default worker job types to include 4 handlers, got %v", jobTypes)
 	}
 }

weed/filer/filer.go

@@ -25,6 +25,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"golang.org/x/sync/singleflight"
 )
 
 const (
@@ -54,6 +55,7 @@ type Filer struct {
 	Signature           int32
 	FilerConf           *FilerConf
 	RemoteStorage       *FilerRemoteStorage
+	lazyFetchGroup      singleflight.Group
 	Dlm                 *lock_manager.DistributedLockManager
 	MaxFilenameLength   uint32
 	deletionQuit        chan struct{}
@@ -375,6 +377,14 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e
 		}
 	}
 
+	if entry == nil && (err == nil || errors.Is(err, filer_pb.ErrNotFound)) {
+		if lazy, lazyErr := f.maybeLazyFetchFromRemote(ctx, p); lazyErr != nil {
+			glog.V(1).InfofCtx(ctx, "FindEntry lazy fetch %s: %v", p, lazyErr)
+		} else if lazy != nil {
+			return lazy, nil
+		}
+	}
+
 	return entry, err
 }
 

weed/filer/filer_lazy_remote.go

@@ -0,0 +1,116 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type lazyFetchContextKey struct{}
+
+// maybeLazyFetchFromRemote is called by FindEntry when the store returns no
+// entry for p. If p is under a remote-storage mount, it stats the remote
+// object, builds a filer Entry from the result, and persists it via
+// CreateEntry with SkipCheckParentDirectory so phantom parent directories
+// under the mount are not required.
+//
+// On a CreateEntry failure after a successful StatFile the in-memory entry is
+// still returned (availability over consistency); the singleflight key is
+// forgotten so the next lookup retries the filer write.
+//
+// Returns nil without error when: p is not under a remote mount; the remote
+// reports the object does not exist; or any other remote error occurs.
+func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (*Entry, error) {
+	// Prevent recursive invocation: CreateEntry calls FindEntry, which would
+	// re-enter this function and deadlock on the singleflight key.
+	if ctx.Value(lazyFetchContextKey{}) != nil {
+		return nil, nil
+	}
+
+	if f.RemoteStorage == nil {
+		return nil, nil
+	}
+
+	mountDir, remoteLoc := f.RemoteStorage.FindMountDirectory(p)
+	if remoteLoc == nil {
+		return nil, nil
+	}
+
+	client, _, found := f.RemoteStorage.FindRemoteStorageClient(p)
+	if !found {
+		return nil, nil
+	}
+
+	relPath := strings.TrimPrefix(string(p), string(mountDir))
+	if relPath != "" && !strings.HasPrefix(relPath, "/") {
+		relPath = "/" + relPath
+	}
+	base := strings.TrimSuffix(remoteLoc.Path, "/")
+	remotePath := "/" + strings.TrimLeft(base+relPath, "/")
+
+	objectLoc := &remote_pb.RemoteStorageLocation{
+		Name:   remoteLoc.Name,
+		Bucket: remoteLoc.Bucket,
+		Path:   remotePath,
+	}
+
+	type lazyFetchResult struct {
+		entry *Entry
+	}
+
+	key := string(p)
+	val, err, _ := f.lazyFetchGroup.Do(key, func() (interface{}, error) {
+		remoteEntry, statErr := client.StatFile(objectLoc)
+		if statErr != nil {
+			if errors.Is(statErr, remote_storage.ErrRemoteObjectNotFound) {
+				glog.V(3).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s not found in remote", p)
+			} else {
+				glog.Warningf("maybeLazyFetchFromRemote: stat %s failed: %v", p, statErr)
+			}
+			return lazyFetchResult{nil}, nil
+		}
+		if remoteEntry == nil {
+			glog.V(3).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s StatFile returned nil entry", p)
+			return lazyFetchResult{nil}, nil
+		}
+
+		mtime := time.Unix(remoteEntry.RemoteMtime, 0)
+		entry := &Entry{
+			FullPath: p,
+			Attr: Attr{
+				Mtime:    mtime,
+				Crtime:   mtime,
+				Mode:     0644,
+				FileSize: uint64(remoteEntry.RemoteSize),
+			},
+			Remote: remoteEntry,
+		}
+
+		persistBaseCtx, cancelPersist := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancelPersist()
+		persistCtx := context.WithValue(persistBaseCtx, lazyFetchContextKey{}, true)
+		saveErr := f.CreateEntry(persistCtx, entry, false, false, nil, true, f.MaxFilenameLength)
+		if saveErr != nil {
+			glog.Warningf("maybeLazyFetchFromRemote: failed to persist filer entry for %s: %v", p, saveErr)
+			f.lazyFetchGroup.Forget(key)
+		}
+
+		return lazyFetchResult{entry}, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	result, ok := val.(lazyFetchResult)
+	if !ok {
+		return nil, fmt.Errorf("maybeLazyFetchFromRemote: unexpected singleflight result type %T for %s", val, p)
+	}
+	return result.entry, nil
+}

weed/filer/filer_lazy_remote_test.go

@@ -0,0 +1,370 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// --- minimal FilerStore stub ---
+
+type stubFilerStore struct {
+	mu      sync.Mutex
+	entries map[string]*Entry
+	insertErr error
+}
+
+func newStubFilerStore() *stubFilerStore {
+	return &stubFilerStore{entries: make(map[string]*Entry)}
+}
+
+func (s *stubFilerStore) GetName() string { return "stub" }
+func (s *stubFilerStore) Initialize(util.Configuration, string) error { return nil }
+func (s *stubFilerStore) Shutdown() {}
+func (s *stubFilerStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (s *stubFilerStore) CommitTransaction(context.Context) error    { return nil }
+func (s *stubFilerStore) RollbackTransaction(context.Context) error  { return nil }
+func (s *stubFilerStore) KvPut(context.Context, []byte, []byte) error { return nil }
+func (s *stubFilerStore) KvGet(context.Context, []byte) ([]byte, error) {
+	return nil, ErrKvNotFound
+}
+func (s *stubFilerStore) KvDelete(context.Context, []byte) error { return nil }
+func (s *stubFilerStore) DeleteFolderChildren(context.Context, util.FullPath) error { return nil }
+func (s *stubFilerStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) {
+	return "", nil
+}
+func (s *stubFilerStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (string, error) {
+	return "", nil
+}
+
+func (s *stubFilerStore) InsertEntry(_ context.Context, entry *Entry) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.insertErr != nil {
+		return s.insertErr
+	}
+	s.entries[string(entry.FullPath)] = entry
+	return nil
+}
+
+func (s *stubFilerStore) UpdateEntry(_ context.Context, entry *Entry) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.entries[string(entry.FullPath)] = entry
+	return nil
+}
+
+func (s *stubFilerStore) FindEntry(_ context.Context, p util.FullPath) (*Entry, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if e, ok := s.entries[string(p)]; ok {
+		return e, nil
+	}
+	return nil, filer_pb.ErrNotFound
+}
+
+func (s *stubFilerStore) DeleteEntry(_ context.Context, p util.FullPath) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	delete(s.entries, string(p))
+	return nil
+}
+
+// --- minimal RemoteStorageClient stub ---
+
+type stubRemoteClient struct {
+	statResult *filer_pb.RemoteEntry
+	statErr    error
+}
+
+func (c *stubRemoteClient) StatFile(*remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
+	return c.statResult, c.statErr
+}
+func (c *stubRemoteClient) Traverse(*remote_pb.RemoteStorageLocation, remote_storage.VisitFunc) error {
+	return nil
+}
+func (c *stubRemoteClient) ReadFile(*remote_pb.RemoteStorageLocation, int64, int64) ([]byte, error) {
+	return nil, nil
+}
+func (c *stubRemoteClient) WriteDirectory(*remote_pb.RemoteStorageLocation, *filer_pb.Entry) error {
+	return nil
+}
+func (c *stubRemoteClient) RemoveDirectory(*remote_pb.RemoteStorageLocation) error { return nil }
+func (c *stubRemoteClient) WriteFile(*remote_pb.RemoteStorageLocation, *filer_pb.Entry, io.Reader) (*filer_pb.RemoteEntry, error) {
+	return nil, nil
+}
+func (c *stubRemoteClient) UpdateFileMetadata(*remote_pb.RemoteStorageLocation, *filer_pb.Entry, *filer_pb.Entry) error {
+	return nil
+}
+func (c *stubRemoteClient) DeleteFile(*remote_pb.RemoteStorageLocation) error { return nil }
+func (c *stubRemoteClient) ListBuckets() ([]*remote_storage.Bucket, error)    { return nil, nil }
+func (c *stubRemoteClient) CreateBucket(string) error                         { return nil }
+func (c *stubRemoteClient) DeleteBucket(string) error                         { return nil }
+
+// --- stub RemoteStorageClientMaker ---
+
+type stubClientMaker struct {
+	client remote_storage.RemoteStorageClient
+}
+
+func (m *stubClientMaker) Make(*remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return m.client, nil
+}
+func (m *stubClientMaker) HasBucket() bool { return true }
+
+// --- test filer factory ---
+
+func newTestFiler(t *testing.T, store *stubFilerStore, rs *FilerRemoteStorage) *Filer {
+	t.Helper()
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	mc := wdclient.NewMasterClient(
+		dialOption, "test", cluster.FilerType,
+		pb.ServerAddress("localhost:0"), "", "",
+		*pb.NewServiceDiscoveryFromMap(map[string]pb.ServerAddress{}),
+	)
+	f := &Filer{
+		RemoteStorage:       rs,
+		Store:               NewFilerStoreWrapper(store),
+		MaxFilenameLength:   255,
+		MasterClient:        mc,
+		fileIdDeletionQueue: util.NewUnboundedQueue(),
+		LocalMetaLogBuffer: log_buffer.NewLogBuffer("test", time.Minute,
+			func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, func() {}),
+	}
+	return f
+}
+
+// registerStubMaker registers a stub RemoteStorageClientMaker for the given
+// type string and returns a cleanup function that restores the previous maker.
+func registerStubMaker(t *testing.T, storageType string, client remote_storage.RemoteStorageClient) func() {
+	t.Helper()
+	prev := remote_storage.RemoteStorageClientMakers[storageType]
+	remote_storage.RemoteStorageClientMakers[storageType] = &stubClientMaker{client: client}
+	return func() {
+		if prev != nil {
+			remote_storage.RemoteStorageClientMakers[storageType] = prev
+		} else {
+			delete(remote_storage.RemoteStorageClientMakers, storageType)
+		}
+	}
+}
+
+// --- tests ---
+
+func TestMaybeLazyFetchFromRemote_HitsRemoteAndPersists(t *testing.T) {
+	const storageType = "stub_lazy_hit"
+	stub := &stubRemoteClient{
+		statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 1234},
+	}
+	defer registerStubMaker(t, storageType, stub)()
+
+	conf := &remote_pb.RemoteConf{Name: "mystore", Type: storageType}
+	rs := NewFilerRemoteStorage()
+	rs.storageNameToConf[conf.Name] = conf
+	rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
+		Name:   "mystore",
+		Bucket: "mybucket",
+		Path:   "/",
+	})
+
+	store := newStubFilerStore()
+	f := newTestFiler(t, store, rs)
+
+	entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/file.txt")
+	require.NoError(t, err)
+	require.NotNil(t, entry)
+	assert.Equal(t, util.FullPath("/buckets/mybucket/file.txt"), entry.FullPath)
+	assert.Equal(t, int64(1234), entry.Remote.RemoteSize)
+	assert.Equal(t, uint64(1234), entry.FileSize)
+
+	// entry must have been persisted in the store
+	stored, sErr := store.FindEntry(context.Background(), "/buckets/mybucket/file.txt")
+	require.NoError(t, sErr)
+	assert.Equal(t, int64(1234), stored.Remote.RemoteSize)
+}
+
+func TestMaybeLazyFetchFromRemote_NotUnderMount(t *testing.T) {
+	rs := NewFilerRemoteStorage()
+	store := newStubFilerStore()
+	f := newTestFiler(t, store, rs)
+
+	entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/not/a/mounted/path.txt")
+	require.NoError(t, err)
+	assert.Nil(t, entry)
+}
+
+func TestMaybeLazyFetchFromRemote_RemoteObjectNotFound(t *testing.T) {
+	const storageType = "stub_lazy_notfound"
+	stub := &stubRemoteClient{statErr: remote_storage.ErrRemoteObjectNotFound}
+	defer registerStubMaker(t, storageType, stub)()
+
+	conf := &remote_pb.RemoteConf{Name: "storenotfound", Type: storageType}
+	rs := NewFilerRemoteStorage()
+	rs.storageNameToConf[conf.Name] = conf
+	rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
+		Name:   "storenotfound",
+		Bucket: "mybucket",
+		Path:   "/",
+	})
+
+	store := newStubFilerStore()
+	f := newTestFiler(t, store, rs)
+
+	entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/missing.txt")
+	require.NoError(t, err)
+	assert.Nil(t, entry)
+}
+
+func TestMaybeLazyFetchFromRemote_CreateEntryFailureReturnsInMemoryEntry(t *testing.T) {
+	const storageType = "stub_lazy_saveerr"
+	stub := &stubRemoteClient{
+		statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 42},
+	}
+	defer registerStubMaker(t, storageType, stub)()
+
+	conf := &remote_pb.RemoteConf{Name: "storesaveerr", Type: storageType}
+	rs := NewFilerRemoteStorage()
+	rs.storageNameToConf[conf.Name] = conf
+	rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
+		Name:   "storesaveerr",
+		Bucket: "mybucket",
+		Path:   "/",
+	})
+
+	store := newStubFilerStore()
+	store.insertErr = errors.New("simulated store failure")
+	f := newTestFiler(t, store, rs)
+
+	// even with a store failure, the in-memory entry should be returned
+	entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/failfile.txt")
+	require.NoError(t, err)
+	require.NotNil(t, entry, "should return in-memory entry even when CreateEntry fails")
+	assert.Equal(t, int64(42), entry.Remote.RemoteSize)
+}
+
+func TestMaybeLazyFetchFromRemote_LongestPrefixMount(t *testing.T) {
+	// Register maker for the root mount
+	const typeRoot = "stub_lp_root"
+	stubRoot := &stubRemoteClient{statResult: &filer_pb.RemoteEntry{RemoteMtime: 1, RemoteSize: 10}}
+	defer registerStubMaker(t, typeRoot, stubRoot)()
+
+	// Register maker for the prefix mount
+	const typePrefix = "stub_lp_prefix"
+	stubPrefix := &stubRemoteClient{statResult: &filer_pb.RemoteEntry{RemoteMtime: 2, RemoteSize: 20}}
+	defer registerStubMaker(t, typePrefix, stubPrefix)()
+
+	rs := NewFilerRemoteStorage()
+	rs.storageNameToConf["rootstore"] = &remote_pb.RemoteConf{Name: "rootstore", Type: typeRoot}
+	rs.storageNameToConf["prefixstore"] = &remote_pb.RemoteConf{Name: "prefixstore", Type: typePrefix}
+
+	rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
+		Name: "rootstore", Bucket: "root-bucket", Path: "/",
+	})
+	rs.mapDirectoryToRemoteStorage("/buckets/mybucket/prefix", &remote_pb.RemoteStorageLocation{
+		Name: "prefixstore", Bucket: "prefix-bucket", Path: "/",
+	})
+
+	store := newStubFilerStore()
+	f := newTestFiler(t, store, rs)
+
+	// path under root mount only
+	entryRoot, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/file.txt")
+	require.NoError(t, err)
+	require.NotNil(t, entryRoot)
+	assert.Equal(t, int64(10), entryRoot.Remote.RemoteSize, "root mount should be used")
+
+	// path under nested (longer) mount — must prefer the longer prefix
+	entryPrefix, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/prefix/file.txt")
+	require.NoError(t, err)
+	require.NotNil(t, entryPrefix)
+	assert.Equal(t, int64(20), entryPrefix.Remote.RemoteSize, "nested mount should win (longest prefix)")
+}
+
+type countingRemoteClient struct {
+	stubRemoteClient
+	statCalls int
+}
+
+func (c *countingRemoteClient) StatFile(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
+	c.statCalls++
+	return c.stubRemoteClient.StatFile(loc)
+}
+
+func TestMaybeLazyFetchFromRemote_ContextGuardPreventsRecursion(t *testing.T) {
+	const storageType = "stub_lazy_guard"
+	countingStub := &countingRemoteClient{
+		stubRemoteClient: stubRemoteClient{
+			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1, RemoteSize: 1},
+		},
+	}
+	defer registerStubMaker(t, storageType, countingStub)()
+
+	conf := &remote_pb.RemoteConf{Name: "guardstore", Type: storageType}
+	rs := NewFilerRemoteStorage()
+	rs.storageNameToConf[conf.Name] = conf
+	rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
+		Name:   "guardstore",
+		Bucket: "mybucket",
+		Path:   "/",
+	})
+
+	store := newStubFilerStore()
+	f := newTestFiler(t, store, rs)
+
+	guardCtx := context.WithValue(context.Background(), lazyFetchContextKey{}, true)
+	entry, err := f.maybeLazyFetchFromRemote(guardCtx, "/buckets/mybucket/file.txt")
+	require.NoError(t, err)
+	assert.Nil(t, entry)
+	assert.Equal(t, 0, countingStub.statCalls, "guard should prevent StatFile from being called")
+}
+
+func TestFindEntry_LazyFetchOnMiss(t *testing.T) {
+	const storageType = "stub_lazy_findentry"
+	stub := &stubRemoteClient{
+		statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 999},
+	}
+	defer registerStubMaker(t, storageType, stub)()
+
+	conf := &remote_pb.RemoteConf{Name: "findentrystore", Type: storageType}
+	rs := NewFilerRemoteStorage()
+	rs.storageNameToConf[conf.Name] = conf
+	rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
+		Name:   "findentrystore",
+		Bucket: "mybucket",
+		Path:   "/",
+	})
+
+	store := newStubFilerStore()
+	f := newTestFiler(t, store, rs)
+
+	// First lookup: store miss → lazy fetch
+	entry, err := f.FindEntry(context.Background(), "/buckets/mybucket/obj.txt")
+	require.NoError(t, err, fmt.Sprintf("unexpected err: %v", err))
+	require.NotNil(t, entry)
+	assert.Equal(t, uint64(999), entry.FileSize)
+
+	// Second lookup: now in store, no remote call needed
+	entry2, err2 := f.FindEntry(context.Background(), "/buckets/mybucket/obj.txt")
+	require.NoError(t, err2)
+	require.NotNil(t, entry2)
+	assert.Equal(t, uint64(999), entry2.FileSize)
+}

weed/filer/remote_storage.go

@@ -82,6 +82,9 @@ func (rs *FilerRemoteStorage) mapDirectoryToRemoteStorage(dir util.FullPath, loc
 	rs.rules.Put([]byte(dir+"/"), loc)
 }
 
+// FindMountDirectory returns the mount directory and location for p. When multiple
+// mounts match (e.g. /buckets/b and /buckets/b/prefix), ptrie MatchPrefix visits
+// shorter prefixes first, so the last match is the longest prefix.
 func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util.FullPath, remoteLocation *remote_pb.RemoteStorageLocation) {
 	rs.rules.MatchPrefix([]byte(p), func(key []byte, value *remote_pb.RemoteStorageLocation) bool {
 		mountDir = util.FullPath(string(key[:len(key)-1]))

weed/filer/remote_storage_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -33,3 +34,37 @@ func TestFilerRemoteStorage_FindRemoteStorageClient(t *testing.T) {
 	_, _, found4 := rs.FindRemoteStorageClient("/a/b/cc")
 	assert.Equal(t, false, found4, "should not find storage client")
 }
+
+func TestFilerRemoteStorage_FindMountDirectory_LongestPrefixWins(t *testing.T) {
+	conf := &remote_pb.RemoteConf{Name: "store", Type: "s3"}
+	rs := NewFilerRemoteStorage()
+	rs.storageNameToConf[conf.Name] = conf
+
+	rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
+		Name:   "store",
+		Bucket: "bucket-root",
+		Path:   "/",
+	})
+	rs.mapDirectoryToRemoteStorage("/buckets/mybucket/prefix", &remote_pb.RemoteStorageLocation{
+		Name:   "store",
+		Bucket: "bucket-prefix",
+		Path:   "/",
+	})
+
+	tests := []struct {
+		path           string
+		wantMount      string
+		wantBucket     string
+	}{
+		{"/buckets/mybucket/file.txt", "/buckets/mybucket", "bucket-root"},
+		{"/buckets/mybucket/prefix/file.txt", "/buckets/mybucket/prefix", "bucket-prefix"},
+		{"/buckets/mybucket/prefix/sub/file.txt", "/buckets/mybucket/prefix", "bucket-prefix"},
+	}
+	for _, tt := range tests {
+		mountDir, loc := rs.FindMountDirectory(util.FullPath(tt.path))
+		assert.Equal(t, util.FullPath(tt.wantMount), mountDir, "mount dir for %s", tt.path)
+		if assert.NotNil(t, loc, "location for %s", tt.path) {
+			assert.Equal(t, tt.wantBucket, loc.Bucket, "bucket for %s", tt.path)
+		}
+	}
+}

weed/pb/master.proto

@@ -61,6 +61,11 @@ service Seaweed {
 
 //////////////////////////////////////////////////
 
+message DiskTag {
+  uint32 disk_id = 1;
+  repeated string tags = 2;
+}
+
 message Heartbeat {
   string ip = 1;
   uint32 port = 2;
@@ -89,6 +94,8 @@ message Heartbeat {
 
   // state flags
   volume_server_pb.VolumeServerState state = 23;
+
+  repeated DiskTag disk_tags = 24;
 }
 
 message HeartbeatResponse {
@@ -292,6 +299,7 @@ message DiskInfo {
   repeated VolumeEcShardInformationMessage ec_shard_infos = 7;
   int64 remote_volume_count = 8;
   uint32 disk_id = 9;
+  repeated string tags = 10;
 }
 message DataNodeInfo {
   string id = 1;
@@ -460,4 +468,4 @@ message RaftLeadershipTransferResponse {
 }
 
 message VolumeGrowResponse {
-}
+}

weed/pb/worker.proto

@@ -314,6 +314,7 @@ message ErasureCodingTaskConfig {
   int32 quiet_for_seconds = 2;      // Minimum quiet time before EC
   int32 min_volume_size_mb = 3;     // Minimum volume size for EC
   string collection_filter = 4;     // Only process volumes from specific collections
+  repeated string preferred_tags = 5; // Disk tags to prioritize for EC shard placement
 }
 
 // BalanceTaskConfig contains balance-specific configuration

weed/pb/worker_pb/worker.pb.go

@@ -2589,6 +2589,7 @@ type ErasureCodingTaskConfig struct {
 	QuietForSeconds  int32                  `protobuf:"varint,2,opt,name=quiet_for_seconds,json=quietForSeconds,proto3" json:"quiet_for_seconds,omitempty"`   // Minimum quiet time before EC
 	MinVolumeSizeMb  int32                  `protobuf:"varint,3,opt,name=min_volume_size_mb,json=minVolumeSizeMb,proto3" json:"min_volume_size_mb,omitempty"` // Minimum volume size for EC
 	CollectionFilter string                 `protobuf:"bytes,4,opt,name=collection_filter,json=collectionFilter,proto3" json:"collection_filter,omitempty"`   // Only process volumes from specific collections
+	PreferredTags    []string               `protobuf:"bytes,5,rep,name=preferred_tags,json=preferredTags,proto3" json:"preferred_tags,omitempty"`            // Disk tags to prioritize for EC shard placement
 	unknownFields    protoimpl.UnknownFields
 	sizeCache        protoimpl.SizeCache
 }
@@ -2651,6 +2652,13 @@ func (x *ErasureCodingTaskConfig) GetCollectionFilter() string {
 	return ""
 }
 
+func (x *ErasureCodingTaskConfig) GetPreferredTags() []string {
+	if x != nil {
+		return x.PreferredTags
+	}
+	return nil
+}
+
 // BalanceTaskConfig contains balance-specific configuration
 type BalanceTaskConfig struct {
 	state              protoimpl.MessageState `protogen:"open.v1"`
@@ -3559,12 +3567,13 @@ const file_worker_proto_rawDesc = "" +
 	"\x10VacuumTaskConfig\x12+\n" +
 	"\x11garbage_threshold\x18\x01 \x01(\x01R\x10garbageThreshold\x12/\n" +
 	"\x14min_volume_age_hours\x18\x02 \x01(\x05R\x11minVolumeAgeHours\x120\n" +
-	"\x14min_interval_seconds\x18\x03 \x01(\x05R\x12minIntervalSeconds\"\xc6\x01\n" +
+	"\x14min_interval_seconds\x18\x03 \x01(\x05R\x12minIntervalSeconds\"\xed\x01\n" +
 	"\x17ErasureCodingTaskConfig\x12%\n" +
 	"\x0efullness_ratio\x18\x01 \x01(\x01R\rfullnessRatio\x12*\n" +
 	"\x11quiet_for_seconds\x18\x02 \x01(\x05R\x0fquietForSeconds\x12+\n" +
 	"\x12min_volume_size_mb\x18\x03 \x01(\x05R\x0fminVolumeSizeMb\x12+\n" +
-	"\x11collection_filter\x18\x04 \x01(\tR\x10collectionFilter\"n\n" +
+	"\x11collection_filter\x18\x04 \x01(\tR\x10collectionFilter\x12%\n" +
+	"\x0epreferred_tags\x18\x05 \x03(\tR\rpreferredTags\"n\n" +
 	"\x11BalanceTaskConfig\x12/\n" +
 	"\x13imbalance_threshold\x18\x01 \x01(\x01R\x12imbalanceThreshold\x12(\n" +
 	"\x10min_server_count\x18\x02 \x01(\x05R\x0eminServerCount\"I\n" +

weed/plugin/worker/admin_script_handler.go

@@ -0,0 +1,635 @@
+package pluginworker
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/shell"
+	"google.golang.org/grpc"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+const (
+	adminScriptJobType        = "admin_script"
+	maxAdminScriptOutputBytes = 16 * 1024
+	defaultAdminScriptRunMins = 17
+	adminScriptDetectTickSecs = 60
+)
+
+const defaultAdminScript = `ec.balance -apply
+fs.log.purge -daysAgo=7
+volume.deleteEmpty -quietFor=24h -apply
+volume.fix.replication -apply
+s3.clean.uploads -timeAgo=24h`
+
+var adminScriptTokenRegex = regexp.MustCompile(`'.*?'|".*?"|\S+`)
+
+type AdminScriptHandler struct {
+	grpcDialOption grpc.DialOption
+}
+
+func NewAdminScriptHandler(grpcDialOption grpc.DialOption) *AdminScriptHandler {
+	return &AdminScriptHandler{grpcDialOption: grpcDialOption}
+}
+
+func (h *AdminScriptHandler) Capability() *plugin_pb.JobTypeCapability {
+	return &plugin_pb.JobTypeCapability{
+		JobType:                 adminScriptJobType,
+		CanDetect:               true,
+		CanExecute:              true,
+		MaxDetectionConcurrency: 1,
+		MaxExecutionConcurrency: 1,
+		DisplayName:             "Admin Script",
+		Description:             "Execute custom admin shell scripts",
+		Weight:                  20,
+	}
+}
+
+func (h *AdminScriptHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
+	return &plugin_pb.JobTypeDescriptor{
+		JobType:           adminScriptJobType,
+		DisplayName:       "Admin Script",
+		Description:       "Run custom admin shell scripts not covered by built-in job types",
+		Icon:              "fas fa-terminal",
+		DescriptorVersion: 1,
+		AdminConfigForm: &plugin_pb.ConfigForm{
+			FormId:      "admin-script-admin",
+			Title:       "Admin Script Configuration",
+			Description: "Define the admin shell script to execute.",
+			Sections: []*plugin_pb.ConfigSection{
+				{
+					SectionId:   "script",
+					Title:       "Script",
+					Description: "Commands run sequentially by the admin script worker.",
+					Fields: []*plugin_pb.ConfigField{
+						{
+							Name:        "script",
+							Label:       "Script",
+							Description: "Admin shell commands to execute (one per line).",
+							HelpText:    "Lock/unlock are handled by the admin server; omit explicit lock/unlock commands.",
+							Placeholder: "volume.balance -apply\nvolume.fix.replication -apply",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXTAREA,
+							Required:    true,
+						},
+						{
+							Name:        "run_interval_minutes",
+							Label:       "Run Interval (minutes)",
+							Description: "Minimum interval between successful admin script runs.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							Required:    true,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}},
+						},
+					},
+				},
+			},
+			DefaultValues: map[string]*plugin_pb.ConfigValue{
+				"script": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: defaultAdminScript},
+				},
+				"run_interval_minutes": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultAdminScriptRunMins},
+				},
+			},
+		},
+		AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
+			Enabled:                       true,
+			DetectionIntervalSeconds:      adminScriptDetectTickSecs,
+			DetectionTimeoutSeconds:       300,
+			MaxJobsPerDetection:           1,
+			GlobalExecutionConcurrency:    1,
+			PerWorkerExecutionConcurrency: 1,
+			RetryLimit:                    0,
+			RetryBackoffSeconds:           30,
+		},
+		WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{},
+	}
+}
+
+func (h *AdminScriptHandler) Detect(ctx context.Context, request *plugin_pb.RunDetectionRequest, sender DetectionSender) error {
+	if request == nil {
+		return fmt.Errorf("run detection request is nil")
+	}
+	if sender == nil {
+		return fmt.Errorf("detection sender is nil")
+	}
+	if request.JobType != "" && request.JobType != adminScriptJobType {
+		return fmt.Errorf("job type %q is not handled by admin_script worker", request.JobType)
+	}
+
+	script := normalizeAdminScript(readStringConfig(request.GetAdminConfigValues(), "script", ""))
+	scriptName := strings.TrimSpace(readStringConfig(request.GetAdminConfigValues(), "script_name", ""))
+	runIntervalMinutes := readAdminScriptRunIntervalMinutes(request.GetAdminConfigValues())
+	if shouldSkipDetectionByInterval(request.GetLastSuccessfulRun(), runIntervalMinutes*60) {
+		_ = sender.SendActivity(buildDetectorActivity(
+			"skipped_by_interval",
+			fmt.Sprintf("ADMIN SCRIPT: Detection skipped due to run interval (%dm)", runIntervalMinutes),
+			map[string]*plugin_pb.ConfigValue{
+				"run_interval_minutes": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(runIntervalMinutes)},
+				},
+			},
+		))
+		if err := sender.SendProposals(&plugin_pb.DetectionProposals{
+			JobType:   adminScriptJobType,
+			Proposals: []*plugin_pb.JobProposal{},
+			HasMore:   false,
+		}); err != nil {
+			return err
+		}
+		return sender.SendComplete(&plugin_pb.DetectionComplete{
+			JobType:        adminScriptJobType,
+			Success:        true,
+			TotalProposals: 0,
+		})
+	}
+
+	commands := parseAdminScriptCommands(script)
+	execCount := countExecutableCommands(commands)
+	if execCount == 0 {
+		_ = sender.SendActivity(buildDetectorActivity(
+			"no_script",
+			"ADMIN SCRIPT: No executable commands configured",
+			map[string]*plugin_pb.ConfigValue{
+				"command_count": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(execCount)},
+				},
+			},
+		))
+		if err := sender.SendProposals(&plugin_pb.DetectionProposals{
+			JobType:   adminScriptJobType,
+			Proposals: []*plugin_pb.JobProposal{},
+			HasMore:   false,
+		}); err != nil {
+			return err
+		}
+		return sender.SendComplete(&plugin_pb.DetectionComplete{
+			JobType:        adminScriptJobType,
+			Success:        true,
+			TotalProposals: 0,
+		})
+	}
+
+	proposal := buildAdminScriptProposal(script, scriptName, execCount)
+	proposals := []*plugin_pb.JobProposal{proposal}
+	hasMore := false
+	maxResults := int(request.MaxResults)
+	if maxResults > 0 && len(proposals) > maxResults {
+		proposals = proposals[:maxResults]
+		hasMore = true
+	}
+
+	if err := sender.SendProposals(&plugin_pb.DetectionProposals{
+		JobType:   adminScriptJobType,
+		Proposals: proposals,
+		HasMore:   hasMore,
+	}); err != nil {
+		return err
+	}
+
+	return sender.SendComplete(&plugin_pb.DetectionComplete{
+		JobType:        adminScriptJobType,
+		Success:        true,
+		TotalProposals: 1,
+	})
+}
+
+func (h *AdminScriptHandler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequest, sender ExecutionSender) error {
+	if request == nil || request.Job == nil {
+		return fmt.Errorf("execute job request is nil")
+	}
+	if sender == nil {
+		return fmt.Errorf("execution sender is nil")
+	}
+	if request.Job.JobType != "" && request.Job.JobType != adminScriptJobType {
+		return fmt.Errorf("job type %q is not handled by admin_script worker", request.Job.JobType)
+	}
+
+	script := normalizeAdminScript(readStringConfig(request.Job.Parameters, "script", ""))
+	scriptName := strings.TrimSpace(readStringConfig(request.Job.Parameters, "script_name", ""))
+	if script == "" {
+		script = normalizeAdminScript(readStringConfig(request.GetAdminConfigValues(), "script", ""))
+	}
+	if scriptName == "" {
+		scriptName = strings.TrimSpace(readStringConfig(request.GetAdminConfigValues(), "script_name", ""))
+	}
+
+	commands := parseAdminScriptCommands(script)
+	execCommands := filterExecutableCommands(commands)
+	if len(execCommands) == 0 {
+		return sender.SendCompleted(&plugin_pb.JobCompleted{
+			Success:      false,
+			ErrorMessage: "no executable admin script commands configured",
+		})
+	}
+
+	commandEnv, cancel, err := h.buildAdminScriptCommandEnv(ctx, request.ClusterContext)
+	if err != nil {
+		return sender.SendCompleted(&plugin_pb.JobCompleted{
+			Success:      false,
+			ErrorMessage: err.Error(),
+		})
+	}
+	defer cancel()
+
+	if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
+		JobId:           request.Job.JobId,
+		JobType:         request.Job.JobType,
+		State:           plugin_pb.JobState_JOB_STATE_ASSIGNED,
+		ProgressPercent: 0,
+		Stage:           "assigned",
+		Message:         "admin script job accepted",
+		Activities: []*plugin_pb.ActivityEvent{
+			buildExecutorActivity("assigned", "admin script job accepted"),
+		},
+	}); err != nil {
+		return err
+	}
+
+	output := &limitedBuffer{maxBytes: maxAdminScriptOutputBytes}
+	executed := 0
+	errorMessages := make([]string, 0)
+	executedCommands := make([]string, 0, len(execCommands))
+
+	for _, cmd := range execCommands {
+		if ctx.Err() != nil {
+			errorMessages = append(errorMessages, ctx.Err().Error())
+			break
+		}
+
+		commandLine := formatAdminScriptCommand(cmd)
+		executedCommands = append(executedCommands, commandLine)
+		_, _ = fmt.Fprintf(output, "$ %s\n", commandLine)
+
+		found := false
+		for _, command := range shell.Commands {
+			if command.Name() != cmd.Name {
+				continue
+			}
+			found = true
+			if err := command.Do(cmd.Args, commandEnv, output); err != nil {
+				msg := fmt.Sprintf("%s: %v", cmd.Name, err)
+				errorMessages = append(errorMessages, msg)
+				_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
+					State:           plugin_pb.JobState_JOB_STATE_RUNNING,
+					ProgressPercent: percentProgress(executed+1, len(execCommands)),
+					Stage:           "error",
+					Message:         msg,
+					Activities: []*plugin_pb.ActivityEvent{
+						buildExecutorActivity("error", msg),
+					},
+				})
+			}
+			break
+		}
+
+		if !found {
+			msg := fmt.Sprintf("unknown admin command: %s", cmd.Name)
+			errorMessages = append(errorMessages, msg)
+			_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
+				State:           plugin_pb.JobState_JOB_STATE_RUNNING,
+				ProgressPercent: percentProgress(executed+1, len(execCommands)),
+				Stage:           "error",
+				Message:         msg,
+				Activities: []*plugin_pb.ActivityEvent{
+					buildExecutorActivity("error", msg),
+				},
+			})
+		}
+
+		executed++
+		progress := percentProgress(executed, len(execCommands))
+		_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
+			State:           plugin_pb.JobState_JOB_STATE_RUNNING,
+			ProgressPercent: progress,
+			Stage:           "running",
+			Message:         fmt.Sprintf("executed %d/%d command(s)", executed, len(execCommands)),
+			Activities: []*plugin_pb.ActivityEvent{
+				buildExecutorActivity("running", commandLine),
+			},
+		})
+	}
+
+	scriptHash := hashAdminScript(script)
+	resultSummary := fmt.Sprintf("admin script executed (%d command(s))", executed)
+	if scriptName != "" {
+		resultSummary = fmt.Sprintf("admin script %q executed (%d command(s))", scriptName, executed)
+	}
+
+	outputValues := map[string]*plugin_pb.ConfigValue{
+		"command_count": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(executed)},
+		},
+		"error_count": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(errorMessages))},
+		},
+		"script_hash": {
+			Kind: &plugin_pb.ConfigValue_StringValue{StringValue: scriptHash},
+		},
+	}
+	if scriptName != "" {
+		outputValues["script_name"] = &plugin_pb.ConfigValue{
+			Kind: &plugin_pb.ConfigValue_StringValue{StringValue: scriptName},
+		}
+	}
+	if len(executedCommands) > 0 {
+		outputValues["commands"] = &plugin_pb.ConfigValue{
+			Kind: &plugin_pb.ConfigValue_StringList{
+				StringList: &plugin_pb.StringList{Values: executedCommands},
+			},
+		}
+	}
+	if out := strings.TrimSpace(output.String()); out != "" {
+		outputValues["output"] = &plugin_pb.ConfigValue{
+			Kind: &plugin_pb.ConfigValue_StringValue{StringValue: out},
+		}
+	}
+	if output.truncated {
+		outputValues["output_truncated"] = &plugin_pb.ConfigValue{
+			Kind: &plugin_pb.ConfigValue_BoolValue{BoolValue: true},
+		}
+	}
+
+	success := len(errorMessages) == 0 && ctx.Err() == nil
+	errorMessage := ""
+	if !success {
+		errorMessage = strings.Join(errorMessages, "; ")
+		if ctx.Err() != nil {
+			if errorMessage == "" {
+				errorMessage = ctx.Err().Error()
+			} else {
+				errorMessage = fmt.Sprintf("%s; %s", errorMessage, ctx.Err().Error())
+			}
+		}
+	}
+
+	return sender.SendCompleted(&plugin_pb.JobCompleted{
+		Success:      success,
+		ErrorMessage: errorMessage,
+		Result: &plugin_pb.JobResult{
+			Summary:      resultSummary,
+			OutputValues: outputValues,
+		},
+		Activities: []*plugin_pb.ActivityEvent{
+			buildExecutorActivity("completed", resultSummary),
+		},
+		CompletedAt: timestamppb.Now(),
+	})
+}
+
+func readAdminScriptRunIntervalMinutes(values map[string]*plugin_pb.ConfigValue) int {
+	runIntervalMinutes := int(readInt64Config(values, "run_interval_minutes", defaultAdminScriptRunMins))
+	if runIntervalMinutes <= 0 {
+		return defaultAdminScriptRunMins
+	}
+	return runIntervalMinutes
+}
+
+type adminScriptCommand struct {
+	Name string
+	Args []string
+	Raw  string
+}
+
+func normalizeAdminScript(script string) string {
+	script = strings.ReplaceAll(script, "\r\n", "\n")
+	return strings.TrimSpace(script)
+}
+
+func parseAdminScriptCommands(script string) []adminScriptCommand {
+	script = normalizeAdminScript(script)
+	if script == "" {
+		return nil
+	}
+	lines := strings.Split(script, "\n")
+	commands := make([]adminScriptCommand, 0)
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		for _, chunk := range strings.Split(line, ";") {
+			chunk = strings.TrimSpace(chunk)
+			if chunk == "" {
+				continue
+			}
+			parts := adminScriptTokenRegex.FindAllString(chunk, -1)
+			if len(parts) == 0 {
+				continue
+			}
+			args := make([]string, 0, len(parts)-1)
+			for _, arg := range parts[1:] {
+				args = append(args, strings.Trim(arg, "\"'"))
+			}
+			commands = append(commands, adminScriptCommand{
+				Name: strings.TrimSpace(parts[0]),
+				Args: args,
+				Raw:  chunk,
+			})
+		}
+	}
+	return commands
+}
+
+func filterExecutableCommands(commands []adminScriptCommand) []adminScriptCommand {
+	exec := make([]adminScriptCommand, 0, len(commands))
+	for _, cmd := range commands {
+		if cmd.Name == "" {
+			continue
+		}
+		if isAdminScriptLockCommand(cmd.Name) {
+			continue
+		}
+		exec = append(exec, cmd)
+	}
+	return exec
+}
+
+func countExecutableCommands(commands []adminScriptCommand) int {
+	count := 0
+	for _, cmd := range commands {
+		if cmd.Name == "" {
+			continue
+		}
+		if isAdminScriptLockCommand(cmd.Name) {
+			continue
+		}
+		count++
+	}
+	return count
+}
+
+func isAdminScriptLockCommand(name string) bool {
+	switch strings.ToLower(strings.TrimSpace(name)) {
+	case "lock", "unlock":
+		return true
+	default:
+		return false
+	}
+}
+
+func buildAdminScriptProposal(script, scriptName string, commandCount int) *plugin_pb.JobProposal {
+	scriptHash := hashAdminScript(script)
+	summary := "Run admin script"
+	if scriptName != "" {
+		summary = fmt.Sprintf("Run admin script: %s", scriptName)
+	}
+	detail := fmt.Sprintf("Admin script with %d command(s)", commandCount)
+	proposalID := fmt.Sprintf("admin-script-%s-%d", scriptHash[:8], time.Now().UnixNano())
+
+	labels := map[string]string{
+		"script_hash": scriptHash,
+	}
+	if scriptName != "" {
+		labels["script_name"] = scriptName
+	}
+
+	return &plugin_pb.JobProposal{
+		ProposalId: proposalID,
+		DedupeKey:  "admin-script:" + scriptHash,
+		JobType:    adminScriptJobType,
+		Priority:   plugin_pb.JobPriority_JOB_PRIORITY_NORMAL,
+		Summary:    summary,
+		Detail:     detail,
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"script": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: script},
+			},
+			"script_name": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: scriptName},
+			},
+			"script_hash": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: scriptHash},
+			},
+			"command_count": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(commandCount)},
+			},
+		},
+		Labels: labels,
+	}
+}
+
+func (h *AdminScriptHandler) buildAdminScriptCommandEnv(
+	ctx context.Context,
+	clusterContext *plugin_pb.ClusterContext,
+) (*shell.CommandEnv, context.CancelFunc, error) {
+	if clusterContext == nil {
+		return nil, nil, fmt.Errorf("cluster context is required")
+	}
+
+	masters := normalizeAddressList(clusterContext.MasterGrpcAddresses)
+	if len(masters) == 0 {
+		return nil, nil, fmt.Errorf("missing master addresses for admin script")
+	}
+
+	filerGroup := ""
+	mastersValue := strings.Join(masters, ",")
+	options := shell.ShellOptions{
+		Masters:        &mastersValue,
+		GrpcDialOption: h.grpcDialOption,
+		FilerGroup:     &filerGroup,
+		Directory:      "/",
+	}
+
+	filers := normalizeAddressList(clusterContext.FilerGrpcAddresses)
+	if len(filers) > 0 {
+		options.FilerAddress = pb.ServerAddress(filers[0])
+	} else {
+		glog.V(1).Infof("admin script worker missing filer address; filer-dependent commands may fail")
+	}
+
+	commandEnv := shell.NewCommandEnv(&options)
+	commandEnv.ForceNoLock()
+
+	ctx, cancel := context.WithCancel(ctx)
+	go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
+
+	return commandEnv, cancel, nil
+}
+
+func normalizeAddressList(addresses []string) []string {
+	normalized := make([]string, 0, len(addresses))
+	seen := make(map[string]struct{}, len(addresses))
+	for _, address := range addresses {
+		address = strings.TrimSpace(address)
+		if address == "" {
+			continue
+		}
+		if _, exists := seen[address]; exists {
+			continue
+		}
+		seen[address] = struct{}{}
+		normalized = append(normalized, address)
+	}
+	return normalized
+}
+
+func hashAdminScript(script string) string {
+	sum := sha256.Sum256([]byte(script))
+	return hex.EncodeToString(sum[:])
+}
+
+func formatAdminScriptCommand(cmd adminScriptCommand) string {
+	if len(cmd.Args) == 0 {
+		return cmd.Name
+	}
+	return fmt.Sprintf("%s %s", cmd.Name, strings.Join(cmd.Args, " "))
+}
+
+func percentProgress(done, total int) float64 {
+	if total <= 0 {
+		return 0
+	}
+	if done < 0 {
+		done = 0
+	}
+	if done > total {
+		done = total
+	}
+	return float64(done) / float64(total) * 100
+}
+
+type limitedBuffer struct {
+	buf       bytes.Buffer
+	maxBytes  int
+	truncated bool
+}
+
+func (b *limitedBuffer) Write(p []byte) (int, error) {
+	if b == nil {
+		return len(p), nil
+	}
+	if b.maxBytes <= 0 {
+		b.truncated = true
+		return len(p), nil
+	}
+	remaining := b.maxBytes - b.buf.Len()
+	if remaining <= 0 {
+		b.truncated = true
+		return len(p), nil
+	}
+	if len(p) > remaining {
+		_, _ = b.buf.Write(p[:remaining])
+		b.truncated = true
+		return len(p), nil
+	}
+	_, _ = b.buf.Write(p)
+	return len(p), nil
+}
+
+func (b *limitedBuffer) String() string {
+	if b == nil {
+		return ""
+	}
+	return b.buf.String()
+}

weed/plugin/worker/admin_script_handler_test.go

@@ -0,0 +1,100 @@
+package pluginworker
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func TestAdminScriptDescriptorDefaults(t *testing.T) {
+	descriptor := NewAdminScriptHandler(nil).Descriptor()
+	if descriptor == nil {
+		t.Fatalf("expected descriptor")
+	}
+	if descriptor.AdminRuntimeDefaults == nil {
+		t.Fatalf("expected admin runtime defaults")
+	}
+	if descriptor.AdminRuntimeDefaults.DetectionIntervalSeconds != adminScriptDetectTickSecs {
+		t.Fatalf("unexpected detection interval seconds: got=%d want=%d",
+			descriptor.AdminRuntimeDefaults.DetectionIntervalSeconds, adminScriptDetectTickSecs)
+	}
+	if descriptor.AdminConfigForm == nil {
+		t.Fatalf("expected admin config form")
+	}
+	runInterval := readInt64Config(descriptor.AdminConfigForm.DefaultValues, "run_interval_minutes", 0)
+	if runInterval != defaultAdminScriptRunMins {
+		t.Fatalf("unexpected run_interval_minutes default: got=%d want=%d", runInterval, defaultAdminScriptRunMins)
+	}
+	script := readStringConfig(descriptor.AdminConfigForm.DefaultValues, "script", "")
+	if strings.TrimSpace(script) == "" {
+		t.Fatalf("expected non-empty default script")
+	}
+}
+
+func TestAdminScriptDetectSkipsByRunInterval(t *testing.T) {
+	handler := NewAdminScriptHandler(nil)
+	sender := &recordingDetectionSender{}
+	err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType:           adminScriptJobType,
+		LastSuccessfulRun: timestamppb.New(time.Now().Add(-2 * time.Minute)),
+		AdminConfigValues: map[string]*plugin_pb.ConfigValue{
+			"script": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: defaultAdminScript},
+			},
+			"run_interval_minutes": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 17},
+			},
+		},
+	}, sender)
+	if err != nil {
+		t.Fatalf("detect returned err = %v", err)
+	}
+	if sender.proposals == nil {
+		t.Fatalf("expected proposals message")
+	}
+	if len(sender.proposals.Proposals) != 0 {
+		t.Fatalf("expected zero proposals, got %d", len(sender.proposals.Proposals))
+	}
+	if sender.complete == nil || !sender.complete.Success {
+		t.Fatalf("expected successful completion message")
+	}
+	if len(sender.events) == 0 {
+		t.Fatalf("expected detector activity events")
+	}
+	if !strings.Contains(sender.events[0].Message, "run interval") {
+		t.Fatalf("unexpected skip message: %q", sender.events[0].Message)
+	}
+}
+
+func TestAdminScriptDetectCreatesProposalWhenIntervalElapsed(t *testing.T) {
+	handler := NewAdminScriptHandler(nil)
+	sender := &recordingDetectionSender{}
+	err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType:           adminScriptJobType,
+		LastSuccessfulRun: timestamppb.New(time.Now().Add(-20 * time.Minute)),
+		AdminConfigValues: map[string]*plugin_pb.ConfigValue{
+			"script": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: defaultAdminScript},
+			},
+			"run_interval_minutes": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 17},
+			},
+		},
+	}, sender)
+	if err != nil {
+		t.Fatalf("detect returned err = %v", err)
+	}
+	if sender.proposals == nil {
+		t.Fatalf("expected proposals message")
+	}
+	if len(sender.proposals.Proposals) != 1 {
+		t.Fatalf("expected one proposal, got %d", len(sender.proposals.Proposals))
+	}
+	if sender.complete == nil || !sender.complete.Success || sender.complete.TotalProposals != 1 {
+		t.Fatalf("unexpected completion message: %+v", sender.complete)
+	}
+}

weed/plugin/worker/erasure_coding_handler.go

@@ -11,6 +11,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 	ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
 	erasurecodingtask "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
@@ -128,6 +129,14 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
 							Required:    true,
 							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
 						},
+						{
+							Name:        "preferred_tags",
+							Label:       "Preferred Tags",
+							Description: "Comma-separated disk tags to prioritize for EC shard placement, ordered by preference.",
+							Placeholder: "fast,ssd",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
+						},
 					},
 				},
 			},
@@ -144,6 +153,9 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
 				"min_interval_seconds": {
 					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 60},
 				},
+				"preferred_tags": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
+				},
 			},
 		},
 		AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
@@ -169,6 +181,9 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
 			"min_interval_seconds": {
 				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 60},
 			},
+			"preferred_tags": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
+			},
 		},
 	}
 }
@@ -601,6 +616,7 @@ func deriveErasureCodingWorkerConfig(values map[string]*plugin_pb.ConfigValue) *
 	if minIntervalSeconds < 0 {
 		minIntervalSeconds = 0
 	}
+	taskConfig.PreferredTags = util.NormalizeTagList(readStringListConfig(values, "preferred_tags"))
 
 	return &erasureCodingWorkerConfig{
 		TaskConfig:         taskConfig,

weed/server/master_grpc_server.go

@@ -162,6 +162,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 		}
 
 		dn.AdjustMaxVolumeCounts(heartbeat.MaxVolumeCounts)
+		dn.UpdateDiskTags(heartbeat.DiskTags)
 
 		glog.V(4).Infof("master received heartbeat %s", heartbeat.String())
 		stats.MasterReceivedHeartbeatCounter.WithLabelValues("total").Inc()

weed/server/volume_server.go

@@ -58,7 +58,7 @@ type VolumeServer struct {
 
 func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	port int, grpcPort int, publicUrl string, id string,
-	folders []string, maxCounts []int32, minFreeSpaces []util.MinFreeSpace, diskTypes []types.DiskType,
+	folders []string, maxCounts []int32, minFreeSpaces []util.MinFreeSpace, diskTypes []types.DiskType, diskTags [][]string,
 	idxFolder string,
 	needleMapKind storage.NeedleMapKind,
 	masterNodes []pb.ServerAddress, pulsePeriod time.Duration,
@@ -118,7 +118,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 
 	vs.checkWithMaster()
 
-	vs.store = storage.NewStore(vs.grpcDialOption, ip, port, grpcPort, publicUrl, id, folders, maxCounts, minFreeSpaces, idxFolder, vs.needleMapKind, diskTypes, ldbTimeout)
+	vs.store = storage.NewStore(vs.grpcDialOption, ip, port, grpcPort, publicUrl, id, folders, maxCounts, minFreeSpaces, idxFolder, vs.needleMapKind, diskTypes, diskTags, ldbTimeout)
 	vs.guard = security.NewGuard(whiteList, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec)
 
 	handleStaticResources(adminMux)

weed/shell/command_ec_balance.go

@@ -2,9 +2,9 @@ package shell
 
 import (
 	"flag"
-	"fmt"
 	"io"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 )
 
@@ -66,7 +66,7 @@ func (c *commandEcBalance) Do(args []string, commandEnv *CommandEnv, writer io.W
 	} else {
 		collections = append(collections, *collection)
 	}
-	fmt.Printf("balanceEcVolumes collections %+v\n", len(collections))
+	glog.V(1).Infof("balanceEcVolumes collections %+v\n", len(collections))
 
 	rp, err := parseReplicaPlacementArg(commandEnv, *shardReplicaPlacement)
 	if err != nil {

weed/shell/command_ec_common.go

@@ -150,14 +150,14 @@ func parseReplicaPlacementArg(commandEnv *CommandEnv, replicaStr string) (*super
 		if err != nil {
 			return rp, err
 		}
-		fmt.Printf("using replica placement %q for EC volumes\n", rp.String())
+		glog.V(1).Infof("using replica placement %q for EC volumes\n", rp.String())
 	} else {
 		// No replica placement argument provided, resolve from master default settings.
 		rp, err = getDefaultReplicaPlacement(commandEnv)
 		if err != nil {
 			return rp, err
 		}
-		fmt.Printf("using master default replica placement %q for EC volumes\n", rp.String())
+		glog.V(1).Infof("using master default replica placement %q for EC volumes\n", rp.String())
 	}
 
 	return rp, nil
@@ -1628,7 +1628,7 @@ func EcBalance(commandEnv *CommandEnv, collections []string, dc string, ecReplic
 	}
 
 	if len(collections) == 0 {
-		fmt.Printf("WARNING: No collections to balance EC volumes across.\n")
+		glog.V(1).Infof("WARNING: No collections to balance EC volumes across.\n")
 	}
 	for _, c := range collections {
 		if err = ecb.balanceEcVolumes(c); err != nil {

weed/shell/command_ec_decode.go

@@ -35,7 +35,7 @@ func (c *commandEcDecode) Name() string {
 func (c *commandEcDecode) Help() string {
 	return `decode a erasure coded volume into a normal volume
 
-	ec.decode [-collection=""] [-volumeId=<volume_id>] [-diskType=<disk_type>]
+	ec.decode [-collection=""] [-volumeId=<volume_id>] [-diskType=<disk_type>] [-checkMinFreeSpace]
 
 	The -collection parameter supports regular expressions for pattern matching:
 	  - Use exact match: ec.decode -collection="^mybucket$"
@@ -44,6 +44,7 @@ func (c *commandEcDecode) Help() string {
 
 	Options:
 	  -diskType: source disk type where EC shards are stored (hdd, ssd, or empty for default hdd)
+	  -checkMinFreeSpace: check min free space when selecting the decode target (default true)
 
 	Examples:
 	  # Decode EC shards from HDD (default)
@@ -64,6 +65,7 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 	volumeId := decodeCommand.Int("volumeId", 0, "the volume id")
 	collection := decodeCommand.String("collection", "", "the collection name")
 	diskTypeStr := decodeCommand.String("diskType", "", "source disk type where EC shards are stored (hdd, ssd, or empty for default hdd)")
+	checkMinFreeSpace := decodeCommand.Bool("checkMinFreeSpace", true, "check min free space when selecting the decode target")
 	if err = decodeCommand.Parse(args); err != nil {
 		return nil
 	}
@@ -80,10 +82,14 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 	if err != nil {
 		return err
 	}
+	var diskUsageState *decodeDiskUsageState
+	if *checkMinFreeSpace {
+		diskUsageState = newDecodeDiskUsageState(topologyInfo, diskType)
+	}
 
 	// volumeId is provided
 	if vid != 0 {
-		return doEcDecode(commandEnv, topologyInfo, *collection, vid, diskType)
+		return doEcDecode(commandEnv, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState)
 	}
 
 	// apply to all volumes in the collection
@@ -93,7 +99,7 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 	}
 	fmt.Printf("ec decode volumes: %v\n", volumeIds)
 	for _, vid := range volumeIds {
-		if err = doEcDecode(commandEnv, topologyInfo, *collection, vid, diskType); err != nil {
+		if err = doEcDecode(commandEnv, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState); err != nil {
 			return err
 		}
 	}
@@ -101,7 +107,7 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 	return nil
 }
 
-func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collection string, vid needle.VolumeId, diskType types.DiskType) (err error) {
+func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collection string, vid needle.VolumeId, diskType types.DiskType, checkMinFreeSpace bool, diskUsageState *decodeDiskUsageState) (err error) {
 
 	if !commandEnv.isLocked() {
 		return fmt.Errorf("lock is lost")
@@ -112,8 +118,36 @@ func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collec
 
 	fmt.Printf("ec volume %d shard locations: %+v\n", vid, nodeToEcShardsInfo)
 
+	if len(nodeToEcShardsInfo) == 0 {
+		return fmt.Errorf("no EC shards found for volume %d (diskType %s)", vid, diskType.ReadableString())
+	}
+
+	var originalShardCounts map[pb.ServerAddress]int
+	if diskUsageState != nil {
+		originalShardCounts = make(map[pb.ServerAddress]int, len(nodeToEcShardsInfo))
+		for location, si := range nodeToEcShardsInfo {
+			originalShardCounts[location] = si.Count()
+		}
+	}
+
+	var eligibleTargets map[pb.ServerAddress]struct{}
+	if checkMinFreeSpace {
+		if diskUsageState == nil {
+			return fmt.Errorf("min free space checking requires disk usage state")
+		}
+		eligibleTargets = make(map[pb.ServerAddress]struct{})
+		for location := range nodeToEcShardsInfo {
+			if freeCount, found := diskUsageState.freeVolumeCount(location); found && freeCount > 0 {
+				eligibleTargets[location] = struct{}{}
+			}
+		}
+		if len(eligibleTargets) == 0 {
+			return fmt.Errorf("no eligible target datanodes with free volume slots for volume %d (diskType %s); use -checkMinFreeSpace=false to override", vid, diskType.ReadableString())
+		}
+	}
+
 	// collect ec shards to the server with most space
-	targetNodeLocation, err := collectEcShards(commandEnv, nodeToEcShardsInfo, collection, vid)
+	targetNodeLocation, err := collectEcShards(commandEnv, nodeToEcShardsInfo, collection, vid, eligibleTargets)
 	if err != nil {
 		return fmt.Errorf("collectEcShards for volume %d: %v", vid, err)
 	}
@@ -124,7 +158,13 @@ func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collec
 		// Special case: if the EC index has no live entries, decoding is a no-op.
 		// Just purge EC shards and return success without generating/mounting an empty volume.
 		if isEcDecodeEmptyVolumeErr(err) {
-			return unmountAndDeleteEcShards(commandEnv.option.GrpcDialOption, collection, nodeToEcShardsInfo, vid)
+			if err := unmountAndDeleteEcShards(commandEnv.option.GrpcDialOption, collection, nodeToEcShardsInfo, vid); err != nil {
+				return err
+			}
+			if diskUsageState != nil {
+				diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, false)
+			}
+			return nil
 		}
 		return fmt.Errorf("generate normal volume %d on %s: %v", vid, targetNodeLocation, err)
 	}
@@ -134,6 +174,9 @@ func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collec
 	if err != nil {
 		return fmt.Errorf("delete ec shards for volume %d: %v", vid, err)
 	}
+	if diskUsageState != nil {
+		diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, true)
+	}
 
 	return nil
 }
@@ -207,11 +250,16 @@ func generateNormalVolume(grpcDialOption grpc.DialOption, vid needle.VolumeId, c
 
 }
 
-func collectEcShards(commandEnv *CommandEnv, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, collection string, vid needle.VolumeId) (targetNodeLocation pb.ServerAddress, err error) {
+func collectEcShards(commandEnv *CommandEnv, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, collection string, vid needle.VolumeId, eligibleTargets map[pb.ServerAddress]struct{}) (targetNodeLocation pb.ServerAddress, err error) {
 
-	maxShardCount := 0
+	maxShardCount := -1
 	existingShardsInfo := erasure_coding.NewShardsInfo()
 	for loc, si := range nodeToShardsInfo {
+		if eligibleTargets != nil {
+			if _, ok := eligibleTargets[loc]; !ok {
+				continue
+			}
+		}
 		toBeCopiedShardCount := si.MinusParityShards().Count()
 		if toBeCopiedShardCount > maxShardCount {
 			maxShardCount = toBeCopiedShardCount
@@ -219,6 +267,9 @@ func collectEcShards(commandEnv *CommandEnv, nodeToShardsInfo map[pb.ServerAddre
 			existingShardsInfo = si
 		}
 	}
+	if targetNodeLocation == "" {
+		return "", fmt.Errorf("no eligible target datanodes available to decode volume %d", vid)
+	}
 
 	fmt.Printf("collectEcShards: ec volume %d collect shards to %s from: %+v\n", vid, targetNodeLocation, nodeToShardsInfo)
 
@@ -326,3 +377,58 @@ func collectEcNodeShardsInfo(topoInfo *master_pb.TopologyInfo, vid needle.Volume
 
 	return res
 }
+
+type decodeDiskUsageState struct {
+	byNode map[pb.ServerAddress]*decodeDiskUsageCounts
+}
+
+type decodeDiskUsageCounts struct {
+	maxVolumeCount    int64
+	volumeCount       int64
+	remoteVolumeCount int64
+	ecShardCount      int64
+}
+
+func newDecodeDiskUsageState(topoInfo *master_pb.TopologyInfo, diskType types.DiskType) *decodeDiskUsageState {
+	state := &decodeDiskUsageState{byNode: make(map[pb.ServerAddress]*decodeDiskUsageCounts)}
+	eachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		if diskInfo, found := dn.DiskInfos[string(diskType)]; found {
+			state.byNode[pb.NewServerAddressFromDataNode(dn)] = &decodeDiskUsageCounts{
+				maxVolumeCount:    diskInfo.MaxVolumeCount,
+				volumeCount:       diskInfo.VolumeCount,
+				remoteVolumeCount: diskInfo.RemoteVolumeCount,
+				ecShardCount:      int64(countShards(diskInfo.EcShardInfos)),
+			}
+		}
+	})
+	return state
+}
+
+func (state *decodeDiskUsageState) freeVolumeCount(location pb.ServerAddress) (int64, bool) {
+	if state == nil {
+		return 0, false
+	}
+	usage, found := state.byNode[location]
+	if !found {
+		return 0, false
+	}
+	free := usage.maxVolumeCount - (usage.volumeCount - usage.remoteVolumeCount)
+	free -= (usage.ecShardCount + int64(erasure_coding.DataShardsCount) - 1) / int64(erasure_coding.DataShardsCount)
+	return free, true
+}
+
+func (state *decodeDiskUsageState) applyDecode(targetNodeLocation pb.ServerAddress, shardCounts map[pb.ServerAddress]int, createdVolume bool) {
+	if state == nil {
+		return
+	}
+	for location, shardCount := range shardCounts {
+		if usage, found := state.byNode[location]; found {
+			usage.ecShardCount -= int64(shardCount)
+		}
+	}
+	if createdVolume {
+		if usage, found := state.byNode[targetNodeLocation]; found {
+			usage.volumeCount++
+		}
+	}
+}

weed/shell/command_sleep.go

@@ -0,0 +1,43 @@
+package shell
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+	"time"
+)
+
+func init() {
+	Commands = append(Commands, &commandSleep{})
+}
+
+// =========== Sleep ==============
+type commandSleep struct {
+}
+
+func (c *commandSleep) Name() string {
+	return "sleep"
+}
+
+func (c *commandSleep) Help() string {
+	return `sleep for N seconds (useful to simulate long running jobs)
+
+	sleep 5
+`
+}
+
+func (c *commandSleep) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandSleep) Do(args []string, _ *CommandEnv, _ io.Writer) error {
+	if len(args) == 0 {
+		return fmt.Errorf("sleep requires a seconds argument")
+	}
+	seconds, err := strconv.Atoi(args[0])
+	if err != nil || seconds <= 0 {
+		return fmt.Errorf("sleep duration must be a positive integer, got %q", args[0])
+	}
+	time.Sleep(time.Duration(seconds) * time.Second)
+	return nil
+}

weed/shell/commands.go

@@ -38,6 +38,7 @@ type CommandEnv struct {
 	option       *ShellOptions
 	locker       *exclusive_locks.ExclusiveLocker
 	noLock       bool
+	forceNoLock  bool
 	verbose      bool
 }
 
@@ -71,6 +72,9 @@ func (ce *CommandEnv) isDirectory(path string) bool {
 
 func (ce *CommandEnv) confirmIsLocked(args []string) error {
 
+	if ce.noLock || ce.forceNoLock {
+		return nil
+	}
 	if ce.locker.IsLocked() {
 		return nil
 	}
@@ -80,11 +84,25 @@ func (ce *CommandEnv) confirmIsLocked(args []string) error {
 
 }
 
+func (ce *CommandEnv) SetNoLock(noLock bool) {
+	if ce == nil {
+		return
+	}
+	ce.noLock = noLock
+}
+
+func (ce *CommandEnv) ForceNoLock() {
+	if ce == nil {
+		return
+	}
+	ce.forceNoLock = true
+}
+
 func (ce *CommandEnv) isLocked() bool {
 	if ce == nil {
 		return true
 	}
-	if ce.noLock {
+	if ce.noLock || ce.forceNoLock {
 		return true
 	}
 	return ce.locker.IsLocked()

weed/storage/disk_location.go

@@ -31,6 +31,7 @@ type DiskLocation struct {
 	DirectoryUuid          string
 	IdxDirectory           string
 	DiskType               types.DiskType
+	Tags                   []string
 	MaxVolumeCount         int32
 	OriginalMaxVolumeCount int32
 	MinFreeSpace           util.MinFreeSpace
@@ -76,7 +77,7 @@ func writeNewUuid(fileName string) (string, error) {
 	return dirUuidString, nil
 }
 
-func NewDiskLocation(dir string, maxVolumeCount int32, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType) *DiskLocation {
+func NewDiskLocation(dir string, maxVolumeCount int32, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType, tags []string) *DiskLocation {
 	glog.V(4).Infof("Added new Disk %s: maxVolumes=%d", dir, maxVolumeCount)
 	dir = util.ResolvePath(dir)
 	if idxDir == "" {
@@ -88,11 +89,18 @@ func NewDiskLocation(dir string, maxVolumeCount int32, minFreeSpace util.MinFree
 	if err != nil {
 		glog.Fatalf("cannot generate uuid of dir %s: %v", dir, err)
 	}
+	// Defensive copy of tags to prevent external mutation
+	var copiedTags []string
+	if len(tags) > 0 {
+		copiedTags = make([]string, len(tags))
+		copy(copiedTags, tags)
+	}
 	location := &DiskLocation{
 		Directory:              dir,
 		DirectoryUuid:          dirUuid,
 		IdxDirectory:           idxDir,
 		DiskType:               diskType,
+		Tags:                   copiedTags,
 		MaxVolumeCount:         maxVolumeCount,
 		OriginalMaxVolumeCount: maxVolumeCount,
 		MinFreeSpace:           minFreeSpace,

weed/storage/erasure_coding/shard_distribution.go

@@ -0,0 +1,367 @@
+package erasure_coding
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	"google.golang.org/grpc"
+)
+
+func ensureLogger(logger logger) logger {
+	if logger == nil {
+		return &glogFallbackLogger{}
+	}
+	return logger
+}
+
+type logger interface {
+	Info(string, ...interface{})
+	Warning(string, ...interface{})
+	Error(string, ...interface{})
+}
+
+type withFieldLogger interface {
+	WithFields(map[string]interface{}) logger
+}
+
+func withFields(log logger, fields map[string]interface{}) logger {
+	if fields == nil {
+		return log
+	}
+	if wf, ok := log.(withFieldLogger); ok {
+		return wf.WithFields(fields)
+	}
+
+	val := reflect.ValueOf(log)
+	method := val.MethodByName("WithFields")
+	if !method.IsValid() {
+		return log
+	}
+	methodType := method.Type()
+	if methodType.NumIn() != 1 || methodType.NumOut() != 1 {
+		return log
+	}
+	argType := methodType.In(0)
+	mapType := reflect.TypeOf(map[string]interface{}{})
+	var arg reflect.Value
+	if mapType.AssignableTo(argType) {
+		arg = reflect.ValueOf(fields)
+	} else if mapType.ConvertibleTo(argType) {
+		arg = reflect.ValueOf(fields).Convert(argType)
+	} else {
+		return log
+	}
+	result := method.Call([]reflect.Value{arg})[0]
+	if result.IsValid() && result.CanInterface() {
+		if enhanced, ok := result.Interface().(logger); ok {
+			return enhanced
+		}
+	}
+	return log
+}
+
+type glogFallbackLogger struct{}
+
+func (g *glogFallbackLogger) Info(msg string, args ...interface{}) {
+	if len(args) > 0 {
+		glog.Infof(msg, args...)
+		return
+	}
+	glog.Info(msg)
+}
+
+func (g *glogFallbackLogger) Warning(msg string, args ...interface{}) {
+	if len(args) > 0 {
+		glog.Warningf(msg, args...)
+		return
+	}
+	glog.Warning(msg)
+}
+
+func (g *glogFallbackLogger) Error(msg string, args ...interface{}) {
+	if len(args) > 0 {
+		glog.Errorf(msg, args...)
+		return
+	}
+	glog.Error(msg)
+}
+
+// DistributeEcShards distributes locally generated EC shards to destination servers.
+// Returns the shard assignment map used for mounting.
+func DistributeEcShards(volumeID uint32, collection string, targets []*worker_pb.TaskTarget, shardFiles map[string]string, dialOption grpc.DialOption, logger logger) (map[string][]string, error) {
+	if len(targets) == 0 {
+		return nil, fmt.Errorf("no targets specified for EC shard distribution")
+	}
+
+	if len(shardFiles) == 0 {
+		return nil, fmt.Errorf("no shard files available for distribution")
+	}
+
+	log := ensureLogger(logger)
+
+	shardAssignment := make(map[string][]string)
+
+	for _, target := range targets {
+		if len(target.ShardIds) == 0 {
+			continue
+		}
+
+		var assignedShards []string
+		for _, shardId := range target.ShardIds {
+			shardType := fmt.Sprintf("ec%02d", shardId)
+			assignedShards = append(assignedShards, shardType)
+		}
+
+		if len(assignedShards) > 0 {
+			if _, hasEcx := shardFiles["ecx"]; hasEcx {
+				assignedShards = append(assignedShards, "ecx")
+			}
+			if _, hasEcj := shardFiles["ecj"]; hasEcj {
+				assignedShards = append(assignedShards, "ecj")
+			}
+			if _, hasVif := shardFiles["vif"]; hasVif {
+				assignedShards = append(assignedShards, "vif")
+			}
+		}
+
+		existing := shardAssignment[target.Node]
+		if len(existing) == 0 {
+			shardAssignment[target.Node] = assignedShards
+			continue
+		}
+
+		seen := make(map[string]struct{}, len(existing))
+		for _, shard := range existing {
+			seen[shard] = struct{}{}
+		}
+		for _, shard := range assignedShards {
+			if _, ok := seen[shard]; ok {
+				continue
+			}
+			seen[shard] = struct{}{}
+			existing = append(existing, shard)
+		}
+		shardAssignment[target.Node] = existing
+	}
+
+	if len(shardAssignment) == 0 {
+		return nil, fmt.Errorf("no shard assignments found from planning phase")
+	}
+
+	for destNode, assignedShards := range shardAssignment {
+		withFields(log, map[string]interface{}{
+			"destination":     destNode,
+			"assigned_shards": len(assignedShards),
+			"shard_types":     assignedShards,
+		}).Info("Starting shard distribution to destination server")
+
+		var transferredBytes int64
+		for _, shardType := range assignedShards {
+			filePath, exists := shardFiles[shardType]
+			if !exists {
+				return nil, fmt.Errorf("shard file %s not found for destination %s", shardType, destNode)
+			}
+
+			if info, err := os.Stat(filePath); err == nil {
+				transferredBytes += info.Size()
+				withFields(log, map[string]interface{}{
+					"destination": destNode,
+					"shard_type":  shardType,
+					"file_path":   filePath,
+					"size_bytes":  info.Size(),
+					"size_kb":     float64(info.Size()) / 1024,
+				}).Info("Starting shard file transfer")
+			}
+
+			if err := sendShardFileToDestination(volumeID, collection, dialOption, destNode, filePath, shardType); err != nil {
+				return nil, fmt.Errorf("failed to send %s to %s: %w", shardType, destNode, err)
+			}
+
+			withFields(log, map[string]interface{}{
+				"destination": destNode,
+				"shard_type":  shardType,
+			}).Info("Shard file transfer completed")
+		}
+
+		withFields(log, map[string]interface{}{
+			"destination":        destNode,
+			"shards_transferred": len(assignedShards),
+			"total_bytes":        transferredBytes,
+			"total_mb":           float64(transferredBytes) / (1024 * 1024),
+		}).Info("All shards distributed to destination server")
+	}
+
+	glog.V(1).Infof("Successfully distributed EC shards to %d destinations", len(shardAssignment))
+	return shardAssignment, nil
+}
+
+// MountEcShards mounts EC shards on destination servers using an assignment map.
+func MountEcShards(volumeID uint32, collection string, shardAssignment map[string][]string, dialOption grpc.DialOption, logger logger) error {
+	if shardAssignment == nil {
+		return fmt.Errorf("shard assignment not available for mounting")
+	}
+
+	log := ensureLogger(logger)
+
+	var mountErrors []error
+	for destNode, assignedShards := range shardAssignment {
+		var shardIds []uint32
+		var metadataFiles []string
+
+		for _, shardType := range assignedShards {
+			if strings.HasPrefix(shardType, "ec") && len(shardType) == 4 {
+				var shardId uint32
+				if _, err := fmt.Sscanf(shardType[2:], "%d", &shardId); err == nil {
+					shardIds = append(shardIds, shardId)
+				}
+			} else {
+				metadataFiles = append(metadataFiles, shardType)
+			}
+		}
+
+		withFields(log, map[string]interface{}{
+			"destination":    destNode,
+			"shard_ids":      shardIds,
+			"shard_count":    len(shardIds),
+			"metadata_files": metadataFiles,
+		}).Info("Starting EC shard mount operation")
+
+		if len(shardIds) == 0 {
+			withFields(log, map[string]interface{}{
+				"destination":    destNode,
+				"metadata_files": metadataFiles,
+			}).Info("No EC shards to mount (only metadata files)")
+			continue
+		}
+
+		err := operation.WithVolumeServerClient(false, pb.ServerAddress(destNode), dialOption,
+			func(client volume_server_pb.VolumeServerClient) error {
+				_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
+					VolumeId:   volumeID,
+					Collection: collection,
+					ShardIds:   shardIds,
+				})
+				return mountErr
+			})
+
+		if err != nil {
+			mountErrors = append(mountErrors, fmt.Errorf("mount %s shards %v: %w", destNode, shardIds, err))
+			withFields(log, map[string]interface{}{
+				"destination": destNode,
+				"shard_ids":   shardIds,
+				"error":       err.Error(),
+			}).Error("Failed to mount EC shards")
+		} else {
+			withFields(log, map[string]interface{}{
+				"destination": destNode,
+				"shard_ids":   shardIds,
+				"volume_id":   volumeID,
+				"collection":  collection,
+			}).Info("Successfully mounted EC shards")
+		}
+	}
+
+	if len(mountErrors) > 0 {
+		return errors.Join(mountErrors...)
+	}
+	return nil
+}
+
+// sendShardFileToDestination sends a single shard file to a destination server using ReceiveFile API.
+func sendShardFileToDestination(volumeID uint32, collection string, dialOption grpc.DialOption, destServer, filePath, shardType string) error {
+	return operation.WithVolumeServerClient(false, pb.ServerAddress(destServer), dialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			file, err := os.Open(filePath)
+			if err != nil {
+				return fmt.Errorf("failed to open shard file %s: %v", filePath, err)
+			}
+			defer file.Close()
+
+			fileInfo, err := file.Stat()
+			if err != nil {
+				return fmt.Errorf("failed to get file info for %s: %v", filePath, err)
+			}
+
+			var ext string
+			var shardId uint32
+			if shardType == "ecx" {
+				ext = ".ecx"
+				shardId = 0
+			} else if shardType == "ecj" {
+				ext = ".ecj"
+				shardId = 0
+			} else if shardType == "vif" {
+				ext = ".vif"
+				shardId = 0
+			} else if strings.HasPrefix(shardType, "ec") && len(shardType) == 4 {
+				ext = "." + shardType
+				fmt.Sscanf(shardType[2:], "%d", &shardId)
+			} else {
+				return fmt.Errorf("unknown shard type: %s", shardType)
+			}
+
+			stream, err := client.ReceiveFile(context.Background())
+			if err != nil {
+				return fmt.Errorf("failed to create receive stream: %v", err)
+			}
+
+			err = stream.Send(&volume_server_pb.ReceiveFileRequest{
+				Data: &volume_server_pb.ReceiveFileRequest_Info{
+					Info: &volume_server_pb.ReceiveFileInfo{
+						VolumeId:   volumeID,
+						Ext:        ext,
+						Collection: collection,
+						IsEcVolume: true,
+						ShardId:    shardId,
+						FileSize:   uint64(fileInfo.Size()),
+					},
+				},
+			})
+			if err != nil {
+				return fmt.Errorf("failed to send file info: %v", err)
+			}
+
+			buffer := make([]byte, 64*1024)
+			for {
+				n, readErr := file.Read(buffer)
+				if n > 0 {
+					err = stream.Send(&volume_server_pb.ReceiveFileRequest{
+						Data: &volume_server_pb.ReceiveFileRequest_FileContent{
+							FileContent: buffer[:n],
+						},
+					})
+					if err != nil {
+						return fmt.Errorf("failed to send file content: %v", err)
+					}
+				}
+				if readErr == io.EOF {
+					break
+				}
+				if readErr != nil {
+					return fmt.Errorf("failed to read file: %v", readErr)
+				}
+			}
+
+			resp, err := stream.CloseAndRecv()
+			if err != nil {
+				return fmt.Errorf("failed to close stream: %v", err)
+			}
+
+			if resp.Error != "" {
+				return fmt.Errorf("server error: %s", resp.Error)
+			}
+
+			glog.V(2).Infof("Successfully sent %s (%d bytes) to %s", shardType, resp.BytesWritten, destServer)
+			return nil
+		})
+}

weed/storage/store.go

@@ -92,6 +92,7 @@ func NewStore(
 	idxFolder string,
 	needleMapKind NeedleMapKind,
 	diskTypes []DiskType,
+	diskTags [][]string,
 	ldbTimeout int64,
 ) (s *Store) {
 	s = &Store{
@@ -113,7 +114,11 @@ func NewStore(
 
 	var wg sync.WaitGroup
 	for i := 0; i < len(dirnames); i++ {
-		location := NewDiskLocation(dirnames[i], int32(maxVolumeCounts[i]), minFreeSpaces[i], idxFolder, diskTypes[i])
+		var tags []string
+		if i < len(diskTags) {
+			tags = diskTags[i]
+		}
+		location := NewDiskLocation(dirnames[i], int32(maxVolumeCounts[i]), minFreeSpaces[i], idxFolder, diskTypes[i], tags)
 		s.Locations = append(s.Locations, location)
 		stats.VolumeServerMaxVolumeCounter.Add(float64(maxVolumeCounts[i]))
 
@@ -474,6 +479,13 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
 	for _, loc := range s.Locations {
 		uuidList = append(uuidList, loc.DirectoryUuid)
 	}
+	var diskTags []*master_pb.DiskTag
+	for diskID, loc := range s.Locations {
+		diskTags = append(diskTags, &master_pb.DiskTag{
+			DiskId: uint32(diskID),
+			Tags:   append([]string(nil), loc.Tags...),
+		})
+	}
 
 	for col, size := range collectionVolumeSize {
 		stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "normal").Set(float64(size))
@@ -504,6 +516,7 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
 		HasNoVolumes:    len(volumeMessages) == 0,
 		HasNoEcShards:   len(ecVolumeMessages) == 0,
 		LocationUuids:   uuidList,
+		DiskTags:        diskTags,
 	}
 
 }

weed/storage/store_load_balancing_test.go

@@ -32,7 +32,7 @@ func newTestStore(t *testing.T, numDirs int) *Store {
 	}
 
 	store := NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "",
-		dirs, maxCounts, minFreeSpaces, "", NeedleMapInMemory, diskTypes, 3)
+		dirs, maxCounts, minFreeSpaces, "", NeedleMapInMemory, diskTypes, nil, 3)
 
 	// Consume channel messages to prevent blocking
 	done := make(chan bool)

weed/topology/data_node.go

@@ -25,6 +25,7 @@ type DataNode struct {
 	IsTerminating bool
 
 	MaintenanceMode bool
+	diskTags        map[uint32][]string
 }
 
 func NewDataNode(id string) *DataNode {
@@ -291,9 +292,41 @@ func (dn *DataNode) ToDataNodeInfo() *master_pb.DataNodeInfo {
 		disk := c.(*Disk)
 		m.DiskInfos[string(disk.Id())] = disk.ToDiskInfo()
 	}
+
+	dn.RLock()
+	diskTags := make(map[uint32][]string, len(dn.diskTags))
+	for diskID, tags := range dn.diskTags {
+		diskTags[diskID] = append([]string(nil), tags...)
+	}
+	dn.RUnlock()
+	for _, diskInfo := range m.DiskInfos {
+		if diskInfo == nil {
+			continue
+		}
+		if tags, found := diskTags[diskInfo.DiskId]; found {
+			diskInfo.Tags = append([]string(nil), tags...)
+		}
+	}
 	return m
 }
 
+func (dn *DataNode) UpdateDiskTags(tags []*master_pb.DiskTag) {
+	if len(tags) == 0 {
+		return
+	}
+	dn.Lock()
+	if dn.diskTags == nil {
+		dn.diskTags = make(map[uint32][]string, len(tags))
+	}
+	for _, tagInfo := range tags {
+		if tagInfo == nil {
+			continue
+		}
+		dn.diskTags[tagInfo.DiskId] = append([]string(nil), tagInfo.Tags...)
+	}
+	dn.Unlock()
+}
+
 // GetVolumeIds returns the human readable volume ids limited to count of max 100.
 func (dn *DataNode) GetVolumeIds() string {
 	dn.RLock()

weed/util/tag.go

@@ -0,0 +1,25 @@
+package util
+
+import "strings"
+
+// NormalizeTagList normalizes a list of tags by converting to lowercase,
+// trimming whitespace, removing duplicates, and filtering empty strings.
+func NormalizeTagList(tags []string) []string {
+	normalized := make([]string, 0, len(tags))
+	seen := make(map[string]struct{}, len(tags))
+	for _, tag := range tags {
+		tag = strings.ToLower(strings.TrimSpace(tag))
+		if tag == "" {
+			continue
+		}
+		if _, exists := seen[tag]; exists {
+			continue
+		}
+		seen[tag] = struct{}{}
+		normalized = append(normalized, tag)
+	}
+	if len(normalized) == 0 {
+		return []string{}
+	}
+	return normalized
+}

weed/wdclient/masterclient.go

@@ -11,6 +11,8 @@ import (
 	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -26,6 +28,22 @@ type masterVolumeProvider struct {
 	masterClient *MasterClient
 }
 
+func isCanceledErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+		return true
+	}
+	if statusErr, ok := status.FromError(err); ok {
+		switch statusErr.Code() {
+		case codes.Canceled, codes.DeadlineExceeded:
+			return true
+		}
+	}
+	return false
+}
+
 // LookupVolumeIds queries the master for volume locations (fallback when cache misses)
 // Returns partial results with aggregated errors for volumes that failed
 func (p *masterVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []string) (map[string][]Location, error) {
@@ -194,8 +212,13 @@ func (mc *MasterClient) tryConnectToMaster(ctx context.Context, master pb.Server
 
 		resp, err := stream.Recv()
 		if err != nil {
-			glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
-			stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
+			canceled := isCanceledErr(err) || ctx.Err() != nil
+			if canceled {
+				glog.V(1).Infof("%s.%s masterClient stream closed from %s: %v", mc.FilerGroup, mc.clientType, master, err)
+			} else {
+				glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
+				stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
+			}
 			return err
 		}
 
@@ -219,8 +242,13 @@ func (mc *MasterClient) tryConnectToMaster(ctx context.Context, master pb.Server
 		for {
 			resp, err := stream.Recv()
 			if err != nil {
-				glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
-				stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
+				canceled := isCanceledErr(err) || ctx.Err() != nil
+				if canceled {
+					glog.V(1).Infof("%s.%s masterClient stream closed from %s: %v", mc.FilerGroup, mc.clientType, master, err)
+				} else {
+					glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
+					stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
+				}
 				return err
 			}
 
@@ -252,12 +280,20 @@ func (mc *MasterClient) tryConnectToMaster(ctx context.Context, master pb.Server
 				mc.OnPeerUpdateLock.RUnlock()
 			}
 			if err := ctx.Err(); err != nil {
-				glog.V(0).Infof("Connection attempt to master stopped: %v", err)
+				if isCanceledErr(err) {
+					glog.V(1).Infof("Connection attempt to master stopped: %v", err)
+				} else {
+					glog.V(0).Infof("Connection attempt to master stopped: %v", err)
+				}
 				return err
 			}
 		}
 	})
 	if gprcErr != nil {
+		if isCanceledErr(gprcErr) || ctx.Err() != nil {
+			glog.V(1).Infof("%s.%s masterClient connection closed to %v: %v", mc.FilerGroup, mc.clientType, master, gprcErr)
+			return nextHintedLeader
+		}
 		stats.MasterClientConnectCounter.WithLabelValues(stats.Failed).Inc()
 		glog.V(1).Infof("%s.%s masterClient failed to connect with master %v: %v", mc.FilerGroup, mc.clientType, master, gprcErr)
 	}
@@ -387,7 +423,11 @@ func (mc *MasterClient) KeepConnectedToMaster(ctx context.Context) {
 	for {
 		select {
 		case <-ctx.Done():
-			glog.V(0).Infof("Connection to masters stopped: %v", ctx.Err())
+			if isCanceledErr(ctx.Err()) {
+				glog.V(1).Infof("Connection to masters stopped: %v", ctx.Err())
+			} else {
+				glog.V(0).Infof("Connection to masters stopped: %v", ctx.Err())
+			}
 			return
 		default:
 			reconnectStart := time.Now()

weed/worker/tasks/erasure_coding/config.go

@@ -12,10 +12,11 @@ import (
 // Config extends BaseConfig with erasure coding specific settings
 type Config struct {
 	base.BaseConfig
-	QuietForSeconds  int     `json:"quiet_for_seconds"`
-	FullnessRatio    float64 `json:"fullness_ratio"`
-	CollectionFilter string  `json:"collection_filter"`
-	MinSizeMB        int     `json:"min_size_mb"`
+	QuietForSeconds  int      `json:"quiet_for_seconds"`
+	FullnessRatio    float64  `json:"fullness_ratio"`
+	CollectionFilter string   `json:"collection_filter"`
+	MinSizeMB        int      `json:"min_size_mb"`
+	PreferredTags    []string `json:"preferred_tags"`
 }
 
 // NewDefaultConfig creates a new default erasure coding configuration
@@ -30,6 +31,7 @@ func NewDefaultConfig() *Config {
 		FullnessRatio:    0.8, // 80%
 		CollectionFilter: "",
 		MinSizeMB:        30, // 30MB (more reasonable than 100MB)
+		PreferredTags:    nil,
 	}
 }
 
@@ -142,12 +144,27 @@ func GetConfigSpec() base.ConfigSpec {
 				InputType:    "number",
 				CSSClasses:   "form-control",
 			},
+			{
+				Name:         "preferred_tags",
+				JSONName:     "preferred_tags",
+				Type:         config.FieldTypeString,
+				DefaultValue: "",
+				Required:     false,
+				DisplayName:  "Preferred Disk Tags",
+				Description:  "Comma-separated disk tags to prioritize for EC shard placement",
+				HelpText:     "EC shards will be placed on disks with these tags first, then fall back to other disks if needed",
+				Placeholder:  "fast,ssd",
+				InputType:    "text",
+				CSSClasses:   "form-control",
+			},
 		},
 	}
 }
 
 // ToTaskPolicy converts configuration to a TaskPolicy protobuf message
 func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
+	// Defensive copy of PreferredTags to prevent external mutation
+	preferredTagsCopy := append([]string(nil), c.PreferredTags...)
 	return &worker_pb.TaskPolicy{
 		Enabled:               c.Enabled,
 		MaxConcurrent:         int32(c.MaxConcurrent),
@@ -159,6 +176,7 @@ func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
 				QuietForSeconds:  int32(c.QuietForSeconds),
 				MinVolumeSizeMb:  int32(c.MinSizeMB),
 				CollectionFilter: c.CollectionFilter,
+				PreferredTags:    preferredTagsCopy,
 			},
 		},
 	}
@@ -181,6 +199,7 @@ func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
 		c.QuietForSeconds = int(ecConfig.QuietForSeconds)
 		c.MinSizeMB = int(ecConfig.MinVolumeSizeMb)
 		c.CollectionFilter = ecConfig.CollectionFilter
+		c.PreferredTags = append([]string(nil), ecConfig.PreferredTags...)
 	}
 
 	return nil

weed/worker/tasks/erasure_coding/detection.go

@@ -11,9 +11,10 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/placement"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
 	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
-	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
+	workerutil "github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
 	"github.com/seaweedfs/seaweedfs/weed/worker/types"
 )
 
@@ -148,7 +149,7 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste
 
 				glog.Infof("EC Detection: ActiveTopology available, planning destinations for volume %d", metric.VolumeID)
 				if planner == nil {
-					planner = newECPlacementPlanner(clusterInfo.ActiveTopology)
+					planner = newECPlacementPlanner(clusterInfo.ActiveTopology, ecConfig.PreferredTags)
 				}
 				multiPlan, err := planECDestinations(planner, metric, ecConfig)
 				if err != nil {
@@ -344,21 +345,27 @@ type ecPlacementPlanner struct {
 	candidates     []*placement.DiskCandidate
 	candidateByKey map[string]*placement.DiskCandidate
 	diskStates     map[string]*ecDiskState
+	diskTags       map[string][]string
+	preferredTags  []string
 }
 
-func newECPlacementPlanner(activeTopology *topology.ActiveTopology) *ecPlacementPlanner {
+func newECPlacementPlanner(activeTopology *topology.ActiveTopology, preferredTags []string) *ecPlacementPlanner {
 	if activeTopology == nil {
 		return nil
 	}
 
 	disks := activeTopology.GetDisksWithEffectiveCapacity(topology.TaskTypeErasureCoding, "", 0)
 	candidates := diskInfosToCandidates(disks)
+	tagsByKey := collectDiskTags(disks)
+	normalizedPreferredTags := util.NormalizeTagList(preferredTags)
 	if len(candidates) == 0 {
 		return &ecPlacementPlanner{
 			activeTopology: activeTopology,
 			candidates:     candidates,
 			candidateByKey: map[string]*placement.DiskCandidate{},
 			diskStates:     map[string]*ecDiskState{},
+			diskTags:       tagsByKey,
+			preferredTags:  normalizedPreferredTags,
 		}
 	}
 
@@ -377,6 +384,8 @@ func newECPlacementPlanner(activeTopology *topology.ActiveTopology) *ecPlacement
 		candidates:     candidates,
 		candidateByKey: candidateByKey,
 		diskStates:     diskStates,
+		diskTags:       tagsByKey,
+		preferredTags:  normalizedPreferredTags,
 	}
 }
 
@@ -397,11 +406,21 @@ func (p *ecPlacementPlanner) selectDestinations(sourceRack, sourceDC string, sha
 		PreferDifferentRacks:   true,
 	}
 
-	result, err := placement.SelectDestinations(p.candidates, config)
-	if err != nil {
-		return nil, err
+	var lastErr error
+	for _, candidates := range p.buildCandidateSets(shardsNeeded) {
+		if len(candidates) == 0 {
+			continue
+		}
+		result, err := placement.SelectDestinations(candidates, config)
+		if err == nil {
+			return result.SelectedDisks, nil
+		}
+		lastErr = err
 	}
-	return result.SelectedDisks, nil
+	if lastErr == nil {
+		lastErr = fmt.Errorf("no EC placement candidates available")
+	}
+	return nil, lastErr
 }
 
 func (p *ecPlacementPlanner) applyTaskReservations(volumeSize int64, sources []topology.TaskSourceSpec, destinations []topology.TaskDestinationSpec) {
@@ -501,6 +520,77 @@ func ecDiskKey(nodeID string, diskID uint32) string {
 	return fmt.Sprintf("%s:%d", nodeID, diskID)
 }
 
+func collectDiskTags(disks []*topology.DiskInfo) map[string][]string {
+	tagMap := make(map[string][]string, len(disks))
+	for _, disk := range disks {
+		if disk == nil || disk.DiskInfo == nil {
+			continue
+		}
+		key := ecDiskKey(disk.NodeID, disk.DiskID)
+		tags := util.NormalizeTagList(disk.DiskInfo.Tags)
+		if len(tags) > 0 {
+			tagMap[key] = tags
+		}
+	}
+	return tagMap
+}
+
+func diskHasTag(tags []string, tag string) bool {
+	if tag == "" || len(tags) == 0 {
+		return false
+	}
+	for _, candidate := range tags {
+		if candidate == tag {
+			return true
+		}
+	}
+	return false
+}
+
+// buildCandidateSets builds tiered candidate sets for preferred-tag prioritized placement.
+// For a planner with preferredTags, it accumulates disks matching each tag in order into
+// progressively larger tiers. It emits a candidate set once a tier reaches shardsNeeded,
+// then continues accumulating for subsequent tags. Finally, it falls back to the full
+// p.candidates set if preferred-tag tiers are insufficient. This ensures tagged disks
+// are selected first before falling back to all available candidates.
+func (p *ecPlacementPlanner) buildCandidateSets(shardsNeeded int) [][]*placement.DiskCandidate {
+	if p == nil {
+		return nil
+	}
+	if len(p.preferredTags) == 0 {
+		return [][]*placement.DiskCandidate{p.candidates}
+	}
+	selected := make(map[string]bool, len(p.candidates))
+	var tier []*placement.DiskCandidate
+	var candidateSets [][]*placement.DiskCandidate
+	for _, tag := range p.preferredTags {
+		for _, candidate := range p.candidates {
+			key := ecDiskKey(candidate.NodeID, candidate.DiskID)
+			if selected[key] {
+				continue
+			}
+			if diskHasTag(p.diskTags[key], tag) {
+				selected[key] = true
+				tier = append(tier, candidate)
+			}
+		}
+		if shardsNeeded > 0 && len(tier) >= shardsNeeded {
+			candidateSets = append(candidateSets, append([]*placement.DiskCandidate(nil), tier...))
+		}
+	}
+	// Defensive check: selectDestinations always ensures shardsNeeded > 0 before calling
+	// buildCandidateSets, but this branch handles direct callers and edge cases.
+	if shardsNeeded <= 0 && len(tier) > 0 {
+		candidateSets = append(candidateSets, append([]*placement.DiskCandidate(nil), tier...))
+	}
+	if len(tier) < len(p.candidates) {
+		candidateSets = append(candidateSets, p.candidates)
+	} else if len(candidateSets) == 0 {
+		candidateSets = append(candidateSets, p.candidates)
+	}
+	return candidateSets
+}
+
 // planECDestinations plans the destinations for erasure coding operation
 // This function implements EC destination planning logic directly in the detection phase
 func planECDestinations(planner *ecPlacementPlanner, metric *types.VolumeHealthMetrics, ecConfig *Config) (*topology.MultiDestinationPlan, error) {
@@ -550,7 +640,7 @@ func planECDestinations(planner *ecPlacementPlanner, metric *types.VolumeHealthM
 
 	for _, disk := range selectedDisks {
 		// Get the target server address
-		targetAddress, err := util.ResolveServerAddress(disk.NodeID, planner.activeTopology)
+		targetAddress, err := workerutil.ResolveServerAddress(disk.NodeID, planner.activeTopology)
 		if err != nil {
 			return nil, fmt.Errorf("failed to resolve address for target server %s: %v", disk.NodeID, err)
 		}
@@ -654,7 +744,7 @@ func convertTaskSourcesToProtobuf(sources []topology.TaskSourceSpec, volumeID ui
 	var protobufSources []*worker_pb.TaskSource
 
 	for _, source := range sources {
-		serverAddress, err := util.ResolveServerAddress(source.ServerID, activeTopology)
+		serverAddress, err := workerutil.ResolveServerAddress(source.ServerID, activeTopology)
 		if err != nil {
 			return nil, fmt.Errorf("failed to resolve address for source server %s: %v", source.ServerID, err)
 		}

weed/worker/tasks/erasure_coding/detection_test.go

@@ -17,7 +17,7 @@ import (
 func TestECPlacementPlannerApplyReservations(t *testing.T) {
 	activeTopology := buildActiveTopology(t, 1, []string{"hdd"}, 10, 0)
 
-	planner := newECPlacementPlanner(activeTopology)
+	planner := newECPlacementPlanner(activeTopology, nil)
 	require.NotNil(t, planner)
 
 	key := ecDiskKey("10.0.0.1:8080", 0)
@@ -47,7 +47,7 @@ func TestECPlacementPlannerApplyReservations(t *testing.T) {
 
 func TestPlanECDestinationsUsesPlanner(t *testing.T) {
 	activeTopology := buildActiveTopology(t, 7, []string{"hdd", "ssd"}, 100, 0)
-	planner := newECPlacementPlanner(activeTopology)
+	planner := newECPlacementPlanner(activeTopology, nil)
 	require.NotNil(t, planner)
 
 	metric := &types.VolumeHealthMetrics{
@@ -63,6 +63,70 @@ func TestPlanECDestinationsUsesPlanner(t *testing.T) {
 	assert.Equal(t, erasure_coding.TotalShardsCount, len(plan.Plans))
 }
 
+func TestECPlacementPlannerPrefersTaggedDisks(t *testing.T) {
+	activeTopology := buildActiveTopology(t, 3, []string{"hdd"}, 10, 0)
+	topo := activeTopology.GetTopologyInfo()
+	for _, dc := range topo.DataCenterInfos {
+		for _, rack := range dc.RackInfos {
+			for k, node := range rack.DataNodeInfos {
+				for diskType := range node.DiskInfos {
+					if k < 2 {
+						node.DiskInfos[diskType].Tags = []string{"fast"}
+					} else {
+						node.DiskInfos[diskType].Tags = []string{"slow"}
+					}
+				}
+			}
+		}
+	}
+	require.NoError(t, activeTopology.UpdateTopology(topo))
+
+	planner := newECPlacementPlanner(activeTopology, []string{"fast"})
+	require.NotNil(t, planner)
+
+	selected, err := planner.selectDestinations("", "", 2)
+	require.NoError(t, err)
+	require.Len(t, selected, 2)
+
+	for _, candidate := range selected {
+		key := ecDiskKey(candidate.NodeID, candidate.DiskID)
+		assert.True(t, diskHasTag(planner.diskTags[key], "fast"))
+	}
+}
+
+func TestECPlacementPlannerFallsBackWhenTagsInsufficient(t *testing.T) {
+	activeTopology := buildActiveTopology(t, 3, []string{"hdd"}, 10, 0)
+	topo := activeTopology.GetTopologyInfo()
+	for _, dc := range topo.DataCenterInfos {
+		for _, rack := range dc.RackInfos {
+			for i, node := range rack.DataNodeInfos {
+				for diskType := range node.DiskInfos {
+					if i == 0 {
+						node.DiskInfos[diskType].Tags = []string{"fast"}
+					}
+				}
+			}
+		}
+	}
+	require.NoError(t, activeTopology.UpdateTopology(topo))
+
+	planner := newECPlacementPlanner(activeTopology, []string{"fast"})
+	require.NotNil(t, planner)
+
+	selected, err := planner.selectDestinations("", "", 3)
+	require.NoError(t, err)
+	require.Len(t, selected, 3)
+
+	taggedCount := 0
+	for _, candidate := range selected {
+		key := ecDiskKey(candidate.NodeID, candidate.DiskID)
+		if diskHasTag(planner.diskTags[key], "fast") {
+			taggedCount++
+		}
+	}
+	assert.Less(t, taggedCount, len(selected))
+}
+
 func TestDetectionContextCancellation(t *testing.T) {
 	activeTopology := buildActiveTopology(t, 5, []string{"hdd", "ssd"}, 50, 0)
 	clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
@@ -88,7 +152,7 @@ func TestDetectionMaxResultsHonorsLimit(t *testing.T) {
 
 func TestPlanECDestinationsFailsWithInsufficientCapacity(t *testing.T) {
 	activeTopology := buildActiveTopology(t, 1, []string{"hdd"}, 1, 1)
-	planner := newECPlacementPlanner(activeTopology)
+	planner := newECPlacementPlanner(activeTopology, nil)
 	require.NotNil(t, planner)
 
 	metric := &types.VolumeHealthMetrics{

weed/worker/tasks/erasure_coding/ec_task.go

@@ -413,6 +413,16 @@ func (t *ErasureCodingTask) generateEcShardsLocally(localFiles map[string]string
 		}).Info("EC index file generated")
 	}
 
+	ecjFile := baseName + ".ecj"
+	if info, err := os.Stat(ecjFile); err == nil {
+		shardFiles["ecj"] = ecjFile
+		t.GetLogger().WithFields(map[string]interface{}{
+			"file_type":  "ecj",
+			"file_path":  ecjFile,
+			"size_bytes": info.Size(),
+		}).Info("EC journal file generated")
+	}
+
 	// Generate .vif file (volume info)
 	vifFile := baseName + ".vif"
 	volumeInfo := &volume_server_pb.VolumeInfo{
@@ -444,262 +454,17 @@ func (t *ErasureCodingTask) generateEcShardsLocally(localFiles map[string]string
 // distributeEcShards distributes locally generated EC shards to destination servers
 // using pre-assigned shard IDs from planning phase
 func (t *ErasureCodingTask) distributeEcShards(shardFiles map[string]string) error {
-	if len(t.targets) == 0 {
-		return fmt.Errorf("no targets specified for EC shard distribution")
+	assignment, err := erasure_coding.DistributeEcShards(t.volumeID, t.collection, t.targets, shardFiles, t.grpcDialOption, t.GetLogger())
+	if err != nil {
+		return err
 	}
-
-	if len(shardFiles) == 0 {
-		return fmt.Errorf("no shard files available for distribution")
-	}
-
-	// Build shard assignment from pre-assigned target shard IDs (from planning phase)
-	shardAssignment := make(map[string][]string)
-
-	for _, target := range t.targets {
-		if len(target.ShardIds) == 0 {
-			continue // Skip targets with no assigned shards
-		}
-
-		var assignedShards []string
-
-		// Convert shard IDs to shard file names (e.g., 0 → "ec00", 1 → "ec01")
-		for _, shardId := range target.ShardIds {
-			shardType := fmt.Sprintf("ec%02d", shardId)
-			assignedShards = append(assignedShards, shardType)
-		}
-
-		// Add metadata files (.ecx, .vif) to targets that have shards
-		if len(assignedShards) > 0 {
-			if _, hasEcx := shardFiles["ecx"]; hasEcx {
-				assignedShards = append(assignedShards, "ecx")
-			}
-			if _, hasVif := shardFiles["vif"]; hasVif {
-				assignedShards = append(assignedShards, "vif")
-			}
-		}
-
-		shardAssignment[target.Node] = assignedShards
-	}
-
-	if len(shardAssignment) == 0 {
-		return fmt.Errorf("no shard assignments found from planning phase")
-	}
-
-	// Store assignment for use during mounting
-	t.shardAssignment = shardAssignment
-
-	// Send assigned shards to each destination
-	for destNode, assignedShards := range shardAssignment {
-		t.GetLogger().WithFields(map[string]interface{}{
-			"destination":     destNode,
-			"assigned_shards": len(assignedShards),
-			"shard_types":     assignedShards,
-		}).Info("Starting shard distribution to destination server")
-
-		// Send only the assigned shards to this destination
-		var transferredBytes int64
-		for _, shardType := range assignedShards {
-			filePath, exists := shardFiles[shardType]
-			if !exists {
-				return fmt.Errorf("shard file %s not found for destination %s", shardType, destNode)
-			}
-
-			// Log file size before transfer
-			if info, err := os.Stat(filePath); err == nil {
-				transferredBytes += info.Size()
-				t.GetLogger().WithFields(map[string]interface{}{
-					"destination": destNode,
-					"shard_type":  shardType,
-					"file_path":   filePath,
-					"size_bytes":  info.Size(),
-					"size_kb":     float64(info.Size()) / 1024,
-				}).Info("Starting shard file transfer")
-			}
-
-			if err := t.sendShardFileToDestination(destNode, filePath, shardType); err != nil {
-				return fmt.Errorf("failed to send %s to %s: %v", shardType, destNode, err)
-			}
-
-			t.GetLogger().WithFields(map[string]interface{}{
-				"destination": destNode,
-				"shard_type":  shardType,
-			}).Info("Shard file transfer completed")
-		}
-
-		// Log summary for this destination
-		t.GetLogger().WithFields(map[string]interface{}{
-			"destination":        destNode,
-			"shards_transferred": len(assignedShards),
-			"total_bytes":        transferredBytes,
-			"total_mb":           float64(transferredBytes) / (1024 * 1024),
-		}).Info("All shards distributed to destination server")
-	}
-
-	glog.V(1).Infof("Successfully distributed EC shards to %d destinations", len(shardAssignment))
+	t.shardAssignment = assignment
 	return nil
 }
 
-// sendShardFileToDestination sends a single shard file to a destination server using ReceiveFile API
-func (t *ErasureCodingTask) sendShardFileToDestination(destServer, filePath, shardType string) error {
-	return operation.WithVolumeServerClient(false, pb.ServerAddress(destServer), t.grpcDialOption,
-		func(client volume_server_pb.VolumeServerClient) error {
-			// Open the local shard file
-			file, err := os.Open(filePath)
-			if err != nil {
-				return fmt.Errorf("failed to open shard file %s: %v", filePath, err)
-			}
-			defer file.Close()
-
-			// Get file size
-			fileInfo, err := file.Stat()
-			if err != nil {
-				return fmt.Errorf("failed to get file info for %s: %v", filePath, err)
-			}
-
-			// Determine file extension and shard ID
-			var ext string
-			var shardId uint32
-			if shardType == "ecx" {
-				ext = ".ecx"
-				shardId = 0 // ecx file doesn't have a specific shard ID
-			} else if shardType == "vif" {
-				ext = ".vif"
-				shardId = 0 // vif file doesn't have a specific shard ID
-			} else if strings.HasPrefix(shardType, "ec") && len(shardType) == 4 {
-				// EC shard file like "ec00", "ec01", etc.
-				ext = "." + shardType
-				fmt.Sscanf(shardType[2:], "%d", &shardId)
-			} else {
-				return fmt.Errorf("unknown shard type: %s", shardType)
-			}
-
-			// Create streaming client
-			stream, err := client.ReceiveFile(context.Background())
-			if err != nil {
-				return fmt.Errorf("failed to create receive stream: %v", err)
-			}
-
-			// Send file info first
-			err = stream.Send(&volume_server_pb.ReceiveFileRequest{
-				Data: &volume_server_pb.ReceiveFileRequest_Info{
-					Info: &volume_server_pb.ReceiveFileInfo{
-						VolumeId:   t.volumeID,
-						Ext:        ext,
-						Collection: t.collection,
-						IsEcVolume: true,
-						ShardId:    shardId,
-						FileSize:   uint64(fileInfo.Size()),
-					},
-				},
-			})
-			if err != nil {
-				return fmt.Errorf("failed to send file info: %v", err)
-			}
-
-			// Send file content in chunks
-			buffer := make([]byte, 64*1024) // 64KB chunks
-			for {
-				n, readErr := file.Read(buffer)
-				if n > 0 {
-					err = stream.Send(&volume_server_pb.ReceiveFileRequest{
-						Data: &volume_server_pb.ReceiveFileRequest_FileContent{
-							FileContent: buffer[:n],
-						},
-					})
-					if err != nil {
-						return fmt.Errorf("failed to send file content: %v", err)
-					}
-				}
-				if readErr == io.EOF {
-					break
-				}
-				if readErr != nil {
-					return fmt.Errorf("failed to read file: %v", readErr)
-				}
-			}
-
-			// Close stream and get response
-			resp, err := stream.CloseAndRecv()
-			if err != nil {
-				return fmt.Errorf("failed to close stream: %v", err)
-			}
-
-			if resp.Error != "" {
-				return fmt.Errorf("server error: %s", resp.Error)
-			}
-
-			glog.V(2).Infof("Successfully sent %s (%d bytes) to %s", shardType, resp.BytesWritten, destServer)
-			return nil
-		})
-}
-
 // mountEcShards mounts EC shards on destination servers
 func (t *ErasureCodingTask) mountEcShards() error {
-	if t.shardAssignment == nil {
-		return fmt.Errorf("shard assignment not available for mounting")
-	}
-
-	// Mount only assigned shards on each destination
-	for destNode, assignedShards := range t.shardAssignment {
-		// Convert shard names to shard IDs for mounting
-		var shardIds []uint32
-		var metadataFiles []string
-
-		for _, shardType := range assignedShards {
-			// Skip metadata files (.ecx, .vif) - only mount EC shards
-			if strings.HasPrefix(shardType, "ec") && len(shardType) == 4 {
-				// Parse shard ID from "ec00", "ec01", etc.
-				var shardId uint32
-				if _, err := fmt.Sscanf(shardType[2:], "%d", &shardId); err == nil {
-					shardIds = append(shardIds, shardId)
-				}
-			} else {
-				metadataFiles = append(metadataFiles, shardType)
-			}
-		}
-
-		t.GetLogger().WithFields(map[string]interface{}{
-			"destination":    destNode,
-			"shard_ids":      shardIds,
-			"shard_count":    len(shardIds),
-			"metadata_files": metadataFiles,
-		}).Info("Starting EC shard mount operation")
-
-		if len(shardIds) == 0 {
-			t.GetLogger().WithFields(map[string]interface{}{
-				"destination":    destNode,
-				"metadata_files": metadataFiles,
-			}).Info("No EC shards to mount (only metadata files)")
-			continue
-		}
-
-		err := operation.WithVolumeServerClient(false, pb.ServerAddress(destNode), t.grpcDialOption,
-			func(client volume_server_pb.VolumeServerClient) error {
-				_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
-					VolumeId:   t.volumeID,
-					Collection: t.collection,
-					ShardIds:   shardIds,
-				})
-				return mountErr
-			})
-
-		if err != nil {
-			t.GetLogger().WithFields(map[string]interface{}{
-				"destination": destNode,
-				"shard_ids":   shardIds,
-				"error":       err.Error(),
-			}).Error("Failed to mount EC shards")
-		} else {
-			t.GetLogger().WithFields(map[string]interface{}{
-				"destination": destNode,
-				"shard_ids":   shardIds,
-				"volume_id":   t.volumeID,
-				"collection":  t.collection,
-			}).Info("Successfully mounted EC shards")
-		}
-	}
-
-	return nil
+	return erasure_coding.MountEcShards(t.volumeID, t.collection, t.shardAssignment, t.grpcDialOption, t.GetLogger())
 }
 
 // deleteOriginalVolume deletes the original volume and all its replicas from all servers