mount: add unit tests for refresh event buffering

Test all key scenarios for the cache refresh mechanism:

- Delete during refresh: subscription delete is buffered and replayed,
  preventing ghost entries from the stale snapshot
- Create during refresh: subscription create is buffered and replayed,
  preventing lost entries
- Stale entries removed: entries absent from the filer snapshot are
  cleaned from the cache
- Local delete buffered: direct DeleteEntry during refresh is both
  applied immediately and replayed after commit
- Local create buffered: direct InsertEntry during refresh is preserved
  after snapshot commit
- Cancel discards buffer: CancelRefresh drops buffered events and
  resumes normal event processing
- Concurrent deletes (issue #8442 scenario): 1000 concurrent delete
  events racing with a stale snapshot all resolve correctly

docker/compose/master-cloud.toml

@@ -7,6 +7,7 @@
 
 [master.maintenance]
 # periodically run these scripts are the same as running them from 'weed shell'
+# Scripts are skipped while an admin server is connected.
 scripts = """
   lock
   ec.encode -fullPercent=95 -quietFor=1h

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.13"
+appVersion: "4.15"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.413
+version: 4.15.0

k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml

@@ -51,7 +51,7 @@ metadata:
   annotations:
     "helm.sh/hook": post-install,post-upgrade
     "helm.sh/hook-weight": "-5"
-    "helm.sh/hook-delete-policy": hook-succeeded
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
 spec:
   template:
     metadata:
@@ -92,6 +92,7 @@ spec:
           - "/bin/sh"
           - "-ec"
           - |
+            set -o pipefail
             wait_for_service() {
               local url=$1
               local max_attempts=60  # 5 minutes total (5s * 60)
@@ -117,8 +118,7 @@ spec:
             wait_for_service "http://$WEED_CLUSTER_SW_MASTER{{ .Values.master.readinessProbe.httpGet.path }}"
             wait_for_service "http://$WEED_CLUSTER_SW_FILER{{ .Values.filer.readinessProbe.httpGet.path }}"
             {{- end }}
-          set -o pipefail
-          {{- range $createBuckets }}
+            {{- range $createBuckets }}
             {{- $bucketName := .name }}
             {{- $bucketLock := or .lock .objectLock .withLock }}
             bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; }

test/erasure_coding/admin_dockertest/ec_integration_test.go

@@ -135,7 +135,7 @@ func ensureEnvironment(t *testing.T) {
 	// 6. Start Admin
 	os.RemoveAll(filepath.Join("tmp", "admin"))
 	os.MkdirAll(filepath.Join("tmp", "admin"), 0755)
-	startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin", "-scheduler.idleSleep=2")
+	startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin")
 	waitForUrl(t, AdminUrl+"/health", 60)
 
 	t.Log("Environment started successfully")
@@ -153,6 +153,38 @@ func waitForUrl(t *testing.T, url string, retries int) {
 	t.Fatalf("Timeout waiting for %s", url)
 }
 
+func fetchJSON(url string, out interface{}) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
+	}
+	return json.NewDecoder(resp.Body).Decode(out)
+}
+
+func mapField(obj map[string]interface{}, key string) (interface{}, bool) {
+	if obj == nil {
+		return nil, false
+	}
+	if value, ok := obj[key]; ok {
+		return value, true
+	}
+	return nil, false
+}
+
+func mapFieldAny(obj map[string]interface{}, keys ...string) (interface{}, bool) {
+	for _, key := range keys {
+		if value, ok := mapField(obj, key); ok {
+			return value, true
+		}
+	}
+	return nil, false
+}
+
 func TestEcEndToEnd(t *testing.T) {
 	defer cleanup()
 	ensureEnvironment(t)
@@ -162,6 +194,28 @@ func TestEcEndToEnd(t *testing.T) {
 	// 1. Configure plugin job types for fast EC detection/execution.
 	t.Log("Configuring plugin job types via API...")
 
+	schedulerConfig := map[string]interface{}{
+		"idle_sleep_seconds": 1,
+	}
+	jsonBody, err := json.Marshal(schedulerConfig)
+	if err != nil {
+		t.Fatalf("Failed to marshal scheduler config: %v", err)
+	}
+	req, err := http.NewRequest("PUT", AdminUrl+"/api/plugin/scheduler-config", bytes.NewBuffer(jsonBody))
+	if err != nil {
+		t.Fatalf("Failed to create scheduler config request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("Failed to update scheduler config: %v", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		t.Fatalf("Failed to update scheduler config (status %d): %s", resp.StatusCode, string(body))
+	}
+	resp.Body.Close()
+
 	// Disable volume balance to reduce interference for this EC-focused test.
 	balanceConfig := map[string]interface{}{
 		"job_type": "volume_balance",
@@ -169,16 +223,16 @@ func TestEcEndToEnd(t *testing.T) {
 			"enabled": false,
 		},
 	}
-	jsonBody, err := json.Marshal(balanceConfig)
+	jsonBody, err = json.Marshal(balanceConfig)
 	if err != nil {
 		t.Fatalf("Failed to marshal volume_balance config: %v", err)
 	}
-	req, err := http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/volume_balance/config", bytes.NewBuffer(jsonBody))
+	req, err = http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/volume_balance/config", bytes.NewBuffer(jsonBody))
 	if err != nil {
 		t.Fatalf("Failed to create volume_balance config request: %v", err)
 	}
 	req.Header.Set("Content-Type", "application/json")
-	resp, err := client.Do(req)
+	resp, err = client.Do(req)
 	if err != nil {
 		t.Fatalf("Failed to update volume_balance config: %v", err)
 	}
@@ -275,6 +329,7 @@ func TestEcEndToEnd(t *testing.T) {
 	startTime := time.Now()
 	ecVerified := false
 	var lastBody []byte
+	debugTick := 0
 
 	for time.Since(startTime) < 300*time.Second {
 		// 3.1 Check Master Topology
@@ -300,25 +355,104 @@ func TestEcEndToEnd(t *testing.T) {
 			}
 		}
 
-		// 3.2 Debug: Check workers and jobs
-		wResp, wErr := http.Get(AdminUrl + "/api/plugin/workers")
+		// 3.2 Debug: Check workers, jobs, and scheduler status
+		debugTick++
+
+		var workers []map[string]interface{}
 		workerCount := 0
-		if wErr == nil {
-			var workers []interface{}
-			json.NewDecoder(wResp.Body).Decode(&workers)
-			wResp.Body.Close()
+		ecDetectorCount := 0
+		ecExecutorCount := 0
+		if err := fetchJSON(AdminUrl+"/api/plugin/workers", &workers); err == nil {
 			workerCount = len(workers)
+			for _, worker := range workers {
+				capsValue, ok := mapFieldAny(worker, "capabilities", "Capabilities")
+				if !ok {
+					continue
+				}
+				caps, ok := capsValue.(map[string]interface{})
+				if !ok {
+					continue
+				}
+				if capValue, ok := caps["erasure_coding"].(map[string]interface{}); ok {
+					if capValue["can_detect"] == true {
+						ecDetectorCount++
+					}
+					if capValue["can_execute"] == true {
+						ecExecutorCount++
+					}
+				}
+			}
 		}
 
-		tResp, tErr := http.Get(AdminUrl + "/api/plugin/jobs?limit=1000")
+		var tasks []map[string]interface{}
 		taskCount := 0
-		if tErr == nil {
-			var tasks []interface{}
-			json.NewDecoder(tResp.Body).Decode(&tasks)
-			tResp.Body.Close()
+		ecTaskCount := 0
+		ecTaskStates := map[string]int{}
+		if err := fetchJSON(AdminUrl+"/api/plugin/jobs?limit=1000", &tasks); err == nil {
 			taskCount = len(tasks)
+			for _, task := range tasks {
+				jobType, _ := task["job_type"].(string)
+				state, _ := task["state"].(string)
+				if jobType == "erasure_coding" {
+					ecTaskCount++
+					ecTaskStates[state]++
+				}
+			}
+		}
+
+		t.Logf("Waiting for EC... (Workers: %d det=%d exec=%d, Tasks: %d ec=%d, EC States: %+v)",
+			workerCount, ecDetectorCount, ecExecutorCount, taskCount, ecTaskCount, ecTaskStates)
+
+		if debugTick%3 == 0 {
+			var pluginStatus map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/status", &pluginStatus); err == nil {
+				t.Logf("Plugin status: enabled=%v worker_count=%v worker_grpc_port=%v configured=%v",
+					pluginStatus["enabled"], pluginStatus["worker_count"], pluginStatus["worker_grpc_port"], pluginStatus["configured"])
+			}
+
+			var schedulerStatus map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-status", &schedulerStatus); err == nil {
+				if schedValue, ok := schedulerStatus["scheduler"].(map[string]interface{}); ok {
+					t.Logf("Scheduler status: current_job_type=%v phase=%v last_iteration_had_jobs=%v idle_sleep_seconds=%v last_iteration_done_at=%v next_detection_at=%v",
+						schedValue["current_job_type"], schedValue["current_phase"],
+						schedValue["last_iteration_had_jobs"], schedValue["idle_sleep_seconds"], schedValue["last_iteration_done_at"], schedValue["next_detection_at"])
+				} else {
+					t.Logf("Scheduler status: %v", schedulerStatus)
+				}
+			}
+
+			var schedulerStates []map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-states", &schedulerStates); err == nil {
+				for _, state := range schedulerStates {
+					if state["job_type"] == "erasure_coding" {
+						t.Logf("EC scheduler state: enabled=%v detection_in_flight=%v detector_available=%v executor_workers=%v next_detection_at=%v last_run_status=%v last_run_started_at=%v last_run_completed_at=%v",
+							state["enabled"], state["detection_in_flight"], state["detector_available"],
+							state["executor_worker_count"], state["next_detection_at"], state["last_run_status"],
+							state["last_run_started_at"], state["last_run_completed_at"])
+						break
+					}
+				}
+			}
+
+			var jobTypes []map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/job-types", &jobTypes); err == nil {
+				var names []string
+				for _, jobType := range jobTypes {
+					if name, ok := jobType["job_type"].(string); ok && name != "" {
+						names = append(names, name)
+					}
+				}
+				t.Logf("Plugin job types: %v", names)
+			}
+
+			var activities []map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/activities?job_type=erasure_coding&limit=5", &activities); err == nil {
+				for i := len(activities) - 1; i >= 0; i-- {
+					act := activities[i]
+					t.Logf("EC activity: stage=%v message=%v occurred_at=%v", act["stage"], act["message"], act["occurred_at"])
+				}
+			}
 		}
-		t.Logf("Waiting for EC... (Workers: %d, Active Tasks: %d)", workerCount, taskCount)
 
 		time.Sleep(10 * time.Second)
 	}

test/s3/normal/get_object_attributes_test.go

@@ -0,0 +1,573 @@
+package example
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	v1credentials "github.com/aws/aws-sdk-go/aws/credentials"
+	v1signer "github.com/aws/aws-sdk-go/aws/signer/v4"
+	v1s3 "github.com/aws/aws-sdk-go/service/s3"
+	v2aws "github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	v2s3 "github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newS3V2Client creates an AWS SDK v2 S3 client from the test cluster.
+func newS3V2Client(cluster *TestCluster) *v2s3.Client {
+	return v2s3.New(v2s3.Options{
+		Region:       testRegion,
+		BaseEndpoint: v2aws.String(cluster.s3Endpoint),
+		Credentials:  v2aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(testAccessKey, testSecretKey, "")),
+		UsePathStyle: true,
+	})
+}
+
+func TestGetObjectAttributes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	t.Run("Basic", func(t *testing.T) {
+		testGetObjectAttributesBasic(t, cluster)
+	})
+	t.Run("MultipartObject", func(t *testing.T) {
+		testGetObjectAttributesMultipart(t, cluster)
+	})
+	t.Run("SelectiveAttributes", func(t *testing.T) {
+		testGetObjectAttributesSelective(t, cluster)
+	})
+	t.Run("InvalidAttribute", func(t *testing.T) {
+		testGetObjectAttributesInvalid(t, cluster)
+	})
+	t.Run("NonExistentObject", func(t *testing.T) {
+		testGetObjectAttributesNotFound(t, cluster)
+	})
+	t.Run("VersionedObject", func(t *testing.T) {
+		testGetObjectAttributesVersioned(t, cluster)
+	})
+	t.Run("ConditionalHeaders", func(t *testing.T) {
+		testGetObjectAttributesConditionalHeaders(t, cluster)
+	})
+	t.Run("VersionedConditionalHeaders", func(t *testing.T) {
+		testGetObjectAttributesVersionedConditionalHeaders(t, cluster)
+	})
+}
+
+func testGetObjectAttributesBasic(t *testing.T, cluster *TestCluster) {
+	bucketName := createTestBucket(t, cluster, "test-goa-basic-")
+	objectKey := "test-object.txt"
+	objectData := "Hello, GetObjectAttributes!"
+
+	_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader([]byte(objectData)),
+	})
+	require.NoError(t, err)
+
+	client := newS3V2Client(cluster)
+	resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String(objectKey),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesEtag,
+			types.ObjectAttributesStorageClass,
+			types.ObjectAttributesObjectSize,
+			types.ObjectAttributesObjectParts,
+		},
+	})
+	require.NoError(t, err)
+
+	// ETag should be present and non-empty
+	require.NotNil(t, resp.ETag)
+	assert.NotEmpty(t, *resp.ETag)
+	assert.False(t, strings.Contains(*resp.ETag, `"`), "ETag in XML body should not have quotes")
+
+	// ObjectSize should match
+	require.NotNil(t, resp.ObjectSize)
+	assert.Equal(t, int64(len(objectData)), *resp.ObjectSize)
+
+	// StorageClass should be STANDARD (default)
+	assert.Equal(t, "STANDARD", string(resp.StorageClass))
+
+	// ObjectParts should be nil for non-multipart objects
+	assert.Nil(t, resp.ObjectParts)
+
+	// LastModified header should be present
+	assert.NotNil(t, resp.LastModified)
+
+	t.Logf("Basic GetObjectAttributes passed: ETag=%s, Size=%d, StorageClass=%s",
+		*resp.ETag, *resp.ObjectSize, resp.StorageClass)
+}
+
+func testGetObjectAttributesMultipart(t *testing.T, cluster *TestCluster) {
+	bucketName := createTestBucket(t, cluster, "test-goa-mp-")
+	objectKey := "test-multipart.bin"
+
+	// Create a 2-part multipart upload
+	part1Data := bytes.Repeat([]byte("A"), 5*1024*1024) // 5MB (minimum part size)
+	part2Data := bytes.Repeat([]byte("B"), 3*1024*1024) // 3MB
+
+	initResp, err := cluster.s3Client.CreateMultipartUpload(&v1s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	uploadID := initResp.UploadId
+
+	part1Resp, err := cluster.s3Client.UploadPart(&v1s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		PartNumber: aws.Int64(1),
+		UploadId:   uploadID,
+		Body:       bytes.NewReader(part1Data),
+	})
+	require.NoError(t, err)
+
+	part2Resp, err := cluster.s3Client.UploadPart(&v1s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		PartNumber: aws.Int64(2),
+		UploadId:   uploadID,
+		Body:       bytes.NewReader(part2Data),
+	})
+	require.NoError(t, err)
+
+	_, err = cluster.s3Client.CompleteMultipartUpload(&v1s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: uploadID,
+		MultipartUpload: &v1s3.CompletedMultipartUpload{
+			Parts: []*v1s3.CompletedPart{
+				{ETag: part1Resp.ETag, PartNumber: aws.Int64(1)},
+				{ETag: part2Resp.ETag, PartNumber: aws.Int64(2)},
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	// Wait briefly for metadata to settle
+	time.Sleep(200 * time.Millisecond)
+
+	client := newS3V2Client(cluster)
+	resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String(objectKey),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesObjectParts,
+			types.ObjectAttributesObjectSize,
+		},
+	})
+	require.NoError(t, err)
+
+	require.NotNil(t, resp.ObjectSize)
+	assert.Equal(t, int64(len(part1Data)+len(part2Data)), *resp.ObjectSize)
+
+	require.NotNil(t, resp.ObjectParts, "ObjectParts should be present for multipart objects")
+	assert.Equal(t, int32(2), *resp.ObjectParts.TotalPartsCount)
+	require.Len(t, resp.ObjectParts.Parts, 2)
+	assert.Equal(t, int32(1), *resp.ObjectParts.Parts[0].PartNumber)
+	assert.Equal(t, int64(len(part1Data)), *resp.ObjectParts.Parts[0].Size)
+	assert.Equal(t, int32(2), *resp.ObjectParts.Parts[1].PartNumber)
+	assert.Equal(t, int64(len(part2Data)), *resp.ObjectParts.Parts[1].Size)
+
+	// Test pagination: MaxParts=1
+	resp2, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket:  v2aws.String(bucketName),
+		Key:     v2aws.String(objectKey),
+		MaxParts: v2aws.Int32(1),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesObjectParts,
+		},
+	})
+	require.NoError(t, err)
+	require.NotNil(t, resp2.ObjectParts)
+	assert.Len(t, resp2.ObjectParts.Parts, 1)
+	assert.True(t, *resp2.ObjectParts.IsTruncated)
+	assert.Equal(t, int32(2), *resp2.ObjectParts.TotalPartsCount)
+
+	t.Logf("Multipart GetObjectAttributes passed: %d parts, total size %d",
+		*resp.ObjectParts.TotalPartsCount, *resp.ObjectSize)
+}
+
+func testGetObjectAttributesSelective(t *testing.T, cluster *TestCluster) {
+	bucketName := createTestBucket(t, cluster, "test-goa-sel-")
+	objectKey := "test-selective.txt"
+	objectData := "Selective attributes test"
+
+	_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader([]byte(objectData)),
+	})
+	require.NoError(t, err)
+
+	client := newS3V2Client(cluster)
+
+	// Request only ETag
+	resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String(objectKey),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesEtag,
+		},
+	})
+	require.NoError(t, err)
+	require.NotNil(t, resp.ETag)
+	assert.NotEmpty(t, *resp.ETag)
+	assert.Nil(t, resp.ObjectSize, "ObjectSize should not be present when not requested")
+	assert.Empty(t, string(resp.StorageClass), "StorageClass should not be present when not requested")
+	assert.Nil(t, resp.ObjectParts, "ObjectParts should not be present when not requested")
+
+	t.Logf("Selective GetObjectAttributes passed: ETag=%s", *resp.ETag)
+}
+
+func testGetObjectAttributesInvalid(t *testing.T, cluster *TestCluster) {
+	bucketName := createTestBucket(t, cluster, "test-goa-inv-")
+	objectKey := "test-object.txt"
+
+	_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader([]byte("test")),
+	})
+	require.NoError(t, err)
+
+	// Use raw HTTP to send an invalid attribute name since the SDK validates
+	reqURL := fmt.Sprintf("%s/%s/%s?attributes", cluster.s3Endpoint, bucketName, objectKey)
+	req, err := http.NewRequest("GET", reqURL, nil)
+	require.NoError(t, err)
+	req.Header.Set("X-Amz-Object-Attributes", "InvalidAttr")
+
+	signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
+	_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
+	require.NoError(t, err)
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	io.Copy(io.Discard, resp.Body)
+
+	assert.Equal(t, 400, resp.StatusCode)
+	t.Logf("Invalid attribute test passed: got %d", resp.StatusCode)
+}
+
+func testGetObjectAttributesNotFound(t *testing.T, cluster *TestCluster) {
+	bucketName := createTestBucket(t, cluster, "test-goa-nf-")
+
+	client := newS3V2Client(cluster)
+	_, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String("nonexistent-key"),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesEtag,
+		},
+	})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "NoSuchKey")
+
+	t.Logf("NotFound GetObjectAttributes passed")
+}
+
+func testGetObjectAttributesVersioned(t *testing.T, cluster *TestCluster) {
+	client := newS3V2Client(cluster)
+	bucketName := createTestBucket(t, cluster, "test-goa-ver-")
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.Background(), &v2s3.PutBucketVersioningInput{
+		Bucket: v2aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err)
+	time.Sleep(200 * time.Millisecond)
+
+	// Put two versions of the same object
+	v1Data := "version 1 content"
+	putResp1, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String("versioned-key"),
+		Body:   strings.NewReader(v1Data),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, putResp1.VersionId)
+	versionId1 := *putResp1.VersionId
+
+	v2Data := "version 2 content - longer"
+	putResp2, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String("versioned-key"),
+		Body:   strings.NewReader(v2Data),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, putResp2.VersionId)
+	versionId2 := *putResp2.VersionId
+
+	assert.NotEqual(t, versionId1, versionId2, "versions should differ")
+
+	// GetObjectAttributes for latest version (v2)
+	resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String("versioned-key"),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesObjectSize,
+			types.ObjectAttributesEtag,
+		},
+	})
+	require.NoError(t, err)
+	require.NotNil(t, resp.ObjectSize)
+	assert.Equal(t, int64(len(v2Data)), *resp.ObjectSize)
+	require.NotNil(t, resp.VersionId)
+	assert.Equal(t, versionId2, *resp.VersionId)
+
+	// GetObjectAttributes for specific older version (v1)
+	resp1, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket:    v2aws.String(bucketName),
+		Key:       v2aws.String("versioned-key"),
+		VersionId: v2aws.String(versionId1),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesObjectSize,
+			types.ObjectAttributesEtag,
+		},
+	})
+	require.NoError(t, err)
+	require.NotNil(t, resp1.ObjectSize)
+	assert.Equal(t, int64(len(v1Data)), *resp1.ObjectSize)
+	require.NotNil(t, resp1.VersionId)
+	assert.Equal(t, versionId1, *resp1.VersionId)
+
+	t.Logf("Versioned GetObjectAttributes passed: v1 size=%d (id=%s), v2 size=%d (id=%s)",
+		*resp1.ObjectSize, versionId1, *resp.ObjectSize, versionId2)
+}
+
+// signedGetObjectAttributes creates a signed GET request for ?attributes with custom headers.
+func signedGetObjectAttributes(t *testing.T, cluster *TestCluster, bucketName, objectKey string, extraHeaders map[string]string) *http.Response {
+	reqURL := fmt.Sprintf("%s/%s/%s?attributes", cluster.s3Endpoint, bucketName, objectKey)
+	req, err := http.NewRequest("GET", reqURL, nil)
+	require.NoError(t, err)
+	req.Header.Set("X-Amz-Object-Attributes", "ETag,ObjectSize")
+	for k, v := range extraHeaders {
+		req.Header.Set(k, v)
+	}
+	signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
+	_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
+	require.NoError(t, err)
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	return resp
+}
+
+func testGetObjectAttributesConditionalHeaders(t *testing.T, cluster *TestCluster) {
+	bucketName := createTestBucket(t, cluster, "test-goa-cond-")
+	objectKey := "cond-test.txt"
+
+	_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader([]byte("conditional headers test")),
+	})
+	require.NoError(t, err)
+
+	// Get the ETag and Last-Modified for the object
+	headResp, err := cluster.s3Client.HeadObject(&v1s3.HeadObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	etag := aws.StringValue(headResp.ETag)
+	lastModified := headResp.LastModified
+	require.NotNil(t, lastModified)
+
+	pastDate := lastModified.Add(-1 * time.Hour).UTC().Format(http.TimeFormat)
+	futureDate := lastModified.Add(1 * time.Hour).UTC().Format(http.TimeFormat)
+
+	// RFC 7232: If-Match true + If-Unmodified-Since false => 200 OK
+	// If-Unmodified-Since is ignored when If-Match is present
+	t.Run("IfMatch_true_IfUnmodifiedSince_false", func(t *testing.T) {
+		resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
+			"If-Match":            etag,
+			"If-Unmodified-Since": pastDate, // object was modified after this => false
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 200, resp.StatusCode,
+			"If-Match=true should return 200 even when If-Unmodified-Since=false (RFC 7232 Section 3.4)")
+	})
+
+	// RFC 7232: If-None-Match false + If-Modified-Since true => 304 Not Modified
+	// If-Modified-Since is ignored when If-None-Match is present
+	t.Run("IfNoneMatch_false_IfModifiedSince_true", func(t *testing.T) {
+		resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
+			"If-None-Match":     etag,
+			"If-Modified-Since": pastDate, // object was modified after this => true
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 304, resp.StatusCode,
+			"If-None-Match=false (ETag match) should return 304 even when If-Modified-Since=true (RFC 7232 Section 3.3)")
+	})
+
+	// If-Match succeeds, If-Unmodified-Since also succeeds => 200
+	t.Run("IfMatch_true_IfUnmodifiedSince_true", func(t *testing.T) {
+		resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
+			"If-Match":            etag,
+			"If-Unmodified-Since": futureDate,
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 200, resp.StatusCode)
+	})
+
+	// If-None-Match passes (ETag differs), If-Modified-Since ignored => 200
+	// Per RFC 7232, If-Modified-Since is ignored when If-None-Match is present
+	t.Run("IfNoneMatch_true_IfModifiedSince_ignored", func(t *testing.T) {
+		resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
+			"If-None-Match":     `"nonexistent-etag"`,
+			"If-Modified-Since": futureDate, // would fail alone, but is ignored
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 200, resp.StatusCode,
+			"If-None-Match=true means If-Modified-Since is ignored, should return 200 (RFC 7232 Section 3.3)")
+	})
+
+	// If-Match fails => 412 regardless of If-Unmodified-Since
+	t.Run("IfMatch_false", func(t *testing.T) {
+		resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
+			"If-Match":            `"wrong-etag"`,
+			"If-Unmodified-Since": futureDate,
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 412, resp.StatusCode)
+	})
+
+	t.Logf("Conditional headers tests passed")
+}
+
+// signedGetObjectAttributesVersioned creates a signed GET request for ?attributes&versionId=... with custom headers.
+func signedGetObjectAttributesVersioned(t *testing.T, cluster *TestCluster, bucketName, objectKey, versionId string, extraHeaders map[string]string) *http.Response {
+	reqURL := fmt.Sprintf("%s/%s/%s?attributes&versionId=%s", cluster.s3Endpoint, bucketName, objectKey, versionId)
+	req, err := http.NewRequest("GET", reqURL, nil)
+	require.NoError(t, err)
+	req.Header.Set("X-Amz-Object-Attributes", "ETag,ObjectSize")
+	for k, v := range extraHeaders {
+		req.Header.Set(k, v)
+	}
+	signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
+	_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
+	require.NoError(t, err)
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	return resp
+}
+
+func testGetObjectAttributesVersionedConditionalHeaders(t *testing.T, cluster *TestCluster) {
+	client := newS3V2Client(cluster)
+	bucketName := createTestBucket(t, cluster, "test-goa-vcond-")
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.Background(), &v2s3.PutBucketVersioningInput{
+		Bucket: v2aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err)
+	time.Sleep(200 * time.Millisecond)
+
+	// Put two versions with different content (different ETags)
+	v1Data := "version 1 - original"
+	putResp1, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String("vcond-key"),
+		Body:   strings.NewReader(v1Data),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, putResp1.VersionId)
+	vid1 := *putResp1.VersionId
+
+	v2Data := "version 2 - updated content"
+	putResp2, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String("vcond-key"),
+		Body:   strings.NewReader(v2Data),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, putResp2.VersionId)
+	vid2 := *putResp2.VersionId
+
+	// Get ETags for each version
+	headV1, err := client.HeadObject(context.Background(), &v2s3.HeadObjectInput{
+		Bucket:    v2aws.String(bucketName),
+		Key:       v2aws.String("vcond-key"),
+		VersionId: v2aws.String(vid1),
+	})
+	require.NoError(t, err)
+	etagV1 := *headV1.ETag
+
+	headV2, err := client.HeadObject(context.Background(), &v2s3.HeadObjectInput{
+		Bucket:    v2aws.String(bucketName),
+		Key:       v2aws.String("vcond-key"),
+		VersionId: v2aws.String(vid2),
+	})
+	require.NoError(t, err)
+	etagV2 := *headV2.ETag
+	require.NotEqual(t, etagV1, etagV2, "versions should have different ETags")
+
+	// If-Match with v1's ETag + versionId=v1 => 200
+	// Before the fix, this would fail with 412 because conditional headers
+	// were evaluated against the latest version (v2) whose ETag differs
+	t.Run("IfMatch_v1_etag_versionId_v1", func(t *testing.T) {
+		resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
+			"If-Match": etagV1,
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 200, resp.StatusCode,
+			"If-Match with v1 ETag targeting versionId=v1 should return 200")
+	})
+
+	// If-Match with v2's ETag + versionId=v1 => 412
+	// The ETag doesn't match v1, so this should fail
+	t.Run("IfMatch_v2_etag_versionId_v1", func(t *testing.T) {
+		resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
+			"If-Match": etagV2,
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 412, resp.StatusCode,
+			"If-Match with v2 ETag targeting versionId=v1 should return 412")
+	})
+
+	// If-None-Match with v1's ETag + versionId=v1 => 304
+	t.Run("IfNoneMatch_v1_etag_versionId_v1", func(t *testing.T) {
+		resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
+			"If-None-Match": etagV1,
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 304, resp.StatusCode,
+			"If-None-Match with v1 ETag targeting versionId=v1 should return 304")
+	})
+
+	t.Logf("Versioned conditional headers tests passed: vid1=%s, vid2=%s", vid1, vid2)
+}

test/s3/policy/policy_test.go

@@ -422,6 +422,218 @@ func TestS3MultipartOperationsInheritPutObjectPermissions(t *testing.T) {
 	require.Equal(t, 0, len(listUploadsOut.Uploads))
 }
 
+// TestS3IAMManagedPolicyLifecycle is an end-to-end integration test covering the
+// user-reported use case in https://github.com/seaweedfs/seaweedfs/issues/8506
+// where managed policy operations (GetPolicy, ListPolicies, DeletePolicy,
+// AttachUserPolicy, DetachUserPolicy) returned 500 errors.
+func TestS3IAMManagedPolicyLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	iamClient := newIAMClient(t, cluster.s3Endpoint)
+
+	// Step 1: Create a user (this already worked per the issue)
+	userName := uniqueName("lifecycle-user")
+	_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+	require.NoError(t, err, "CreateUser should succeed")
+
+	// Step 2: Create a managed policy via IAM API
+	policyName := uniqueName("lifecycle-policy")
+	policyArn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
+	policyDoc := `{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Effect": "Allow",
+			"Action": ["s3:GetObject", "s3:PutObject"],
+			"Resource": "arn:aws:s3:::*"
+		}]
+	}`
+	createOut, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
+		PolicyName:     aws.String(policyName),
+		PolicyDocument: aws.String(policyDoc),
+	})
+	require.NoError(t, err, "CreatePolicy should succeed")
+	require.NotNil(t, createOut.Policy)
+	require.Equal(t, policyName, *createOut.Policy.PolicyName)
+
+	// Step 3: ListPolicies — should include the created policy (was returning 500)
+	listOut, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
+	require.NoError(t, err, "ListPolicies should succeed (was returning 500)")
+	require.True(t, managedPolicyContains(listOut.Policies, policyName),
+		"ListPolicies should contain the newly created policy")
+
+	// Step 4: GetPolicy by ARN — should return the policy (was returning 500)
+	getOut, err := iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
+	require.NoError(t, err, "GetPolicy should succeed (was returning 500)")
+	require.NotNil(t, getOut.Policy)
+	require.Equal(t, policyName, *getOut.Policy.PolicyName)
+	require.Equal(t, policyArn, *getOut.Policy.Arn)
+
+	// Step 5: AttachUserPolicy — should succeed (was returning 500)
+	_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
+		UserName:  aws.String(userName),
+		PolicyArn: aws.String(policyArn),
+	})
+	require.NoError(t, err, "AttachUserPolicy should succeed (was returning 500)")
+
+	// Step 6: ListAttachedUserPolicies — verify the policy is attached
+	attachedOut, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err, "ListAttachedUserPolicies should succeed")
+	require.True(t, attachedPolicyContains(attachedOut.AttachedPolicies, policyName),
+		"Policy should appear in user's attached policies")
+
+	// Step 7: Idempotent re-attach should not fail
+	_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
+		UserName:  aws.String(userName),
+		PolicyArn: aws.String(policyArn),
+	})
+	require.NoError(t, err, "Re-attaching same policy should be idempotent")
+
+	// Step 8: DeletePolicy while attached — should fail with DeleteConflict (AWS behavior)
+	_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
+	require.Error(t, err, "DeletePolicy should fail while policy is attached")
+	var awsErr awserr.Error
+	require.True(t, errors.As(err, &awsErr))
+	require.Equal(t, iam.ErrCodeDeleteConflictException, awsErr.Code(),
+		"Should return DeleteConflict when deleting attached policy")
+
+	// Step 9: DetachUserPolicy
+	_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
+		UserName:  aws.String(userName),
+		PolicyArn: aws.String(policyArn),
+	})
+	require.NoError(t, err, "DetachUserPolicy should succeed")
+
+	// Verify detached
+	attachedOut, err = iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+	require.False(t, attachedPolicyContains(attachedOut.AttachedPolicies, policyName),
+		"Policy should no longer appear in user's attached policies after detach")
+
+	// Step 10: DeletePolicy — should now succeed (was returning XML parsing error)
+	_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
+	require.NoError(t, err, "DeletePolicy should succeed after detach (was returning XML parsing error)")
+
+	// Step 11: Verify the policy is gone
+	listOut, err = iamClient.ListPolicies(&iam.ListPoliciesInput{})
+	require.NoError(t, err)
+	require.False(t, managedPolicyContains(listOut.Policies, policyName),
+		"Deleted policy should not appear in ListPolicies")
+
+	_, err = iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
+	require.Error(t, err, "GetPolicy should fail for deleted policy")
+	require.True(t, errors.As(err, &awsErr))
+	require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+}
+
+// TestS3IAMManagedPolicyErrorCases covers error cases from the user-reported issue:
+// invalid ARNs, missing policies, and missing users.
+func TestS3IAMManagedPolicyErrorCases(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	iamClient := newIAMClient(t, cluster.s3Endpoint)
+
+	t.Run("GetPolicy with nonexistent ARN returns NoSuchEntity", func(t *testing.T) {
+		_, err := iamClient.GetPolicy(&iam.GetPolicyInput{
+			PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
+		})
+		require.Error(t, err)
+		var awsErr awserr.Error
+		require.True(t, errors.As(err, &awsErr))
+		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+
+	t.Run("DeletePolicy with nonexistent ARN returns NoSuchEntity", func(t *testing.T) {
+		_, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{
+			PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
+		})
+		require.Error(t, err)
+		var awsErr awserr.Error
+		require.True(t, errors.As(err, &awsErr))
+		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+
+	t.Run("AttachUserPolicy with nonexistent policy returns NoSuchEntity", func(t *testing.T) {
+		userName := uniqueName("err-user")
+		_, err := iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+		require.NoError(t, err)
+
+		_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
+			UserName:  aws.String(userName),
+			PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
+		})
+		require.Error(t, err)
+		var awsErr awserr.Error
+		require.True(t, errors.As(err, &awsErr))
+		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+
+	t.Run("AttachUserPolicy with nonexistent user returns NoSuchEntity", func(t *testing.T) {
+		policyName := uniqueName("err-policy")
+		_, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`),
+		})
+		require.NoError(t, err)
+
+		_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
+			UserName:  aws.String("nonexistent-user"),
+			PolicyArn: aws.String(fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)),
+		})
+		require.Error(t, err)
+		var awsErr awserr.Error
+		require.True(t, errors.As(err, &awsErr))
+		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+
+	t.Run("DetachUserPolicy that is not attached returns NoSuchEntity", func(t *testing.T) {
+		userName := uniqueName("detach-user")
+		_, err := iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+		require.NoError(t, err)
+
+		policyName := uniqueName("detach-policy")
+		_, err = iamClient.CreatePolicy(&iam.CreatePolicyInput{
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`),
+		})
+		require.NoError(t, err)
+
+		_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
+			UserName:  aws.String(userName),
+			PolicyArn: aws.String(fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)),
+		})
+		require.Error(t, err)
+		var awsErr awserr.Error
+		require.True(t, errors.As(err, &awsErr))
+		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+
+	t.Run("ListAttachedUserPolicies for nonexistent user returns NoSuchEntity", func(t *testing.T) {
+		_, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
+			UserName: aws.String("nonexistent-user"),
+		})
+		require.Error(t, err)
+		var awsErr awserr.Error
+		require.True(t, errors.As(err, &awsErr))
+		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+}
+
 func execShell(t *testing.T, weedCmd, master, filer, shellCmd string) string {
 	// weed shell -master=... -filer=...
 	args := []string{"shell", "-master=" + master, "-filer=" + filer}

weed/admin/dash/admin_lock_manager.go

@@ -4,12 +4,13 @@ import (
 	"sync"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
 )
 
 const (
-	adminLockName       = "shell"
+	adminLockName       = cluster.AdminShellLockName
 	adminLockClientName = "admin-plugin"
 )
 

weed/admin/dash/admin_presence_lock.go

@@ -0,0 +1,61 @@
+package dash
+
+import (
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
+)
+
+const adminPresenceClientName = "admin-server"
+
+type adminPresenceLock struct {
+	locker *exclusive_locks.ExclusiveLocker
+	stopCh chan struct{}
+}
+
+func newAdminPresenceLock(masterClient *wdclient.MasterClient) *adminPresenceLock {
+	if masterClient == nil {
+		return nil
+	}
+	return &adminPresenceLock{
+		locker: exclusive_locks.NewExclusiveLocker(masterClient, cluster.AdminServerPresenceLockName),
+		stopCh: make(chan struct{}),
+	}
+}
+
+func (l *adminPresenceLock) Start() {
+	if l == nil || l.locker == nil {
+		return
+	}
+	l.locker.SetMessage("admin server connected")
+	go func() {
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		for {
+			if !l.locker.IsLocked() {
+				l.locker.RequestLock(adminPresenceClientName)
+			}
+			select {
+			case <-l.stopCh:
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+}
+
+func (l *adminPresenceLock) Stop() {
+	if l == nil {
+		return
+	}
+	select {
+	case <-l.stopCh:
+	default:
+		close(l.stopCh)
+	}
+	if l.locker != nil {
+		l.locker.ReleaseLock()
+	}
+}

weed/admin/dash/admin_server.go

@@ -11,6 +11,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
 	adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin"
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	clustermaintenance "github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
 	"github.com/seaweedfs/seaweedfs/weed/credential"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -99,6 +100,7 @@ type AdminServer struct {
 	maintenanceManager *maintenance.MaintenanceManager
 	plugin             *adminplugin.Plugin
 	pluginLock         *AdminLockManager
+	adminPresenceLock  *adminPresenceLock
 	expireJobHandler   func(jobID string, reason string) (*adminplugin.TrackedJob, bool, error)
 
 	// Topic retention purger
@@ -118,7 +120,7 @@ type AdminServer struct {
 
 // Type definitions moved to types.go
 
-func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string, icebergPort int, idleSleepDuration time.Duration) *AdminServer {
+func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string, icebergPort int) *AdminServer {
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.admin")
 
 	// Create master client with multiple master support
@@ -137,6 +139,10 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 	go masterClient.KeepConnectedToMaster(ctx)
 
 	lockManager := NewAdminLockManager(masterClient, adminLockClientName)
+	presenceLock := newAdminPresenceLock(masterClient)
+	if presenceLock != nil {
+		presenceLock.Start()
+	}
 
 	server := &AdminServer{
 		masterClient:                  masterClient,
@@ -150,6 +156,7 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 		s3TablesManager:               newS3TablesManager(),
 		icebergPort:                   icebergPort,
 		pluginLock:                    lockManager,
+		adminPresenceLock:             presenceLock,
 	}
 
 	// Initialize topic retention purger
@@ -228,24 +235,19 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 		}()
 	}
 
-	plugin, err := adminplugin.New(adminplugin.Options{
-		DataDir:           dataDir,
-		IdleSleepDuration: idleSleepDuration,
+	pluginOpts := adminplugin.Options{
+		DataDir: dataDir,
 		ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
 			return server.buildDefaultPluginClusterContext(), nil
 		},
-		LockManager: lockManager,
-	})
+		LockManager:            lockManager,
+		ConfigDefaultsProvider: server.enrichConfigDefaults,
+	}
+	plugin, err := adminplugin.New(pluginOpts)
 	if err != nil && dataDir != "" {
 		glog.Warningf("Failed to initialize plugin with dataDir=%q: %v. Falling back to in-memory plugin state.", dataDir, err)
-		plugin, err = adminplugin.New(adminplugin.Options{
-			DataDir:           "",
-			IdleSleepDuration: idleSleepDuration,
-			ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
-				return server.buildDefaultPluginClusterContext(), nil
-			},
-			LockManager: lockManager,
-		})
+		pluginOpts.DataDir = ""
+		plugin, err = adminplugin.New(pluginOpts)
 	}
 	if err != nil {
 		glog.Errorf("Failed to initialize plugin: %v", err)
@@ -269,6 +271,89 @@ func (s *AdminServer) loadTaskConfigurationsFromPersistence() {
 	configUpdateRegistry.UpdateAllConfigs(s.configPersistence)
 }
 
+// enrichConfigDefaults is called by the plugin when bootstrapping a job type's
+// default config from its descriptor. For admin_script, it fetches maintenance
+// scripts from the master and uses them as the script default.
+//
+// MIGRATION: This exists to help users migrate from master.toml [master.maintenance]
+// to the admin script plugin worker. Remove after March 2027.
+func (s *AdminServer) enrichConfigDefaults(cfg *plugin_pb.PersistedJobTypeConfig) *plugin_pb.PersistedJobTypeConfig {
+	if cfg.JobType != "admin_script" {
+		return cfg
+	}
+
+	var maintenanceScripts string
+	var sleepMinutes uint32
+	err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		maintenanceScripts = resp.MaintenanceScripts
+		sleepMinutes = resp.MaintenanceSleepMinutes
+		return nil
+	})
+	if err != nil {
+		glog.V(1).Infof("Could not fetch master configuration for admin_script defaults: %v", err)
+		return cfg
+	}
+
+	script := cleanMaintenanceScript(maintenanceScripts)
+	if script == "" {
+		return cfg
+	}
+
+	interval := int64(sleepMinutes)
+	if interval <= 0 {
+		interval = clustermaintenance.DefaultMaintenanceSleepMinutes
+	}
+
+	glog.V(0).Infof("Enriching admin_script defaults from master maintenance scripts (interval=%dm)", interval)
+
+	if cfg.AdminConfigValues == nil {
+		cfg.AdminConfigValues = make(map[string]*plugin_pb.ConfigValue)
+	}
+	cfg.AdminConfigValues["script"] = &plugin_pb.ConfigValue{
+		Kind: &plugin_pb.ConfigValue_StringValue{StringValue: script},
+	}
+	cfg.AdminConfigValues["run_interval_minutes"] = &plugin_pb.ConfigValue{
+		Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: interval},
+	}
+	cfg.UpdatedBy = "master_migration"
+
+	return cfg
+}
+
+// cleanMaintenanceScript strips lock/unlock commands and normalizes a
+// maintenance script string for use with the admin script plugin worker.
+//
+// MIGRATION: Used by enrichConfigDefaults. Remove after March 2027.
+func cleanMaintenanceScript(script string) string {
+	script = strings.ReplaceAll(script, "\r\n", "\n")
+	var lines []string
+	for _, line := range strings.Split(script, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" || strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+		// Strip inline comments (e.g., "lock # migration note")
+		if idx := strings.Index(trimmed, "#"); idx >= 0 {
+			trimmed = strings.TrimSpace(trimmed[:idx])
+			if trimmed == "" {
+				continue
+			}
+		}
+		firstToken := strings.ToLower(strings.Fields(trimmed)[0])
+		if firstToken == "lock" || firstToken == "unlock" {
+			continue
+		}
+		lines = append(lines, trimmed)
+	}
+	return strings.Join(lines, "\n")
+}
+
 // GetCredentialManager returns the credential manager
 func (s *AdminServer) GetCredentialManager() *credential.CredentialManager {
 	return s.credentialManager
@@ -1288,6 +1373,9 @@ func (s *AdminServer) Shutdown() {
 
 	// Stop maintenance manager
 	s.StopMaintenanceManager()
+	if s.adminPresenceLock != nil {
+		s.adminPresenceLock.Stop()
+	}
 
 	if s.plugin != nil {
 		s.plugin.Shutdown()

weed/admin/dash/admin_server_seed_test.go

@@ -0,0 +1,77 @@
+// MIGRATION: Tests for enrichConfigDefaults helpers. Remove after March 2027.
+package dash
+
+import "testing"
+
+func TestCleanMaintenanceScript(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "empty",
+			input:    "",
+			expected: "",
+		},
+		{
+			name:     "only lock unlock",
+			input:    "  lock\n  unlock\n",
+			expected: "",
+		},
+		{
+			name:     "strips lock and unlock",
+			input:    "  lock\n  ec.balance -apply\n  volume.fix.replication -apply\n  unlock\n",
+			expected: "ec.balance -apply\nvolume.fix.replication -apply",
+		},
+		{
+			name:     "case insensitive lock",
+			input:    "Lock\nec.balance -apply\nUNLOCK",
+			expected: "ec.balance -apply",
+		},
+		{
+			name:     "preserves comments removal",
+			input:    "lock\n# a comment\nec.balance -apply\nunlock",
+			expected: "ec.balance -apply",
+		},
+		{
+			name:     "no lock unlock present",
+			input:    "ec.balance -apply\nvolume.fix.replication -apply",
+			expected: "ec.balance -apply\nvolume.fix.replication -apply",
+		},
+		{
+			name:     "windows line endings",
+			input:    "lock\r\nec.balance -apply\r\nunlock\r\n",
+			expected: "ec.balance -apply",
+		},
+		{
+			name:     "lock with inline comment",
+			input:    "lock # migration\nec.balance -apply\nunlock # done",
+			expected: "ec.balance -apply",
+		},
+		{
+			name:     "command with inline comment preserved",
+			input:    "lock\nec.balance -apply # rebalance shards\nunlock",
+			expected: "ec.balance -apply",
+		},
+		{
+			name:     "only inline comment after stripping",
+			input:    "# full line comment\n  # indented comment\n",
+			expected: "",
+		},
+		{
+			name:     "typical master default",
+			input:    "\n  lock\n  ec.encode -fullPercent=95 -quietFor=1h\n  ec.rebuild -apply\n  ec.balance -apply\n  fs.log.purge -daysAgo=7\n  volume.deleteEmpty -quietFor=24h -apply\n  volume.balance -apply\n  volume.fix.replication -apply\n  s3.clean.uploads -timeAgo=24h\n  unlock\n",
+			expected: "ec.encode -fullPercent=95 -quietFor=1h\nec.rebuild -apply\nec.balance -apply\nfs.log.purge -daysAgo=7\nvolume.deleteEmpty -quietFor=24h -apply\nvolume.balance -apply\nvolume.fix.replication -apply\ns3.clean.uploads -timeAgo=24h",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := cleanMaintenanceScript(tt.input)
+			if got != tt.expected {
+				t.Errorf("cleanMaintenanceScript(%q) = %q, want %q", tt.input, got, tt.expected)
+			}
+		})
+	}
+}

weed/admin/dash/plugin_api.go

@@ -235,6 +235,53 @@ func (s *AdminServer) GetPluginSchedulerStatusAPI(w http.ResponseWriter, r *http
 	writeJSON(w, http.StatusOK, response)
 }
 
+// GetPluginSchedulerConfigAPI returns scheduler configuration.
+func (s *AdminServer) GetPluginSchedulerConfigAPI(w http.ResponseWriter, r *http.Request) {
+	pluginSvc := s.GetPlugin()
+	if pluginSvc == nil {
+		writeJSONError(w, http.StatusNotFound, "plugin is not enabled")
+		return
+	}
+
+	writeJSON(w, http.StatusOK, pluginSvc.GetSchedulerConfig())
+}
+
+// UpdatePluginSchedulerConfigAPI updates scheduler configuration.
+func (s *AdminServer) UpdatePluginSchedulerConfigAPI(w http.ResponseWriter, r *http.Request) {
+	pluginSvc := s.GetPlugin()
+	if pluginSvc == nil {
+		writeJSONError(w, http.StatusNotFound, "plugin is not enabled")
+		return
+	}
+
+	var req struct {
+		IdleSleepSeconds *int32 `json:"idle_sleep_seconds"`
+	}
+
+	if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
+		if errors.Is(err, io.EOF) {
+			writeJSONError(w, http.StatusBadRequest, "request body is required")
+			return
+		}
+		writeJSONError(w, http.StatusBadRequest, "invalid request body: "+err.Error())
+		return
+	}
+	if req.IdleSleepSeconds == nil {
+		writeJSONError(w, http.StatusBadRequest, "idle_sleep_seconds is required")
+		return
+	}
+
+	updated, err := pluginSvc.UpdateSchedulerConfig(plugin.SchedulerConfig{
+		IdleSleepSeconds: *req.IdleSleepSeconds,
+	})
+	if err != nil {
+		writeJSONError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	writeJSON(w, http.StatusOK, updated)
+}
+
 // RequestPluginJobTypeSchemaAPI asks a worker for one job type schema.
 func (s *AdminServer) RequestPluginJobTypeSchemaAPI(w http.ResponseWriter, r *http.Request) {
 	jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
@@ -867,6 +914,9 @@ func applyDescriptorDefaultsToPersistedConfig(
 		if runtime.PerWorkerExecutionConcurrency <= 0 {
 			runtime.PerWorkerExecutionConcurrency = defaults.PerWorkerExecutionConcurrency
 		}
+		if runtime.JobTypeMaxRuntimeSeconds <= 0 {
+			runtime.JobTypeMaxRuntimeSeconds = defaults.JobTypeMaxRuntimeSeconds
+		}
 		if runtime.RetryBackoffSeconds <= 0 {
 			runtime.RetryBackoffSeconds = defaults.RetryBackoffSeconds
 		}

weed/admin/handlers/admin_handlers.go

@@ -229,6 +229,8 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
 	pluginApi.HandleFunc("/status", h.adminServer.GetPluginStatusAPI).Methods(http.MethodGet)
 	pluginApi.HandleFunc("/workers", h.adminServer.GetPluginWorkersAPI).Methods(http.MethodGet)
 	pluginApi.HandleFunc("/job-types", h.adminServer.GetPluginJobTypesAPI).Methods(http.MethodGet)
+	pluginApi.HandleFunc("/scheduler-config", h.adminServer.GetPluginSchedulerConfigAPI).Methods(http.MethodGet)
+	pluginApi.Handle("/scheduler-config", wrapWrite(h.adminServer.UpdatePluginSchedulerConfigAPI)).Methods(http.MethodPut)
 	pluginApi.HandleFunc("/jobs", h.adminServer.GetPluginJobsAPI).Methods(http.MethodGet)
 	pluginApi.HandleFunc("/jobs/{jobId}", h.adminServer.GetPluginJobAPI).Methods(http.MethodGet)
 	pluginApi.HandleFunc("/jobs/{jobId}/detail", h.adminServer.GetPluginJobDetailAPI).Methods(http.MethodGet)

weed/admin/maintenance/maintenance_integration.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/admin/topology"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
 	"github.com/seaweedfs/seaweedfs/weed/worker/types"
@@ -229,6 +230,12 @@ func (s *MaintenanceIntegration) ScanWithTaskDetectors(volumeMetrics []*types.Vo
 			continue
 		}
 
+		// Cancel stale pending tasks for this type before re-detection
+		maintenanceType := s.taskTypeMap[taskType]
+		if cancelled := s.maintenanceQueue.CancelPendingTasksByType(maintenanceType); cancelled > 0 {
+			glog.Infof("Cancelled %d stale pending %s tasks before re-detection", cancelled, taskType)
+		}
+
 		glog.V(2).Infof("Running detection for task type: %s", taskType)
 
 		results, err := detector.ScanForTasks(filteredMetrics, clusterInfo)
@@ -528,10 +535,15 @@ func (s *MaintenanceIntegration) SyncTask(task *MaintenanceTask) {
 		// Volume size is not currently used for Balance/Vacuum impact and is not stored in MaintenanceTask
 		sourceImpact, targetImpact := topology.CalculateTaskStorageImpact(topology.TaskType(string(taskType)), 0)
 
-		// Use unified sources and targets from TaskParams
+		// Use unified sources and targets from TaskParams.
+		// Task protos store ServerAddresses (with gRPC port, e.g., "host:port.grpcPort")
+		// but the topology indexes disks by NodeId (e.g., "host:port").
+		// Strip the gRPC port suffix via ToHttpAddress() to match the topology key.
 		for _, src := range task.TypedParams.Sources {
+			resolvedSrc := pb.ServerAddress(src.Node).ToHttpAddress()
+			glog.V(2).Infof("SyncTask %s: source proto Node=%q resolved to %q, diskId=%d", task.ID, src.Node, resolvedSrc, src.DiskId)
 			sources = append(sources, topology.TaskSource{
-				SourceServer:  src.Node,
+				SourceServer:  resolvedSrc,
 				SourceDisk:    src.DiskId,
 				StorageChange: sourceImpact,
 			})
@@ -539,8 +551,10 @@ func (s *MaintenanceIntegration) SyncTask(task *MaintenanceTask) {
 			estimatedSize += int64(src.EstimatedSize)
 		}
 		for _, target := range task.TypedParams.Targets {
+			resolvedTarget := pb.ServerAddress(target.Node).ToHttpAddress()
+			glog.V(2).Infof("SyncTask %s: target proto Node=%q resolved to %q, diskId=%d", task.ID, target.Node, resolvedTarget, target.DiskId)
 			destinations = append(destinations, topology.TaskDestination{
-				TargetServer:  target.Node,
+				TargetServer:  resolvedTarget,
 				TargetDisk:    target.DiskId,
 				StorageChange: targetImpact,
 			})

weed/admin/maintenance/maintenance_queue.go

@@ -4,6 +4,7 @@ import (
 	"crypto/rand"
 	"fmt"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -230,6 +231,46 @@ func (mq *MaintenanceQueue) hasDuplicateTask(newTask *MaintenanceTask) bool {
 	return false
 }
 
+// CancelPendingTasksByType cancels all pending tasks of a given type.
+// This is called before each detection cycle to ensure stale proposals
+// from previous cycles are cleaned up before creating new ones.
+func (mq *MaintenanceQueue) CancelPendingTasksByType(taskType MaintenanceTaskType) int {
+	mq.mutex.Lock()
+
+	var remaining []*MaintenanceTask
+	var cancelledSnapshots []*MaintenanceTask
+	cancelled := 0
+	for _, task := range mq.pendingTasks {
+		if task.Type == taskType {
+			task.Status = TaskStatusCancelled
+			now := time.Now()
+			task.CompletedAt = &now
+			cancelled++
+			cancelledSnapshots = append(cancelledSnapshots, snapshotTask(task))
+			glog.V(1).Infof("Cancelled stale pending task %s (%s) for volume %d before re-detection",
+				task.ID, task.Type, task.VolumeID)
+
+			// Release capacity in ActiveTopology and remove pending operation
+			if mq.integration != nil {
+				if at := mq.integration.GetActiveTopology(); at != nil {
+					_ = at.CompleteTask(task.ID)
+				}
+			}
+			mq.removePendingOperation(task.ID)
+		} else {
+			remaining = append(remaining, task)
+		}
+	}
+	mq.pendingTasks = remaining
+	mq.mutex.Unlock()
+
+	// Persist cancelled state outside the lock to avoid blocking
+	for _, snapshot := range cancelledSnapshots {
+		mq.saveTaskState(snapshot)
+	}
+	return cancelled
+}
+
 // AddTasksFromResults converts detection results to tasks and adds them to the queue
 func (mq *MaintenanceQueue) AddTasksFromResults(results []*TaskDetectionResult) {
 	for _, result := range results {
@@ -455,8 +496,8 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 		task.Status = TaskStatusFailed
 		task.Error = error
 
-		// Check if task should be retried
-		if task.RetryCount < task.MaxRetries {
+		// Check if task should be retried (skip retry for permanent errors)
+		if task.RetryCount < task.MaxRetries && !isNonRetriableError(error) {
 			// Record unassignment due to failure/retry
 			if task.WorkerID != "" && len(task.AssignmentHistory) > 0 {
 				lastAssignment := task.AssignmentHistory[len(task.AssignmentHistory)-1]
@@ -559,6 +600,12 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 	}
 }
 
+// isNonRetriableError returns true for errors that will never succeed on retry,
+// such as when the volume doesn't exist on the source server.
+func isNonRetriableError(errMsg string) bool {
+	return strings.Contains(errMsg, "not found")
+}
+
 // UpdateTaskProgress updates the progress of a running task
 func (mq *MaintenanceQueue) UpdateTaskProgress(taskID string, progress float64) {
 	mq.mutex.Lock()

weed/admin/plugin/config_store.go

@@ -30,6 +30,7 @@ const (
 	runsJSONFileName        = "runs.json"
 	trackedJobsJSONFileName = "tracked_jobs.json"
 	activitiesJSONFileName  = "activities.json"
+	schedulerJSONFileName   = "scheduler.json"
 	defaultDirPerm          = 0o755
 	defaultFilePerm         = 0o644
 )
@@ -53,6 +54,7 @@ type ConfigStore struct {
 	memTrackedJobs []TrackedJob
 	memActivities  []JobActivity
 	memJobDetails  map[string]TrackedJob
+	memScheduler   *SchedulerConfig
 }
 
 func NewConfigStore(adminDataDir string) (*ConfigStore, error) {
@@ -93,6 +95,60 @@ func (s *ConfigStore) BaseDir() string {
 	return s.baseDir
 }
 
+func (s *ConfigStore) LoadSchedulerConfig() (*SchedulerConfig, error) {
+	s.mu.RLock()
+	if !s.configured {
+		cfg := s.memScheduler
+		s.mu.RUnlock()
+		if cfg == nil {
+			return nil, nil
+		}
+		clone := *cfg
+		return &clone, nil
+	}
+	s.mu.RUnlock()
+
+	path := filepath.Join(s.baseDir, schedulerJSONFileName)
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("read scheduler config: %w", err)
+	}
+
+	var cfg SchedulerConfig
+	if err := json.Unmarshal(data, &cfg); err != nil {
+		return nil, fmt.Errorf("unmarshal scheduler config: %w", err)
+	}
+	return &cfg, nil
+}
+
+func (s *ConfigStore) SaveSchedulerConfig(config *SchedulerConfig) error {
+	if config == nil {
+		return fmt.Errorf("scheduler config is nil")
+	}
+	normalized := normalizeSchedulerConfig(*config)
+
+	s.mu.Lock()
+	if !s.configured {
+		s.memScheduler = &normalized
+		s.mu.Unlock()
+		return nil
+	}
+	s.mu.Unlock()
+
+	payload, err := json.MarshalIndent(normalized, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal scheduler config: %w", err)
+	}
+	path := filepath.Join(s.baseDir, schedulerJSONFileName)
+	if err := os.WriteFile(path, payload, defaultFilePerm); err != nil {
+		return fmt.Errorf("save scheduler config: %w", err)
+	}
+	return nil
+}
+
 func (s *ConfigStore) SaveDescriptor(jobType string, descriptor *plugin_pb.JobTypeDescriptor) error {
 	if descriptor == nil {
 		return fmt.Errorf("descriptor is nil")
@@ -200,6 +256,53 @@ func (s *ConfigStore) SaveJobTypeConfig(config *plugin_pb.PersistedJobTypeConfig
 	return nil
 }
 
+// SaveJobTypeConfigIfNotExists atomically checks whether a config for the
+// given job type already exists and only persists config when none is found.
+// Returns true if the config was saved, false if a config already existed.
+func (s *ConfigStore) SaveJobTypeConfigIfNotExists(config *plugin_pb.PersistedJobTypeConfig) (bool, error) {
+	if config == nil {
+		return false, fmt.Errorf("job type config is nil")
+	}
+	if config.JobType == "" {
+		return false, fmt.Errorf("job type config has empty job_type")
+	}
+	sanitizedJobType, err := sanitizeJobType(config.JobType)
+	if err != nil {
+		return false, err
+	}
+	config.JobType = sanitizedJobType
+
+	clone := proto.Clone(config).(*plugin_pb.PersistedJobTypeConfig)
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if !s.configured {
+		if _, exists := s.memConfigs[config.JobType]; exists {
+			return false, nil
+		}
+		s.memConfigs[config.JobType] = clone
+		return true, nil
+	}
+
+	pbPath := filepath.Join(s.baseDir, jobTypesDirName, config.JobType, configPBFileName)
+	if _, statErr := os.Stat(pbPath); statErr == nil {
+		return false, nil
+	}
+
+	jobTypeDir, err := s.ensureJobTypeDir(config.JobType)
+	if err != nil {
+		return false, err
+	}
+
+	jsonPath := filepath.Join(jobTypeDir, configJSONFileName)
+	if err := writeProtoFiles(clone, filepath.Join(jobTypeDir, configPBFileName), jsonPath); err != nil {
+		return false, fmt.Errorf("save job type config for %s: %w", config.JobType, err)
+	}
+
+	return true, nil
+}
+
 func (s *ConfigStore) LoadJobTypeConfig(jobType string) (*plugin_pb.PersistedJobTypeConfig, error) {
 	if _, err := sanitizeJobType(jobType); err != nil {
 		return nil, err

weed/admin/plugin/config_store_test.go

@@ -208,6 +208,81 @@ func TestConfigStoreMonitorStateRoundTrip(t *testing.T) {
 	}
 }
 
+func TestConfigStoreSaveJobTypeConfigIfNotExists(t *testing.T) {
+	t.Parallel()
+
+	t.Run("in-memory", func(t *testing.T) {
+		t.Parallel()
+		store, err := NewConfigStore("")
+		if err != nil {
+			t.Fatalf("NewConfigStore: %v", err)
+		}
+		testSaveJobTypeConfigIfNotExists(t, store)
+	})
+
+	t.Run("on-disk", func(t *testing.T) {
+		t.Parallel()
+		store, err := NewConfigStore(t.TempDir())
+		if err != nil {
+			t.Fatalf("NewConfigStore: %v", err)
+		}
+		testSaveJobTypeConfigIfNotExists(t, store)
+	})
+}
+
+func testSaveJobTypeConfigIfNotExists(t *testing.T, store *ConfigStore) {
+	t.Helper()
+
+	cfg := &plugin_pb.PersistedJobTypeConfig{
+		JobType:      "admin_script",
+		AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true},
+	}
+
+	// First call should save.
+	saved, err := store.SaveJobTypeConfigIfNotExists(cfg)
+	if err != nil {
+		t.Fatalf("first SaveJobTypeConfigIfNotExists: %v", err)
+	}
+	if !saved {
+		t.Fatal("expected first call to save the config")
+	}
+
+	// Second call with same job type should not save.
+	saved, err = store.SaveJobTypeConfigIfNotExists(&plugin_pb.PersistedJobTypeConfig{
+		JobType:      "admin_script",
+		AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: false},
+	})
+	if err != nil {
+		t.Fatalf("second SaveJobTypeConfigIfNotExists: %v", err)
+	}
+	if saved {
+		t.Fatal("expected second call to be a no-op")
+	}
+
+	// Verify the original config was preserved.
+	loaded, err := store.LoadJobTypeConfig("admin_script")
+	if err != nil {
+		t.Fatalf("LoadJobTypeConfig: %v", err)
+	}
+	if loaded == nil {
+		t.Fatal("expected config to exist")
+	}
+	if !loaded.AdminRuntime.Enabled {
+		t.Fatal("expected original config (Enabled=true) to be preserved")
+	}
+
+	// Different job type should still save.
+	saved, err = store.SaveJobTypeConfigIfNotExists(&plugin_pb.PersistedJobTypeConfig{
+		JobType: "vacuum",
+	})
+	if err != nil {
+		t.Fatalf("SaveJobTypeConfigIfNotExists for different type: %v", err)
+	}
+	if !saved {
+		t.Fatal("expected save for a different job type")
+	}
+}
+
 func TestConfigStoreJobDetailRoundTrip(t *testing.T) {
 	t.Parallel()
 

weed/admin/plugin/plugin.go

@@ -32,9 +32,13 @@ type Options struct {
 	OutgoingBufferSize     int
 	SendTimeout            time.Duration
 	SchedulerTick          time.Duration
-	IdleSleepDuration      time.Duration
 	ClusterContextProvider func(context.Context) (*plugin_pb.ClusterContext, error)
 	LockManager            LockManager
+	// ConfigDefaultsProvider is an optional callback invoked when a job type's
+	// config is being bootstrapped from its descriptor defaults. It can enrich
+	// or replace the default config before it is persisted. If nil, descriptor
+	// defaults are used as-is.
+	ConfigDefaultsProvider func(config *plugin_pb.PersistedJobTypeConfig) *plugin_pb.PersistedJobTypeConfig
 }
 
 // JobTypeInfo contains metadata about a plugin job type.
@@ -54,16 +58,13 @@ type Plugin struct {
 	sendTimeout    time.Duration
 
 	schedulerTick          time.Duration
-	idleSleepDuration      time.Duration
 	clusterContextProvider func(context.Context) (*plugin_pb.ClusterContext, error)
+	configDefaultsProvider func(config *plugin_pb.PersistedJobTypeConfig) *plugin_pb.PersistedJobTypeConfig
 	lockManager            LockManager
 
-	schedulerMu                sync.Mutex
-	schedulerPhase             string
-	currentJobType             string
-	iterationStartedAt         time.Time
-	lastIterationEndedAt       time.Time
-	lastIterationWorkDetected  bool
+	schedulerMu       sync.Mutex
+	nextDetectionAt   map[string]time.Time
+	detectionInFlight map[string]bool
 
 	detectorLeaseMu sync.Mutex
 	detectorLeases  map[string]string
@@ -73,6 +74,13 @@ type Plugin struct {
 	adminScriptRunMu          sync.RWMutex
 	schedulerDetectionMu      sync.Mutex
 	schedulerDetection        map[string]*schedulerDetectionInfo
+	schedulerRunMu            sync.Mutex
+	schedulerRun              map[string]*schedulerRunInfo
+	schedulerLoopMu           sync.Mutex
+	schedulerLoopState        schedulerLoopState
+	schedulerConfigMu         sync.RWMutex
+	schedulerConfig           SchedulerConfig
+	schedulerWakeCh           chan struct{}
 
 	dedupeMu           sync.Mutex
 	recentDedupeByType map[string]map[string]time.Time
@@ -151,10 +159,6 @@ func New(options Options) (*Plugin, error) {
 	if schedulerTick <= 0 {
 		schedulerTick = defaultSchedulerTick
 	}
-	idleSleepDuration := options.IdleSleepDuration
-	if idleSleepDuration <= 0 {
-		idleSleepDuration = defaultIdleSleepDuration
-	}
 
 	plugin := &Plugin{
 		store:                     store,
@@ -162,25 +166,43 @@ func New(options Options) (*Plugin, error) {
 		outgoingBuffer:            bufferSize,
 		sendTimeout:               sendTimeout,
 		schedulerTick:             schedulerTick,
-		idleSleepDuration:         idleSleepDuration,
 		clusterContextProvider:    options.ClusterContextProvider,
+		configDefaultsProvider:    options.ConfigDefaultsProvider,
 		lockManager:               options.LockManager,
-		schedulerPhase:            "idle",
 		sessions:                  make(map[string]*streamSession),
 		pendingSchema:             make(map[string]chan *plugin_pb.ConfigSchemaResponse),
 		pendingDetection:          make(map[string]*pendingDetectionState),
 		pendingExecution:          make(map[string]chan *plugin_pb.JobCompleted),
+		nextDetectionAt:           make(map[string]time.Time),
+		detectionInFlight:         make(map[string]bool),
 		detectorLeases:            make(map[string]string),
 		schedulerExecReservations: make(map[string]int),
 		schedulerDetection:        make(map[string]*schedulerDetectionInfo),
+		schedulerRun:              make(map[string]*schedulerRunInfo),
 		recentDedupeByType:        make(map[string]map[string]time.Time),
 		jobs:                      make(map[string]*TrackedJob),
 		activities:                make([]JobActivity, 0, 256),
 		persistTicker:             time.NewTicker(2 * time.Second),
+		schedulerWakeCh:           make(chan struct{}, 1),
 		shutdownCh:                make(chan struct{}),
 	}
 	plugin.ctx, plugin.ctxCancel = context.WithCancel(context.Background())
 
+	if cfg, err := plugin.store.LoadSchedulerConfig(); err != nil {
+		glog.Warningf("Plugin failed to load scheduler config: %v", err)
+		plugin.schedulerConfig = DefaultSchedulerConfig()
+	} else if cfg == nil {
+		defaults := DefaultSchedulerConfig()
+		plugin.schedulerConfig = defaults
+		if plugin.store.IsConfigured() {
+			if err := plugin.store.SaveSchedulerConfig(&defaults); err != nil {
+				glog.Warningf("Plugin failed to persist scheduler defaults: %v", err)
+			}
+		}
+	} else {
+		plugin.schedulerConfig = normalizeSchedulerConfig(*cfg)
+	}
+
 	if err := plugin.loadPersistedMonitorState(); err != nil {
 		glog.Warningf("Plugin failed to load persisted monitoring state: %v", err)
 	}
@@ -380,9 +402,14 @@ func (r *Plugin) LoadJobTypeConfig(jobType string) (*plugin_pb.PersistedJobTypeC
 }
 
 func (r *Plugin) SaveJobTypeConfig(config *plugin_pb.PersistedJobTypeConfig) error {
-	return r.store.SaveJobTypeConfig(config)
+	if err := r.store.SaveJobTypeConfig(config); err != nil {
+		return err
+	}
+	r.wakeScheduler()
+	return nil
 }
 
+
 func (r *Plugin) LoadDescriptor(jobType string) (*plugin_pb.JobTypeDescriptor, error) {
 	return r.store.LoadDescriptor(jobType)
 }
@@ -399,6 +426,31 @@ func (r *Plugin) BaseDir() string {
 	return r.store.BaseDir()
 }
 
+func (r *Plugin) GetSchedulerConfig() SchedulerConfig {
+	if r == nil {
+		return DefaultSchedulerConfig()
+	}
+	r.schedulerConfigMu.RLock()
+	cfg := r.schedulerConfig
+	r.schedulerConfigMu.RUnlock()
+	return normalizeSchedulerConfig(cfg)
+}
+
+func (r *Plugin) UpdateSchedulerConfig(cfg SchedulerConfig) (SchedulerConfig, error) {
+	if r == nil {
+		return DefaultSchedulerConfig(), fmt.Errorf("plugin is not initialized")
+	}
+	normalized := normalizeSchedulerConfig(cfg)
+	if err := r.store.SaveSchedulerConfig(&normalized); err != nil {
+		return SchedulerConfig{}, err
+	}
+	r.schedulerConfigMu.Lock()
+	r.schedulerConfig = normalized
+	r.schedulerConfigMu.Unlock()
+	r.wakeScheduler()
+	return normalized, nil
+}
+
 func (r *Plugin) acquireAdminLock(reason string) (func(), error) {
 	if r == nil || r.lockManager == nil {
 		return func() {}, nil
@@ -921,6 +973,7 @@ func (r *Plugin) handleWorkerMessage(workerID string, message *plugin_pb.WorkerT
 	switch body := message.Body.(type) {
 	case *plugin_pb.WorkerToAdminMessage_Hello:
 		r.registry.UpsertFromHello(body.Hello)
+		r.wakeScheduler()
 	case *plugin_pb.WorkerToAdminMessage_Heartbeat:
 		r.registry.UpdateHeartbeat(workerID, body.Heartbeat)
 	case *plugin_pb.WorkerToAdminMessage_ConfigSchemaResponse:
@@ -990,14 +1043,6 @@ func (r *Plugin) ensureJobTypeConfigFromDescriptor(jobType string, descriptor *p
 		return nil
 	}
 
-	existing, err := r.store.LoadJobTypeConfig(jobType)
-	if err != nil {
-		return err
-	}
-	if existing != nil {
-		return nil
-	}
-
 	workerDefaults := CloneConfigValueMap(descriptor.WorkerDefaultValues)
 	if len(workerDefaults) == 0 && descriptor.WorkerConfigForm != nil {
 		workerDefaults = CloneConfigValueMap(descriptor.WorkerConfigForm.DefaultValues)
@@ -1020,6 +1065,7 @@ func (r *Plugin) ensureJobTypeConfigFromDescriptor(jobType string, descriptor *p
 			PerWorkerExecutionConcurrency: defaults.PerWorkerExecutionConcurrency,
 			RetryLimit:                    defaults.RetryLimit,
 			RetryBackoffSeconds:           defaults.RetryBackoffSeconds,
+			JobTypeMaxRuntimeSeconds:      defaults.JobTypeMaxRuntimeSeconds,
 		}
 	}
 
@@ -1033,7 +1079,22 @@ func (r *Plugin) ensureJobTypeConfigFromDescriptor(jobType string, descriptor *p
 		UpdatedBy:          "plugin",
 	}
 
-	return r.store.SaveJobTypeConfig(cfg)
+	// Check existence first to avoid calling configDefaultsProvider unnecessarily
+	// (e.g., it may make a blocking gRPC call to fetch master config).
+	existing, err := r.store.LoadJobTypeConfig(jobType)
+	if err != nil {
+		return err
+	}
+	if existing != nil {
+		return nil
+	}
+
+	if r.configDefaultsProvider != nil {
+		cfg = r.configDefaultsProvider(cfg)
+	}
+
+	_, err = r.store.SaveJobTypeConfigIfNotExists(cfg)
+	return err
 }
 
 func (r *Plugin) handleDetectionProposals(workerID string, message *plugin_pb.DetectionProposals) {

weed/admin/plugin/plugin_monitor.go

@@ -861,6 +861,79 @@ func (r *Plugin) trackExecutionQueued(job *plugin_pb.JobSpec) {
 	})
 }
 
+func (r *Plugin) cancelQueuedJob(job *plugin_pb.JobSpec, cause error) {
+	reason := "job canceled"
+	if cause != nil {
+		reason = cause.Error()
+	}
+	r.markJobCanceled(job, reason)
+}
+
+func (r *Plugin) markJobCanceled(job *plugin_pb.JobSpec, reason string) {
+	if job == nil || strings.TrimSpace(job.JobId) == "" {
+		return
+	}
+
+	now := time.Now().UTC()
+	if strings.TrimSpace(reason) == "" {
+		reason = "job canceled"
+	}
+
+	r.jobsMu.Lock()
+	tracked := r.jobs[job.JobId]
+	if tracked == nil {
+		tracked = &TrackedJob{
+			JobID:     job.JobId,
+			CreatedAt: timeToPtr(now),
+		}
+		r.jobs[job.JobId] = tracked
+	}
+
+	if job.JobType != "" {
+		tracked.JobType = job.JobType
+	}
+	tracked.State = StateCanceled
+	tracked.Stage = "canceled"
+	tracked.Message = reason
+	tracked.ErrorMessage = reason
+	tracked.Progress = 0
+	if tracked.CreatedAt == nil || tracked.CreatedAt.IsZero() {
+		tracked.CreatedAt = timeToPtr(now)
+	}
+	tracked.UpdatedAt = timeToPtr(now)
+	tracked.CompletedAt = timeToPtr(now)
+	trackedSnapshot := cloneTrackedJob(*tracked)
+	r.pruneTrackedJobsLocked()
+	r.dirtyJobs = true
+	r.jobsMu.Unlock()
+
+	r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
+		detail.JobID = job.JobId
+		if job.JobType != "" {
+			detail.JobType = job.JobType
+		}
+		detail.State = trackedSnapshot.State
+		detail.Stage = trackedSnapshot.Stage
+		detail.Message = trackedSnapshot.Message
+		detail.ErrorMessage = trackedSnapshot.ErrorMessage
+		detail.Progress = trackedSnapshot.Progress
+		if detail.CreatedAt == nil || detail.CreatedAt.IsZero() {
+			detail.CreatedAt = trackedSnapshot.CreatedAt
+		}
+		detail.UpdatedAt = trackedSnapshot.UpdatedAt
+		detail.CompletedAt = trackedSnapshot.CompletedAt
+	})
+
+	r.appendActivity(JobActivity{
+		JobID:      job.JobId,
+		JobType:    job.JobType,
+		Source:     "admin_scheduler",
+		Message:    reason,
+		Stage:      "canceled",
+		OccurredAt: timeToPtr(now),
+	})
+}
+
 func (r *Plugin) trackExecutionCompletion(completed *plugin_pb.JobCompleted) *TrackedJob {
 	if completed == nil || strings.TrimSpace(completed.JobId) == "" {
 		return nil

weed/admin/plugin/plugin_scheduler.go

@@ -13,14 +13,17 @@ import (
 	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
-var errExecutorAtCapacity = errors.New("executor is at capacity")
+var (
+	errExecutorAtCapacity = errors.New("executor is at capacity")
+	errSchedulerShutdown  = errors.New("scheduler shutdown")
+)
 
 const (
 	defaultSchedulerTick                       = 5 * time.Second
-	defaultIdleSleepDuration                   = 17 * time.Minute
-	defaultMaxJobTypeDuration                  = 30 * time.Minute
+	defaultScheduledDetectionInterval          = 300 * time.Second
 	defaultScheduledDetectionTimeout           = 45 * time.Second
 	defaultScheduledExecutionTimeout           = 90 * time.Second
+	defaultScheduledJobTypeMaxRuntime          = 30 * time.Minute
 	defaultScheduledMaxResults           int32 = 1000
 	defaultScheduledExecutionConcurrency       = 1
 	defaultScheduledPerWorkerConcurrency       = 1
@@ -32,9 +35,10 @@ const (
 )
 
 type schedulerPolicy struct {
+	DetectionInterval      time.Duration
 	DetectionTimeout       time.Duration
 	ExecutionTimeout       time.Duration
-	MaxJobTypeDuration     time.Duration
+	JobTypeMaxRuntime      time.Duration
 	RetryBackoff           time.Duration
 	MaxResults             int32
 	ExecutionConcurrency   int
@@ -43,62 +47,73 @@ type schedulerPolicy struct {
 	ExecutorReserveBackoff time.Duration
 }
 
-func (r *Plugin) setSchedulerPhase(phase, jobType string) {
-	r.schedulerMu.Lock()
-	r.schedulerPhase = phase
-	r.currentJobType = jobType
-	r.schedulerMu.Unlock()
-}
-
 func (r *Plugin) schedulerLoop() {
 	defer r.wg.Done()
-
-	// Try once immediately on startup.
-	workDetected := r.runSchedulerIteration()
-
 	for {
-		if workDetected {
-			// Immediate re-iteration when work was found.
-			select {
-			case <-r.shutdownCh:
-				return
-			default:
-			}
-		} else {
-			r.setSchedulerPhase("idle", "")
-			if !waitForShutdownOrTimer(r.shutdownCh, r.idleSleepDuration) {
-				return
-			}
-		}
-
 		select {
 		case <-r.shutdownCh:
 			return
 		default:
 		}
 
-		workDetected = r.runSchedulerIteration()
+		hadJobs := r.runSchedulerIteration()
+		r.recordSchedulerIterationComplete(hadJobs)
+
+		if hadJobs {
+			continue
+		}
+
+		r.setSchedulerLoopState("", "sleeping")
+		idleSleep := r.GetSchedulerConfig().IdleSleepDuration()
+		if nextRun := r.earliestNextDetectionAt(); !nextRun.IsZero() {
+			if until := time.Until(nextRun); until <= 0 {
+				idleSleep = 0
+			} else if until < idleSleep {
+				idleSleep = until
+			}
+		}
+		if idleSleep <= 0 {
+			continue
+		}
+
+		timer := time.NewTimer(idleSleep)
+		select {
+		case <-r.shutdownCh:
+			timer.Stop()
+			return
+		case <-r.schedulerWakeCh:
+			if !timer.Stop() {
+				<-timer.C
+			}
+			continue
+		case <-timer.C:
+		}
 	}
 }
 
 func (r *Plugin) runSchedulerIteration() bool {
-	now := time.Now().UTC()
-
-	r.schedulerMu.Lock()
-	r.iterationStartedAt = now
-	r.schedulerMu.Unlock()
-
-	r.expireStaleJobs(now)
+	r.expireStaleJobs(time.Now().UTC())
 
 	jobTypes := r.registry.DetectableJobTypes()
 	if len(jobTypes) == 0 {
-		r.finishIteration(false)
+		r.setSchedulerLoopState("", "idle")
 		return false
 	}
 
+	r.setSchedulerLoopState("", "waiting_for_lock")
+	releaseLock, err := r.acquireAdminLock("plugin scheduler iteration")
+	if err != nil {
+		glog.Warningf("Plugin scheduler failed to acquire lock: %v", err)
+		r.setSchedulerLoopState("", "idle")
+		return false
+	}
+	if releaseLock != nil {
+		defer releaseLock()
+	}
+
 	active := make(map[string]struct{}, len(jobTypes))
-	enabledJobTypes := make([]string, 0, len(jobTypes))
-	policies := make(map[string]schedulerPolicy, len(jobTypes))
+	schedulerIdleSleep := r.GetSchedulerConfig().IdleSleepDuration()
+	hadJobs := false
 
 	for _, jobType := range jobTypes {
 		active[jobType] = struct{}{}
@@ -112,90 +127,42 @@ func (r *Plugin) runSchedulerIteration() bool {
 			r.clearSchedulerJobType(jobType)
 			continue
 		}
+		initialDelay := time.Duration(0)
+		if runInfo := r.snapshotSchedulerRun(jobType); runInfo.lastRunStartedAt.IsZero() {
+			initialDelay = schedulerIdleSleep / 2
+		}
+		if !r.markDetectionDue(jobType, policy.DetectionInterval, initialDelay) {
+			continue
+		}
 
-		enabledJobTypes = append(enabledJobTypes, jobType)
-		policies[jobType] = policy
+		detected := r.runJobTypeIteration(jobType, policy)
+		if detected {
+			hadJobs = true
+		}
 	}
 
+	r.pruneSchedulerState(active)
 	r.pruneDetectorLeases(active)
-
-	if len(enabledJobTypes) == 0 {
-		r.finishIteration(false)
-		return false
-	}
-
-	// Acquire the lock ONCE for the entire iteration.
-	r.setSchedulerPhase("acquiring_lock", "")
-	releaseLock, lockErr := r.acquireAdminLock("plugin scheduler iteration")
-	if lockErr != nil {
-		glog.Warningf("Plugin scheduler failed to acquire lock: %v", lockErr)
-		r.appendActivity(JobActivity{
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduler iteration aborted: failed to acquire lock: %v", lockErr),
-			Stage:      "failed",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-		r.finishIteration(false)
-		return false
-	}
-	defer releaseLock()
-
-	// Load cluster context ONCE for all job types.
-	clusterContext, err := r.loadSchedulerClusterContext()
-	if err != nil {
-		r.appendActivity(JobActivity{
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduler iteration aborted: %v", err),
-			Stage:      "failed",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-		r.finishIteration(false)
-		return false
-	}
-
-	// Process each job type sequentially.
-	anyWorkDetected := false
-	for _, jobType := range enabledJobTypes {
-		select {
-		case <-r.shutdownCh:
-			r.finishIteration(anyWorkDetected)
-			return anyWorkDetected
-		default:
-		}
-
-		policy := policies[jobType]
-		r.setSchedulerPhase("processing", jobType)
-
-		if r.runJobTypeIteration(jobType, policy, clusterContext) {
-			anyWorkDetected = true
-		}
-	}
-
-	r.finishIteration(anyWorkDetected)
-	return anyWorkDetected
+	r.setSchedulerLoopState("", "idle")
+	return hadJobs
 }
 
-func (r *Plugin) finishIteration(workDetected bool) {
-	now := time.Now().UTC()
-	r.schedulerMu.Lock()
-	r.lastIterationEndedAt = now
-	r.lastIterationWorkDetected = workDetected
-	r.currentJobType = ""
-	r.schedulerPhase = "idle"
-	r.schedulerMu.Unlock()
+func (r *Plugin) wakeScheduler() {
+	if r == nil {
+		return
+	}
+	select {
+	case r.schedulerWakeCh <- struct{}{}:
+	default:
+	}
 }
 
-func (r *Plugin) runJobTypeIteration(
-	jobType string,
-	policy schedulerPolicy,
-	clusterContext *plugin_pb.ClusterContext,
-) bool {
-	budget := policy.MaxJobTypeDuration
-	if budget <= 0 {
-		budget = defaultMaxJobTypeDuration
-	}
-	ctx, cancel := context.WithTimeout(r.ctx, budget)
-	defer cancel()
+func (r *Plugin) runJobTypeIteration(jobType string, policy schedulerPolicy) bool {
+	r.recordSchedulerRunStart(jobType)
+	r.clearWaitingJobQueue(jobType)
+	r.setSchedulerLoopState(jobType, "detecting")
+	r.markJobTypeInFlight(jobType)
+	defer r.finishDetection(jobType)
 
 	start := time.Now().UTC()
 	r.appendActivity(JobActivity{
@@ -215,25 +182,70 @@ func (r *Plugin) runJobTypeIteration(
 			Stage:      "skipped_waiting_backlog",
 			OccurredAt: timeToPtr(time.Now().UTC()),
 		})
+		r.recordSchedulerRunComplete(jobType, "skipped")
 		return false
 	}
 
-	detectionTimeout := policy.DetectionTimeout
-	if detectionTimeout <= 0 {
-		detectionTimeout = defaultScheduledDetectionTimeout
+	maxRuntime := policy.JobTypeMaxRuntime
+	if maxRuntime <= 0 {
+		maxRuntime = defaultScheduledJobTypeMaxRuntime
 	}
-	detCtx, detCancel := context.WithTimeout(ctx, detectionTimeout)
-	proposals, err := r.RunDetection(detCtx, jobType, clusterContext, policy.MaxResults)
-	detCancel()
+	jobCtx, cancel := context.WithTimeout(context.Background(), maxRuntime)
+	defer cancel()
+
+	clusterContext, err := r.loadSchedulerClusterContext(jobCtx)
 	if err != nil {
 		r.recordSchedulerDetectionError(jobType, err)
 		r.appendActivity(JobActivity{
 			JobType:    jobType,
 			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection failed: %v", err),
+			Message:    fmt.Sprintf("scheduled detection aborted: %v", err),
 			Stage:      "failed",
 			OccurredAt: timeToPtr(time.Now().UTC()),
 		})
+		r.recordSchedulerRunComplete(jobType, "error")
+		return false
+	}
+
+	detectionTimeout := policy.DetectionTimeout
+	remaining := time.Until(start.Add(maxRuntime))
+	if remaining <= 0 {
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    "scheduled run timed out before detection",
+			Stage:      "timeout",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		r.recordSchedulerRunComplete(jobType, "timeout")
+		return false
+	}
+	if detectionTimeout <= 0 {
+		detectionTimeout = defaultScheduledDetectionTimeout
+	}
+	if detectionTimeout > remaining {
+		detectionTimeout = remaining
+	}
+
+	detectCtx, cancelDetect := context.WithTimeout(jobCtx, detectionTimeout)
+	proposals, err := r.RunDetection(detectCtx, jobType, clusterContext, policy.MaxResults)
+	cancelDetect()
+	if err != nil {
+		r.recordSchedulerDetectionError(jobType, err)
+		stage := "failed"
+		status := "error"
+		if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+			stage = "timeout"
+			status = "timeout"
+		}
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduled detection failed: %v", err),
+			Stage:      stage,
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		r.recordSchedulerRunComplete(jobType, status)
 		return false
 	}
 
@@ -246,6 +258,8 @@ func (r *Plugin) runJobTypeIteration(
 	})
 	r.recordSchedulerDetectionSuccess(jobType, len(proposals))
 
+	detected := len(proposals) > 0
+
 	filteredByActive, skippedActive := r.filterProposalsWithActiveJobs(jobType, proposals)
 	if skippedActive > 0 {
 		r.appendActivity(JobActivity{
@@ -258,7 +272,8 @@ func (r *Plugin) runJobTypeIteration(
 	}
 
 	if len(filteredByActive) == 0 {
-		return false
+		r.recordSchedulerRunComplete(jobType, "success")
+		return detected
 	}
 
 	filtered := r.filterScheduledProposals(filteredByActive)
@@ -273,11 +288,51 @@ func (r *Plugin) runJobTypeIteration(
 	}
 
 	if len(filtered) == 0 {
-		return false
+		r.recordSchedulerRunComplete(jobType, "success")
+		return detected
 	}
 
-	r.dispatchScheduledProposals(ctx, jobType, filtered, clusterContext, policy)
-	return true
+	r.setSchedulerLoopState(jobType, "executing")
+
+	remaining = time.Until(start.Add(maxRuntime))
+	if remaining <= 0 {
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    "scheduled execution skipped: job type max runtime reached",
+			Stage:      "timeout",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		r.recordSchedulerRunComplete(jobType, "timeout")
+		return detected
+	}
+
+	execPolicy := policy
+	if execPolicy.ExecutionTimeout <= 0 {
+		execPolicy.ExecutionTimeout = defaultScheduledExecutionTimeout
+	}
+	if execPolicy.ExecutionTimeout > remaining {
+		execPolicy.ExecutionTimeout = remaining
+	}
+
+	successCount, errorCount, canceledCount := r.dispatchScheduledProposals(jobCtx, jobType, filtered, clusterContext, execPolicy)
+
+	status := "success"
+	if jobCtx.Err() != nil {
+		status = "timeout"
+	} else if errorCount > 0 || canceledCount > 0 {
+		status = "error"
+	}
+
+	r.appendActivity(JobActivity{
+		JobType:    jobType,
+		Source:     "admin_scheduler",
+		Message:    fmt.Sprintf("scheduled execution finished: success=%d error=%d canceled=%d", successCount, errorCount, canceledCount),
+		Stage:      "executed",
+		OccurredAt: timeToPtr(time.Now().UTC()),
+	})
+	r.recordSchedulerRunComplete(jobType, status)
+	return detected
 }
 
 func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, error) {
@@ -299,9 +354,10 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
 	}
 
 	policy := schedulerPolicy{
+		DetectionInterval:      durationFromSeconds(adminRuntime.DetectionIntervalSeconds, defaultScheduledDetectionInterval),
 		DetectionTimeout:       durationFromSeconds(adminRuntime.DetectionTimeoutSeconds, defaultScheduledDetectionTimeout),
 		ExecutionTimeout:       defaultScheduledExecutionTimeout,
-		MaxJobTypeDuration:     defaultMaxJobTypeDuration,
+		JobTypeMaxRuntime:      durationFromSeconds(adminRuntime.JobTypeMaxRuntimeSeconds, defaultScheduledJobTypeMaxRuntime),
 		RetryBackoff:           durationFromSeconds(adminRuntime.RetryBackoffSeconds, defaultScheduledRetryBackoff),
 		MaxResults:             adminRuntime.MaxJobsPerDetection,
 		ExecutionConcurrency:   int(adminRuntime.GlobalExecutionConcurrency),
@@ -310,6 +366,9 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
 		ExecutorReserveBackoff: 200 * time.Millisecond,
 	}
 
+	if policy.DetectionInterval < r.schedulerTick {
+		policy.DetectionInterval = r.schedulerTick
+	}
 	if policy.MaxResults <= 0 {
 		policy.MaxResults = defaultScheduledMaxResults
 	}
@@ -328,6 +387,9 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
 	if policy.RetryLimit < 0 {
 		policy.RetryLimit = 0
 	}
+	if policy.JobTypeMaxRuntime <= 0 {
+		policy.JobTypeMaxRuntime = defaultScheduledJobTypeMaxRuntime
+	}
 
 	// Plugin protocol currently has only detection timeout in admin settings.
 	execTimeout := time.Duration(adminRuntime.DetectionTimeoutSeconds*2) * time.Second
@@ -346,18 +408,28 @@ func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
 	}
 
 	r.schedulerMu.Lock()
-	currentJobType := r.currentJobType
+	nextDetectionAt := make(map[string]time.Time, len(r.nextDetectionAt))
+	for jobType, nextRun := range r.nextDetectionAt {
+		nextDetectionAt[jobType] = nextRun
+	}
+	detectionInFlight := make(map[string]bool, len(r.detectionInFlight))
+	for jobType, inFlight := range r.detectionInFlight {
+		detectionInFlight[jobType] = inFlight
+	}
 	r.schedulerMu.Unlock()
 
 	states := make([]SchedulerJobTypeState, 0, len(jobTypes))
 	for _, jobTypeInfo := range jobTypes {
 		jobType := jobTypeInfo.JobType
 		state := SchedulerJobTypeState{
-			JobType: jobType,
+			JobType:           jobType,
+			DetectionInFlight: detectionInFlight[jobType],
 		}
 
-		// Mark as in-flight if this job type is being processed right now.
-		state.DetectionInFlight = jobType == currentJobType && currentJobType != ""
+		if nextRun, ok := nextDetectionAt[jobType]; ok && !nextRun.IsZero() {
+			nextRunUTC := nextRun.UTC()
+			state.NextDetectionAt = &nextRunUTC
+		}
 
 		policy, enabled, loadErr := r.loadSchedulerPolicy(jobType)
 
@@ -366,9 +438,10 @@ func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
 		} else {
 			state.Enabled = enabled
 			if enabled {
+				state.DetectionIntervalSeconds = secondsFromDuration(policy.DetectionInterval)
 				state.DetectionTimeoutSeconds = secondsFromDuration(policy.DetectionTimeout)
 				state.ExecutionTimeoutSeconds = secondsFromDuration(policy.ExecutionTimeout)
-				state.MaxJobTypeDurationSeconds = secondsFromDuration(policy.MaxJobTypeDuration)
+				state.JobTypeMaxRuntimeSeconds = secondsFromDuration(policy.JobTypeMaxRuntime)
 				state.MaxJobsPerDetection = policy.MaxResults
 				state.GlobalExecutionConcurrency = policy.ExecutionConcurrency
 				state.PerWorkerExecutionConcurrency = policy.PerWorkerConcurrency
@@ -377,6 +450,19 @@ func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
 			}
 		}
 
+		runInfo := r.snapshotSchedulerRun(jobType)
+		if !runInfo.lastRunStartedAt.IsZero() {
+			at := runInfo.lastRunStartedAt
+			state.LastRunStartedAt = &at
+		}
+		if !runInfo.lastRunCompletedAt.IsZero() {
+			at := runInfo.lastRunCompletedAt
+			state.LastRunCompletedAt = &at
+		}
+		if runInfo.lastRunStatus != "" {
+			state.LastRunStatus = runInfo.lastRunStatus
+		}
+
 		leasedWorkerID := r.getDetectorLease(jobType)
 		if leasedWorkerID != "" {
 			state.DetectorWorkerID = leasedWorkerID
@@ -428,10 +514,84 @@ func deriveSchedulerAdminRuntime(
 		PerWorkerExecutionConcurrency: defaults.PerWorkerExecutionConcurrency,
 		RetryLimit:                    defaults.RetryLimit,
 		RetryBackoffSeconds:           defaults.RetryBackoffSeconds,
+		JobTypeMaxRuntimeSeconds:      defaults.JobTypeMaxRuntimeSeconds,
+	}
+}
+
+func (r *Plugin) markDetectionDue(jobType string, interval, initialDelay time.Duration) bool {
+	now := time.Now().UTC()
+
+	r.schedulerMu.Lock()
+	defer r.schedulerMu.Unlock()
+
+	if r.detectionInFlight[jobType] {
+		return false
+	}
+
+	nextRun, exists := r.nextDetectionAt[jobType]
+	if exists && now.Before(nextRun) {
+		return false
+	}
+	if !exists && initialDelay > 0 {
+		r.nextDetectionAt[jobType] = now.Add(initialDelay)
+		return false
+	}
+
+	r.nextDetectionAt[jobType] = now.Add(interval)
+	r.detectionInFlight[jobType] = true
+	return true
+}
+
+func (r *Plugin) earliestNextDetectionAt() time.Time {
+	if r == nil {
+		return time.Time{}
+	}
+
+	r.schedulerMu.Lock()
+	defer r.schedulerMu.Unlock()
+
+	var earliest time.Time
+	for _, nextRun := range r.nextDetectionAt {
+		if nextRun.IsZero() {
+			continue
+		}
+		if earliest.IsZero() || nextRun.Before(earliest) {
+			earliest = nextRun
+		}
+	}
+
+	return earliest
+}
+
+func (r *Plugin) markJobTypeInFlight(jobType string) {
+	r.schedulerMu.Lock()
+	r.detectionInFlight[jobType] = true
+	r.schedulerMu.Unlock()
+}
+
+func (r *Plugin) finishDetection(jobType string) {
+	r.schedulerMu.Lock()
+	delete(r.detectionInFlight, jobType)
+	r.schedulerMu.Unlock()
+}
+
+func (r *Plugin) pruneSchedulerState(activeJobTypes map[string]struct{}) {
+	r.schedulerMu.Lock()
+	defer r.schedulerMu.Unlock()
+
+	for jobType := range r.nextDetectionAt {
+		if _, ok := activeJobTypes[jobType]; !ok {
+			delete(r.nextDetectionAt, jobType)
+			delete(r.detectionInFlight, jobType)
+		}
 	}
 }
 
 func (r *Plugin) clearSchedulerJobType(jobType string) {
+	r.schedulerMu.Lock()
+	delete(r.nextDetectionAt, jobType)
+	delete(r.detectionInFlight, jobType)
+	r.schedulerMu.Unlock()
 	r.clearDetectorLease(jobType, "")
 }
 
@@ -446,15 +606,18 @@ func (r *Plugin) pruneDetectorLeases(activeJobTypes map[string]struct{}) {
 	}
 }
 
-func (r *Plugin) loadSchedulerClusterContext() (*plugin_pb.ClusterContext, error) {
+func (r *Plugin) loadSchedulerClusterContext(ctx context.Context) (*plugin_pb.ClusterContext, error) {
 	if r.clusterContextProvider == nil {
 		return nil, fmt.Errorf("cluster context provider is not configured")
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultClusterContextTimeout)
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	clusterCtx, cancel := context.WithTimeout(ctx, defaultClusterContextTimeout)
 	defer cancel()
 
-	clusterContext, err := r.clusterContextProvider(ctx)
+	clusterContext, err := r.clusterContextProvider(clusterCtx)
 	if err != nil {
 		return nil, err
 	}
@@ -465,12 +628,16 @@ func (r *Plugin) loadSchedulerClusterContext() (*plugin_pb.ClusterContext, error
 }
 
 func (r *Plugin) dispatchScheduledProposals(
-	parentCtx context.Context,
+	ctx context.Context,
 	jobType string,
 	proposals []*plugin_pb.JobProposal,
 	clusterContext *plugin_pb.ClusterContext,
 	policy schedulerPolicy,
-) {
+) (int, int, int) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
 	jobQueue := make(chan *plugin_pb.JobSpec, len(proposals))
 	for index, proposal := range proposals {
 		job := buildScheduledJobSpec(jobType, proposal, index)
@@ -478,10 +645,7 @@ func (r *Plugin) dispatchScheduledProposals(
 		select {
 		case <-r.shutdownCh:
 			close(jobQueue)
-			return
-		case <-parentCtx.Done():
-			close(jobQueue)
-			return
+			return 0, 0, 0
 		default:
 			jobQueue <- job
 		}
@@ -492,6 +656,7 @@ func (r *Plugin) dispatchScheduledProposals(
 	var statsMu sync.Mutex
 	successCount := 0
 	errorCount := 0
+	canceledCount := 0
 
 	workerCount := policy.ExecutionConcurrency
 	if workerCount < 1 {
@@ -503,30 +668,44 @@ func (r *Plugin) dispatchScheduledProposals(
 		go func() {
 			defer wg.Done()
 
+		jobLoop:
 			for job := range jobQueue {
 				select {
 				case <-r.shutdownCh:
 					return
-				case <-parentCtx.Done():
-					return
 				default:
 				}
 
+				if ctx.Err() != nil {
+					r.cancelQueuedJob(job, ctx.Err())
+					statsMu.Lock()
+					canceledCount++
+					statsMu.Unlock()
+					continue
+				}
+
 				for {
 					select {
 					case <-r.shutdownCh:
 						return
-					case <-parentCtx.Done():
-						return
 					default:
 					}
+					if ctx.Err() != nil {
+						r.cancelQueuedJob(job, ctx.Err())
+						statsMu.Lock()
+						canceledCount++
+						statsMu.Unlock()
+						continue jobLoop
+					}
 
-					executor, release, reserveErr := r.reserveScheduledExecutor(parentCtx, jobType, policy)
+					executor, release, reserveErr := r.reserveScheduledExecutor(ctx, jobType, policy)
 					if reserveErr != nil {
-						select {
-						case <-r.shutdownCh:
-							return
-						default:
+						if ctx.Err() != nil {
+							r.cancelQueuedJob(job, ctx.Err())
+							statsMu.Lock()
+							canceledCount++
+							statsMu.Unlock()
+							continue jobLoop
 						}
 						statsMu.Lock()
 						errorCount++
@@ -541,16 +720,23 @@ func (r *Plugin) dispatchScheduledProposals(
 						break
 					}
 
-					err := r.executeScheduledJobWithExecutor(parentCtx, executor, job, clusterContext, policy)
+					err := r.executeScheduledJobWithExecutor(ctx, executor, job, clusterContext, policy)
 					release()
 					if errors.Is(err, errExecutorAtCapacity) {
 						r.trackExecutionQueued(job)
-						if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
+						if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
 							return
 						}
 						continue
 					}
 					if err != nil {
+						if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+							r.cancelQueuedJob(job, err)
+							statsMu.Lock()
+							canceledCount++
+							statsMu.Unlock()
+							continue jobLoop
+						}
 						statsMu.Lock()
 						errorCount++
 						statsMu.Unlock()
@@ -576,13 +762,16 @@ func (r *Plugin) dispatchScheduledProposals(
 
 	wg.Wait()
 
-	r.appendActivity(JobActivity{
-		JobType:    jobType,
-		Source:     "admin_scheduler",
-		Message:    fmt.Sprintf("scheduled execution finished: success=%d error=%d", successCount, errorCount),
-		Stage:      "executed",
-		OccurredAt: timeToPtr(time.Now().UTC()),
-	})
+	drainErr := ctx.Err()
+	if drainErr == nil {
+		drainErr = errSchedulerShutdown
+	}
+	for job := range jobQueue {
+		r.cancelQueuedJob(job, drainErr)
+		canceledCount++
+	}
+
+	return successCount, errorCount, canceledCount
 }
 
 func (r *Plugin) reserveScheduledExecutor(
@@ -590,19 +779,27 @@ func (r *Plugin) reserveScheduledExecutor(
 	jobType string,
 	policy schedulerPolicy,
 ) (*WorkerSession, func(), error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
 	deadline := time.Now().Add(policy.ExecutionTimeout)
 	if policy.ExecutionTimeout <= 0 {
 		deadline = time.Now().Add(10 * time.Minute) // Default cap
 	}
+	if ctxDeadline, ok := ctx.Deadline(); ok && ctxDeadline.Before(deadline) {
+		deadline = ctxDeadline
+	}
 
 	for {
 		select {
 		case <-r.shutdownCh:
 			return nil, nil, fmt.Errorf("plugin is shutting down")
-		case <-ctx.Done():
-			return nil, nil, ctx.Err()
 		default:
 		}
+		if ctx.Err() != nil {
+			return nil, nil, ctx.Err()
+		}
 
 		if time.Now().After(deadline) {
 			return nil, nil, fmt.Errorf("timed out waiting for executor capacity for %s", jobType)
@@ -610,8 +807,11 @@ func (r *Plugin) reserveScheduledExecutor(
 
 		executors, err := r.registry.ListExecutors(jobType)
 		if err != nil {
-			if !waitForShutdownOrCtx(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
-				return nil, nil, fmt.Errorf("plugin is shutting down or context canceled")
+			if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
+				if ctx.Err() != nil {
+					return nil, nil, ctx.Err()
+				}
+				return nil, nil, fmt.Errorf("plugin is shutting down")
 			}
 			continue
 		}
@@ -624,8 +824,11 @@ func (r *Plugin) reserveScheduledExecutor(
 			return executor, release, nil
 		}
 
-		if !waitForShutdownOrCtx(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
-			return nil, nil, fmt.Errorf("plugin is shutting down or context canceled")
+		if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
+			if ctx.Err() != nil {
+				return nil, nil, ctx.Err()
+			}
+			return nil, nil, fmt.Errorf("plugin is shutting down")
 		}
 	}
 }
@@ -709,7 +912,7 @@ func schedulerWorkerExecutionLimit(executor *WorkerSession, jobType string, poli
 }
 
 func (r *Plugin) executeScheduledJobWithExecutor(
-	parentCtx context.Context,
+	ctx context.Context,
 	executor *WorkerSession,
 	job *plugin_pb.JobSpec,
 	clusterContext *plugin_pb.ClusterContext,
@@ -725,12 +928,17 @@ func (r *Plugin) executeScheduledJobWithExecutor(
 		select {
 		case <-r.shutdownCh:
 			return fmt.Errorf("plugin is shutting down")
-		case <-parentCtx.Done():
-			return parentCtx.Err()
 		default:
 		}
+		if ctx != nil && ctx.Err() != nil {
+			return ctx.Err()
+		}
 
-		execCtx, cancel := context.WithTimeout(parentCtx, policy.ExecutionTimeout)
+		parent := ctx
+		if parent == nil {
+			parent = context.Background()
+		}
+		execCtx, cancel := context.WithTimeout(parent, policy.ExecutionTimeout)
 		_, err := r.executeJobWithExecutor(execCtx, executor, job, clusterContext, int32(attempt))
 		cancel()
 		if err == nil {
@@ -750,8 +958,11 @@ func (r *Plugin) executeScheduledJobWithExecutor(
 				Stage:      "retry",
 				OccurredAt: timeToPtr(time.Now().UTC()),
 			})
-			if !waitForShutdownOrCtx(r.shutdownCh, parentCtx, policy.RetryBackoff) {
-				return fmt.Errorf("plugin is shutting down or context canceled")
+			if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.RetryBackoff) {
+				if ctx != nil && ctx.Err() != nil {
+					return ctx.Err()
+				}
+				return fmt.Errorf("plugin is shutting down")
 			}
 		}
 	}
@@ -796,6 +1007,53 @@ func (r *Plugin) countWaitingTrackedJobs(jobType string) int {
 	return waiting
 }
 
+func (r *Plugin) clearWaitingJobQueue(jobType string) int {
+	normalizedJobType := strings.TrimSpace(jobType)
+	if normalizedJobType == "" {
+		return 0
+	}
+
+	jobIDs := make([]string, 0)
+	seen := make(map[string]struct{})
+
+	r.jobsMu.RLock()
+	for _, job := range r.jobs {
+		if job == nil {
+			continue
+		}
+		if strings.TrimSpace(job.JobType) != normalizedJobType {
+			continue
+		}
+		if !isWaitingTrackedJobState(job.State) {
+			continue
+		}
+		jobID := strings.TrimSpace(job.JobID)
+		if jobID == "" {
+			continue
+		}
+		if _, ok := seen[jobID]; ok {
+			continue
+		}
+		seen[jobID] = struct{}{}
+		jobIDs = append(jobIDs, jobID)
+	}
+	r.jobsMu.RUnlock()
+
+	if len(jobIDs) == 0 {
+		return 0
+	}
+
+	reason := fmt.Sprintf("cleared queued job before %s run", normalizedJobType)
+	for _, jobID := range jobIDs {
+		r.markJobCanceled(&plugin_pb.JobSpec{
+			JobId:   jobID,
+			JobType: normalizedJobType,
+		}, reason)
+	}
+
+	return len(jobIDs)
+}
+
 func waitingBacklogThreshold(policy schedulerPolicy) int {
 	concurrency := policy.ExecutionConcurrency
 	if concurrency <= 0 {
@@ -893,10 +1151,13 @@ func waitForShutdownOrTimer(shutdown <-chan struct{}, duration time.Duration) bo
 	}
 }
 
-func waitForShutdownOrCtx(shutdown <-chan struct{}, ctx context.Context, duration time.Duration) bool {
+func waitForShutdownOrTimerWithContext(shutdown <-chan struct{}, ctx context.Context, duration time.Duration) bool {
 	if duration <= 0 {
 		return true
 	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
 
 	timer := time.NewTimer(duration)
 	defer timer.Stop()

weed/admin/plugin/plugin_scheduler_test.go

@@ -3,7 +3,6 @@ package plugin
 import (
 	"context"
 	"fmt"
-	"sync"
 	"testing"
 	"time"
 
@@ -30,6 +29,7 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
 			PerWorkerExecutionConcurrency: 2,
 			RetryLimit:                    4,
 			RetryBackoffSeconds:           7,
+			JobTypeMaxRuntimeSeconds:      1800,
 		},
 	})
 	if err != nil {
@@ -55,8 +55,8 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
 	if policy.RetryLimit != 4 {
 		t.Fatalf("unexpected retry limit: got=%d", policy.RetryLimit)
 	}
-	if policy.MaxJobTypeDuration != defaultMaxJobTypeDuration {
-		t.Fatalf("unexpected max job type duration: got=%v", policy.MaxJobTypeDuration)
+	if policy.JobTypeMaxRuntime != 30*time.Minute {
+		t.Fatalf("unexpected max runtime: got=%v", policy.JobTypeMaxRuntime)
 	}
 }
 
@@ -80,6 +80,7 @@ func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T
 			PerWorkerExecutionConcurrency: 2,
 			RetryLimit:                    3,
 			RetryBackoffSeconds:           6,
+			JobTypeMaxRuntimeSeconds:      1200,
 		},
 	})
 	if err != nil {
@@ -102,6 +103,9 @@ func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T
 	if policy.PerWorkerConcurrency != 2 {
 		t.Fatalf("unexpected per-worker concurrency: got=%d", policy.PerWorkerConcurrency)
 	}
+	if policy.JobTypeMaxRuntime != 20*time.Minute {
+		t.Fatalf("unexpected max runtime: got=%v", policy.JobTypeMaxRuntime)
+	}
 }
 
 func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) {
@@ -399,6 +403,7 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
 			PerWorkerExecutionConcurrency: 2,
 			RetryLimit:                    1,
 			RetryBackoffSeconds:           9,
+			JobTypeMaxRuntimeSeconds:      900,
 		},
 	})
 	if err != nil {
@@ -412,9 +417,10 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
 		},
 	})
 
-	// Mark this job type as currently processing to test DetectionInFlight.
+	nextDetectionAt := time.Now().UTC().Add(2 * time.Minute).Round(time.Second)
 	pluginSvc.schedulerMu.Lock()
-	pluginSvc.currentJobType = jobType
+	pluginSvc.nextDetectionAt[jobType] = nextDetectionAt
+	pluginSvc.detectionInFlight[jobType] = true
 	pluginSvc.schedulerMu.Unlock()
 
 	states, err := pluginSvc.ListSchedulerStates()
@@ -433,7 +439,16 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
 		t.Fatalf("unexpected policy error: %s", state.PolicyError)
 	}
 	if !state.DetectionInFlight {
-		t.Fatalf("expected detection in flight when current job type matches")
+		t.Fatalf("expected detection in flight")
+	}
+	if state.NextDetectionAt == nil {
+		t.Fatalf("expected next detection time")
+	}
+	if state.NextDetectionAt.Unix() != nextDetectionAt.Unix() {
+		t.Fatalf("unexpected next detection time: got=%v want=%v", state.NextDetectionAt, nextDetectionAt)
+	}
+	if state.DetectionIntervalSeconds != 45 {
+		t.Fatalf("unexpected detection interval: got=%d", state.DetectionIntervalSeconds)
 	}
 	if state.DetectionTimeoutSeconds != 30 {
 		t.Fatalf("unexpected detection timeout: got=%d", state.DetectionTimeoutSeconds)
@@ -441,8 +456,8 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
 	if state.ExecutionTimeoutSeconds != 90 {
 		t.Fatalf("unexpected execution timeout: got=%d", state.ExecutionTimeoutSeconds)
 	}
-	if state.MaxJobTypeDurationSeconds != int32(defaultMaxJobTypeDuration/time.Second) {
-		t.Fatalf("unexpected max job type duration: got=%d", state.MaxJobTypeDurationSeconds)
+	if state.JobTypeMaxRuntimeSeconds != 900 {
+		t.Fatalf("unexpected job type max runtime: got=%d", state.JobTypeMaxRuntimeSeconds)
 	}
 	if state.MaxJobsPerDetection != 80 {
 		t.Fatalf("unexpected max jobs per detection: got=%d", state.MaxJobsPerDetection)
@@ -465,23 +480,6 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
 	if state.ExecutorWorkerCount != 1 {
 		t.Fatalf("unexpected executor worker count: got=%d", state.ExecutorWorkerCount)
 	}
-
-	// Clear the current job type and verify DetectionInFlight is false.
-	pluginSvc.schedulerMu.Lock()
-	pluginSvc.currentJobType = ""
-	pluginSvc.schedulerMu.Unlock()
-
-	states2, err := pluginSvc.ListSchedulerStates()
-	if err != nil {
-		t.Fatalf("ListSchedulerStates (2): %v", err)
-	}
-	state2 := findSchedulerState(states2, jobType)
-	if state2 == nil {
-		t.Fatalf("missing scheduler state for %s (2)", jobType)
-	}
-	if state2.DetectionInFlight {
-		t.Fatalf("expected detection not in flight when current job type is empty")
-	}
 }
 
 func TestListSchedulerStatesShowsDisabledWhenNoPolicy(t *testing.T) {
@@ -596,315 +594,3 @@ func TestPickDetectorReassignsWhenLeaseIsStale(t *testing.T) {
 		t.Fatalf("expected detector lease to be updated to worker-a, got=%s", lease)
 	}
 }
-
-// mockLockManager records lock/release calls for testing.
-type mockLockManager struct {
-	mu           sync.Mutex
-	acquireCount int
-	releaseCount int
-	failAcquire  bool
-}
-
-func (m *mockLockManager) Acquire(reason string) (func(), error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.failAcquire {
-		return nil, fmt.Errorf("mock lock acquisition failed")
-	}
-	m.acquireCount++
-	return func() {
-		m.mu.Lock()
-		m.releaseCount++
-		m.mu.Unlock()
-	}, nil
-}
-
-func (m *mockLockManager) Status() interface{} {
-	return nil
-}
-
-func (m *mockLockManager) getAcquireCount() int {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.acquireCount
-}
-
-func (m *mockLockManager) getReleaseCount() int {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.releaseCount
-}
-
-func TestIterationAcquiresLockOnce(t *testing.T) {
-	t.Parallel()
-
-	lock := &mockLockManager{}
-	pluginSvc, err := New(Options{
-		LockManager: lock,
-	})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	// Register two enabled job types.
-	for _, jt := range []string{"vacuum", "balance"} {
-		err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
-			JobType: jt,
-			AdminRuntime: &plugin_pb.AdminRuntimeConfig{
-				Enabled:                  true,
-				DetectionTimeoutSeconds:  5,
-				MaxJobsPerDetection:      10,
-				GlobalExecutionConcurrency: 1,
-				PerWorkerExecutionConcurrency: 1,
-			},
-		})
-		if err != nil {
-			t.Fatalf("SaveJobTypeConfig(%s): %v", jt, err)
-		}
-		pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-			WorkerId: "worker-" + jt,
-			Capabilities: []*plugin_pb.JobTypeCapability{
-				{JobType: jt, CanDetect: true, CanExecute: true},
-			},
-		})
-	}
-
-	// runSchedulerIteration requires a cluster context provider.
-	pluginSvc.clusterContextProvider = func(_ context.Context) (*plugin_pb.ClusterContext, error) {
-		return &plugin_pb.ClusterContext{}, nil
-	}
-
-	pluginSvc.runSchedulerIteration()
-
-	// Lock should have been acquired exactly once (not per-job-type).
-	if lock.getAcquireCount() != 1 {
-		t.Fatalf("expected 1 lock acquisition, got %d", lock.getAcquireCount())
-	}
-	if lock.getReleaseCount() != 1 {
-		t.Fatalf("expected 1 lock release, got %d", lock.getReleaseCount())
-	}
-}
-
-func TestIterationReturnsfalseWhenLockFails(t *testing.T) {
-	t.Parallel()
-
-	lock := &mockLockManager{failAcquire: true}
-	pluginSvc, err := New(Options{
-		LockManager: lock,
-	})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
-		JobType: "vacuum",
-		AdminRuntime: &plugin_pb.AdminRuntimeConfig{
-			Enabled:                  true,
-			DetectionTimeoutSeconds:  5,
-			MaxJobsPerDetection:      10,
-			GlobalExecutionConcurrency: 1,
-			PerWorkerExecutionConcurrency: 1,
-		},
-	})
-	if err != nil {
-		t.Fatalf("SaveJobTypeConfig: %v", err)
-	}
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "vacuum", CanDetect: true, CanExecute: true},
-		},
-	})
-
-	pluginSvc.clusterContextProvider = func(_ context.Context) (*plugin_pb.ClusterContext, error) {
-		return &plugin_pb.ClusterContext{}, nil
-	}
-
-	result := pluginSvc.runSchedulerIteration()
-	if result {
-		t.Fatalf("expected false when lock acquisition fails")
-	}
-}
-
-func TestSchedulerPhaseTracking(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	// Initial phase should be idle.
-	pluginSvc.schedulerMu.Lock()
-	phase := pluginSvc.schedulerPhase
-	pluginSvc.schedulerMu.Unlock()
-
-	if phase != "idle" {
-		t.Fatalf("expected initial phase to be idle, got=%s", phase)
-	}
-
-	pluginSvc.setSchedulerPhase("processing", "vacuum")
-
-	pluginSvc.schedulerMu.Lock()
-	phase = pluginSvc.schedulerPhase
-	jobType := pluginSvc.currentJobType
-	pluginSvc.schedulerMu.Unlock()
-
-	if phase != "processing" {
-		t.Fatalf("expected phase processing, got=%s", phase)
-	}
-	if jobType != "vacuum" {
-		t.Fatalf("expected current job type vacuum, got=%s", jobType)
-	}
-
-	pluginSvc.finishIteration(true)
-
-	pluginSvc.schedulerMu.Lock()
-	phase = pluginSvc.schedulerPhase
-	jobType = pluginSvc.currentJobType
-	workDetected := pluginSvc.lastIterationWorkDetected
-	lastEnded := pluginSvc.lastIterationEndedAt
-	pluginSvc.schedulerMu.Unlock()
-
-	if phase != "idle" {
-		t.Fatalf("expected phase idle after finish, got=%s", phase)
-	}
-	if jobType != "" {
-		t.Fatalf("expected empty job type after finish, got=%s", jobType)
-	}
-	if !workDetected {
-		t.Fatalf("expected last iteration work detected to be true")
-	}
-	if lastEnded.IsZero() {
-		t.Fatalf("expected last iteration ended at to be set")
-	}
-}
-
-func TestGetSchedulerStatusIncludesIterationFields(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{
-		IdleSleepDuration: 10 * time.Minute,
-	})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	now := time.Now().UTC()
-	pluginSvc.schedulerMu.Lock()
-	pluginSvc.schedulerPhase = "processing"
-	pluginSvc.currentJobType = "vacuum"
-	pluginSvc.iterationStartedAt = now.Add(-5 * time.Second)
-	pluginSvc.lastIterationEndedAt = now.Add(-20 * time.Second)
-	pluginSvc.lastIterationWorkDetected = true
-	pluginSvc.schedulerMu.Unlock()
-
-	status := pluginSvc.GetSchedulerStatus()
-
-	if status.Phase != "processing" {
-		t.Fatalf("expected phase processing, got=%s", status.Phase)
-	}
-	if status.CurrentJobType != "vacuum" {
-		t.Fatalf("expected current job type vacuum, got=%s", status.CurrentJobType)
-	}
-	if status.IdleSleepSeconds != 600 {
-		t.Fatalf("expected idle sleep 600s, got=%d", status.IdleSleepSeconds)
-	}
-	if status.IterationStartedAt == nil {
-		t.Fatalf("expected iteration started at to be set")
-	}
-	if status.LastIterationEndedAt == nil {
-		t.Fatalf("expected last iteration ended at to be set")
-	}
-	if !status.LastIterationWorkDetected {
-		t.Fatalf("expected last iteration work detected to be true")
-	}
-	// SchedulerTickSeconds should match IdleSleepSeconds for backward compat.
-	if status.SchedulerTickSeconds != 600 {
-		t.Fatalf("expected scheduler tick seconds to match idle sleep, got=%d", status.SchedulerTickSeconds)
-	}
-}
-
-func TestGracefulShutdownDuringIteration(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{
-		IdleSleepDuration: time.Millisecond,
-		ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
-			return &plugin_pb.ClusterContext{}, nil
-		},
-	})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-
-	// Register an enabled job type so the scheduler loop has work to consider.
-	err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
-		JobType: "vacuum",
-		AdminRuntime: &plugin_pb.AdminRuntimeConfig{
-			Enabled:                       true,
-			DetectionTimeoutSeconds:       5,
-			MaxJobsPerDetection:           10,
-			GlobalExecutionConcurrency:    1,
-			PerWorkerExecutionConcurrency: 1,
-		},
-	})
-	if err != nil {
-		t.Fatalf("SaveJobTypeConfig: %v", err)
-	}
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "vacuum", CanDetect: true, CanExecute: true},
-		},
-	})
-
-	// Shutdown while the scheduler loop is actively iterating.
-	done := make(chan struct{})
-	go func() {
-		pluginSvc.Shutdown()
-		close(done)
-	}()
-
-	select {
-	case <-done:
-		// Good — clean shutdown.
-	case <-time.After(5 * time.Second):
-		t.Fatalf("shutdown did not complete in time")
-	}
-}
-
-func TestIdleSleepDurationDefault(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	if pluginSvc.idleSleepDuration != defaultIdleSleepDuration {
-		t.Fatalf("expected default idle sleep %v, got=%v", defaultIdleSleepDuration, pluginSvc.idleSleepDuration)
-	}
-}
-
-func TestIdleSleepDurationCustom(t *testing.T) {
-	t.Parallel()
-
-	customDuration := 5 * time.Minute
-	pluginSvc, err := New(Options{
-		IdleSleepDuration: customDuration,
-	})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	if pluginSvc.idleSleepDuration != customDuration {
-		t.Fatalf("expected custom idle sleep %v, got=%v", customDuration, pluginSvc.idleSleepDuration)
-	}
-}

weed/admin/plugin/scheduler_config.go

@@ -0,0 +1,31 @@
+package plugin
+
+import "time"
+
+const (
+	defaultSchedulerIdleSleep = 613 * time.Second
+)
+
+type SchedulerConfig struct {
+	IdleSleepSeconds int32 `json:"idle_sleep_seconds"`
+}
+
+func DefaultSchedulerConfig() SchedulerConfig {
+	return SchedulerConfig{
+		IdleSleepSeconds: int32(defaultSchedulerIdleSleep / time.Second),
+	}
+}
+
+func normalizeSchedulerConfig(cfg SchedulerConfig) SchedulerConfig {
+	if cfg.IdleSleepSeconds <= 0 {
+		return DefaultSchedulerConfig()
+	}
+	return cfg
+}
+
+func (c SchedulerConfig) IdleSleepDuration() time.Duration {
+	if c.IdleSleepSeconds <= 0 {
+		return defaultSchedulerIdleSleep
+	}
+	return time.Duration(c.IdleSleepSeconds) * time.Second
+}

weed/admin/plugin/scheduler_status.go

@@ -7,17 +7,17 @@ import (
 )
 
 type SchedulerStatus struct {
-	Now                       time.Time                `json:"now"`
-	SchedulerTickSeconds      int                      `json:"scheduler_tick_seconds"`
-	IdleSleepSeconds          int                      `json:"idle_sleep_seconds"`
-	Phase                     string                   `json:"phase"`
-	CurrentJobType            string                   `json:"current_job_type,omitempty"`
-	IterationStartedAt        *time.Time               `json:"iteration_started_at,omitempty"`
-	LastIterationEndedAt      *time.Time               `json:"last_iteration_ended_at,omitempty"`
-	LastIterationWorkDetected bool                     `json:"last_iteration_work_detected"`
-	Waiting                   []SchedulerWaitingStatus `json:"waiting,omitempty"`
-	InProcessJobs             []SchedulerJobStatus     `json:"in_process_jobs,omitempty"`
-	JobTypes                  []SchedulerJobTypeStatus `json:"job_types,omitempty"`
+	Now                  time.Time                `json:"now"`
+	SchedulerTickSeconds int                      `json:"scheduler_tick_seconds"`
+	IdleSleepSeconds     int                      `json:"idle_sleep_seconds,omitempty"`
+	NextDetectionAt      *time.Time               `json:"next_detection_at,omitempty"`
+	CurrentJobType       string                   `json:"current_job_type,omitempty"`
+	CurrentPhase         string                   `json:"current_phase,omitempty"`
+	LastIterationHadJobs bool                     `json:"last_iteration_had_jobs,omitempty"`
+	LastIterationDoneAt  *time.Time               `json:"last_iteration_done_at,omitempty"`
+	Waiting              []SchedulerWaitingStatus `json:"waiting,omitempty"`
+	InProcessJobs        []SchedulerJobStatus     `json:"in_process_jobs,omitempty"`
+	JobTypes             []SchedulerJobTypeStatus `json:"job_types,omitempty"`
 }
 
 type SchedulerWaitingStatus struct {
@@ -42,14 +42,15 @@ type SchedulerJobStatus struct {
 }
 
 type SchedulerJobTypeStatus struct {
-	JobType                      string     `json:"job_type"`
-	Enabled                      bool       `json:"enabled"`
-	DetectionInFlight            bool       `json:"detection_in_flight"`
-	MaxJobTypeDurationSeconds    int32      `json:"max_job_type_duration_seconds,omitempty"`
-	LastDetectedAt               *time.Time `json:"last_detected_at,omitempty"`
-	LastDetectedCount            int        `json:"last_detected_count,omitempty"`
-	LastDetectionError           string     `json:"last_detection_error,omitempty"`
-	LastDetectionSkipped         string     `json:"last_detection_skipped,omitempty"`
+	JobType                  string     `json:"job_type"`
+	Enabled                  bool       `json:"enabled"`
+	DetectionInFlight        bool       `json:"detection_in_flight"`
+	NextDetectionAt          *time.Time `json:"next_detection_at,omitempty"`
+	DetectionIntervalSeconds int32      `json:"detection_interval_seconds,omitempty"`
+	LastDetectedAt           *time.Time `json:"last_detected_at,omitempty"`
+	LastDetectedCount        int        `json:"last_detected_count,omitempty"`
+	LastDetectionError       string     `json:"last_detection_error,omitempty"`
+	LastDetectionSkipped     string     `json:"last_detection_skipped,omitempty"`
 }
 
 type schedulerDetectionInfo struct {
@@ -61,6 +62,19 @@ type schedulerDetectionInfo struct {
 	lastSkippedReason string
 }
 
+type schedulerRunInfo struct {
+	lastRunStartedAt   time.Time
+	lastRunCompletedAt time.Time
+	lastRunStatus      string
+}
+
+type schedulerLoopState struct {
+	currentJobType         string
+	currentPhase           string
+	lastIterationHadJobs   bool
+	lastIterationCompleted time.Time
+}
+
 func (r *Plugin) recordSchedulerDetectionSuccess(jobType string, count int) {
 	if r == nil {
 		return
@@ -127,31 +141,105 @@ func (r *Plugin) snapshotSchedulerDetection(jobType string) schedulerDetectionIn
 	return *info
 }
 
+func (r *Plugin) recordSchedulerRunStart(jobType string) {
+	if r == nil {
+		return
+	}
+	r.schedulerRunMu.Lock()
+	defer r.schedulerRunMu.Unlock()
+	info := r.schedulerRun[jobType]
+	if info == nil {
+		info = &schedulerRunInfo{}
+		r.schedulerRun[jobType] = info
+	}
+	info.lastRunStartedAt = time.Now().UTC()
+	info.lastRunStatus = ""
+}
+
+func (r *Plugin) recordSchedulerRunComplete(jobType, status string) {
+	if r == nil {
+		return
+	}
+	r.schedulerRunMu.Lock()
+	defer r.schedulerRunMu.Unlock()
+	info := r.schedulerRun[jobType]
+	if info == nil {
+		info = &schedulerRunInfo{}
+		r.schedulerRun[jobType] = info
+	}
+	info.lastRunCompletedAt = time.Now().UTC()
+	info.lastRunStatus = status
+}
+
+func (r *Plugin) snapshotSchedulerRun(jobType string) schedulerRunInfo {
+	if r == nil {
+		return schedulerRunInfo{}
+	}
+	r.schedulerRunMu.Lock()
+	defer r.schedulerRunMu.Unlock()
+	info := r.schedulerRun[jobType]
+	if info == nil {
+		return schedulerRunInfo{}
+	}
+	return *info
+}
+
+func (r *Plugin) setSchedulerLoopState(jobType, phase string) {
+	if r == nil {
+		return
+	}
+	r.schedulerLoopMu.Lock()
+	r.schedulerLoopState.currentJobType = jobType
+	r.schedulerLoopState.currentPhase = phase
+	r.schedulerLoopMu.Unlock()
+}
+
+func (r *Plugin) recordSchedulerIterationComplete(hadJobs bool) {
+	if r == nil {
+		return
+	}
+	r.schedulerLoopMu.Lock()
+	r.schedulerLoopState.lastIterationHadJobs = hadJobs
+	r.schedulerLoopState.lastIterationCompleted = time.Now().UTC()
+	r.schedulerLoopMu.Unlock()
+}
+
+func (r *Plugin) snapshotSchedulerLoopState() schedulerLoopState {
+	if r == nil {
+		return schedulerLoopState{}
+	}
+	r.schedulerLoopMu.Lock()
+	defer r.schedulerLoopMu.Unlock()
+	return r.schedulerLoopState
+}
+
 func (r *Plugin) GetSchedulerStatus() SchedulerStatus {
 	now := time.Now().UTC()
-
-	r.schedulerMu.Lock()
-	phase := r.schedulerPhase
-	currentJobType := r.currentJobType
-	iterationStartedAt := r.iterationStartedAt
-	lastIterationEndedAt := r.lastIterationEndedAt
-	lastIterationWorkDetected := r.lastIterationWorkDetected
-	r.schedulerMu.Unlock()
-
+	loopState := r.snapshotSchedulerLoopState()
+	schedulerConfig := r.GetSchedulerConfig()
 	status := SchedulerStatus{
-		Now:                       now,
-		SchedulerTickSeconds:      int(secondsFromDuration(r.idleSleepDuration)),
-		IdleSleepSeconds:          int(secondsFromDuration(r.idleSleepDuration)),
-		Phase:                     phase,
-		CurrentJobType:            currentJobType,
-		LastIterationWorkDetected: lastIterationWorkDetected,
-		InProcessJobs:             r.listInProcessJobs(now),
+		Now:                  now,
+		SchedulerTickSeconds: int(secondsFromDuration(r.schedulerTick)),
+		InProcessJobs:        r.listInProcessJobs(now),
+		IdleSleepSeconds:     int(schedulerConfig.IdleSleepSeconds),
+		CurrentJobType:       loopState.currentJobType,
+		CurrentPhase:         loopState.currentPhase,
+		LastIterationHadJobs: loopState.lastIterationHadJobs,
 	}
-	if !iterationStartedAt.IsZero() {
-		status.IterationStartedAt = timeToPtr(iterationStartedAt)
+	nextDetectionAt := r.earliestNextDetectionAt()
+	if nextDetectionAt.IsZero() && loopState.currentPhase == "sleeping" && !loopState.lastIterationCompleted.IsZero() {
+		idleSleep := schedulerConfig.IdleSleepDuration()
+		if idleSleep > 0 {
+			nextDetectionAt = loopState.lastIterationCompleted.Add(idleSleep)
+		}
 	}
-	if !lastIterationEndedAt.IsZero() {
-		status.LastIterationEndedAt = timeToPtr(lastIterationEndedAt)
+	if !nextDetectionAt.IsZero() {
+		at := nextDetectionAt
+		status.NextDetectionAt = &at
+	}
+	if !loopState.lastIterationCompleted.IsZero() {
+		at := loopState.lastIterationCompleted
+		status.LastIterationDoneAt = &at
 	}
 
 	states, err := r.ListSchedulerStates()
@@ -167,10 +255,11 @@ func (r *Plugin) GetSchedulerStatus() SchedulerStatus {
 		info := r.snapshotSchedulerDetection(jobType)
 
 		jobStatus := SchedulerJobTypeStatus{
-			JobType:                   jobType,
-			Enabled:                   state.Enabled,
-			DetectionInFlight:         state.DetectionInFlight,
-			MaxJobTypeDurationSeconds: state.MaxJobTypeDurationSeconds,
+			JobType:                  jobType,
+			Enabled:                  state.Enabled,
+			DetectionInFlight:        state.DetectionInFlight,
+			NextDetectionAt:          state.NextDetectionAt,
+			DetectionIntervalSeconds: state.DetectionIntervalSeconds,
 		}
 		if !info.lastDetectedAt.IsZero() {
 			jobStatus.LastDetectedAt = timeToPtr(info.lastDetectedAt)
@@ -186,18 +275,18 @@ func (r *Plugin) GetSchedulerStatus() SchedulerStatus {
 
 		if state.DetectionInFlight {
 			waiting = append(waiting, SchedulerWaitingStatus{
-				Reason:  "processing",
+				Reason:  "detection_in_flight",
 				JobType: jobType,
 			})
+		} else if state.Enabled && state.NextDetectionAt != nil && now.Before(*state.NextDetectionAt) {
+			waiting = append(waiting, SchedulerWaitingStatus{
+				Reason:  "next_detection_at",
+				JobType: jobType,
+				Until:   state.NextDetectionAt,
+			})
 		}
 	}
 
-	if phase == "idle" && !lastIterationEndedAt.IsZero() {
-		waiting = append(waiting, SchedulerWaitingStatus{
-			Reason: "idle_sleep",
-		})
-	}
-
 	sort.Slice(jobTypes, func(i, j int) bool {
 		return jobTypes[i].JobType < jobTypes[j].JobType
 	})

weed/admin/plugin/types.go

@@ -82,21 +82,26 @@ type JobDetail struct {
 }
 
 type SchedulerJobTypeState struct {
-	JobType                       string `json:"job_type"`
-	Enabled                       bool   `json:"enabled"`
-	PolicyError                   string `json:"policy_error,omitempty"`
-	DetectionInFlight             bool   `json:"detection_in_flight"`
-	DetectionTimeoutSeconds       int32  `json:"detection_timeout_seconds,omitempty"`
-	ExecutionTimeoutSeconds       int32  `json:"execution_timeout_seconds,omitempty"`
-	MaxJobTypeDurationSeconds     int32  `json:"max_job_type_duration_seconds,omitempty"`
-	MaxJobsPerDetection           int32  `json:"max_jobs_per_detection,omitempty"`
-	GlobalExecutionConcurrency    int    `json:"global_execution_concurrency,omitempty"`
-	PerWorkerExecutionConcurrency int    `json:"per_worker_execution_concurrency,omitempty"`
-	RetryLimit                    int    `json:"retry_limit,omitempty"`
-	RetryBackoffSeconds           int32  `json:"retry_backoff_seconds,omitempty"`
-	DetectorAvailable             bool   `json:"detector_available"`
-	DetectorWorkerID              string `json:"detector_worker_id,omitempty"`
-	ExecutorWorkerCount           int    `json:"executor_worker_count"`
+	JobType                       string     `json:"job_type"`
+	Enabled                       bool       `json:"enabled"`
+	PolicyError                   string     `json:"policy_error,omitempty"`
+	DetectionInFlight             bool       `json:"detection_in_flight"`
+	NextDetectionAt               *time.Time `json:"next_detection_at,omitempty"`
+	DetectionIntervalSeconds      int32      `json:"detection_interval_seconds,omitempty"`
+	DetectionTimeoutSeconds       int32      `json:"detection_timeout_seconds,omitempty"`
+	ExecutionTimeoutSeconds       int32      `json:"execution_timeout_seconds,omitempty"`
+	JobTypeMaxRuntimeSeconds      int32      `json:"job_type_max_runtime_seconds,omitempty"`
+	MaxJobsPerDetection           int32      `json:"max_jobs_per_detection,omitempty"`
+	GlobalExecutionConcurrency    int        `json:"global_execution_concurrency,omitempty"`
+	PerWorkerExecutionConcurrency int        `json:"per_worker_execution_concurrency,omitempty"`
+	RetryLimit                    int        `json:"retry_limit,omitempty"`
+	RetryBackoffSeconds           int32      `json:"retry_backoff_seconds,omitempty"`
+	DetectorAvailable             bool       `json:"detector_available"`
+	DetectorWorkerID              string     `json:"detector_worker_id,omitempty"`
+	ExecutorWorkerCount           int        `json:"executor_worker_count"`
+	LastRunStartedAt              *time.Time `json:"last_run_started_at,omitempty"`
+	LastRunCompletedAt            *time.Time `json:"last_run_completed_at,omitempty"`
+	LastRunStatus                 string     `json:"last_run_status,omitempty"`
 }
 
 func timeToPtr(t time.Time) *time.Time {

weed/admin/topology/task_management.go

@@ -17,20 +17,30 @@ func (at *ActiveTopology) AssignTask(taskID string) error {
 		return fmt.Errorf("pending task %s not found", taskID)
 	}
 
-	// Check if all destination disks have sufficient capacity to reserve
-	for _, dest := range task.Destinations {
-		targetKey := fmt.Sprintf("%s:%d", dest.TargetServer, dest.TargetDisk)
-		if targetDisk, exists := at.disks[targetKey]; exists {
-			availableCapacity := at.getEffectiveAvailableCapacityUnsafe(targetDisk)
+	// Skip capacity check if topology hasn't been populated yet
+	if len(at.disks) == 0 {
+		glog.Warningf("AssignTask %s: topology has no disks yet, skipping capacity check", taskID)
+	} else {
+		// Check if all destination disks have sufficient capacity to reserve
+		for _, dest := range task.Destinations {
+			targetKey := fmt.Sprintf("%s:%d", dest.TargetServer, dest.TargetDisk)
+			if targetDisk, exists := at.disks[targetKey]; exists {
+				availableCapacity := at.getEffectiveAvailableCapacityUnsafe(targetDisk)
 
-			// Check if we have enough total capacity using the improved unified comparison
-			if !availableCapacity.CanAccommodate(dest.StorageChange) {
-				return fmt.Errorf("insufficient capacity on target disk %s:%d. Available: %+v, Required: %+v",
-					dest.TargetServer, dest.TargetDisk, availableCapacity, dest.StorageChange)
+				// Check if we have enough total capacity using the improved unified comparison
+				if !availableCapacity.CanAccommodate(dest.StorageChange) {
+					return fmt.Errorf("insufficient capacity on target disk %s:%d. Available: %+v, Required: %+v",
+						dest.TargetServer, dest.TargetDisk, availableCapacity, dest.StorageChange)
+				}
+			} else if dest.TargetServer != "" {
+				// Fail fast if destination disk is not found in topology
+				var existingKeys []string
+				for k := range at.disks {
+					existingKeys = append(existingKeys, k)
+				}
+				glog.Warningf("destination disk %s not found in topology. Existing disk keys: %v", targetKey, existingKeys)
+				return fmt.Errorf("destination disk %s not found in topology", targetKey)
 			}
-		} else if dest.TargetServer != "" {
-			// Fail fast if destination disk is not found in topology
-			return fmt.Errorf("destination disk %s not found in topology", targetKey)
 		}
 	}
 

weed/admin/topology/topology_management.go

@@ -87,6 +87,8 @@ func (at *ActiveTopology) UpdateTopology(topologyInfo *master_pb.TopologyInfo) e
 					}
 
 					diskKey := fmt.Sprintf("%s:%d", nodeInfo.Id, diskInfo.DiskId)
+					glog.V(2).Infof("UpdateTopology: adding disk key=%q nodeId=%q diskId=%d diskType=%q address=%q grpcPort=%d volumes=%d maxVolumes=%d",
+						diskKey, nodeInfo.Id, diskInfo.DiskId, diskType, nodeInfo.Address, nodeInfo.GrpcPort, diskInfo.VolumeCount, diskInfo.MaxVolumeCount)
 					node.disks[diskInfo.DiskId] = disk
 					at.disks[diskKey] = disk
 				}

weed/admin/view/app/plugin.templ

@@ -115,42 +115,12 @@ templ Plugin(page string) {
                     </div>
                 </div>
             </div>
-            <div class="row mb-4">
-                <div class="col-12">
-                    <div class="card shadow-sm">
-                        <div class="card-header d-flex justify-content-between align-items-center flex-wrap gap-2">
-                            <h5 class="mb-0"><i class="fas fa-sync-alt me-2"></i>Scheduler Iteration</h5>
-                            <small class="text-muted">Current iteration phase and timing</small>
-                        </div>
-                        <div class="card-body" id="plugin-scheduler-iteration-card">
-                            <div class="row g-3">
-                                <div class="col-md-3 col-sm-6">
-                                    <div class="small text-muted">Phase</div>
-                                    <div class="fw-semibold" id="plugin-scheduler-phase">-</div>
-                                </div>
-                                <div class="col-md-3 col-sm-6">
-                                    <div class="small text-muted">Current Job Type</div>
-                                    <div class="fw-semibold" id="plugin-scheduler-current-jt">-</div>
-                                </div>
-                                <div class="col-md-3 col-sm-6">
-                                    <div class="small text-muted">Idle Sleep</div>
-                                    <div class="fw-semibold" id="plugin-scheduler-idle-sleep">-</div>
-                                </div>
-                                <div class="col-md-3 col-sm-6">
-                                    <div class="small text-muted">Last Iteration</div>
-                                    <div class="fw-semibold" id="plugin-scheduler-last-iteration">-</div>
-                                </div>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
             <div class="row mb-4">
                 <div class="col-12">
                     <div class="card shadow-sm">
                         <div class="card-header d-flex justify-content-between align-items-center flex-wrap gap-2">
                             <h5 class="mb-0"><i class="fas fa-clock me-2"></i>Scheduler State</h5>
-                            <small class="text-muted">Per job type execution limits and status</small>
+                            <small class="text-muted">Sequential scheduler with per-job runtime limits</small>
                         </div>
                         <div class="card-body p-0">
                             <div class="table-responsive">
@@ -160,16 +130,17 @@ templ Plugin(page string) {
                                             <th>Job Type</th>
                                             <th>Enabled</th>
                                             <th>Detector</th>
-                                            <th>Status</th>
-                                            <th>Max Duration</th>
+                                            <th>In Flight</th>
+                                            <th>Max Runtime</th>
                                             <th>Exec Global</th>
                                             <th>Exec/Worker</th>
                                             <th>Executor Workers</th>
                                             <th>Effective Exec</th>
+                                            <th>Last Run</th>
                                         </tr>
                                     </thead>
                                     <tbody id="plugin-scheduler-table-body">
-                                        <tr><td colspan="9" class="text-muted text-center py-3">Loading...</td></tr>
+                                        <tr><td colspan="10" class="text-muted text-center py-3">Loading...</td></tr>
                                     </tbody>
                                 </table>
                             </div>
@@ -177,6 +148,38 @@ templ Plugin(page string) {
                     </div>
                 </div>
             </div>
+            <div class="row mb-4">
+                <div class="col-lg-6 mb-3">
+                    <div class="card shadow-sm h-100">
+                        <div class="card-header d-flex justify-content-between align-items-center">
+                            <h5 class="mb-0"><i class="fas fa-sliders-h me-2"></i>Scheduler Settings</h5>
+                            <small class="text-muted">Global</small>
+                        </div>
+                        <div class="card-body">
+                            <div class="mb-2">
+                                <label class="form-label" for="plugin-scheduler-idle-sleep-overview">Sleep Between Iterations (s)</label>
+                                <input type="number" class="form-control" id="plugin-scheduler-idle-sleep-overview" min="0"/>
+                                <div class="form-text">Used when no jobs are detected.</div>
+                            </div>
+                            <button type="button" class="btn btn-outline-primary" id="plugin-save-scheduler-btn-overview">
+                                <i class="fas fa-save me-1"></i>Save Scheduler Settings
+                            </button>
+                        </div>
+                    </div>
+                </div>
+                <div class="col-lg-6 mb-3">
+                    <div class="card shadow-sm h-100">
+                        <div class="card-header d-flex justify-content-between align-items-center">
+                            <h5 class="mb-0"><i class="fas fa-hourglass-half me-2"></i>Next Run</h5>
+                            <small class="text-muted">Scheduler</small>
+                        </div>
+                        <div class="card-body">
+                            <div class="h5 mb-1 plugin-scheduler-next-run">-</div>
+                            <div class="text-muted small plugin-scheduler-next-run-meta">Not scheduled</div>
+                        </div>
+                    </div>
+                </div>
+            </div>
             <div class="row mb-4">
                 <div class="col-12">
                     <div class="card shadow-sm">
@@ -271,14 +274,14 @@ templ Plugin(page string) {
                                         <input class="form-check-input" type="checkbox" id="plugin-admin-enabled"/>
                                     </div>
                                 </div>
-                                <div class="col-12">
-                                    <label class="form-label" for="plugin-admin-detection-interval">Detection Interval (s)</label>
-                                    <input type="number" class="form-control" id="plugin-admin-detection-interval" min="0"/>
-                                </div>
                                 <div class="col-12">
                                     <label class="form-label" for="plugin-admin-detection-timeout">Detection Timeout (s)</label>
                                     <input type="number" class="form-control" id="plugin-admin-detection-timeout" min="0"/>
                                 </div>
+                                <div class="col-12">
+                                    <label class="form-label" for="plugin-admin-max-runtime">Job Type Max Runtime (s)</label>
+                                    <input type="number" class="form-control" id="plugin-admin-max-runtime" min="0"/>
+                                </div>
                                 <div class="col-12">
                                     <label class="form-label" for="plugin-admin-max-results">Max Jobs / Detection</label>
                                     <input type="number" class="form-control" id="plugin-admin-max-results" min="0"/>
@@ -302,6 +305,33 @@ templ Plugin(page string) {
                             </div>
                         </div>
                     </div>
+
+                    <div class="card shadow-sm mt-3">
+                        <div class="card-header">
+                            <h5 class="mb-0"><i class="fas fa-clock me-2"></i>Scheduler Settings</h5>
+                        </div>
+                        <div class="card-body">
+                            <div class="mb-3">
+                                <label class="form-label" for="plugin-scheduler-idle-sleep">Sleep Between Iterations (s)</label>
+                                <input type="number" class="form-control" id="plugin-scheduler-idle-sleep" min="0"/>
+                                <div class="form-text">Used when no jobs are detected.</div>
+                            </div>
+                            <button type="button" class="btn btn-outline-primary" id="plugin-save-scheduler-btn">
+                                <i class="fas fa-save me-1"></i>Save Scheduler Settings
+                            </button>
+                        </div>
+                    </div>
+
+                    <div class="card shadow-sm mt-3">
+                        <div class="card-header d-flex justify-content-between align-items-center">
+                            <h5 class="mb-0"><i class="fas fa-hourglass-half me-2"></i>Next Run</h5>
+                            <small class="text-muted">Scheduler</small>
+                        </div>
+                        <div class="card-body">
+                            <div class="h5 mb-1 plugin-scheduler-next-run">-</div>
+                            <div class="text-muted small plugin-scheduler-next-run-meta">Not scheduled</div>
+                        </div>
+                    </div>
                 </div>
             </div>
 
@@ -601,6 +631,9 @@ templ Plugin(page string) {
                 jobs: [],
                 activities: [],
                 schedulerStates: [],
+                schedulerStatus: null,
+                schedulerConfig: null,
+                schedulerConfigLoaded: false,
                 allJobs: [],
                 allActivities: [],
                 loadedJobType: '',
@@ -1461,7 +1494,7 @@ templ Plugin(page string) {
 
                 var states = Array.isArray(state.schedulerStates) ? state.schedulerStates : [];
                 if (!states.length) {
-                    tbody.innerHTML = '<tr><td colspan="9" class="text-muted text-center py-3">No scheduler state available</td></tr>';
+                    tbody.innerHTML = '<tr><td colspan="10" class="text-muted text-center py-3">No scheduler state available</td></tr>';
                     return;
                 }
 
@@ -1471,8 +1504,8 @@ templ Plugin(page string) {
                     var enabled = !!item.enabled;
                     var inFlight = !!item.detection_in_flight;
                     var detector = item.detector_available ? textOrDash(item.detector_worker_id) : 'No detector';
-                    var maxDurationSeconds = Number(item.max_job_type_duration_seconds || 0);
-                    var maxDurationText = maxDurationSeconds > 0 ? (String(Math.round(maxDurationSeconds / 60)) + 'm') : '-';
+                    var maxRuntimeSeconds = Number(item.job_type_max_runtime_seconds || 0);
+                    var maxRuntimeText = maxRuntimeSeconds > 0 ? (String(maxRuntimeSeconds) + 's') : '-';
                     var globalExec = Number(item.global_execution_concurrency || 0);
                     var perWorkerExec = Number(item.per_worker_execution_concurrency || 0);
                     var executorWorkers = Number(item.executor_worker_count || 0);
@@ -1481,9 +1514,16 @@ templ Plugin(page string) {
                     var perWorkerExecText = enabled ? String(perWorkerExec) : '-';
                     var executorWorkersText = enabled ? String(executorWorkers) : '-';
                     var effectiveExecText = enabled ? String(effectiveExec) : '-';
+                    var lastRunStatus = textOrDash(item.last_run_status);
+                    var lastRunTime = parseTime(item.last_run_completed_at);
+                    var lastRunText = '-';
+                    if (lastRunStatus !== '-' || lastRunTime) {
+                        var statusLabel = lastRunStatus !== '-' ? lastRunStatus : 'run';
+                        lastRunText = lastRunTime ? (statusLabel + ' @ ' + lastRunTime) : statusLabel;
+                    }
 
                     var enabledBadge = enabled ? '<span class="badge bg-success">Enabled</span>' : '<span class="badge bg-secondary">Disabled</span>';
-                    var statusBadge = inFlight ? '<span class="badge bg-warning text-dark">Processing</span>' : '<span class="badge bg-light text-dark">Idle</span>';
+                    var inFlightBadge = inFlight ? '<span class="badge bg-warning text-dark">Yes</span>' : '<span class="badge bg-light text-dark">No</span>';
                     var policyErrorHtml = '';
                     if (item.policy_error) {
                         policyErrorHtml = '<div><small class="text-danger">' + escapeHtml(String(item.policy_error)) + '</small></div>';
@@ -1493,18 +1533,39 @@ templ Plugin(page string) {
                         '<td>' + escapeHtml(textOrDash(item.job_type)) + policyErrorHtml + '</td>' +
                         '<td>' + enabledBadge + '</td>' +
                         '<td><small>' + escapeHtml(detector) + '</small></td>' +
-                        '<td>' + statusBadge + '</td>' +
-                        '<td><small>' + escapeHtml(maxDurationText) + '</small></td>' +
+                        '<td>' + inFlightBadge + '</td>' +
+                        '<td><small>' + escapeHtml(maxRuntimeText) + '</small></td>' +
                         '<td><small>' + escapeHtml(globalExecText) + '</small></td>' +
                         '<td><small>' + escapeHtml(perWorkerExecText) + '</small></td>' +
                         '<td><small>' + escapeHtml(executorWorkersText) + '</small></td>' +
                         '<td><small>' + escapeHtml(effectiveExecText) + '</small></td>' +
+                        '<td><small>' + escapeHtml(lastRunText) + '</small></td>' +
                         '</tr>';
                 }
 
                 tbody.innerHTML = rows;
             }
 
+            function renderSchedulerStatus() {
+                var valueNodes = document.querySelectorAll('.plugin-scheduler-next-run');
+                if (!valueNodes.length) {
+                    return;
+                }
+                var metaNodes = document.querySelectorAll('.plugin-scheduler-next-run-meta');
+                var status = state.schedulerStatus || {};
+                var nextRun = parseTime(status.next_detection_at);
+                var display = nextRun || '-';
+                valueNodes.forEach(function(node) {
+                    node.textContent = display;
+                });
+                var metaText = nextRun ? 'Local time' : 'Not scheduled';
+                if (metaNodes.length) {
+                    metaNodes.forEach(function(node) {
+                        node.textContent = metaText;
+                    });
+                }
+            }
+
             function renderWorkers() {
                 var tbody = document.getElementById('plugin-workers-table-body');
                 if (!state.workers.length) {
@@ -2400,8 +2461,8 @@ templ Plugin(page string) {
                 }
 
                 document.getElementById('plugin-admin-enabled').checked = pickBool('enabled');
-                document.getElementById('plugin-admin-detection-interval').value = String(pickNumber('detection_interval_seconds'));
                 document.getElementById('plugin-admin-detection-timeout').value = String(pickNumber('detection_timeout_seconds'));
+                document.getElementById('plugin-admin-max-runtime').value = String(pickNumber('job_type_max_runtime_seconds'));
                 document.getElementById('plugin-admin-max-results').value = String(pickNumber('max_jobs_per_detection'));
                 document.getElementById('plugin-admin-global-exec').value = String(pickNumber('global_execution_concurrency'));
                 document.getElementById('plugin-admin-per-worker-exec').value = String(pickNumber('per_worker_execution_concurrency'));
@@ -2410,6 +2471,9 @@ templ Plugin(page string) {
             }
 
             function collectAdminSettings() {
+                var existingRuntime = (state.config && state.config.admin_runtime) ? state.config.admin_runtime : {};
+                var existingDetectionInterval = Number(existingRuntime.detection_interval_seconds || 0);
+
                 function getInt(id) {
                     var raw = String(document.getElementById(id).value || '').trim();
                     if (!raw) {
@@ -2424,8 +2488,9 @@ templ Plugin(page string) {
 
                 return {
                     enabled: !!document.getElementById('plugin-admin-enabled').checked,
-                    detection_interval_seconds: getInt('plugin-admin-detection-interval'),
+                    detection_interval_seconds: existingDetectionInterval,
                     detection_timeout_seconds: getInt('plugin-admin-detection-timeout'),
+                    job_type_max_runtime_seconds: getInt('plugin-admin-max-runtime'),
                     max_jobs_per_detection: getInt('plugin-admin-max-results'),
                     global_execution_concurrency: getInt('plugin-admin-global-exec'),
                     per_worker_execution_concurrency: getInt('plugin-admin-per-worker-exec'),
@@ -2741,6 +2806,75 @@ templ Plugin(page string) {
                 }
             }
 
+            async function loadSchedulerConfig(forceRefresh) {
+                if (state.schedulerConfigLoaded && !forceRefresh) {
+                    return;
+                }
+                var idleInputs = [
+                    document.getElementById('plugin-scheduler-idle-sleep'),
+                    document.getElementById('plugin-scheduler-idle-sleep-overview'),
+                ].filter(Boolean);
+                if (idleInputs.length === 0) {
+                    return;
+                }
+                try {
+                    var cfg = await pluginRequest('GET', '/api/plugin/scheduler-config');
+                    state.schedulerConfig = cfg || {};
+                    state.schedulerConfigLoaded = true;
+                    var idleSeconds = Number((cfg && cfg.idle_sleep_seconds) || 0);
+                    idleInputs.forEach(function(input) {
+                        input.value = idleSeconds > 0 ? String(idleSeconds) : '';
+                    });
+                } catch (e) {
+                    notify('Failed to load scheduler config: ' + e.message, 'error');
+                }
+            }
+
+            async function saveSchedulerConfig(sourceInput) {
+                var idleInputs = [
+                    document.getElementById('plugin-scheduler-idle-sleep'),
+                    document.getElementById('plugin-scheduler-idle-sleep-overview'),
+                ].filter(Boolean);
+                if (idleInputs.length === 0) {
+                    return;
+                }
+                var raw = '';
+                if (sourceInput) {
+                    raw = String(sourceInput.value || '').trim();
+                }
+                if (!raw) {
+                    for (var i = 0; i < idleInputs.length; i++) {
+                        if (idleInputs[i] === sourceInput) {
+                            continue;
+                        }
+                        var candidate = String(idleInputs[i].value || '').trim();
+                        if (candidate) {
+                            raw = candidate;
+                            break;
+                        }
+                    }
+                }
+                var parsed = raw ? parseInt(raw, 10) : 0;
+                if (Number.isNaN(parsed) || parsed < 0) {
+                    notify('Invalid idle sleep value', 'error');
+                    return;
+                }
+                try {
+                    var updated = await pluginRequest('PUT', '/api/plugin/scheduler-config', {
+                        idle_sleep_seconds: parsed,
+                    });
+                    state.schedulerConfig = updated || {};
+                    state.schedulerConfigLoaded = true;
+                    var idleSeconds = Number((updated && updated.idle_sleep_seconds) || 0);
+                    idleInputs.forEach(function(input) {
+                        input.value = idleSeconds > 0 ? String(idleSeconds) : '';
+                    });
+                    notify('Scheduler settings saved', 'success');
+                } catch (e) {
+                    notify('Failed to save scheduler config: ' + e.message, 'error');
+                }
+            }
+
             function getMaxResults() {
                 var raw = String(document.getElementById('plugin-admin-max-results').value || '').trim();
                 if (!raw) {
@@ -2808,37 +2942,6 @@ templ Plugin(page string) {
                 }
             }
 
-            function renderSchedulerIterationStatus(schedulerStatus) {
-                var phaseEl = document.getElementById('plugin-scheduler-phase');
-                var currentJtEl = document.getElementById('plugin-scheduler-current-jt');
-                var idleSleepEl = document.getElementById('plugin-scheduler-idle-sleep');
-                var lastIterEl = document.getElementById('plugin-scheduler-last-iteration');
-                if (!phaseEl) {
-                    return;
-                }
-
-                var sched = (schedulerStatus && schedulerStatus.scheduler) ? schedulerStatus.scheduler : {};
-                var phase = String(sched.phase || 'unknown');
-                var phaseColors = { idle: 'bg-secondary', acquiring_lock: 'bg-info', processing: 'bg-primary' };
-                var phaseColor = phaseColors[phase] || 'bg-dark';
-                phaseEl.innerHTML = '<span class="badge ' + phaseColor + '">' + escapeHtml(phase) + '</span>';
-                currentJtEl.textContent = sched.current_job_type || '-';
-
-                var idleSleep = Number(sched.idle_sleep_seconds || 0);
-                idleSleepEl.textContent = idleSleep > 0 ? (String(Math.round(idleSleep / 60)) + ' min') : '-';
-
-                var lastEnded = parseTime(sched.last_iteration_ended_at);
-                var workDetected = !!sched.last_iteration_work_detected;
-                if (lastEnded) {
-                    var workBadge = workDetected
-                        ? '<span class="badge bg-success ms-1">work found</span>'
-                        : '<span class="badge bg-light text-dark ms-1">no work</span>';
-                    lastIterEl.innerHTML = '<small>' + escapeHtml(lastEnded) + '</small>' + workBadge;
-                } else {
-                    lastIterEl.textContent = '-';
-                }
-            }
-
             async function refreshJobsAndActivities() {
                 var executionStateFilter = document.getElementById('plugin-monitor-job-state-filter');
 
@@ -2847,17 +2950,16 @@ templ Plugin(page string) {
                 var allJobsPromise = pluginRequest('GET', '/api/plugin/jobs?limit=500');
                 var allActivitiesPromise = pluginRequest('GET', '/api/plugin/activities?limit=500');
                 var schedulerPromise = pluginRequest('GET', '/api/plugin/scheduler-states');
-                var schedulerStatusPromise = pluginRequest('GET', '/api/plugin/scheduler-status').catch(function(e) { return e; });
+                var schedulerStatusPromise = pluginRequest('GET', '/api/plugin/scheduler-status');
 
                 var allJobs = await allJobsPromise;
                 var allActivities = await allActivitiesPromise;
                 var schedulerStates = await schedulerPromise;
-                var schedulerStatusResult = await schedulerStatusPromise;
                 var schedulerStatus = null;
-                if (schedulerStatusResult instanceof Error) {
-                    console.error('Failed to fetch scheduler status:', schedulerStatusResult);
-                } else {
-                    schedulerStatus = schedulerStatusResult;
+                try {
+                    schedulerStatus = await schedulerStatusPromise;
+                } catch (e) {
+                    schedulerStatus = null;
                 }
 
                 state.jobs = Array.isArray(allJobs) ? allJobs : [];
@@ -2865,12 +2967,13 @@ templ Plugin(page string) {
                 state.allJobs = state.jobs;
                 state.allActivities = state.activities;
                 state.schedulerStates = Array.isArray(schedulerStates) ? schedulerStates : [];
+                state.schedulerStatus = schedulerStatus && schedulerStatus.scheduler ? schedulerStatus.scheduler : null;
                 renderQueueJobs();
                 renderDetectionJobs();
                 renderExecutionJobs();
                 renderExecutionActivities();
                 renderSchedulerStates();
-                renderSchedulerIterationStatus(schedulerStatus);
+                renderSchedulerStatus();
                 renderStatus();
                 renderJobTypeSummary();
             }
@@ -2948,6 +3051,19 @@ templ Plugin(page string) {
                     saveConfig();
                 });
 
+                var saveSchedulerBtn = document.getElementById('plugin-save-scheduler-btn');
+                if (saveSchedulerBtn) {
+                    saveSchedulerBtn.addEventListener('click', function() {
+                        saveSchedulerConfig(document.getElementById('plugin-scheduler-idle-sleep'));
+                    });
+                }
+                var saveSchedulerBtnOverview = document.getElementById('plugin-save-scheduler-btn-overview');
+                if (saveSchedulerBtnOverview) {
+                    saveSchedulerBtnOverview.addEventListener('click', function() {
+                        saveSchedulerConfig(document.getElementById('plugin-scheduler-idle-sleep-overview'));
+                    });
+                }
+
                 document.getElementById('plugin-trigger-detection-btn').addEventListener('click', function() {
                     runDetection();
                 });
@@ -3032,6 +3148,7 @@ templ Plugin(page string) {
                 ensureActiveNavigation();
                 renderNavigationState();
                 await refreshAll();
+                await loadSchedulerConfig(false);
 
                 state.refreshTimer = setInterval(function() {
                     refreshAll();

weed/admin/view/layout/layout.templ

@@ -51,14 +51,13 @@ templ Layout(view ViewContext, content templ.Component) {
     <link rel="stylesheet" href="/static/css/admin.css">
 </head>
 <body>
-    <div class="container-fluid">
+    <div class="container-fluid p-0">
         <!-- Header -->
         <header class="navbar navbar-expand-lg navbar-dark bg-primary sticky-top">
             <div class="container-fluid">
                 <a class="navbar-brand fw-bold" href="/admin">
                     <i class="fas fa-server me-2"></i>
                     SeaweedFS Admin
-                    <span class="badge bg-warning text-dark ms-2">ALPHA</span>
                 </a>
                 
                 <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarNav">
@@ -245,19 +244,6 @@ templ Layout(view ViewContext, content templ.Component) {
                                 </div>
                             }
                         </li>
-                        <!-- Commented out for later -->
-                        <!--
-                        <li class="nav-item">
-                            <a class="nav-link" href="/metrics">
-                                <i class="fas fa-chart-line me-2"></i>Metrics
-                            </a>
-                        </li>
-                        <li class="nav-item">
-                            <a class="nav-link" href="/logs">
-                                <i class="fas fa-file-alt me-2"></i>Logs
-                            </a>
-                        </li>
-                        -->
                     </ul>
 
                     <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted">
@@ -324,7 +310,7 @@ templ Layout(view ViewContext, content templ.Component) {
             </div>
 
             <!-- Main content -->
-            <main class="col-md-9 ms-sm-auto col-lg-10 px-md-4">
+            <main class="col-md-9 ms-sm-auto col-lg-10 px-3 px-md-4">
                 <div class="pt-3">
                     @content
                 </div>

weed/admin/view/layout/layout_templ.go

@@ -71,14 +71,14 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\"><link rel=\"icon\" href=\"/static/favicon.ico\" type=\"image/x-icon\"><!-- Bootstrap CSS --><link href=\"/static/css/bootstrap.min.css\" rel=\"stylesheet\"><!-- Font Awesome CSS --><link href=\"/static/css/fontawesome.min.css\" rel=\"stylesheet\"><!-- HTMX --><script src=\"/static/js/htmx.min.js\"></script><!-- Custom CSS --><link rel=\"stylesheet\" href=\"/static/css/admin.css\"></head><body><div class=\"container-fluid\"><!-- Header --><header class=\"navbar navbar-expand-lg navbar-dark bg-primary sticky-top\"><div class=\"container-fluid\"><a class=\"navbar-brand fw-bold\" href=\"/admin\"><i class=\"fas fa-server me-2\"></i> SeaweedFS Admin <span class=\"badge bg-warning text-dark ms-2\">ALPHA</span></a> <button class=\"navbar-toggler\" type=\"button\" data-bs-toggle=\"collapse\" data-bs-target=\"#navbarNav\"><span class=\"navbar-toggler-icon\"></span></button><div class=\"collapse navbar-collapse\" id=\"navbarNav\"><ul class=\"navbar-nav ms-auto\"><li class=\"nav-item dropdown\"><a class=\"nav-link dropdown-toggle\" href=\"#\" role=\"button\" data-bs-toggle=\"dropdown\"><i class=\"fas fa-user me-1\"></i>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\"><link rel=\"icon\" href=\"/static/favicon.ico\" type=\"image/x-icon\"><!-- Bootstrap CSS --><link href=\"/static/css/bootstrap.min.css\" rel=\"stylesheet\"><!-- Font Awesome CSS --><link href=\"/static/css/fontawesome.min.css\" rel=\"stylesheet\"><!-- HTMX --><script src=\"/static/js/htmx.min.js\"></script><!-- Custom CSS --><link rel=\"stylesheet\" href=\"/static/css/admin.css\"></head><body><div class=\"container-fluid p-0\"><!-- Header --><header class=\"navbar navbar-expand-lg navbar-dark bg-primary sticky-top\"><div class=\"container-fluid\"><a class=\"navbar-brand fw-bold\" href=\"/admin\"><i class=\"fas fa-server me-2\"></i> SeaweedFS Admin</a> <button class=\"navbar-toggler\" type=\"button\" data-bs-toggle=\"collapse\" data-bs-target=\"#navbarNav\"><span class=\"navbar-toggler-icon\"></span></button><div class=\"collapse navbar-collapse\" id=\"navbarNav\"><ul class=\"navbar-nav ms-auto\"><li class=\"nav-item dropdown\"><a class=\"nav-link dropdown-toggle\" href=\"#\" role=\"button\" data-bs-toggle=\"dropdown\"><i class=\"fas fa-user me-1\"></i>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var3 string
 		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(username)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 72, Col: 73}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 71, Col: 73}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
 		if templ_7745c5c3_Err != nil {
@@ -113,7 +113,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 		var templ_7745c5c3_Var6 string
 		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", isClusterPage))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 99, Col: 207}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 98, Col: 207}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {
@@ -170,7 +170,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 		var templ_7745c5c3_Var11 string
 		templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", isStoragePage))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 124, Col: 207}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 123, Col: 207}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
 		if templ_7745c5c3_Err != nil {
@@ -254,7 +254,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "</li><!-- Commented out for later --><!--\n                        <li class=\"nav-item\">\n                            <a class=\"nav-link\" href=\"/metrics\">\n                                <i class=\"fas fa-chart-line me-2\"></i>Metrics\n                            </a>\n                        </li>\n                        <li class=\"nav-item\">\n                            <a class=\"nav-link\" href=\"/logs\">\n                                <i class=\"fas fa-file-alt me-2\"></i>Logs\n                            </a>\n                        </li>\n                        --></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>WORKERS</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\">")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "</li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>WORKERS</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\">")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
@@ -329,7 +329,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "</li></ul></div></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-md-4\"><div class=\"pt-3\">")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "</li></ul></div></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-3 px-md-4\"><div class=\"pt-3\">")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
@@ -344,7 +344,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 		var templ_7745c5c3_Var14 string
 		templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 339, Col: 60}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 325, Col: 60}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
 		if templ_7745c5c3_Err != nil {
@@ -357,7 +357,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 		var templ_7745c5c3_Var15 string
 		templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 339, Col: 102}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 325, Col: 102}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
 		if templ_7745c5c3_Err != nil {
@@ -409,7 +409,7 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
 		var templ_7745c5c3_Var17 string
 		templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(title)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 367, Col: 17}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 353, Col: 17}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 		if templ_7745c5c3_Err != nil {
@@ -422,7 +422,7 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
 		var templ_7745c5c3_Var18 string
 		templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(title)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 381, Col: 57}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 367, Col: 57}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 		if templ_7745c5c3_Err != nil {
@@ -440,7 +440,7 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
 			var templ_7745c5c3_Var19 string
 			templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 388, Col: 45}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 374, Col: 45}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 			if templ_7745c5c3_Err != nil {
@@ -458,7 +458,7 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
 		var templ_7745c5c3_Var20 string
 		templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(csrfToken)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 393, Col: 84}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 379, Col: 84}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
 		if templ_7745c5c3_Err != nil {

weed/cluster/admin_locks.go

@@ -0,0 +1,6 @@
+package cluster
+
+const (
+	AdminShellLockName          = "shell"
+	AdminServerPresenceLockName = "admin-server"
+)

weed/cluster/maintenance/maintenance_config.go

@@ -1,5 +1,7 @@
 package maintenance
 
+const DefaultMaintenanceSleepMinutes = 17
+
 const DefaultMasterMaintenanceScripts = `
   lock
   ec.encode -fullPercent=95 -quietFor=1h

weed/command/admin.go

@@ -45,7 +45,6 @@ type AdminOptions struct {
 	readOnlyPassword *string
 	dataDir          *string
 	icebergPort      *int
-	idleSleepSeconds *int
 }
 
 func init() {
@@ -61,7 +60,6 @@ func init() {
 	a.readOnlyUser = cmdAdmin.Flag.String("readOnlyUser", "", "read-only user username (optional, for view-only access)")
 	a.readOnlyPassword = cmdAdmin.Flag.String("readOnlyPassword", "", "read-only user password (optional, for view-only access; requires adminPassword to be set)")
 	a.icebergPort = cmdAdmin.Flag.Int("iceberg.port", 8181, "Iceberg REST Catalog port (0 to hide in UI)")
-	a.idleSleepSeconds = cmdAdmin.Flag.Int("scheduler.idleSleep", 0, "scheduler idle sleep in seconds between iterations when no work is found (0 = default 17 minutes)")
 }
 
 var cmdAdmin = &Command{
@@ -292,11 +290,7 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
 	}
 
 	// Create admin server (plugin is always enabled)
-	var idleSleep time.Duration
-	if options.idleSleepSeconds != nil && *options.idleSleepSeconds > 0 {
-		idleSleep = time.Duration(*options.idleSleepSeconds) * time.Second
-	}
-	adminServer := dash.NewAdminServer(*options.master, nil, dataDir, icebergPort, idleSleep)
+	adminServer := dash.NewAdminServer(*options.master, nil, dataDir, icebergPort)
 
 	// Show discovered filers
 	filers := adminServer.GetAllFilers()

weed/command/scaffold/master.toml

@@ -6,6 +6,7 @@
 
 [master.maintenance]
 # periodically run these scripts are the same as running them from 'weed shell'
+# Scripts are skipped while an admin server is connected.
 scripts = """{{DEFAULT_MAINTENANCE_SCRIPTS}}"""
 sleep_minutes = 17          # sleep minutes between each script execution
 

weed/iamapi/iamapi_handlers.go

@@ -36,6 +36,8 @@ func writeIamErrorResponse(w http.ResponseWriter, r *http.Request, iamError *Iam
 		s3err.WriteXMLResponse(w, r, http.StatusNotFound, errorResp)
 	case iam.ErrCodeMalformedPolicyDocumentException, iam.ErrCodeInvalidInputException:
 		s3err.WriteXMLResponse(w, r, http.StatusBadRequest, errorResp)
+	case iam.ErrCodeDeleteConflictException:
+		s3err.WriteXMLResponse(w, r, http.StatusConflict, errorResp)
 	case iam.ErrCodeServiceFailureException:
 		// We do not want to expose internal server error to the client
 		s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, internalErrorResponse)

weed/iamapi/iamapi_management_handlers.go

@@ -41,6 +41,20 @@ const (
 
 var policyLock = sync.RWMutex{}
 
+const policyArnPrefix = "arn:aws:iam:::policy/"
+
+// parsePolicyArn validates an IAM policy ARN and extracts the policy name.
+func parsePolicyArn(policyArn string) (string, *IamError) {
+	if !strings.HasPrefix(policyArn, policyArnPrefix) {
+		return "", &IamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("invalid policy ARN: %s", policyArn)}
+	}
+	policyName := strings.TrimPrefix(policyArn, policyArnPrefix)
+	if policyName == "" {
+		return "", &IamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("invalid policy ARN: %s", policyArn)}
+	}
+	return policyName, nil
+}
+
 // userPolicyKey returns a namespaced key for inline user policies to prevent collision with managed policies.
 // getOrCreateUserPolicies returns the policy map for a user, creating it if needed.
 // Returns a pointer to the user's policy map from Policies.InlinePolicies.
@@ -234,16 +248,19 @@ func (iama *IamApiServer) CreatePolicy(s3cfg *iam_pb.S3ApiConfiguration, values
 	if err != nil {
 		return CreatePolicyResponse{}, &IamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err}
 	}
-	policyId := Hash(&policyDocumentString)
+	policyId := Hash(&policyName)
 	arn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
 	resp.CreatePolicyResult.Policy.PolicyName = &policyName
 	resp.CreatePolicyResult.Policy.Arn = &arn
 	resp.CreatePolicyResult.Policy.PolicyId = &policyId
 	policies := Policies{}
 	// Note: Lock is already held by DoActions, no need to acquire here
-	if err = iama.s3ApiConfig.GetPolicies(&policies); err != nil {
+	if err = iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
 		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
 	}
+	if policies.Policies == nil {
+		policies.Policies = make(map[string]policy_engine.PolicyDocument)
+	}
 	policies.Policies[policyName] = policyDocument
 	if err = iama.s3ApiConfig.PutPolicies(&policies); err != nil {
 		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
@@ -265,43 +282,43 @@ func (iama *IamApiServer) PutUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values
 	if err != nil {
 		return PutUserPolicyResponse{}, &IamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err}
 	}
-	actions, err := GetActions(&policyDocument)
-	if err != nil {
+	if _, err := GetActions(&policyDocument); err != nil {
 		return PutUserPolicyResponse{}, &IamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err}
 	}
 
+	// Verify the user exists before persisting the policy
+	var targetIdent *iam_pb.Identity
+	for _, ident := range s3cfg.Identities {
+		if ident.Name == userName {
+			targetIdent = ident
+			break
+		}
+	}
+	if targetIdent == nil {
+		return PutUserPolicyResponse{}, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("the user with name %s cannot be found", userName)}
+	}
+
 	// Persist inline policy to storage using per-user indexed structure
 	policies := Policies{}
 	if err = iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
 		return PutUserPolicyResponse{}, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
 	}
 
-	// Get or create user's policy map
 	userPolicies := policies.getOrCreateUserPolicies(userName)
 	userPolicies[policyName] = policyDocument
-	// policies.InlinePolicies[userName] now contains the updated map
 
 	if err = iama.s3ApiConfig.PutPolicies(&policies); err != nil {
 		return PutUserPolicyResponse{}, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
 	}
 
-	// Compute aggregated actions from all user's inline policies, passing the local policies
-	// to avoid redundant I/O (reuses the just-written Policies map)
-	aggregatedActions, computeErr := computeAggregatedActionsForUser(iama, userName, &policies)
+	// Recompute aggregated actions (inline + managed)
+	aggregatedActions, computeErr := computeAllActionsForUser(iama, userName, &policies, targetIdent)
 	if computeErr != nil {
-		glog.Warningf("Failed to compute aggregated actions for user %s: %v", userName, computeErr)
-		aggregatedActions = actions // Fall back to current policy's actions
+		glog.Warningf("Failed to compute aggregated actions for user %s: %v; keeping existing actions", userName, computeErr)
+	} else {
+		targetIdent.Actions = aggregatedActions
 	}
-
-	glog.V(3).Infof("PutUserPolicy: aggregated actions=%v", aggregatedActions)
-	for _, ident := range s3cfg.Identities {
-		if userName != ident.Name {
-			continue
-		}
-		ident.Actions = aggregatedActions
-		return resp, nil
-	}
-	return PutUserPolicyResponse{}, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("the user with name %s cannot be found", userName)}
+	return resp, nil
 }
 
 func (iama *IamApiServer) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp GetUserPolicyResponse, err *IamError) {
@@ -419,17 +436,247 @@ func (iama *IamApiServer) DeleteUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, val
 		}
 	}
 
-	// Recompute aggregated actions from remaining inline policies (passing policies to avoid redundant GetPolicies)
-	aggregatedActions, computeErr := computeAggregatedActionsForUser(iama, userName, &policies)
+	// Recompute aggregated actions from remaining inline + managed policies
+	aggregatedActions, computeErr := computeAllActionsForUser(iama, userName, &policies, targetIdent)
 	if computeErr != nil {
-		glog.Warningf("Failed to recompute aggregated actions for user %s: %v", userName, computeErr)
+		glog.Warningf("Failed to recompute aggregated actions for user %s: %v; keeping existing actions", userName, computeErr)
+	} else {
+		targetIdent.Actions = aggregatedActions
+	}
+	return resp, nil
+}
+
+// GetPolicy retrieves a managed policy by ARN.
+func (iama *IamApiServer) GetPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp GetPolicyResponse, iamError *IamError) {
+	policyArn := values.Get("PolicyArn")
+	policyName, iamError := parsePolicyArn(policyArn)
+	if iamError != nil {
+		return resp, iamError
 	}
 
-	// Update the found identity's actions
-	targetIdent.Actions = aggregatedActions
+	policies := Policies{}
+	if err := iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+
+	if _, exists := policies.Policies[policyName]; !exists {
+		return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("policy %s not found", policyName)}
+	}
+
+	policyId := Hash(&policyName)
+	resp.GetPolicyResult.Policy.PolicyName = &policyName
+	resp.GetPolicyResult.Policy.Arn = &policyArn
+	resp.GetPolicyResult.Policy.PolicyId = &policyId
 	return resp, nil
 }
 
+// DeletePolicy removes a managed policy. Rejects deletion if the policy is still attached to any user
+// (matching AWS IAM behavior: must detach before deleting).
+func (iama *IamApiServer) DeletePolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp DeletePolicyResponse, iamError *IamError) {
+	policyArn := values.Get("PolicyArn")
+	policyName, iamError := parsePolicyArn(policyArn)
+	if iamError != nil {
+		return resp, iamError
+	}
+
+	policies := Policies{}
+	if err := iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+
+	if _, exists := policies.Policies[policyName]; !exists {
+		return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("policy %s not found", policyName)}
+	}
+
+	// Reject deletion if the policy is still attached to any user
+	for _, ident := range s3cfg.Identities {
+		for _, name := range ident.PolicyNames {
+			if name == policyName {
+				return resp, &IamError{
+					Code:  iam.ErrCodeDeleteConflictException,
+					Error: fmt.Errorf("policy %s is still attached to user %s", policyName, ident.Name),
+				}
+			}
+		}
+	}
+
+	delete(policies.Policies, policyName)
+	if err := iama.s3ApiConfig.PutPolicies(&policies); err != nil {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+
+	return resp, nil
+}
+
+// ListPolicies lists all managed policies.
+func (iama *IamApiServer) ListPolicies(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp ListPoliciesResponse, iamError *IamError) {
+	policies := Policies{}
+	if err := iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+
+	for policyName := range policies.Policies {
+		name := policyName
+		arn := fmt.Sprintf("arn:aws:iam:::policy/%s", name)
+		policyId := Hash(&name)
+		resp.ListPoliciesResult.Policies = append(resp.ListPoliciesResult.Policies, &iam.Policy{
+			PolicyName: &name,
+			Arn:        &arn,
+			PolicyId:   &policyId,
+		})
+	}
+	return resp, nil
+}
+
+// AttachUserPolicy attaches a managed policy to a user.
+func (iama *IamApiServer) AttachUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp AttachUserPolicyResponse, iamError *IamError) {
+	userName := values.Get("UserName")
+	policyArn := values.Get("PolicyArn")
+	policyName, iamError := parsePolicyArn(policyArn)
+	if iamError != nil {
+		return resp, iamError
+	}
+
+	// Verify managed policy exists
+	policies := Policies{}
+	if err := iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+	if _, exists := policies.Policies[policyName]; !exists {
+		return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("policy %s not found", policyName)}
+	}
+
+	// Find user and attach policy
+	for _, ident := range s3cfg.Identities {
+		if ident.Name != userName {
+			continue
+		}
+		// Check if already attached
+		for _, name := range ident.PolicyNames {
+			if name == policyName {
+				return resp, nil // Already attached, idempotent
+			}
+		}
+		prevPolicyNames := ident.PolicyNames
+		ident.PolicyNames = append(ident.PolicyNames, policyName)
+
+		// Recompute aggregated actions (inline + managed)
+		aggregatedActions, err := computeAllActionsForUser(iama, userName, &policies, ident)
+		if err != nil {
+			// Roll back PolicyNames to keep identity consistent
+			ident.PolicyNames = prevPolicyNames
+			return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: fmt.Errorf("failed to compute actions after attaching policy: %w", err)}
+		}
+		ident.Actions = aggregatedActions
+		return resp, nil
+	}
+	return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(USER_DOES_NOT_EXIST, userName)}
+}
+
+// DetachUserPolicy detaches a managed policy from a user.
+func (iama *IamApiServer) DetachUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp DetachUserPolicyResponse, iamError *IamError) {
+	userName := values.Get("UserName")
+	policyArn := values.Get("PolicyArn")
+	policyName, iamError := parsePolicyArn(policyArn)
+	if iamError != nil {
+		return resp, iamError
+	}
+
+	for _, ident := range s3cfg.Identities {
+		if ident.Name != userName {
+			continue
+		}
+		// Find and remove policy name from the list
+		prevPolicyNames := make([]string, len(ident.PolicyNames))
+		copy(prevPolicyNames, ident.PolicyNames)
+
+		found := false
+		for i, name := range ident.PolicyNames {
+			if name == policyName {
+				ident.PolicyNames = append(ident.PolicyNames[:i], ident.PolicyNames[i+1:]...)
+				found = true
+				break
+			}
+		}
+		if !found {
+			return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("policy %s is not attached to user %s", policyName, userName)}
+		}
+
+		// Recompute aggregated actions (inline + managed)
+		policies := Policies{}
+		if err := iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+			// Roll back PolicyNames on storage error
+			ident.PolicyNames = prevPolicyNames
+			return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+		}
+		aggregatedActions, err := computeAllActionsForUser(iama, userName, &policies, ident)
+		if err != nil {
+			// Roll back PolicyNames to keep identity consistent
+			ident.PolicyNames = prevPolicyNames
+			return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: fmt.Errorf("failed to compute actions after detaching policy: %w", err)}
+		}
+		ident.Actions = aggregatedActions
+		return resp, nil
+	}
+	return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(USER_DOES_NOT_EXIST, userName)}
+}
+
+// ListAttachedUserPolicies lists the managed policies attached to a user.
+func (iama *IamApiServer) ListAttachedUserPolicies(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp ListAttachedUserPoliciesResponse, iamError *IamError) {
+	userName := values.Get("UserName")
+	for _, ident := range s3cfg.Identities {
+		if ident.Name != userName {
+			continue
+		}
+		for _, policyName := range ident.PolicyNames {
+			name := policyName
+			arn := fmt.Sprintf("arn:aws:iam:::policy/%s", name)
+			resp.ListAttachedUserPoliciesResult.AttachedPolicies = append(
+				resp.ListAttachedUserPoliciesResult.AttachedPolicies,
+				&iam.AttachedPolicy{PolicyName: &name, PolicyArn: &arn},
+			)
+		}
+		return resp, nil
+	}
+	return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(USER_DOES_NOT_EXIST, userName)}
+}
+
+// computeAllActionsForUser computes the union of actions from both inline and managed policies.
+func computeAllActionsForUser(iama *IamApiServer, userName string, policies *Policies, ident *iam_pb.Identity) ([]string, error) {
+	actionSet := make(map[string]bool)
+	var aggregatedActions []string
+
+	addUniqueActions := func(actions []string) {
+		for _, action := range actions {
+			if !actionSet[action] {
+				actionSet[action] = true
+				aggregatedActions = append(aggregatedActions, action)
+			}
+		}
+	}
+
+	// Include inline policy actions
+	inlineActions, err := computeAggregatedActionsForUser(iama, userName, policies)
+	if err != nil {
+		return nil, err
+	}
+	addUniqueActions(inlineActions)
+
+	// Include managed policy actions
+	for _, policyName := range ident.PolicyNames {
+		if policyDoc, exists := policies.Policies[policyName]; exists {
+			actions, err := GetActions(&policyDoc)
+			if err != nil {
+				glog.Warningf("Failed to get actions from managed policy '%s' for user %s: %v", policyName, userName, err)
+				continue
+			}
+			addUniqueActions(actions)
+		}
+	}
+
+	return aggregatedActions, nil
+}
+
 func GetActions(policy *policy_engine.PolicyDocument) ([]string, error) {
 	var actions []string
 
@@ -708,6 +955,46 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 			writeIamErrorResponse(w, r, iamError)
 			return
 		}
+	case "GetPolicy":
+		response, iamError = iama.GetPolicy(s3cfg, values)
+		if iamError != nil {
+			writeIamErrorResponse(w, r, iamError)
+			return
+		}
+		changed = false
+	case "DeletePolicy":
+		response, iamError = iama.DeletePolicy(s3cfg, values)
+		if iamError != nil {
+			writeIamErrorResponse(w, r, iamError)
+			return
+		}
+		changed = false
+	case "ListPolicies":
+		response, iamError = iama.ListPolicies(s3cfg, values)
+		if iamError != nil {
+			writeIamErrorResponse(w, r, iamError)
+			return
+		}
+		changed = false
+	case "AttachUserPolicy":
+		response, iamError = iama.AttachUserPolicy(s3cfg, values)
+		if iamError != nil {
+			writeIamErrorResponse(w, r, iamError)
+			return
+		}
+	case "DetachUserPolicy":
+		response, iamError = iama.DetachUserPolicy(s3cfg, values)
+		if iamError != nil {
+			writeIamErrorResponse(w, r, iamError)
+			return
+		}
+	case "ListAttachedUserPolicies":
+		response, iamError = iama.ListAttachedUserPolicies(s3cfg, values)
+		if iamError != nil {
+			writeIamErrorResponse(w, r, iamError)
+			return
+		}
+		changed = false
 	default:
 		errNotImplemented := s3err.GetAPIError(s3err.ErrNotImplemented)
 		errorResponse := ErrorResponse{}

weed/iamapi/iamapi_management_handlers_test.go

@@ -5,6 +5,7 @@ import (
 	"net/url"
 	"testing"
 
+	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
@@ -261,3 +262,299 @@ func TestMultipleInlinePoliciesAggregateActions(t *testing.T) {
 		assert.True(t, actionSet[expectedAction], "Expected action '%s' not found in aggregated actions. Got: %v", expectedAction, aliceIdent.Actions)
 	}
 }
+
+// newTestIamApiServer creates a minimal IamApiServer for unit testing with only s3ApiConfig set.
+// Other fields (iam, masterClient, etc.) are left nil — tests must not call code paths that use them.
+func newTestIamApiServer(policies Policies) *IamApiServer {
+	return &IamApiServer{
+		s3ApiConfig: &mockIamS3ApiConfig{policies: policies},
+	}
+}
+
+func TestGetPolicy(t *testing.T) {
+	policyDoc := policy_engine.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+				Resource: policy_engine.NewStringOrStringSlice("arn:aws:s3:::*"),
+			},
+		},
+	}
+	iama := newTestIamApiServer(Policies{
+		Policies: map[string]policy_engine.PolicyDocument{"my-policy": policyDoc},
+	})
+	s3cfg := &iam_pb.S3ApiConfiguration{}
+
+	// Success case
+	values := url.Values{"PolicyArn": []string{"arn:aws:iam:::policy/my-policy"}}
+	resp, iamErr := iama.GetPolicy(s3cfg, values)
+	assert.Nil(t, iamErr)
+	assert.Equal(t, "my-policy", *resp.GetPolicyResult.Policy.PolicyName)
+	assert.Equal(t, "arn:aws:iam:::policy/my-policy", *resp.GetPolicyResult.Policy.Arn)
+	policyName := "my-policy"
+	expectedId := Hash(&policyName)
+	assert.Equal(t, expectedId, *resp.GetPolicyResult.Policy.PolicyId)
+
+	// Not found case
+	values = url.Values{"PolicyArn": []string{"arn:aws:iam:::policy/nonexistent"}}
+	_, iamErr = iama.GetPolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code)
+
+	// Invalid ARN
+	values = url.Values{"PolicyArn": []string{"invalid-arn"}}
+	_, iamErr = iama.GetPolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeInvalidInputException, iamErr.Code)
+
+	// Empty ARN
+	values = url.Values{"PolicyArn": []string{""}}
+	_, iamErr = iama.GetPolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeInvalidInputException, iamErr.Code)
+}
+
+func TestDeletePolicy(t *testing.T) {
+	policyDoc := policy_engine.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+				Resource: policy_engine.NewStringOrStringSlice("arn:aws:s3:::*"),
+			},
+		},
+	}
+	mock := &mockIamS3ApiConfig{policies: Policies{
+		Policies: map[string]policy_engine.PolicyDocument{"my-policy": policyDoc},
+	}}
+	iama := &IamApiServer{s3ApiConfig: mock}
+
+	// Reject deletion when policy is attached to a user (AWS-compatible behavior)
+	s3cfg := &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{
+			Name:        "alice",
+			PolicyNames: []string{"my-policy"},
+		}},
+	}
+	values := url.Values{"PolicyArn": []string{"arn:aws:iam:::policy/my-policy"}}
+	_, iamErr := iama.DeletePolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeDeleteConflictException, iamErr.Code)
+
+	// Succeed when no users are attached
+	s3cfgEmpty := &iam_pb.S3ApiConfiguration{}
+	_, iamErr = iama.DeletePolicy(s3cfgEmpty, values)
+	assert.Nil(t, iamErr)
+
+	// Verify deleted
+	_, iamErr = iama.GetPolicy(s3cfgEmpty, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code)
+}
+
+func TestListPolicies(t *testing.T) {
+	iama := newTestIamApiServer(Policies{})
+	s3cfg := &iam_pb.S3ApiConfiguration{}
+
+	// Empty case
+	resp, iamErr := iama.ListPolicies(s3cfg, url.Values{})
+	assert.Nil(t, iamErr)
+	assert.Empty(t, resp.ListPoliciesResult.Policies)
+
+	// Populated case
+	policyDoc := policy_engine.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+				Resource: policy_engine.NewStringOrStringSlice("arn:aws:s3:::*"),
+			},
+		},
+	}
+	iama = newTestIamApiServer(Policies{
+		Policies: map[string]policy_engine.PolicyDocument{
+			"policy-a": policyDoc,
+			"policy-b": policyDoc,
+		},
+	})
+
+	resp, iamErr = iama.ListPolicies(s3cfg, url.Values{})
+	assert.Nil(t, iamErr)
+	assert.Equal(t, 2, len(resp.ListPoliciesResult.Policies))
+	for _, p := range resp.ListPoliciesResult.Policies {
+		name := *p.PolicyName
+		expectedId := Hash(&name)
+		assert.Equal(t, expectedId, *p.PolicyId, "PolicyId should be Hash(policyName) for %s", name)
+	}
+}
+
+func TestAttachUserPolicy(t *testing.T) {
+	policyDoc := policy_engine.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+				Resource: policy_engine.NewStringOrStringSlice("arn:aws:s3:::*"),
+			},
+		},
+	}
+	iama := newTestIamApiServer(Policies{
+		Policies: map[string]policy_engine.PolicyDocument{"my-policy": policyDoc},
+	})
+	s3cfg := &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice"}},
+	}
+
+	// Success case
+	values := url.Values{
+		"UserName":  []string{"alice"},
+		"PolicyArn": []string{"arn:aws:iam:::policy/my-policy"},
+	}
+	_, iamErr := iama.AttachUserPolicy(s3cfg, values)
+	assert.Nil(t, iamErr)
+	assert.Contains(t, s3cfg.Identities[0].PolicyNames, "my-policy")
+	// Verify actions were computed from the managed policy
+	assert.Greater(t, len(s3cfg.Identities[0].Actions), 0)
+
+	// Idempotent re-attach
+	_, iamErr = iama.AttachUserPolicy(s3cfg, values)
+	assert.Nil(t, iamErr)
+	// Should still have exactly one entry
+	count := 0
+	for _, name := range s3cfg.Identities[0].PolicyNames {
+		if name == "my-policy" {
+			count++
+		}
+	}
+	assert.Equal(t, 1, count)
+
+	// Policy not found
+	values = url.Values{
+		"UserName":  []string{"alice"},
+		"PolicyArn": []string{"arn:aws:iam:::policy/nonexistent"},
+	}
+	_, iamErr = iama.AttachUserPolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code)
+
+	// User not found
+	values = url.Values{
+		"UserName":  []string{"bob"},
+		"PolicyArn": []string{"arn:aws:iam:::policy/my-policy"},
+	}
+	_, iamErr = iama.AttachUserPolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code)
+}
+
+func TestManagedPolicyActionsPreservedAcrossInlineMutations(t *testing.T) {
+	managedPolicyDoc := policy_engine.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+				Resource: policy_engine.NewStringOrStringSlice("arn:aws:s3:::*"),
+			},
+		},
+	}
+	iama := newTestIamApiServer(Policies{
+		Policies: map[string]policy_engine.PolicyDocument{"my-policy": managedPolicyDoc},
+	})
+	s3cfg := &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice"}},
+	}
+
+	// Attach managed policy
+	_, iamErr := iama.AttachUserPolicy(s3cfg, url.Values{
+		"UserName":  []string{"alice"},
+		"PolicyArn": []string{"arn:aws:iam:::policy/my-policy"},
+	})
+	assert.Nil(t, iamErr)
+	assert.Contains(t, s3cfg.Identities[0].Actions, "Read", "Managed policy should grant Read action")
+
+	// Add an inline policy
+	inlinePolicyJSON := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::bucket-x/*"}]}`
+	_, iamErr = iama.PutUserPolicy(s3cfg, url.Values{
+		"UserName":       []string{"alice"},
+		"PolicyName":     []string{"inline-write"},
+		"PolicyDocument": []string{inlinePolicyJSON},
+	})
+	assert.Nil(t, iamErr)
+
+	// Should have both managed (Read) and inline (Write:bucket-x/*) actions
+	actionSet := make(map[string]bool)
+	for _, a := range s3cfg.Identities[0].Actions {
+		actionSet[a] = true
+	}
+	assert.True(t, actionSet["Read"], "Managed policy Read action should persist after PutUserPolicy")
+	assert.True(t, actionSet["Write:bucket-x/*"], "Inline policy Write action should be present")
+
+	// Delete the inline policy
+	_, iamErr = iama.DeleteUserPolicy(s3cfg, url.Values{
+		"UserName":   []string{"alice"},
+		"PolicyName": []string{"inline-write"},
+	})
+	assert.Nil(t, iamErr)
+
+	// Managed policy actions should still be present
+	assert.Contains(t, s3cfg.Identities[0].PolicyNames, "my-policy", "Managed policy should still be attached")
+	assert.Contains(t, s3cfg.Identities[0].Actions, "Read", "Managed policy Read action should persist after DeleteUserPolicy")
+}
+
+func TestDetachUserPolicy(t *testing.T) {
+	policyDoc := policy_engine.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+				Resource: policy_engine.NewStringOrStringSlice("arn:aws:s3:::*"),
+			},
+		},
+	}
+	iama := newTestIamApiServer(Policies{
+		Policies: map[string]policy_engine.PolicyDocument{"my-policy": policyDoc},
+	})
+	s3cfg := &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice", PolicyNames: []string{"my-policy"}}},
+	}
+
+	values := url.Values{
+		"UserName":  []string{"alice"},
+		"PolicyArn": []string{"arn:aws:iam:::policy/my-policy"},
+	}
+	_, iamErr := iama.DetachUserPolicy(s3cfg, values)
+	assert.Nil(t, iamErr)
+	assert.Empty(t, s3cfg.Identities[0].PolicyNames)
+
+	// Detach again should fail (not attached)
+	_, iamErr = iama.DetachUserPolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code)
+}
+
+func TestListAttachedUserPolicies(t *testing.T) {
+	iama := newTestIamApiServer(Policies{})
+	s3cfg := &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice", PolicyNames: []string{"policy-a", "policy-b"}}},
+	}
+
+	values := url.Values{"UserName": []string{"alice"}}
+	resp, iamErr := iama.ListAttachedUserPolicies(s3cfg, values)
+	assert.Nil(t, iamErr)
+	assert.Equal(t, 2, len(resp.ListAttachedUserPoliciesResult.AttachedPolicies))
+	assert.Equal(t, "policy-a", *resp.ListAttachedUserPoliciesResult.AttachedPolicies[0].PolicyName)
+	assert.Equal(t, "arn:aws:iam:::policy/policy-a", *resp.ListAttachedUserPoliciesResult.AttachedPolicies[0].PolicyArn)
+
+	// User not found
+	values = url.Values{"UserName": []string{"bob"}}
+	_, iamErr = iama.ListAttachedUserPolicies(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code)
+}

weed/iamapi/iamapi_response.go

@@ -22,8 +22,14 @@ type (
 	UpdateAccessKeyResponse      = iamlib.UpdateAccessKeyResponse
 	PutUserPolicyResponse        = iamlib.PutUserPolicyResponse
 	DeleteUserPolicyResponse     = iamlib.DeleteUserPolicyResponse
-	GetUserPolicyResponse        = iamlib.GetUserPolicyResponse
-	ErrorResponse                = iamlib.ErrorResponse
+	GetUserPolicyResponse            = iamlib.GetUserPolicyResponse
+	GetPolicyResponse                = iamlib.GetPolicyResponse
+	DeletePolicyResponse             = iamlib.DeletePolicyResponse
+	ListPoliciesResponse             = iamlib.ListPoliciesResponse
+	AttachUserPolicyResponse         = iamlib.AttachUserPolicyResponse
+	DetachUserPolicyResponse         = iamlib.DetachUserPolicyResponse
+	ListAttachedUserPoliciesResponse = iamlib.ListAttachedUserPoliciesResponse
+	ErrorResponse                    = iamlib.ErrorResponse
 	ServiceAccountInfo           = iamlib.ServiceAccountInfo
 	CreateServiceAccountResponse = iamlib.CreateServiceAccountResponse
 	DeleteServiceAccountResponse = iamlib.DeleteServiceAccountResponse

weed/mount/meta_cache/meta_cache.go

@@ -18,6 +18,20 @@ import (
 // need to have logic similar to FilerStoreWrapper
 // e.g. fill fileId field for chunks
 
+// bufferedEvent represents a subscription event captured during a directory refresh.
+type bufferedEvent struct {
+	oldPath  util.FullPath
+	newEntry *filer.Entry
+}
+
+// refreshState tracks events that arrive while a directory is being refreshed
+// from the filer. After the refresh snapshot is applied, buffered events are
+// replayed so that creates, deletes, and updates that raced with the snapshot
+// are not lost.
+type refreshState struct {
+	events []bufferedEvent
+}
+
 type MetaCache struct {
 	root         util.FullPath
 	localStore   filer.VirtualFilerStore
@@ -29,6 +43,7 @@ type MetaCache struct {
 	invalidateFunc    func(fullpath util.FullPath, entry *filer_pb.Entry)
 	onDirectoryUpdate func(dir util.FullPath)
 	visitGroup        singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path
+	refreshing        map[util.FullPath]*refreshState
 }
 
 func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath,
@@ -45,6 +60,7 @@ func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPat
 		invalidateFunc: func(fullpath util.FullPath, entry *filer_pb.Entry) {
 			invalidateFunc(fullpath, entry)
 		},
+		refreshing: make(map[util.FullPath]*refreshState),
 	}
 }
 
@@ -69,6 +85,11 @@ func openMetaStore(dbFolder string) (*leveldb.LevelDBStore, filer.VirtualFilerSt
 func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer.Entry) error {
 	mc.Lock()
 	defer mc.Unlock()
+	// Buffer the insert if the parent directory is being refreshed
+	dir, _ := entry.DirAndName()
+	if state := mc.isRefreshingDir(util.FullPath(dir)); state != nil {
+		state.events = append(state.events, bufferedEvent{newEntry: entry})
+	}
 	return mc.doInsertEntry(ctx, entry)
 }
 
@@ -76,16 +97,33 @@ func (mc *MetaCache) doInsertEntry(ctx context.Context, entry *filer.Entry) erro
 	return mc.localStore.InsertEntry(ctx, entry)
 }
 
-// doBatchInsertEntries inserts multiple entries using LevelDB's batch write.
-// This is more efficient than inserting entries one by one.
-func (mc *MetaCache) doBatchInsertEntries(ctx context.Context, entries []*filer.Entry) error {
-	return mc.leveldbStore.BatchInsertEntries(ctx, entries)
-}
-
 func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry) error {
 	mc.Lock()
 	defer mc.Unlock()
 
+	// If the affected directory is being refreshed, buffer the event
+	// instead of applying it. It will be replayed after the snapshot commits.
+	if oldPath != "" {
+		dir, _ := oldPath.DirAndName()
+		if state := mc.isRefreshingDir(util.FullPath(dir)); state != nil {
+			state.events = append(state.events, bufferedEvent{oldPath, newEntry})
+			return nil
+		}
+	}
+	if newEntry != nil {
+		newDir, _ := newEntry.DirAndName()
+		if state := mc.isRefreshingDir(util.FullPath(newDir)); state != nil {
+			state.events = append(state.events, bufferedEvent{oldPath, newEntry})
+			return nil
+		}
+	}
+
+	return mc.doAtomicUpdateEntryFromFiler(ctx, oldPath, newEntry)
+}
+
+// doAtomicUpdateEntryFromFiler is the core logic for applying a filer event
+// to the local cache. Caller must hold mc.Lock().
+func (mc *MetaCache) doAtomicUpdateEntryFromFiler(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry) error {
 	entry, err := mc.localStore.FindEntry(ctx, oldPath)
 	if err != nil && err != filer_pb.ErrNotFound {
 		glog.Errorf("Metacache: find entry error: %v", err)
@@ -104,8 +142,6 @@ func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath uti
 				}
 			}
 		}
-	} else {
-		// println("unknown old directory:", oldDir)
 	}
 
 	if newEntry != nil {
@@ -143,6 +179,14 @@ func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *fi
 func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) {
 	mc.Lock()
 	defer mc.Unlock()
+	// Buffer the delete if the parent directory is being refreshed
+	dir, _ := fp.DirAndName()
+	if state := mc.isRefreshingDir(util.FullPath(dir)); state != nil {
+		state.events = append(state.events, bufferedEvent{oldPath: fp})
+	}
+	// Always apply the delete directly as well, so the entry is removed
+	// immediately for the current node's view. CommitRefresh's snapshot
+	// may re-insert it, but the buffered event replay will re-delete it.
 	return mc.localStore.DeleteEntry(ctx, fp)
 }
 func (mc *MetaCache) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
@@ -151,6 +195,58 @@ func (mc *MetaCache) DeleteFolderChildren(ctx context.Context, fp util.FullPath)
 	return mc.localStore.DeleteFolderChildren(ctx, fp)
 }
 
+// BeginRefresh starts buffering subscription events for dirPath.
+// While a refresh is active, AtomicUpdateEntryFromFiler will buffer events
+// instead of applying them, so they can be replayed after the snapshot is committed.
+func (mc *MetaCache) BeginRefresh(dirPath util.FullPath) {
+	mc.Lock()
+	defer mc.Unlock()
+	mc.refreshing[dirPath] = &refreshState{}
+}
+
+// CommitRefresh atomically replaces a directory's cached entries with the
+// filer snapshot, then replays any subscription events that were buffered
+// during the refresh. This ensures no creates, deletes, or updates are lost.
+func (mc *MetaCache) CommitRefresh(ctx context.Context, dirPath util.FullPath, entries []*filer.Entry) error {
+	mc.Lock()
+	defer mc.Unlock()
+
+	// Clear stale entries and insert the fresh snapshot
+	if err := mc.localStore.DeleteFolderChildren(ctx, dirPath); err != nil {
+		return err
+	}
+	if len(entries) > 0 {
+		if err := mc.leveldbStore.BatchInsertEntries(ctx, entries); err != nil {
+			return err
+		}
+	}
+
+	// Replay buffered events so mutations that raced with the snapshot are applied
+	state := mc.refreshing[dirPath]
+	delete(mc.refreshing, dirPath)
+	if state != nil {
+		for _, ev := range state.events {
+			if err := mc.doAtomicUpdateEntryFromFiler(ctx, ev.oldPath, ev.newEntry); err != nil {
+				glog.Warningf("replay buffered event for %s: %v", dirPath, err)
+			}
+		}
+	}
+	return nil
+}
+
+// CancelRefresh discards the refresh state without replaying buffered events.
+func (mc *MetaCache) CancelRefresh(dirPath util.FullPath) {
+	mc.Lock()
+	defer mc.Unlock()
+	delete(mc.refreshing, dirPath)
+}
+
+// isRefreshing returns the refresh state for the directory containing fp,
+// or nil if no refresh is active. Caller must hold mc.Lock().
+func (mc *MetaCache) isRefreshingDir(dirPath util.FullPath) *refreshState {
+	return mc.refreshing[dirPath]
+}
+
 func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) error {
 	mc.RLock()
 	defer mc.RUnlock()

weed/mount/meta_cache/meta_cache_init.go

@@ -49,11 +49,6 @@ func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.Full
 	return g.Wait()
 }
 
-// batchInsertSize is the number of entries to accumulate before flushing to LevelDB.
-// 100 provides a balance between memory usage (~100 Entry pointers) and write efficiency
-// (fewer disk syncs). Larger values reduce I/O overhead but increase memory and latency.
-const batchInsertSize = 100
-
 func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerClient, path util.FullPath) error {
 	// Use singleflight to deduplicate concurrent requests for the same path
 	_, err, _ := mc.visitGroup.Do(string(path), func() (interface{}, error) {
@@ -69,42 +64,36 @@ func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerCl
 
 		glog.V(4).Infof("ReadDirAllEntries %s ...", path)
 
-		// Collect entries in batches for efficient LevelDB writes
-		var batch []*filer.Entry
+		// Start buffering subscription events for this directory.
+		// Any events that arrive during the fetch will be replayed
+		// after the snapshot is committed, preventing lost mutations.
+		mc.BeginRefresh(path)
+
+		// Collect all entries from the filer. No lock is held during
+		// network I/O so subscription events can still be buffered.
+		var allEntries []*filer.Entry
 
 		fetchErr := util.Retry("ReadDirAllEntries", func() error {
-			batch = nil // Reset batch on retry, allow GC of previous entries
+			allEntries = nil // Reset on retry
 			return filer_pb.ReadDirAllEntries(ctx, client, path, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
 				entry := filer.FromPbEntry(string(path), pbEntry)
 				if IsHiddenSystemEntry(string(path), entry.Name()) {
 					return nil
 				}
-
-				batch = append(batch, entry)
-
-				// Flush batch when it reaches the threshold
-				// Don't rely on isLast here - hidden entries may cause early return
-				if len(batch) >= batchInsertSize {
-					// No lock needed - LevelDB Write() is thread-safe
-					if err := mc.doBatchInsertEntries(ctx, batch); err != nil {
-						return fmt.Errorf("batch insert for %s: %w", path, err)
-					}
-					// Create new slice to allow GC of flushed entries
-					batch = make([]*filer.Entry, 0, batchInsertSize)
-				}
+				allEntries = append(allEntries, entry)
 				return nil
 			})
 		})
 
 		if fetchErr != nil {
+			mc.CancelRefresh(path)
 			return nil, fmt.Errorf("list %s: %w", path, fetchErr)
 		}
 
-		// Flush any remaining entries in the batch
-		if len(batch) > 0 {
-			if err := mc.doBatchInsertEntries(ctx, batch); err != nil {
-				return nil, fmt.Errorf("batch insert remaining for %s: %w", path, err)
-			}
+		// Atomically replace cached entries with the snapshot and replay
+		// any buffered events that arrived during the fetch.
+		if err := mc.CommitRefresh(ctx, path, allEntries); err != nil {
+			return nil, fmt.Errorf("commit refresh for %s: %w", path, err)
 		}
 		mc.markCachedFn(path)
 		return nil, nil

weed/mount/meta_cache/meta_cache_test.go

@@ -0,0 +1,340 @@
+package meta_cache
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func newTestMetaCache(t *testing.T) (*MetaCache, map[util.FullPath]bool) {
+	t.Helper()
+	uidGidMapper, err := NewUidGidMapper("", "")
+	if err != nil {
+		t.Fatalf("create uid/gid mapper: %v", err)
+	}
+	cached := make(map[util.FullPath]bool)
+	mc := NewMetaCache(
+		filepath.Join(t.TempDir(), "meta"),
+		uidGidMapper,
+		util.FullPath("/"),
+		func(path util.FullPath) { cached[path] = true },
+		func(path util.FullPath) bool { return cached[path] },
+		func(util.FullPath, *filer_pb.Entry) {},
+		nil,
+	)
+	t.Cleanup(func() { mc.Shutdown() })
+	return mc, cached
+}
+
+func makeEntry(dir, name string) *filer.Entry {
+	return &filer.Entry{
+		FullPath: util.NewFullPath(dir, name),
+		Attr:     filer.Attr{Mode: 0644},
+	}
+}
+
+func listEntries(t *testing.T, mc *MetaCache, dir string) []string {
+	t.Helper()
+	var names []string
+	err := mc.ListDirectoryEntries(context.Background(), util.FullPath(dir), "", false, 10000, func(entry *filer.Entry) (bool, error) {
+		names = append(names, entry.Name())
+		return true, nil
+	})
+	if err != nil {
+		t.Fatalf("list %s: %v", dir, err)
+	}
+	return names
+}
+
+// TestCommitRefreshDeleteDuringRefresh verifies that a delete event arriving
+// during a directory refresh is not overwritten by the stale snapshot.
+func TestCommitRefreshDeleteDuringRefresh(t *testing.T) {
+	mc, cached := newTestMetaCache(t)
+	ctx := context.Background()
+	dir := util.FullPath("/testdir")
+	cached[dir] = true
+
+	// Pre-populate the cache with 3 entries
+	for _, name := range []string{"a", "b", "c"} {
+		if err := mc.InsertEntry(ctx, makeEntry("/testdir", name)); err != nil {
+			t.Fatalf("insert %s: %v", name, err)
+		}
+	}
+
+	// Start a refresh — simulate fetching a snapshot that includes all 3
+	mc.BeginRefresh(dir)
+
+	// While refresh is in progress, a subscription event deletes "b"
+	if err := mc.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath("/testdir", "b"), nil); err != nil {
+		t.Fatalf("atomic delete b: %v", err)
+	}
+
+	// Commit the snapshot (which still includes "b") — the buffered delete
+	// should be replayed, removing "b" from the final result
+	snapshot := []*filer.Entry{
+		makeEntry("/testdir", "a"),
+		makeEntry("/testdir", "b"),
+		makeEntry("/testdir", "c"),
+	}
+	if err := mc.CommitRefresh(ctx, dir, snapshot); err != nil {
+		t.Fatalf("commit refresh: %v", err)
+	}
+
+	names := listEntries(t, mc, "/testdir")
+	expected := map[string]bool{"a": true, "c": true}
+	if len(names) != len(expected) {
+		t.Fatalf("expected %v, got %v", expected, names)
+	}
+	for _, n := range names {
+		if !expected[n] {
+			t.Errorf("unexpected entry %q in listing", n)
+		}
+	}
+}
+
+// TestCommitRefreshCreateDuringRefresh verifies that a create event arriving
+// during a directory refresh is preserved after the snapshot is applied.
+func TestCommitRefreshCreateDuringRefresh(t *testing.T) {
+	mc, cached := newTestMetaCache(t)
+	ctx := context.Background()
+	dir := util.FullPath("/testdir")
+	cached[dir] = true
+
+	// Pre-populate with "a"
+	if err := mc.InsertEntry(ctx, makeEntry("/testdir", "a")); err != nil {
+		t.Fatalf("insert a: %v", err)
+	}
+
+	// Start refresh — snapshot will have "a" only
+	mc.BeginRefresh(dir)
+
+	// During refresh, a subscription event creates "d"
+	newEntry := makeEntry("/testdir", "d")
+	if err := mc.AtomicUpdateEntryFromFiler(ctx, "", newEntry); err != nil {
+		t.Fatalf("atomic create d: %v", err)
+	}
+
+	// Commit snapshot (only "a") — buffered create of "d" should be replayed
+	snapshot := []*filer.Entry{makeEntry("/testdir", "a")}
+	if err := mc.CommitRefresh(ctx, dir, snapshot); err != nil {
+		t.Fatalf("commit refresh: %v", err)
+	}
+
+	names := listEntries(t, mc, "/testdir")
+	expected := map[string]bool{"a": true, "d": true}
+	if len(names) != len(expected) {
+		t.Fatalf("expected %v, got %v", expected, names)
+	}
+	for _, n := range names {
+		if !expected[n] {
+			t.Errorf("unexpected entry %q", n)
+		}
+	}
+}
+
+// TestCommitRefreshStaleEntriesRemoved verifies that entries deleted on the
+// filer (absent from the snapshot) are removed from the local cache.
+func TestCommitRefreshStaleEntriesRemoved(t *testing.T) {
+	mc, cached := newTestMetaCache(t)
+	ctx := context.Background()
+	dir := util.FullPath("/testdir")
+	cached[dir] = true
+
+	// Pre-populate with a, b, c
+	for _, name := range []string{"a", "b", "c"} {
+		if err := mc.InsertEntry(ctx, makeEntry("/testdir", name)); err != nil {
+			t.Fatalf("insert %s: %v", name, err)
+		}
+	}
+
+	// Refresh with snapshot that only has "a" and "c" — "b" was deleted on filer
+	mc.BeginRefresh(dir)
+	snapshot := []*filer.Entry{
+		makeEntry("/testdir", "a"),
+		makeEntry("/testdir", "c"),
+	}
+	if err := mc.CommitRefresh(ctx, dir, snapshot); err != nil {
+		t.Fatalf("commit refresh: %v", err)
+	}
+
+	names := listEntries(t, mc, "/testdir")
+	for _, n := range names {
+		if n == "b" {
+			t.Errorf("stale entry 'b' should have been removed by refresh")
+		}
+	}
+	if len(names) != 2 {
+		t.Fatalf("expected [a, c], got %v", names)
+	}
+}
+
+// TestCommitRefreshLocalDeleteBuffered verifies that a local DeleteEntry
+// during refresh is both applied immediately and replayed after commit.
+func TestCommitRefreshLocalDeleteBuffered(t *testing.T) {
+	mc, cached := newTestMetaCache(t)
+	ctx := context.Background()
+	dir := util.FullPath("/testdir")
+	cached[dir] = true
+
+	// Pre-populate
+	for _, name := range []string{"a", "b"} {
+		if err := mc.InsertEntry(ctx, makeEntry("/testdir", name)); err != nil {
+			t.Fatalf("insert %s: %v", name, err)
+		}
+	}
+
+	mc.BeginRefresh(dir)
+
+	// Local delete of "a" — simulates Unlink calling DeleteEntry directly
+	if err := mc.DeleteEntry(ctx, util.NewFullPath("/testdir", "a")); err != nil {
+		t.Fatalf("delete a: %v", err)
+	}
+
+	// Verify "a" is gone immediately (before commit)
+	_, err := mc.FindEntry(ctx, util.NewFullPath("/testdir", "a"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("expected ErrNotFound for deleted entry, got: %v", err)
+	}
+
+	// Commit with snapshot that includes "a" — the buffered delete replays
+	snapshot := []*filer.Entry{
+		makeEntry("/testdir", "a"),
+		makeEntry("/testdir", "b"),
+	}
+	if err := mc.CommitRefresh(ctx, dir, snapshot); err != nil {
+		t.Fatalf("commit refresh: %v", err)
+	}
+
+	// "a" should still be gone after commit
+	names := listEntries(t, mc, "/testdir")
+	for _, n := range names {
+		if n == "a" {
+			t.Errorf("locally deleted entry 'a' reappeared after commit")
+		}
+	}
+}
+
+// TestCommitRefreshLocalCreateBuffered verifies that a local InsertEntry
+// during refresh is preserved after the snapshot is applied.
+func TestCommitRefreshLocalCreateBuffered(t *testing.T) {
+	mc, cached := newTestMetaCache(t)
+	ctx := context.Background()
+	dir := util.FullPath("/testdir")
+	cached[dir] = true
+
+	mc.BeginRefresh(dir)
+
+	// Local create during refresh
+	if err := mc.InsertEntry(ctx, makeEntry("/testdir", "new")); err != nil {
+		t.Fatalf("insert new: %v", err)
+	}
+
+	// Commit with empty snapshot — the buffered create should still appear
+	if err := mc.CommitRefresh(ctx, dir, nil); err != nil {
+		t.Fatalf("commit refresh: %v", err)
+	}
+
+	names := listEntries(t, mc, "/testdir")
+	if len(names) != 1 || names[0] != "new" {
+		t.Fatalf("expected [new], got %v", names)
+	}
+}
+
+// TestCancelRefreshDiscardsBuffer verifies that CancelRefresh discards
+// buffered events and resumes normal event processing.
+func TestCancelRefreshDiscardsBuffer(t *testing.T) {
+	mc, cached := newTestMetaCache(t)
+	ctx := context.Background()
+	dir := util.FullPath("/testdir")
+	cached[dir] = true
+
+	if err := mc.InsertEntry(ctx, makeEntry("/testdir", "a")); err != nil {
+		t.Fatalf("insert a: %v", err)
+	}
+
+	mc.BeginRefresh(dir)
+
+	// Buffer a delete event
+	if err := mc.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath("/testdir", "a"), nil); err != nil {
+		t.Fatalf("atomic delete: %v", err)
+	}
+
+	// Cancel — buffered events are discarded
+	mc.CancelRefresh(dir)
+
+	// "a" should still be in cache since the buffered delete was discarded
+	entry, err := mc.FindEntry(ctx, util.NewFullPath("/testdir", "a"))
+	if err != nil {
+		t.Fatalf("find a: %v", err)
+	}
+	if entry == nil {
+		t.Fatal("entry 'a' should still exist after cancel")
+	}
+
+	// After cancel, events should apply normally (not buffered)
+	if err := mc.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath("/testdir", "a"), nil); err != nil {
+		t.Fatalf("atomic delete after cancel: %v", err)
+	}
+	_, err = mc.FindEntry(ctx, util.NewFullPath("/testdir", "a"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("expected ErrNotFound after direct delete, got: %v", err)
+	}
+}
+
+// TestConcurrentDeletesDuringRefresh simulates the scenario from issue #8442:
+// multiple concurrent deletes racing with a directory refresh.
+func TestConcurrentDeletesDuringRefresh(t *testing.T) {
+	mc, cached := newTestMetaCache(t)
+	ctx := context.Background()
+	dir := util.FullPath("/testdir")
+	cached[dir] = true
+
+	const numFiles = 1000
+
+	// Pre-populate
+	for i := 0; i < numFiles; i++ {
+		name := fmt.Sprintf("file_%04d", i)
+		if err := mc.InsertEntry(ctx, makeEntry("/testdir", name)); err != nil {
+			t.Fatalf("insert %s: %v", name, err)
+		}
+	}
+
+	// Build snapshot (taken before deletes happen)
+	var snapshot []*filer.Entry
+	for i := 0; i < numFiles; i++ {
+		snapshot = append(snapshot, makeEntry("/testdir", fmt.Sprintf("file_%04d", i)))
+	}
+
+	mc.BeginRefresh(dir)
+
+	// Concurrently delete all files via subscription events (simulates
+	// deletes from two mount nodes)
+	var wg sync.WaitGroup
+	for i := 0; i < numFiles; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			name := fmt.Sprintf("file_%04d", idx)
+			fp := util.NewFullPath("/testdir", name)
+			mc.AtomicUpdateEntryFromFiler(ctx, fp, nil)
+		}(i)
+	}
+	wg.Wait()
+
+	// Commit with the stale snapshot — all deletes should be replayed
+	if err := mc.CommitRefresh(ctx, dir, snapshot); err != nil {
+		t.Fatalf("commit refresh: %v", err)
+	}
+
+	names := listEntries(t, mc, "/testdir")
+	if len(names) != 0 {
+		t.Errorf("expected 0 entries after all deletes, got %d: first few: %v",
+			len(names), names[:min(5, len(names))])
+	}
+}

weed/pb/master.proto

@@ -383,6 +383,9 @@ message GetMasterConfigurationResponse {
   string leader = 5;
   uint32 volume_size_limit_m_b = 6;
   bool volume_preallocate = 7;
+  // MIGRATION: fields 8-9 help migrate master.toml [master.maintenance] to admin script plugin. Remove after March 2027.
+  string maintenance_scripts = 8;
+  uint32 maintenance_sleep_minutes = 9;
 }
 
 message ListClusterNodesRequest {

weed/pb/master_pb/master.pb.go

@@ -2973,16 +2973,18 @@ func (*GetMasterConfigurationRequest) Descriptor() ([]byte, []int) {
 }
 
 type GetMasterConfigurationResponse struct {
-	state                  protoimpl.MessageState `protogen:"open.v1"`
-	MetricsAddress         string                 `protobuf:"bytes,1,opt,name=metrics_address,json=metricsAddress,proto3" json:"metrics_address,omitempty"`
-	MetricsIntervalSeconds uint32                 `protobuf:"varint,2,opt,name=metrics_interval_seconds,json=metricsIntervalSeconds,proto3" json:"metrics_interval_seconds,omitempty"`
-	StorageBackends        []*StorageBackend      `protobuf:"bytes,3,rep,name=storage_backends,json=storageBackends,proto3" json:"storage_backends,omitempty"`
-	DefaultReplication     string                 `protobuf:"bytes,4,opt,name=default_replication,json=defaultReplication,proto3" json:"default_replication,omitempty"`
-	Leader                 string                 `protobuf:"bytes,5,opt,name=leader,proto3" json:"leader,omitempty"`
-	VolumeSizeLimitMB      uint32                 `protobuf:"varint,6,opt,name=volume_size_limit_m_b,json=volumeSizeLimitMB,proto3" json:"volume_size_limit_m_b,omitempty"`
-	VolumePreallocate      bool                   `protobuf:"varint,7,opt,name=volume_preallocate,json=volumePreallocate,proto3" json:"volume_preallocate,omitempty"`
-	unknownFields          protoimpl.UnknownFields
-	sizeCache              protoimpl.SizeCache
+	state                   protoimpl.MessageState `protogen:"open.v1"`
+	MetricsAddress          string                 `protobuf:"bytes,1,opt,name=metrics_address,json=metricsAddress,proto3" json:"metrics_address,omitempty"`
+	MetricsIntervalSeconds  uint32                 `protobuf:"varint,2,opt,name=metrics_interval_seconds,json=metricsIntervalSeconds,proto3" json:"metrics_interval_seconds,omitempty"`
+	StorageBackends         []*StorageBackend      `protobuf:"bytes,3,rep,name=storage_backends,json=storageBackends,proto3" json:"storage_backends,omitempty"`
+	DefaultReplication      string                 `protobuf:"bytes,4,opt,name=default_replication,json=defaultReplication,proto3" json:"default_replication,omitempty"`
+	Leader                  string                 `protobuf:"bytes,5,opt,name=leader,proto3" json:"leader,omitempty"`
+	VolumeSizeLimitMB       uint32                 `protobuf:"varint,6,opt,name=volume_size_limit_m_b,json=volumeSizeLimitMB,proto3" json:"volume_size_limit_m_b,omitempty"`
+	VolumePreallocate       bool                   `protobuf:"varint,7,opt,name=volume_preallocate,json=volumePreallocate,proto3" json:"volume_preallocate,omitempty"`
+	MaintenanceScripts      string                 `protobuf:"bytes,8,opt,name=maintenance_scripts,json=maintenanceScripts,proto3" json:"maintenance_scripts,omitempty"`
+	MaintenanceSleepMinutes uint32                 `protobuf:"varint,9,opt,name=maintenance_sleep_minutes,json=maintenanceSleepMinutes,proto3" json:"maintenance_sleep_minutes,omitempty"`
+	unknownFields           protoimpl.UnknownFields
+	sizeCache               protoimpl.SizeCache
 }
 
 func (x *GetMasterConfigurationResponse) Reset() {
@@ -3064,6 +3066,20 @@ func (x *GetMasterConfigurationResponse) GetVolumePreallocate() bool {
 	return false
 }
 
+func (x *GetMasterConfigurationResponse) GetMaintenanceScripts() string {
+	if x != nil {
+		return x.MaintenanceScripts
+	}
+	return ""
+}
+
+func (x *GetMasterConfigurationResponse) GetMaintenanceSleepMinutes() uint32 {
+	if x != nil {
+		return x.MaintenanceSleepMinutes
+	}
+	return 0
+}
+
 type ListClusterNodesRequest struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	ClientType    string                 `protobuf:"bytes,1,opt,name=client_type,json=clientType,proto3" json:"client_type,omitempty"`
@@ -4529,7 +4545,7 @@ const file_master_proto_rawDesc = "" +
 	" \x01(\bR\n" +
 	"isReadonly\"\x1c\n" +
 	"\x1aVolumeMarkReadonlyResponse\"\x1f\n" +
-	"\x1dGetMasterConfigurationRequest\"\xf3\x02\n" +
+	"\x1dGetMasterConfigurationRequest\"\xe0\x03\n" +
 	"\x1eGetMasterConfigurationResponse\x12'\n" +
 	"\x0fmetrics_address\x18\x01 \x01(\tR\x0emetricsAddress\x128\n" +
 	"\x18metrics_interval_seconds\x18\x02 \x01(\rR\x16metricsIntervalSeconds\x12D\n" +
@@ -4537,7 +4553,9 @@ const file_master_proto_rawDesc = "" +
 	"\x13default_replication\x18\x04 \x01(\tR\x12defaultReplication\x12\x16\n" +
 	"\x06leader\x18\x05 \x01(\tR\x06leader\x120\n" +
 	"\x15volume_size_limit_m_b\x18\x06 \x01(\rR\x11volumeSizeLimitMB\x12-\n" +
-	"\x12volume_preallocate\x18\a \x01(\bR\x11volumePreallocate\"q\n" +
+	"\x12volume_preallocate\x18\a \x01(\bR\x11volumePreallocate\x12/\n" +
+	"\x13maintenance_scripts\x18\b \x01(\tR\x12maintenanceScripts\x12:\n" +
+	"\x19maintenance_sleep_minutes\x18\t \x01(\rR\x17maintenanceSleepMinutes\"q\n" +
 	"\x17ListClusterNodesRequest\x12\x1f\n" +
 	"\vclient_type\x18\x01 \x01(\tR\n" +
 	"clientType\x12\x1f\n" +

weed/pb/plugin.proto

@@ -232,6 +232,7 @@ message AdminRuntimeDefaults {
   int32 per_worker_execution_concurrency = 6;
   int32 retry_limit = 7;
   int32 retry_backoff_seconds = 8;
+  int32 job_type_max_runtime_seconds = 9;
 }
 
 message AdminRuntimeConfig {
@@ -243,6 +244,7 @@ message AdminRuntimeConfig {
   int32 per_worker_execution_concurrency = 6;
   int32 retry_limit = 7;
   int32 retry_backoff_seconds = 8;
+  int32 job_type_max_runtime_seconds = 9;
 }
 
 message RunDetectionRequest {

weed/pb/plugin_pb/plugin.pb.go

@@ -2492,6 +2492,7 @@ type AdminRuntimeDefaults struct {
 	PerWorkerExecutionConcurrency int32                  `protobuf:"varint,6,opt,name=per_worker_execution_concurrency,json=perWorkerExecutionConcurrency,proto3" json:"per_worker_execution_concurrency,omitempty"`
 	RetryLimit                    int32                  `protobuf:"varint,7,opt,name=retry_limit,json=retryLimit,proto3" json:"retry_limit,omitempty"`
 	RetryBackoffSeconds           int32                  `protobuf:"varint,8,opt,name=retry_backoff_seconds,json=retryBackoffSeconds,proto3" json:"retry_backoff_seconds,omitempty"`
+	JobTypeMaxRuntimeSeconds      int32                  `protobuf:"varint,9,opt,name=job_type_max_runtime_seconds,json=jobTypeMaxRuntimeSeconds,proto3" json:"job_type_max_runtime_seconds,omitempty"`
 	unknownFields                 protoimpl.UnknownFields
 	sizeCache                     protoimpl.SizeCache
 }
@@ -2582,6 +2583,13 @@ func (x *AdminRuntimeDefaults) GetRetryBackoffSeconds() int32 {
 	return 0
 }
 
+func (x *AdminRuntimeDefaults) GetJobTypeMaxRuntimeSeconds() int32 {
+	if x != nil {
+		return x.JobTypeMaxRuntimeSeconds
+	}
+	return 0
+}
+
 type AdminRuntimeConfig struct {
 	state                         protoimpl.MessageState `protogen:"open.v1"`
 	Enabled                       bool                   `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"`
@@ -2592,6 +2600,7 @@ type AdminRuntimeConfig struct {
 	PerWorkerExecutionConcurrency int32                  `protobuf:"varint,6,opt,name=per_worker_execution_concurrency,json=perWorkerExecutionConcurrency,proto3" json:"per_worker_execution_concurrency,omitempty"`
 	RetryLimit                    int32                  `protobuf:"varint,7,opt,name=retry_limit,json=retryLimit,proto3" json:"retry_limit,omitempty"`
 	RetryBackoffSeconds           int32                  `protobuf:"varint,8,opt,name=retry_backoff_seconds,json=retryBackoffSeconds,proto3" json:"retry_backoff_seconds,omitempty"`
+	JobTypeMaxRuntimeSeconds      int32                  `protobuf:"varint,9,opt,name=job_type_max_runtime_seconds,json=jobTypeMaxRuntimeSeconds,proto3" json:"job_type_max_runtime_seconds,omitempty"`
 	unknownFields                 protoimpl.UnknownFields
 	sizeCache                     protoimpl.SizeCache
 }
@@ -2682,6 +2691,13 @@ func (x *AdminRuntimeConfig) GetRetryBackoffSeconds() int32 {
 	return 0
 }
 
+func (x *AdminRuntimeConfig) GetJobTypeMaxRuntimeSeconds() int32 {
+	if x != nil {
+		return x.JobTypeMaxRuntimeSeconds
+	}
+	return 0
+}
+
 type RunDetectionRequest struct {
 	state              protoimpl.MessageState  `protogen:"open.v1"`
 	RequestId          string                  `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
@@ -4075,7 +4091,7 @@ const file_plugin_proto_rawDesc = "" +
 	"\x06fields\x18\x01 \x03(\v2\x1c.plugin.ValueMap.FieldsEntryR\x06fields\x1aN\n" +
 	"\vFieldsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
-	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\"\xbf\x03\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\"\xff\x03\n" +
 	"\x14AdminRuntimeDefaults\x12\x18\n" +
 	"\aenabled\x18\x01 \x01(\bR\aenabled\x12<\n" +
 	"\x1adetection_interval_seconds\x18\x02 \x01(\x05R\x18detectionIntervalSeconds\x12:\n" +
@@ -4085,7 +4101,8 @@ const file_plugin_proto_rawDesc = "" +
 	" per_worker_execution_concurrency\x18\x06 \x01(\x05R\x1dperWorkerExecutionConcurrency\x12\x1f\n" +
 	"\vretry_limit\x18\a \x01(\x05R\n" +
 	"retryLimit\x122\n" +
-	"\x15retry_backoff_seconds\x18\b \x01(\x05R\x13retryBackoffSeconds\"\xbd\x03\n" +
+	"\x15retry_backoff_seconds\x18\b \x01(\x05R\x13retryBackoffSeconds\x12>\n" +
+	"\x1cjob_type_max_runtime_seconds\x18\t \x01(\x05R\x18jobTypeMaxRuntimeSeconds\"\xfd\x03\n" +
 	"\x12AdminRuntimeConfig\x12\x18\n" +
 	"\aenabled\x18\x01 \x01(\bR\aenabled\x12<\n" +
 	"\x1adetection_interval_seconds\x18\x02 \x01(\x05R\x18detectionIntervalSeconds\x12:\n" +
@@ -4095,7 +4112,8 @@ const file_plugin_proto_rawDesc = "" +
 	" per_worker_execution_concurrency\x18\x06 \x01(\x05R\x1dperWorkerExecutionConcurrency\x12\x1f\n" +
 	"\vretry_limit\x18\a \x01(\x05R\n" +
 	"retryLimit\x122\n" +
-	"\x15retry_backoff_seconds\x18\b \x01(\x05R\x13retryBackoffSeconds\"\xef\x05\n" +
+	"\x15retry_backoff_seconds\x18\b \x01(\x05R\x13retryBackoffSeconds\x12>\n" +
+	"\x1cjob_type_max_runtime_seconds\x18\t \x01(\x05R\x18jobTypeMaxRuntimeSeconds\"\xef\x05\n" +
 	"\x13RunDetectionRequest\x12\x1d\n" +
 	"\n" +
 	"request_id\x18\x01 \x01(\tR\trequestId\x12\x19\n" +

weed/pb/server_address.go

@@ -86,6 +86,23 @@ func (sa ServerAddress) ToGrpcAddress() string {
 	return ServerToGrpcAddress(string(sa))
 }
 
+// ToHost returns the host part only, without any port information.
+func (sa ServerAddress) ToHost() string {
+	httpAddr := sa.ToHttpAddress()
+
+	host, _, err := net.SplitHostPort(httpAddr)
+	if err == nil {
+		return host
+	}
+
+	// Fallback: if parsing fails, it's likely a host without a port.
+	// Handle bracketed IPv6 (e.g., "[::1]" without port) by trimming brackets.
+	if strings.HasPrefix(httpAddr, "[") && strings.HasSuffix(httpAddr, "]") {
+		return httpAddr[1 : len(httpAddr)-1]
+	}
+	return httpAddr
+}
+
 // LookUp may return an error for some records along with successful lookups - make sure you do not
 // discard `addresses` even if `err == nil`
 func (r ServerSrvAddress) LookUp() (addresses []ServerAddress, err error) {

weed/pb/server_address_test.go

@@ -35,6 +35,64 @@ func TestServerAddresses_ToAddressMapOrSrv_shouldHandleIPPortList(t *testing.T)
 	}
 }
 
+func TestServerAddress_ToHost(t *testing.T) {
+	testCases := []struct {
+		name     string
+		address  ServerAddress
+		expected string
+	}{
+		{
+			name:     "hostname with port",
+			address:  ServerAddress("master.example.com:9333"),
+			expected: "master.example.com",
+		},
+		{
+			name:     "IPv4 with port",
+			address:  ServerAddress("192.168.1.1:9333"),
+			expected: "192.168.1.1",
+		},
+		{
+			name:     "IPv6 with port",
+			address:  ServerAddress("[2001:db8::1]:9333"),
+			expected: "2001:db8::1",
+		},
+		{
+			name:     "hostname without port",
+			address:  ServerAddress("master.example.com"),
+			expected: "master.example.com",
+		},
+		{
+			name:     "hostname with port.grpcPort",
+			address:  ServerAddress("master.example.com:443.10443"),
+			expected: "master.example.com",
+		},
+		{
+			name:     "IPv4 with port.grpcPort",
+			address:  ServerAddress("192.168.1.1:8080.18080"),
+			expected: "192.168.1.1",
+		},
+		{
+			name:     "IPv6 with port.grpcPort",
+			address:  ServerAddress("[2001:db8::1]:8080.18080"),
+			expected: "2001:db8::1",
+		},
+		{
+			name:     "bracketed IPv6 without port",
+			address:  ServerAddress("[2001:db8::1]"),
+			expected: "2001:db8::1",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := tc.address.ToHost()
+			if got != tc.expected {
+				t.Errorf("ServerAddress(%q).ToHost() = %q, want %q", tc.address, got, tc.expected)
+			}
+		})
+	}
+}
+
 func TestIPv6ServerAddressFormatting(t *testing.T) {
 	testCases := []struct {
 		name         string

weed/plugin/worker/admin_script_handler.go

@@ -111,6 +111,7 @@ func (h *AdminScriptHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
 			PerWorkerExecutionConcurrency: 1,
 			RetryLimit:                    0,
 			RetryBackoffSeconds:           30,
+			JobTypeMaxRuntimeSeconds:      1800,
 		},
 		WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{},
 	}

weed/plugin/worker/erasure_coding_handler.go

@@ -167,6 +167,7 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
 			PerWorkerExecutionConcurrency: 4,
 			RetryLimit:                    1,
 			RetryBackoffSeconds:           30,
+			JobTypeMaxRuntimeSeconds:      1800,
 		},
 		WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{
 			"quiet_for_seconds": {

weed/plugin/worker/vacuum_handler.go

@@ -152,6 +152,7 @@ func (h *VacuumHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
 			PerWorkerExecutionConcurrency: 4,
 			RetryLimit:                    1,
 			RetryBackoffSeconds:           10,
+			JobTypeMaxRuntimeSeconds:      1800,
 		},
 		WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{
 			"garbage_threshold": {

weed/plugin/worker/volume_balance_handler.go

@@ -144,6 +144,7 @@ func (h *VolumeBalanceHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
 			PerWorkerExecutionConcurrency: 4,
 			RetryLimit:                    1,
 			RetryBackoffSeconds:           15,
+			JobTypeMaxRuntimeSeconds:      1800,
 		},
 		WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{
 			"imbalance_threshold": {

weed/s3api/auth_credentials.go

@@ -21,6 +21,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
@@ -67,6 +68,10 @@ type IdentityAccessManagement struct {
 	// Bucket policy engine for evaluating bucket policies
 	policyEngine *BucketPolicyEngine
 
+	// Cached policy engine for IAM policy fallback evaluation.
+	// Keyed by policy name, kept in sync by PutPolicy/DeletePolicy.
+	iamPolicyEngine *policy_engine.PolicyEngine
+
 	// background polling
 	stopChan     chan struct{}
 	shutdownOnce sync.Once
@@ -658,6 +663,7 @@ func (iam *IdentityAccessManagement) ReplaceS3ApiConfiguration(config *iam_pb.S3
 	iam.nameToIdentity = nameToIdentity
 	iam.accessKeyIdent = accessKeyIdent
 	iam.policies = policies
+	iam.rebuildIAMPolicyEngineLocked()
 
 	// Re-add environment-based identities that were preserved
 	for _, envIdent := range envIdentities {
@@ -914,6 +920,7 @@ func (iam *IdentityAccessManagement) MergeS3ApiConfiguration(config *iam_pb.S3Ap
 	iam.nameToIdentity = nameToIdentity
 	iam.accessKeyIdent = accessKeyIdent
 	iam.policies = policies
+	iam.rebuildIAMPolicyEngineLocked()
 	// Update authentication state based on whether identities exist
 	// Once enabled, keep it enabled (one-way toggle)
 	authJustEnabled := iam.updateAuthenticationState(len(identities))
@@ -1658,6 +1665,51 @@ func determineIAMAuthPath(sessionToken, principal, principalArn string) iamAuthP
 	return iamAuthPathNone
 }
 
+// evaluateIAMPolicies evaluates attached IAM policies for a user identity.
+// Returns true if any matching statement explicitly allows the action.
+// Uses the cached iamPolicyEngine to avoid re-parsing policy JSON on every request.
+func (iam *IdentityAccessManagement) evaluateIAMPolicies(r *http.Request, identity *Identity, action Action, bucket, object string) bool {
+	if identity == nil || len(identity.PolicyNames) == 0 {
+		return false
+	}
+
+	iam.m.RLock()
+	engine := iam.iamPolicyEngine
+	iam.m.RUnlock()
+
+	if engine == nil {
+		return false
+	}
+
+	resource := buildResourceARN(bucket, object)
+	principal := buildPrincipalARN(identity, r)
+	s3Action := ResolveS3Action(r, string(action), bucket, object)
+	explicitAllow := false
+	conditions := policy_engine.ExtractConditionValuesFromRequest(r)
+	for k, v := range policy_engine.ExtractPrincipalVariables(principal) {
+		conditions[k] = v
+	}
+
+	for _, policyName := range identity.PolicyNames {
+		result := engine.EvaluatePolicy(policyName, &policy_engine.PolicyEvaluationArgs{
+			Action:     s3Action,
+			Resource:   resource,
+			Principal:  principal,
+			Conditions: conditions,
+			Claims:     identity.Claims,
+		})
+
+		if result == policy_engine.PolicyResultDeny {
+			return false
+		}
+		if result == policy_engine.PolicyResultAllow {
+			explicitAllow = true
+		}
+	}
+
+	return explicitAllow
+}
+
 // VerifyActionPermission checks if the identity is allowed to perform the action on the resource.
 // It handles both traditional identities (via Actions) and IAM/STS identities (via Policy).
 func (iam *IdentityAccessManagement) VerifyActionPermission(r *http.Request, identity *Identity, action Action, bucket, object string) s3err.ErrorCode {
@@ -1679,6 +1731,7 @@ func (iam *IdentityAccessManagement) VerifyActionPermission(r *http.Request, ide
 		return iam.authorizeWithIAM(r, identity, action, bucket, object)
 	}
 
+	// Traditional actions-based authorization from static S3 config.
 	if len(identity.Actions) > 0 {
 		if !identity.CanDo(action, bucket, object) {
 			return s3err.ErrAccessDenied
@@ -1686,6 +1739,14 @@ func (iam *IdentityAccessManagement) VerifyActionPermission(r *http.Request, ide
 		return s3err.ErrNone
 	}
 
+	// IAM policy fallback for identities with attached policies but without IAM integration.
+	if len(identity.PolicyNames) > 0 {
+		if iam.evaluateIAMPolicies(r, identity, action, bucket, object) {
+			return s3err.ErrNone
+		}
+		return s3err.ErrAccessDenied
+	}
+
 	return s3err.ErrAccessDenied
 }
 
@@ -1752,6 +1813,12 @@ func (iam *IdentityAccessManagement) PutPolicy(name string, content string) erro
 		iam.policies = make(map[string]*iam_pb.Policy)
 	}
 	iam.policies[name] = &iam_pb.Policy{Name: name, Content: content}
+	iam.ensureIAMPolicyEngine()
+	// Remove old entry first so that a parse failure doesn't leave a stale allow.
+	_ = iam.iamPolicyEngine.DeleteBucketPolicy(name)
+	if err := iam.iamPolicyEngine.SetBucketPolicy(name, content); err != nil {
+		glog.Warningf("IAM policy %q is stored but could not be compiled for cache: %v", name, err)
+	}
 	return nil
 }
 
@@ -1770,9 +1837,36 @@ func (iam *IdentityAccessManagement) DeletePolicy(name string) error {
 	iam.m.Lock()
 	defer iam.m.Unlock()
 	delete(iam.policies, name)
+	if iam.iamPolicyEngine != nil {
+		_ = iam.iamPolicyEngine.DeleteBucketPolicy(name)
+	}
 	return nil
 }
 
+// ensureIAMPolicyEngine lazily initializes the shared IAM policy engine.
+// Must be called with iam.m held.
+func (iam *IdentityAccessManagement) ensureIAMPolicyEngine() {
+	if iam.iamPolicyEngine == nil {
+		iam.iamPolicyEngine = policy_engine.NewPolicyEngine()
+	}
+}
+
+// rebuildIAMPolicyEngineLocked rebuilds the entire IAM policy engine cache
+// from the current policies map. Must be called with iam.m held.
+func (iam *IdentityAccessManagement) rebuildIAMPolicyEngineLocked() {
+	if len(iam.policies) == 0 {
+		iam.iamPolicyEngine = nil
+		return
+	}
+	engine := policy_engine.NewPolicyEngine()
+	for name, p := range iam.policies {
+		if err := engine.SetBucketPolicy(name, p.Content); err != nil {
+			glog.Warningf("IAM policy cache rebuild: skipping invalid policy %q: %v", name, err)
+		}
+	}
+	iam.iamPolicyEngine = engine
+}
+
 // ListPolicies lists all policies
 func (iam *IdentityAccessManagement) ListPolicies() []*iam_pb.Policy {
 	iam.m.RLock()

weed/s3api/auth_credentials_test.go

@@ -1,7 +1,9 @@
 package s3api
 
 import (
+	"crypto/tls"
 	"fmt"
+	"net/http"
 	"os"
 	"reflect"
 	"sync"
@@ -9,6 +11,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/credential"
 	. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
 	"github.com/stretchr/testify/assert"
 
@@ -260,6 +263,152 @@ func TestMatchWildcardPattern(t *testing.T) {
 	}
 }
 
+func TestVerifyActionPermissionPolicyFallback(t *testing.T) {
+	buildRequest := func(t *testing.T, method string) *http.Request {
+		t.Helper()
+		req, err := http.NewRequest(method, "http://s3.amazonaws.com/test-bucket/test-object", nil)
+		assert.NoError(t, err)
+		return req
+	}
+
+	t.Run("policy allow grants access", func(t *testing.T) {
+		iam := &IdentityAccessManagement{}
+		err := iam.PutPolicy("allowGet", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}`)
+		assert.NoError(t, err)
+
+		identity := &Identity{
+			Name:        "policy-user",
+			Account:     &AccountAdmin,
+			PolicyNames: []string{"allowGet"},
+		}
+
+		errCode := iam.VerifyActionPermission(buildRequest(t, http.MethodGet), identity, Action(ACTION_READ), "test-bucket", "test-object")
+		assert.Equal(t, s3err.ErrNone, errCode)
+	})
+
+	t.Run("explicit deny overrides allow", func(t *testing.T) {
+		iam := &IdentityAccessManagement{}
+		err := iam.PutPolicy("allowAllGet", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}`)
+		assert.NoError(t, err)
+		err = iam.PutPolicy("denySecret", `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/secret.txt"}]}`)
+		assert.NoError(t, err)
+
+		identity := &Identity{
+			Name:        "policy-user",
+			Account:     &AccountAdmin,
+			PolicyNames: []string{"allowAllGet", "denySecret"},
+		}
+
+		errCode := iam.VerifyActionPermission(buildRequest(t, http.MethodGet), identity, Action(ACTION_READ), "test-bucket", "secret.txt")
+		assert.Equal(t, s3err.ErrAccessDenied, errCode)
+	})
+
+	t.Run("implicit deny when no statement matches", func(t *testing.T) {
+		iam := &IdentityAccessManagement{}
+		err := iam.PutPolicy("allowOtherBucket", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::other-bucket/*"}]}`)
+		assert.NoError(t, err)
+
+		identity := &Identity{
+			Name:        "policy-user",
+			Account:     &AccountAdmin,
+			PolicyNames: []string{"allowOtherBucket"},
+		}
+
+		errCode := iam.VerifyActionPermission(buildRequest(t, http.MethodGet), identity, Action(ACTION_READ), "test-bucket", "test-object")
+		assert.Equal(t, s3err.ErrAccessDenied, errCode)
+	})
+
+	t.Run("invalid policy document does not allow", func(t *testing.T) {
+		iam := &IdentityAccessManagement{}
+		err := iam.PutPolicy("invalidPolicy", "{not-json")
+		assert.NoError(t, err)
+
+		identity := &Identity{
+			Name:        "policy-user",
+			Account:     &AccountAdmin,
+			PolicyNames: []string{"invalidPolicy"},
+		}
+
+		errCode := iam.VerifyActionPermission(buildRequest(t, http.MethodGet), identity, Action(ACTION_READ), "test-bucket", "test-object")
+		assert.Equal(t, s3err.ErrAccessDenied, errCode)
+	})
+
+	t.Run("notresource excludes denied object", func(t *testing.T) {
+		iam := &IdentityAccessManagement{}
+		err := iam.PutPolicy("denyNotResource", `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"s3:GetObject","NotResource":"arn:aws:s3:::test-bucket/public/*"}]}`)
+		assert.NoError(t, err)
+		err = iam.PutPolicy("allowAllGet", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}`)
+		assert.NoError(t, err)
+
+		identity := &Identity{
+			Name:        "policy-user",
+			Account:     &AccountAdmin,
+			PolicyNames: []string{"allowAllGet", "denyNotResource"},
+		}
+
+		errCode := iam.VerifyActionPermission(buildRequest(t, http.MethodGet), identity, Action(ACTION_READ), "test-bucket", "private/secret.txt")
+		assert.Equal(t, s3err.ErrAccessDenied, errCode)
+
+		errCode = iam.VerifyActionPermission(buildRequest(t, http.MethodGet), identity, Action(ACTION_READ), "test-bucket", "public/readme.txt")
+		assert.Equal(t, s3err.ErrNone, errCode)
+	})
+
+	t.Run("condition securetransport enforced", func(t *testing.T) {
+		iam := &IdentityAccessManagement{}
+		err := iam.PutPolicy("allowTLSOnly", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*","Condition":{"Bool":{"aws:SecureTransport":"true"}}}]}`)
+		assert.NoError(t, err)
+
+		identity := &Identity{
+			Name:        "policy-user",
+			Account:     &AccountAdmin,
+			PolicyNames: []string{"allowTLSOnly"},
+		}
+
+		httpReq := buildRequest(t, http.MethodGet)
+		errCode := iam.VerifyActionPermission(httpReq, identity, Action(ACTION_READ), "test-bucket", "test-object")
+		assert.Equal(t, s3err.ErrAccessDenied, errCode)
+
+		httpsReq := buildRequest(t, http.MethodGet)
+		httpsReq.TLS = &tls.ConnectionState{}
+		errCode = iam.VerifyActionPermission(httpsReq, identity, Action(ACTION_READ), "test-bucket", "test-object")
+		assert.Equal(t, s3err.ErrNone, errCode)
+	})
+
+	t.Run("valid policy updated to invalid denies access", func(t *testing.T) {
+		iam := &IdentityAccessManagement{}
+		err := iam.PutPolicy("myPolicy", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}`)
+		assert.NoError(t, err)
+
+		identity := &Identity{
+			Name:        "policy-user",
+			Account:     &AccountAdmin,
+			PolicyNames: []string{"myPolicy"},
+		}
+
+		errCode := iam.VerifyActionPermission(buildRequest(t, http.MethodGet), identity, Action(ACTION_READ), "test-bucket", "test-object")
+		assert.Equal(t, s3err.ErrNone, errCode)
+
+		// Update to invalid JSON — should revoke access.
+		err = iam.PutPolicy("myPolicy", "{broken")
+		assert.NoError(t, err)
+
+		errCode = iam.VerifyActionPermission(buildRequest(t, http.MethodGet), identity, Action(ACTION_READ), "test-bucket", "test-object")
+		assert.Equal(t, s3err.ErrAccessDenied, errCode)
+	})
+
+	t.Run("actions based path still works", func(t *testing.T) {
+		iam := &IdentityAccessManagement{}
+		identity := &Identity{
+			Name:    "legacy-user",
+			Account: &AccountAdmin,
+			Actions: []Action{"Read:test-bucket"},
+		}
+
+		errCode := iam.VerifyActionPermission(buildRequest(t, http.MethodGet), identity, Action(ACTION_READ), "test-bucket", "any-object")
+		assert.Equal(t, s3err.ErrNone, errCode)
+	})
+}
+
 type LoadS3ApiConfigurationTestCase struct {
 	pbAccount   *iam_pb.Account
 	pbIdent     *iam_pb.Identity

weed/s3api/s3_action_resolver.go

@@ -159,6 +159,13 @@ func resolveFromQueryParameters(query url.Values, method string, hasObject bool)
 		}
 	}
 
+	// GetObjectAttributes (object-level only)
+	// Must be checked before versionId, because GET /bucket/key?attributes&versionId=xyz
+	// is a GetObjectAttributes request, not a GetObjectVersion request
+	if hasObject && query.Has("attributes") && method == http.MethodGet {
+		return s3_constants.S3_ACTION_GET_OBJECT_ATTRIBUTES
+	}
+
 	// Versioning operations - distinguish between versionId (specific version) and versions (list versions)
 	// versionId: Used to access/delete a specific version of an object (e.g., GET /bucket/key?versionId=xyz)
 	if query.Has("versionId") {

weed/s3api/s3_action_resolver_test.go

@@ -0,0 +1,62 @@
+package s3api
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+)
+
+func TestResolveS3Action_AttributesBeforeVersionId(t *testing.T) {
+	tests := []struct {
+		name       string
+		query      string
+		method     string
+		baseAction string
+		object     string
+		want       string
+	}{
+		{
+			name:       "attributes only",
+			query:      "attributes",
+			method:     http.MethodGet,
+			baseAction: s3_constants.ACTION_READ,
+			object:     "key",
+			want:       s3_constants.S3_ACTION_GET_OBJECT_ATTRIBUTES,
+		},
+		{
+			name:       "attributes with versionId",
+			query:      "attributes&versionId=abc123",
+			method:     http.MethodGet,
+			baseAction: s3_constants.ACTION_READ,
+			object:     "key",
+			want:       s3_constants.S3_ACTION_GET_OBJECT_ATTRIBUTES,
+		},
+		{
+			name:       "versionId only GET",
+			query:      "versionId=abc123",
+			method:     http.MethodGet,
+			baseAction: s3_constants.ACTION_READ,
+			object:     "key",
+			want:       s3_constants.S3_ACTION_GET_OBJECT_VERSION,
+		},
+		{
+			name:       "versionId only DELETE",
+			query:      "versionId=abc123",
+			method:     http.MethodDelete,
+			baseAction: s3_constants.ACTION_WRITE,
+			object:     "key",
+			want:       s3_constants.S3_ACTION_DELETE_OBJECT_VERSION,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r, _ := http.NewRequest(tt.method, "http://localhost/bucket/"+tt.object+"?"+tt.query, nil)
+			got := ResolveS3Action(r, tt.baseAction, "bucket", tt.object)
+			if got != tt.want {
+				t.Errorf("ResolveS3Action() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}

weed/s3api/s3_constants/header.go

@@ -45,6 +45,12 @@ const (
 	AmzObjectTaggingDirective = "X-Amz-Tagging-Directive"
 	AmzTagCount               = "x-amz-tagging-count"
 
+	// GetObjectAttributes headers
+	AmzObjectAttributes = "X-Amz-Object-Attributes"
+	AmzMaxParts         = "X-Amz-Max-Parts"
+	AmzPartNumberMarker = "X-Amz-Part-Number-Marker"
+	AmzDeleteMarker     = "X-Amz-Delete-Marker"
+
 	SeaweedFSUploadId                = "X-Seaweedfs-Upload-Id"
 	SeaweedFSMultipartPartsCount     = "X-Seaweedfs-Multipart-Parts-Count"
 	SeaweedFSMultipartPartBoundaries = "X-Seaweedfs-Multipart-Part-Boundaries" // JSON: [{part:1,start:0,end:2,etag:"abc"},{part:2,start:2,end:3,etag:"def"}]

weed/s3api/s3_constants/s3_action_strings.go

@@ -8,7 +8,8 @@ const (
 	S3_ACTION_PUT_OBJECT            = "s3:PutObject"
 	S3_ACTION_DELETE_OBJECT         = "s3:DeleteObject"
 	S3_ACTION_DELETE_OBJECT_VERSION = "s3:DeleteObjectVersion"
-	S3_ACTION_GET_OBJECT_VERSION    = "s3:GetObjectVersion"
+	S3_ACTION_GET_OBJECT_VERSION     = "s3:GetObjectVersion"
+	S3_ACTION_GET_OBJECT_ATTRIBUTES = "s3:GetObjectAttributes"
 
 	// Object ACL operations
 	S3_ACTION_GET_OBJECT_ACL = "s3:GetObjectAcl"

weed/s3api/s3api_object_handlers_attributes.go

@@ -0,0 +1,326 @@
+package s3api
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+)
+
+const maxPartsListDefault = 1000
+
+// GetObjectAttributesResponse is the XML response for GetObjectAttributes.
+type GetObjectAttributesResponse struct {
+	XMLName      xml.Name                  `xml:"GetObjectAttributesResponse"`
+	ETag         string                    `xml:"ETag,omitempty"`
+	Checksum     *ObjectAttributesChecksum `xml:"Checksum,omitempty"`
+	ObjectParts  *ObjectAttributesParts    `xml:"ObjectParts,omitempty"`
+	StorageClass string                    `xml:"StorageClass,omitempty"`
+	ObjectSize   *int64                    `xml:"ObjectSize,omitempty"`
+}
+
+// ObjectAttributesChecksum holds checksum info for GetObjectAttributes.
+type ObjectAttributesChecksum struct {
+	ChecksumCRC32  string `xml:"ChecksumCRC32,omitempty"`
+	ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
+	ChecksumSHA1   string `xml:"ChecksumSHA1,omitempty"`
+	ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
+}
+
+// ObjectAttributesParts holds parts info for GetObjectAttributes.
+type ObjectAttributesParts struct {
+	IsTruncated          bool                    `xml:"IsTruncated"`
+	MaxParts             int                     `xml:"MaxParts"`
+	NextPartNumberMarker int                     `xml:"NextPartNumberMarker"`
+	PartNumberMarker     int                     `xml:"PartNumberMarker"`
+	TotalPartsCount      int                     `xml:"PartsCount"`
+	Parts                []*ObjectAttributesPart `xml:"Part"`
+}
+
+// ObjectAttributesPart holds info for a single part.
+type ObjectAttributesPart struct {
+	PartNumber int   `xml:"PartNumber"`
+	Size       int64 `xml:"Size"`
+}
+
+// parseObjectAttributes parses the X-Amz-Object-Attributes header.
+func parseObjectAttributes(h http.Header) map[string]struct{} {
+	attrs := make(map[string]struct{})
+	for _, headerVal := range h.Values(s3_constants.AmzObjectAttributes) {
+		for _, v := range strings.Split(strings.TrimSpace(headerVal), ",") {
+			v = strings.TrimSpace(v)
+			if v != "" {
+				attrs[v] = struct{}{}
+			}
+		}
+	}
+	return attrs
+}
+
+// validateObjectAttributes checks that all requested attributes are valid.
+func validateObjectAttributes(attrs map[string]struct{}) bool {
+	for attr := range attrs {
+		switch attr {
+		case "ETag", "Checksum", "StorageClass", "ObjectSize", "ObjectParts":
+		default:
+			return false
+		}
+	}
+	return true
+}
+
+func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) {
+	bucket, object := s3_constants.GetBucketAndObject(r)
+	glog.V(3).Infof("GetObjectAttributesHandler %s %s", bucket, object)
+
+	// Parse and validate requested attributes
+	requestedAttrs := parseObjectAttributes(r.Header)
+	if len(requestedAttrs) == 0 || !validateObjectAttributes(requestedAttrs) {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidAttributeName)
+		return
+	}
+
+	// Parse optional pagination headers for ObjectParts
+	maxParts := maxPartsListDefault
+	if v := r.Header.Get(s3_constants.AmzMaxParts); v != "" {
+		parsed, err := strconv.Atoi(v)
+		if err != nil || parsed < 0 {
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts)
+			return
+		}
+		maxParts = parsed
+		if maxParts > maxPartsListDefault {
+			maxParts = maxPartsListDefault
+		}
+	}
+	partNumberMarker := 0
+	if v := r.Header.Get(s3_constants.AmzPartNumberMarker); v != "" {
+		parsed, err := strconv.Atoi(v)
+		if err != nil || parsed < 0 {
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPartNumberMarker)
+			return
+		}
+		partNumberMarker = parsed
+	}
+
+	// Check for specific version ID
+	versionId := r.URL.Query().Get("versionId")
+
+	var (
+		entry *filer_pb.Entry
+		err   error
+	)
+
+	versioningConfigured, err := s3a.isVersioningConfigured(bucket)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			return
+		}
+		glog.Errorf("Error checking versioning for bucket %s: %v", bucket, err)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	if versioningConfigured {
+		var targetVersionId string
+
+		if versionId != "" {
+			entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId)
+			if err != nil {
+				s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+				return
+			}
+			targetVersionId = versionId
+		} else {
+			bucketDir := s3a.bucketDir(bucket)
+			normalizedObject := s3_constants.NormalizeObjectKey(object)
+			versionsDir := normalizedObject + s3_constants.VersionsFolder
+
+			versionsEntry, versionsErr := s3a.getEntry(bucketDir, versionsDir)
+			if versionsErr == nil && versionsEntry != nil {
+				entry, err = s3a.getLatestObjectVersion(bucket, object)
+				if err != nil {
+					s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+					return
+				}
+			} else if errors.Is(versionsErr, filer_pb.ErrNotFound) {
+				regularEntry, regularErr := s3a.getEntry(bucketDir, normalizedObject)
+				if regularErr == nil && regularEntry != nil {
+					entry = regularEntry
+					targetVersionId = "null"
+				} else {
+					s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+					return
+				}
+			} else {
+				entry, err = s3a.getLatestObjectVersion(bucket, object)
+				if err != nil {
+					s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+					return
+				}
+			}
+
+			if targetVersionId == "" {
+				if entry.Extended != nil {
+					if versionIdBytes, exists := entry.Extended[s3_constants.ExtVersionIdKey]; exists {
+						targetVersionId = string(versionIdBytes)
+					}
+				}
+				if targetVersionId == "" {
+					targetVersionId = "null"
+				}
+			}
+		}
+
+		// Check for delete marker
+		if entry.Extended != nil {
+			if deleteMarker, exists := entry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
+				w.Header().Set(s3_constants.AmzDeleteMarker, "true")
+				s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+				return
+			}
+		}
+
+		w.Header().Set("x-amz-version-id", targetVersionId)
+	} else {
+		entry, err = s3a.fetchObjectEntry(bucket, object)
+		if err != nil {
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+			return
+		}
+		if entry == nil {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+			return
+		}
+	}
+
+	// Evaluate conditional headers against the resolved entry (after version resolution)
+	// This ensures conditions are checked against the correct version, not always the latest
+	if s3a.hasConditionalHeaders(r) {
+		headers, errCode := parseConditionalHeaders(r)
+		if errCode != s3err.ErrNone {
+			s3err.WriteErrorResponse(w, r, errCode)
+			return
+		}
+		result := s3a.validateConditionalHeadersForReads(r, headers, entry, bucket, object)
+		if result.ErrorCode != s3err.ErrNone {
+			glog.V(3).Infof("GetObjectAttributesHandler: Conditional header check failed for %s/%s with error %v", bucket, object, result.ErrorCode)
+			if result.ErrorCode == s3err.ErrNotModified && result.ETag != "" {
+				w.Header().Set("ETag", result.ETag)
+			}
+			s3err.WriteErrorResponse(w, r, result.ErrorCode)
+			return
+		}
+	}
+
+	if entry == nil {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	// Build response with only requested attributes
+	resp := &GetObjectAttributesResponse{}
+
+	if _, ok := requestedAttrs["ETag"]; ok {
+		etag := s3a.getObjectETag(entry)
+		resp.ETag = strings.Trim(etag, `"`)
+	}
+
+	if _, ok := requestedAttrs["StorageClass"]; ok {
+		storageClass := "STANDARD"
+		if entry.Extended != nil {
+			if sc, exists := entry.Extended[s3_constants.AmzStorageClass]; exists && len(sc) > 0 {
+				storageClass = string(sc)
+			}
+		}
+		resp.StorageClass = storageClass
+	}
+
+	// Checksum: accepted in validation so clients don't get a 400, but SeaweedFS
+	// does not yet store S3 checksums (CRC32, CRC32C, SHA1, SHA256), so
+	// resp.Checksum is intentionally left nil. When checksum storage is added,
+	// populate resp.Checksum here.
+
+	if _, ok := requestedAttrs["ObjectSize"]; ok {
+		var size int64
+		if entry.Attributes != nil {
+			size = int64(entry.Attributes.FileSize)
+		}
+		resp.ObjectSize = &size
+	}
+
+	if _, ok := requestedAttrs["ObjectParts"]; ok {
+		resp.ObjectParts = s3a.buildObjectAttributesParts(entry, maxParts, partNumberMarker)
+	}
+
+	// Set Last-Modified, remove Content-Type
+	if entry.Attributes != nil && entry.Attributes.Mtime > 0 {
+		w.Header().Set("Last-Modified", time.Unix(entry.Attributes.Mtime, 0).UTC().Format(http.TimeFormat))
+	}
+	w.Header().Del("Content-Type")
+
+	writeSuccessResponseXML(w, r, resp)
+}
+
+// buildObjectAttributesParts builds the ObjectParts section from multipart metadata.
+func (s3a *S3ApiServer) buildObjectAttributesParts(entry *filer_pb.Entry, maxParts, partNumberMarker int) *ObjectAttributesParts {
+	if entry.Extended == nil {
+		return nil
+	}
+
+	boundariesJSON, exists := entry.Extended[s3_constants.SeaweedFSMultipartPartBoundaries]
+	if !exists {
+		return nil
+	}
+
+	var boundaries []PartBoundaryInfo
+	if err := json.Unmarshal(boundariesJSON, &boundaries); err != nil {
+		glog.Warningf("GetObjectAttributes: failed to unmarshal part boundaries: %v", err)
+		return nil
+	}
+
+	if len(boundaries) == 0 {
+		return nil
+	}
+
+	parts := &ObjectAttributesParts{
+		PartNumberMarker: partNumberMarker,
+		MaxParts:         maxParts,
+		TotalPartsCount:  len(boundaries),
+	}
+
+	chunks := entry.GetChunks()
+	for _, b := range boundaries {
+		if b.PartNumber <= partNumberMarker {
+			continue
+		}
+		if len(parts.Parts) >= maxParts {
+			parts.IsTruncated = true
+			break
+		}
+
+		var partSize int64
+		if b.StartChunk >= 0 && b.EndChunk >= 0 && b.StartChunk < len(chunks) && b.EndChunk <= len(chunks) && b.StartChunk < b.EndChunk {
+			for ci := b.StartChunk; ci < b.EndChunk; ci++ {
+				partSize += int64(chunks[ci].Size)
+			}
+		}
+
+		parts.NextPartNumberMarker = b.PartNumber
+		parts.Parts = append(parts.Parts, &ObjectAttributesPart{
+			PartNumber: b.PartNumber,
+			Size:       partSize,
+		})
+	}
+
+	return parts
+}

weed/s3api/s3api_object_handlers_put.go

@@ -1658,7 +1658,11 @@ func (s3a *S3ApiServer) validateConditionalHeaders(r *http.Request, headers cond
 	objectExists := entry != nil
 
 	// For PUT requests, all specified conditions must be met.
-	// The evaluation order follows AWS S3 behavior for consistency.
+	// The evaluation order follows RFC 7232 Section 6 and AWS S3 behavior:
+	//   1. If-Match
+	//   2. If-Unmodified-Since (ignored when If-Match is present, per RFC 7232 Section 3.4)
+	//   3. If-None-Match
+	//   4. If-Modified-Since (ignored when If-None-Match is present, per RFC 7232 Section 3.3)
 
 	// 1. Check If-Match
 	if headers.ifMatch != "" {
@@ -1679,7 +1683,9 @@ func (s3a *S3ApiServer) validateConditionalHeaders(r *http.Request, headers cond
 	}
 
 	// 2. Check If-Unmodified-Since
-	if !headers.ifUnmodifiedSince.IsZero() {
+	// Per RFC 7232 Section 3.4: "A recipient MUST ignore If-Unmodified-Since
+	// if the request contains an If-Match header field"
+	if !headers.ifUnmodifiedSince.IsZero() && headers.ifMatch == "" {
 		if objectExists {
 			if entry.Attributes != nil {
 				objectModTime := time.Unix(entry.Attributes.Mtime, 0)
@@ -1713,7 +1719,9 @@ func (s3a *S3ApiServer) validateConditionalHeaders(r *http.Request, headers cond
 	}
 
 	// 4. Check If-Modified-Since
-	if !headers.ifModifiedSince.IsZero() {
+	// Per RFC 7232 Section 3.3: "A recipient MUST ignore If-Modified-Since
+	// if the request contains an If-None-Match header field"
+	if !headers.ifModifiedSince.IsZero() && headers.ifNoneMatch == "" {
 		if objectExists {
 			if entry.Attributes != nil {
 				objectModTime := time.Unix(entry.Attributes.Mtime, 0)
@@ -1802,7 +1810,11 @@ func (s3a *S3ApiServer) validateConditionalHeadersForReads(r *http.Request, head
 	}
 
 	// Object exists - check all conditions
-	// The evaluation order follows AWS S3 behavior for consistency.
+	// The evaluation order follows RFC 7232 Section 6 and AWS S3 behavior:
+	//   1. If-Match
+	//   2. If-Unmodified-Since (ignored when If-Match is present, per RFC 7232 Section 3.4)
+	//   3. If-None-Match
+	//   4. If-Modified-Since (ignored when If-None-Match is present, per RFC 7232 Section 3.3)
 
 	// 1. Check If-Match (412 Precondition Failed if fails)
 	if headers.ifMatch != "" {
@@ -1821,7 +1833,9 @@ func (s3a *S3ApiServer) validateConditionalHeadersForReads(r *http.Request, head
 	}
 
 	// 2. Check If-Unmodified-Since (412 Precondition Failed if fails)
-	if !headers.ifUnmodifiedSince.IsZero() {
+	// Per RFC 7232 Section 3.4: "A recipient MUST ignore If-Unmodified-Since
+	// if the request contains an If-Match header field"
+	if !headers.ifUnmodifiedSince.IsZero() && headers.ifMatch == "" {
 		objectModTime := time.Unix(entry.Attributes.Mtime, 0)
 		if objectModTime.After(headers.ifUnmodifiedSince) {
 			glog.V(3).Infof("validateConditionalHeadersForReads: If-Unmodified-Since failed - object modified after %s", r.Header.Get(s3_constants.IfUnmodifiedSince))
@@ -1848,7 +1862,9 @@ func (s3a *S3ApiServer) validateConditionalHeadersForReads(r *http.Request, head
 	}
 
 	// 4. Check If-Modified-Since (304 Not Modified if fails)
-	if !headers.ifModifiedSince.IsZero() {
+	// Per RFC 7232 Section 3.3: "A recipient MUST ignore If-Modified-Since
+	// if the request contains an If-None-Match header field"
+	if !headers.ifModifiedSince.IsZero() && headers.ifNoneMatch == "" {
 		objectModTime := time.Unix(entry.Attributes.Mtime, 0)
 		if !objectModTime.After(headers.ifModifiedSince) {
 			// Use production getObjectETag method

weed/s3api/s3api_server.go

@@ -605,6 +605,8 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 		// ListMultipartUploads
 		bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListMultipartUploadsHandler, ACTION_READ)), "GET")).Queries("uploads", "")
 
+		// GetObjectAttributes
+		bucket.Methods(http.MethodGet).Path(objectPath).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectAttributesHandler, ACTION_READ)), "GET")).Queries("attributes", "")
 		// GetObjectTagging
 		bucket.Methods(http.MethodGet).Path(objectPath).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectTaggingHandler, ACTION_READ)), "GET")).Queries("tagging", "")
 		// PutObjectTagging

weed/s3api/s3err/s3api_errors.go

@@ -143,6 +143,8 @@ const (
 	// Bucket encryption errors
 	ErrNoSuchBucketEncryptionConfiguration
 	ErrInvalidStorageClass
+
+	ErrInvalidAttributeName
 )
 
 // Error message constants for checksum validation
@@ -600,6 +602,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The storage class you specified is not valid",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+
+	ErrInvalidAttributeName: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid attribute name specified",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.

weed/security/tls.go

@@ -1,9 +1,11 @@
 package security
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"net"
 	"os"
 	"slices"
 	"strings"
@@ -12,6 +14,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
 	"google.golang.org/grpc/security/advancedtls"
@@ -24,6 +27,37 @@ type Authenticator struct {
 	AllowedCommonNames    map[string]bool
 }
 
+// SNIStrippingTransportCredentials wraps another TransportCredentials
+// and strips the port from the authority in ClientHandshake to prevent
+// advancedtls from using the full "host:port" as ServerName in SNI.
+type SNIStrippingTransportCredentials struct {
+	creds credentials.TransportCredentials
+}
+
+func (s *SNIStrippingTransportCredentials) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	host, _, err := net.SplitHostPort(authority)
+	if err == nil {
+		authority = host
+	}
+	return s.creds.ClientHandshake(ctx, authority, rawConn)
+}
+
+func (s *SNIStrippingTransportCredentials) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return s.creds.ServerHandshake(rawConn)
+}
+
+func (s *SNIStrippingTransportCredentials) Info() credentials.ProtocolInfo {
+	return s.creds.Info()
+}
+
+func (s *SNIStrippingTransportCredentials) Clone() credentials.TransportCredentials {
+	return &SNIStrippingTransportCredentials{creds: s.creds.Clone()}
+}
+
+func (s *SNIStrippingTransportCredentials) OverrideServerName(serverNameOverride string) error {
+	return s.creds.OverrideServerName(serverNameOverride)
+}
+
 func LoadServerTLS(config *util.ViperProxy, component string) (grpc.ServerOption, grpc.ServerOption) {
 	if config == nil {
 		return nil, nil
@@ -151,7 +185,8 @@ func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption {
 		glog.Warningf("advancedtls.NewClientCreds(%v) failed: %v", options, err)
 		return grpc.WithTransportCredentials(insecure.NewCredentials())
 	}
-	return grpc.WithTransportCredentials(ta)
+	wrapped := &SNIStrippingTransportCredentials{creds: ta}
+	return grpc.WithTransportCredentials(wrapped)
 }
 
 func LoadClientTLSHTTP(clientCertFile string) *tls.Config {

weed/security/tls_sni_test.go

@@ -0,0 +1,219 @@
+package security
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"math/big"
+	"net"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/security/advancedtls"
+)
+
+func generateSelfSignedCert(t *testing.T) (tls.Certificate, *x509.CertPool) {
+	t.Helper()
+
+	caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("generate CA key: %v", err)
+	}
+	caTemplate := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "Test CA"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(time.Hour),
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+	caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey)
+	if err != nil {
+		t.Fatalf("create CA cert: %v", err)
+	}
+	caCert, err := x509.ParseCertificate(caDER)
+	if err != nil {
+		t.Fatalf("parse CA cert: %v", err)
+	}
+
+	leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("generate leaf key: %v", err)
+	}
+	leafTemplate := &x509.Certificate{
+		SerialNumber: big.NewInt(2),
+		Subject:      pkix.Name{CommonName: "localhost"},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		DNSNames:     []string{"localhost"},
+		IPAddresses:  []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
+	}
+	leafDER, err := x509.CreateCertificate(rand.Reader, leafTemplate, caCert, &leafKey.PublicKey, caKey)
+	if err != nil {
+		t.Fatalf("create leaf cert: %v", err)
+	}
+
+	leafCert := tls.Certificate{
+		Certificate: [][]byte{leafDER},
+		PrivateKey:  leafKey,
+	}
+
+	pool := x509.NewCertPool()
+	pool.AddCert(caCert)
+
+	return leafCert, pool
+}
+
+func startTLSListenerCapturingSNI(t *testing.T, cert tls.Certificate) (string, <-chan string) {
+	t.Helper()
+
+	sniChan := make(chan string, 1)
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) {
+			select {
+			case sniChan <- hello.ServerName:
+				close(sniChan)
+			default:
+			}
+			return nil, nil
+		},
+	}
+
+	ln, err := tls.Listen("tcp", "127.0.0.1:0", tlsConfig)
+	if err != nil {
+		t.Fatalf("tls.Listen: %v", err)
+	}
+	t.Cleanup(func() { ln.Close() })
+
+	go func() {
+		conn, err := ln.Accept()
+		if err != nil {
+			return
+		}
+		defer conn.Close()
+		buf := make([]byte, 1)
+		conn.Read(buf)
+	}()
+
+	return ln.Addr().String(), sniChan
+}
+
+func newSNIStrippingCreds(t *testing.T, cert tls.Certificate, pool *x509.CertPool) credentials.TransportCredentials {
+	t.Helper()
+	clientCreds, err := advancedtls.NewClientCreds(&advancedtls.Options{
+		IdentityOptions: advancedtls.IdentityCertificateOptions{
+			Certificates: []tls.Certificate{cert},
+		},
+		RootOptions: advancedtls.RootCertificateOptions{
+			RootCertificates: pool,
+		},
+		VerificationType: advancedtls.CertVerification,
+		AdditionalPeerVerification: func(params *advancedtls.HandshakeVerificationInfo) (*advancedtls.PostHandshakeVerificationResults, error) {
+			return &advancedtls.PostHandshakeVerificationResults{}, nil
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewClientCreds: %v", err)
+	}
+	return &SNIStrippingTransportCredentials{creds: clientCreds}
+}
+
+func TestSNI_HostnameStripsPort(t *testing.T) {
+	cert, pool := generateSelfSignedCert(t)
+	wrapped := newSNIStrippingCreds(t, cert, pool)
+	addr, sniChan := startTLSListenerCapturingSNI(t, cert)
+
+	conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
+	if err != nil {
+		t.Fatalf("dial: %v", err)
+	}
+	defer conn.Close()
+
+	// gRPC passes "host:port" as authority; SNI wrapper strips the port
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, _, err = wrapped.ClientHandshake(ctx, "localhost:"+portFromAddr(addr), conn)
+	if err != nil {
+		t.Fatalf("ClientHandshake: %v", err)
+	}
+
+	select {
+	case sni := <-sniChan:
+		if sni != "localhost" {
+			t.Errorf("SNI = %q, want %q", sni, "localhost")
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for SNI")
+	}
+}
+
+func TestSNI_IPAddressEmptySNI(t *testing.T) {
+	cert, pool := generateSelfSignedCert(t)
+	wrapped := newSNIStrippingCreds(t, cert, pool)
+	addr, sniChan := startTLSListenerCapturingSNI(t, cert)
+
+	conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
+	if err != nil {
+		t.Fatalf("dial: %v", err)
+	}
+	defer conn.Close()
+
+	// RFC 6066: IP addresses MUST NOT be sent as SNI; Go's TLS sends empty ServerName for IPs
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, _, err = wrapped.ClientHandshake(ctx, "127.0.0.1:"+portFromAddr(addr), conn)
+	if err != nil {
+		t.Fatalf("ClientHandshake: %v", err)
+	}
+
+	select {
+	case sni := <-sniChan:
+		if sni != "" {
+			t.Errorf("SNI = %q, want empty string", sni)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for SNI")
+	}
+}
+
+func TestSNI_IPv6AddressEmptySNI(t *testing.T) {
+	cert, pool := generateSelfSignedCert(t)
+	wrapped := newSNIStrippingCreds(t, cert, pool)
+	addr, sniChan := startTLSListenerCapturingSNI(t, cert)
+
+	conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
+	if err != nil {
+		t.Fatalf("dial: %v", err)
+	}
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, _, err = wrapped.ClientHandshake(ctx, "[::1]:"+portFromAddr(addr), conn)
+	if err != nil {
+		t.Fatalf("ClientHandshake: %v", err)
+	}
+
+	select {
+	case sni := <-sniChan:
+		if sni != "" {
+			t.Errorf("SNI = %q, want empty string", sni)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for SNI")
+	}
+}
+
+func portFromAddr(addr string) string {
+	_, port, _ := net.SplitHostPort(addr)
+	return port
+}

weed/server/master_grpc_server.go

@@ -11,6 +11,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
 
+	"github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
@@ -112,7 +113,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 			// tell the volume servers about the leader
 			newLeader, err := ms.Topo.MaybeLeader()
 			if err != nil || newLeader == "" {
-				glog.Warningf("SendHeartbeat find leader: %v", err)
+				glog.Warningf("SendHeartbeat find leader: %v, %v", newLeader, err)
 				return raft.NotLeaderError
 			}
 			if err := stream.Send(&master_pb.HeartbeatResponse{
@@ -454,14 +455,24 @@ func (ms *MasterServer) GetMasterConfiguration(ctx context.Context, req *master_
 	// tell the volume servers about the leader
 	leader, _ := ms.Topo.Leader()
 
+	// MIGRATION: expose maintenance scripts for admin server seeding. Remove after March 2027.
+	v := util.GetViper()
+	maintenanceScripts := v.GetString("master.maintenance.scripts")
+	maintenanceSleepMinutes := v.GetInt("master.maintenance.sleep_minutes")
+	if maintenanceSleepMinutes <= 0 {
+		maintenanceSleepMinutes = maintenance.DefaultMaintenanceSleepMinutes
+	}
+
 	resp := &master_pb.GetMasterConfigurationResponse{
-		MetricsAddress:         ms.option.MetricsAddress,
-		MetricsIntervalSeconds: uint32(ms.option.MetricsIntervalSec),
-		StorageBackends:        backend.ToPbStorageBackends(),
-		DefaultReplication:     ms.option.DefaultReplicaPlacement,
-		VolumeSizeLimitMB:      uint32(ms.option.VolumeSizeLimitMB),
-		VolumePreallocate:      ms.option.VolumePreallocate,
-		Leader:                 string(leader),
+		MetricsAddress:          ms.option.MetricsAddress,
+		MetricsIntervalSeconds:  uint32(ms.option.MetricsIntervalSec),
+		StorageBackends:         backend.ToPbStorageBackends(),
+		DefaultReplication:      ms.option.DefaultReplicaPlacement,
+		VolumeSizeLimitMB:       uint32(ms.option.VolumeSizeLimitMB),
+		VolumePreallocate:       ms.option.VolumePreallocate,
+		Leader:                  string(leader),
+		MaintenanceScripts:      maintenanceScripts,
+		MaintenanceSleepMinutes: uint32(maintenanceSleepMinutes),
 	}
 
 	return resp, nil

weed/server/master_server.go

@@ -333,17 +333,26 @@ func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
+func (ms *MasterServer) isAdminServerConnected() bool {
+	if ms == nil || ms.adminLocks == nil {
+		return false
+	}
+	_, _, isLocked := ms.adminLocks.isLocked(cluster.AdminServerPresenceLockName)
+	return isLocked
+}
+
 func (ms *MasterServer) startAdminScripts() {
 	v := util.GetViper()
-	v.SetDefault("master.maintenance.scripts", maintenance.DefaultMasterMaintenanceScripts)
 	adminScripts := v.GetString("master.maintenance.scripts")
 	if adminScripts == "" {
 		return
 	}
 	glog.V(0).Infof("adminScripts: %v", adminScripts)
 
-	v.SetDefault("master.maintenance.sleep_minutes", 17)
 	sleepMinutes := v.GetFloat64("master.maintenance.sleep_minutes")
+	if sleepMinutes <= 0 {
+		sleepMinutes = float64(maintenance.DefaultMaintenanceSleepMinutes)
+	}
 
 	scriptLines := strings.Split(adminScripts, "\n")
 	if !strings.Contains(adminScripts, "lock") {
@@ -371,6 +380,10 @@ func (ms *MasterServer) startAdminScripts() {
 		for {
 			time.Sleep(time.Duration(sleepMinutes) * time.Minute)
 			if ms.Topo.IsLeader() && ms.MasterClient.GetMaster(context.Background()) != "" {
+				if ms.isAdminServerConnected() {
+					glog.V(1).Infof("Skipping master maintenance scripts because admin server is connected")
+					continue
+				}
 				shellOptions.FilerAddress = ms.GetOneFiler(cluster.FilerGroupName(*shellOptions.FilerGroup))
 				if shellOptions.FilerAddress == "" {
 					continue

weed/util/version/constants.go

@@ -9,7 +9,7 @@ import (
 
 var (
 	MAJOR_VERSION  = int32(4)
-	MINOR_VERSION  = int32(13)
+	MINOR_VERSION  = int32(15)
 	VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION)
 	VERSION        = util.SizeLimit + " " + VERSION_NUMBER
 	COMMIT         = ""