Update plugin_templ.go

Move "Next Run" from a standalone card into the top row alongside
Workers, Active Jobs, and Activities as a compact stat card.

.github/workflows/container_release_unified.yml

@@ -60,16 +60,16 @@ jobs:
             build_args: TAGS=5BytesOffset
             tag_suffix: _large_disk
             
-          # Full tags - amd64 only
+          # Full tags - multi-arch
           - variant: full
-            platforms: linux/amd64
+            platforms: linux/amd64,linux/arm64
             dockerfile: ./docker/Dockerfile.go_build
             build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
             tag_suffix: _full
-            
-          # Large disk + full tags - amd64 only
+
+          # Large disk + full tags - multi-arch
           - variant: large_disk_full
-            platforms: linux/amd64
+            platforms: linux/amd64,linux/arm64
             dockerfile: ./docker/Dockerfile.go_build
             build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
             tag_suffix: _large_disk_full

docker/compose/master-cloud.toml

@@ -7,6 +7,7 @@
 
 [master.maintenance]
 # periodically run these scripts are the same as running them from 'weed shell'
+# Scripts are skipped while an admin server is connected.
 scripts = """
   lock
   ec.encode -fullPercent=95 -quietFor=1h

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.13"
+appVersion: "4.15"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.413
+version: 4.15.0

k8s/charts/seaweedfs/openshift-values.yaml

@@ -23,8 +23,15 @@
 #
 # Adjust storageClass and sizes to match your cluster's available StorageClasses.
 # On OpenShift you can discover them with: oc get storageclass
-
+global:
+  enableReplication: true
+  #  replication type is XYZ:
+  # X number of replica in other data centers
+  # Y number of replica in other racks in the same data center
+  # Z number of replica in other servers in the same rack
+  replicationPlacement: "000" # no data replica
 master:
+  replicas: 1
   data:
     type: "persistentVolumeClaim"
     size: "10Gi"
@@ -49,6 +56,7 @@ master:
       type: RuntimeDefault
 
 volume:
+  replicas: 1
   dataDirs:
     - name: data1
       type: "persistentVolumeClaim"
@@ -75,6 +83,7 @@ volume:
       type: RuntimeDefault
 
 filer:
+  replicas: 1
   data:
     type: "persistentVolumeClaim"
     size: "25Gi"

k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml

@@ -51,7 +51,7 @@ metadata:
   annotations:
     "helm.sh/hook": post-install,post-upgrade
     "helm.sh/hook-weight": "-5"
-    "helm.sh/hook-delete-policy": hook-succeeded
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
 spec:
   template:
     metadata:
@@ -92,6 +92,7 @@ spec:
           - "/bin/sh"
           - "-ec"
           - |
+            set -o pipefail
             wait_for_service() {
               local url=$1
               local max_attempts=60  # 5 minutes total (5s * 60)
@@ -117,8 +118,7 @@ spec:
             wait_for_service "http://$WEED_CLUSTER_SW_MASTER{{ .Values.master.readinessProbe.httpGet.path }}"
             wait_for_service "http://$WEED_CLUSTER_SW_FILER{{ .Values.filer.readinessProbe.httpGet.path }}"
             {{- end }}
-          set -o pipefail
-          {{- range $createBuckets }}
+            {{- range $createBuckets }}
             {{- $bucketName := .name }}
             {{- $bucketLock := or .lock .objectLock .withLock }}
             bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; }

other/java/client/src/main/proto/filer.proto

@@ -100,10 +100,12 @@ message ListEntriesRequest {
     string startFromFileName = 3;
     bool inclusiveStartFrom = 4;
     uint32 limit = 5;
+    int64 snapshot_ts_ns = 6;
 }
 
 message ListEntriesResponse {
     Entry entry = 1;
+    int64 snapshot_ts_ns = 2;
 }
 
 message RemoteEntry {
@@ -203,6 +205,7 @@ message CreateEntryRequest {
 
 message CreateEntryResponse {
     string error = 1;
+    SubscribeMetadataResponse metadata_event = 2;
 }
 
 message UpdateEntryRequest {
@@ -212,6 +215,7 @@ message UpdateEntryRequest {
     repeated int32 signatures = 4;
 }
 message UpdateEntryResponse {
+    SubscribeMetadataResponse metadata_event = 1;
 }
 
 message AppendToEntryRequest {
@@ -236,6 +240,7 @@ message DeleteEntryRequest {
 
 message DeleteEntryResponse {
     string error = 1;
+    SubscribeMetadataResponse metadata_event = 2;
 }
 
 message AtomicRenameEntryRequest {
@@ -469,6 +474,7 @@ message CacheRemoteObjectToLocalClusterRequest {
 }
 message CacheRemoteObjectToLocalClusterResponse {
     Entry entry = 1;
+    SubscribeMetadataResponse metadata_event = 2;
 }
 
 /////////////////////////

test/erasure_coding/admin_dockertest/ec_integration_test.go

@@ -153,6 +153,38 @@ func waitForUrl(t *testing.T, url string, retries int) {
 	t.Fatalf("Timeout waiting for %s", url)
 }
 
+func fetchJSON(url string, out interface{}) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
+	}
+	return json.NewDecoder(resp.Body).Decode(out)
+}
+
+func mapField(obj map[string]interface{}, key string) (interface{}, bool) {
+	if obj == nil {
+		return nil, false
+	}
+	if value, ok := obj[key]; ok {
+		return value, true
+	}
+	return nil, false
+}
+
+func mapFieldAny(obj map[string]interface{}, keys ...string) (interface{}, bool) {
+	for _, key := range keys {
+		if value, ok := mapField(obj, key); ok {
+			return value, true
+		}
+	}
+	return nil, false
+}
+
 func TestEcEndToEnd(t *testing.T) {
 	defer cleanup()
 	ensureEnvironment(t)
@@ -162,6 +194,28 @@ func TestEcEndToEnd(t *testing.T) {
 	// 1. Configure plugin job types for fast EC detection/execution.
 	t.Log("Configuring plugin job types via API...")
 
+	schedulerConfig := map[string]interface{}{
+		"idle_sleep_seconds": 1,
+	}
+	jsonBody, err := json.Marshal(schedulerConfig)
+	if err != nil {
+		t.Fatalf("Failed to marshal scheduler config: %v", err)
+	}
+	req, err := http.NewRequest("PUT", AdminUrl+"/api/plugin/scheduler-config", bytes.NewBuffer(jsonBody))
+	if err != nil {
+		t.Fatalf("Failed to create scheduler config request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("Failed to update scheduler config: %v", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		t.Fatalf("Failed to update scheduler config (status %d): %s", resp.StatusCode, string(body))
+	}
+	resp.Body.Close()
+
 	// Disable volume balance to reduce interference for this EC-focused test.
 	balanceConfig := map[string]interface{}{
 		"job_type": "volume_balance",
@@ -169,16 +223,16 @@ func TestEcEndToEnd(t *testing.T) {
 			"enabled": false,
 		},
 	}
-	jsonBody, err := json.Marshal(balanceConfig)
+	jsonBody, err = json.Marshal(balanceConfig)
 	if err != nil {
 		t.Fatalf("Failed to marshal volume_balance config: %v", err)
 	}
-	req, err := http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/volume_balance/config", bytes.NewBuffer(jsonBody))
+	req, err = http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/volume_balance/config", bytes.NewBuffer(jsonBody))
 	if err != nil {
 		t.Fatalf("Failed to create volume_balance config request: %v", err)
 	}
 	req.Header.Set("Content-Type", "application/json")
-	resp, err := client.Do(req)
+	resp, err = client.Do(req)
 	if err != nil {
 		t.Fatalf("Failed to update volume_balance config: %v", err)
 	}
@@ -275,6 +329,7 @@ func TestEcEndToEnd(t *testing.T) {
 	startTime := time.Now()
 	ecVerified := false
 	var lastBody []byte
+	debugTick := 0
 
 	for time.Since(startTime) < 300*time.Second {
 		// 3.1 Check Master Topology
@@ -300,25 +355,104 @@ func TestEcEndToEnd(t *testing.T) {
 			}
 		}
 
-		// 3.2 Debug: Check workers and jobs
-		wResp, wErr := http.Get(AdminUrl + "/api/plugin/workers")
+		// 3.2 Debug: Check workers, jobs, and scheduler status
+		debugTick++
+
+		var workers []map[string]interface{}
 		workerCount := 0
-		if wErr == nil {
-			var workers []interface{}
-			json.NewDecoder(wResp.Body).Decode(&workers)
-			wResp.Body.Close()
+		ecDetectorCount := 0
+		ecExecutorCount := 0
+		if err := fetchJSON(AdminUrl+"/api/plugin/workers", &workers); err == nil {
 			workerCount = len(workers)
+			for _, worker := range workers {
+				capsValue, ok := mapFieldAny(worker, "capabilities", "Capabilities")
+				if !ok {
+					continue
+				}
+				caps, ok := capsValue.(map[string]interface{})
+				if !ok {
+					continue
+				}
+				if capValue, ok := caps["erasure_coding"].(map[string]interface{}); ok {
+					if capValue["can_detect"] == true {
+						ecDetectorCount++
+					}
+					if capValue["can_execute"] == true {
+						ecExecutorCount++
+					}
+				}
+			}
 		}
 
-		tResp, tErr := http.Get(AdminUrl + "/api/plugin/jobs?limit=1000")
+		var tasks []map[string]interface{}
 		taskCount := 0
-		if tErr == nil {
-			var tasks []interface{}
-			json.NewDecoder(tResp.Body).Decode(&tasks)
-			tResp.Body.Close()
+		ecTaskCount := 0
+		ecTaskStates := map[string]int{}
+		if err := fetchJSON(AdminUrl+"/api/plugin/jobs?limit=1000", &tasks); err == nil {
 			taskCount = len(tasks)
+			for _, task := range tasks {
+				jobType, _ := task["job_type"].(string)
+				state, _ := task["state"].(string)
+				if jobType == "erasure_coding" {
+					ecTaskCount++
+					ecTaskStates[state]++
+				}
+			}
+		}
+
+		t.Logf("Waiting for EC... (Workers: %d det=%d exec=%d, Tasks: %d ec=%d, EC States: %+v)",
+			workerCount, ecDetectorCount, ecExecutorCount, taskCount, ecTaskCount, ecTaskStates)
+
+		if debugTick%3 == 0 {
+			var pluginStatus map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/status", &pluginStatus); err == nil {
+				t.Logf("Plugin status: enabled=%v worker_count=%v worker_grpc_port=%v configured=%v",
+					pluginStatus["enabled"], pluginStatus["worker_count"], pluginStatus["worker_grpc_port"], pluginStatus["configured"])
+			}
+
+			var schedulerStatus map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-status", &schedulerStatus); err == nil {
+				if schedValue, ok := schedulerStatus["scheduler"].(map[string]interface{}); ok {
+					t.Logf("Scheduler status: current_job_type=%v phase=%v last_iteration_had_jobs=%v idle_sleep_seconds=%v last_iteration_done_at=%v next_detection_at=%v",
+						schedValue["current_job_type"], schedValue["current_phase"],
+						schedValue["last_iteration_had_jobs"], schedValue["idle_sleep_seconds"], schedValue["last_iteration_done_at"], schedValue["next_detection_at"])
+				} else {
+					t.Logf("Scheduler status: %v", schedulerStatus)
+				}
+			}
+
+			var schedulerStates []map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-states", &schedulerStates); err == nil {
+				for _, state := range schedulerStates {
+					if state["job_type"] == "erasure_coding" {
+						t.Logf("EC scheduler state: enabled=%v detection_in_flight=%v detector_available=%v executor_workers=%v next_detection_at=%v last_run_status=%v last_run_started_at=%v last_run_completed_at=%v",
+							state["enabled"], state["detection_in_flight"], state["detector_available"],
+							state["executor_worker_count"], state["next_detection_at"], state["last_run_status"],
+							state["last_run_started_at"], state["last_run_completed_at"])
+						break
+					}
+				}
+			}
+
+			var jobTypes []map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/job-types", &jobTypes); err == nil {
+				var names []string
+				for _, jobType := range jobTypes {
+					if name, ok := jobType["job_type"].(string); ok && name != "" {
+						names = append(names, name)
+					}
+				}
+				t.Logf("Plugin job types: %v", names)
+			}
+
+			var activities []map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/activities?job_type=erasure_coding&limit=5", &activities); err == nil {
+				for i := len(activities) - 1; i >= 0; i-- {
+					act := activities[i]
+					t.Logf("EC activity: stage=%v message=%v occurred_at=%v", act["stage"], act["message"], act["occurred_at"])
+				}
+			}
 		}
-		t.Logf("Waiting for EC... (Workers: %d, Active Tasks: %d)", workerCount, taskCount)
 
 		time.Sleep(10 * time.Second)
 	}

test/s3/normal/get_object_attributes_test.go

@@ -0,0 +1,573 @@
+package example
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	v1credentials "github.com/aws/aws-sdk-go/aws/credentials"
+	v1signer "github.com/aws/aws-sdk-go/aws/signer/v4"
+	v1s3 "github.com/aws/aws-sdk-go/service/s3"
+	v2aws "github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	v2s3 "github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newS3V2Client creates an AWS SDK v2 S3 client from the test cluster.
+func newS3V2Client(cluster *TestCluster) *v2s3.Client {
+	return v2s3.New(v2s3.Options{
+		Region:       testRegion,
+		BaseEndpoint: v2aws.String(cluster.s3Endpoint),
+		Credentials:  v2aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(testAccessKey, testSecretKey, "")),
+		UsePathStyle: true,
+	})
+}
+
+func TestGetObjectAttributes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	t.Run("Basic", func(t *testing.T) {
+		testGetObjectAttributesBasic(t, cluster)
+	})
+	t.Run("MultipartObject", func(t *testing.T) {
+		testGetObjectAttributesMultipart(t, cluster)
+	})
+	t.Run("SelectiveAttributes", func(t *testing.T) {
+		testGetObjectAttributesSelective(t, cluster)
+	})
+	t.Run("InvalidAttribute", func(t *testing.T) {
+		testGetObjectAttributesInvalid(t, cluster)
+	})
+	t.Run("NonExistentObject", func(t *testing.T) {
+		testGetObjectAttributesNotFound(t, cluster)
+	})
+	t.Run("VersionedObject", func(t *testing.T) {
+		testGetObjectAttributesVersioned(t, cluster)
+	})
+	t.Run("ConditionalHeaders", func(t *testing.T) {
+		testGetObjectAttributesConditionalHeaders(t, cluster)
+	})
+	t.Run("VersionedConditionalHeaders", func(t *testing.T) {
+		testGetObjectAttributesVersionedConditionalHeaders(t, cluster)
+	})
+}
+
+func testGetObjectAttributesBasic(t *testing.T, cluster *TestCluster) {
+	bucketName := createTestBucket(t, cluster, "test-goa-basic-")
+	objectKey := "test-object.txt"
+	objectData := "Hello, GetObjectAttributes!"
+
+	_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader([]byte(objectData)),
+	})
+	require.NoError(t, err)
+
+	client := newS3V2Client(cluster)
+	resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String(objectKey),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesEtag,
+			types.ObjectAttributesStorageClass,
+			types.ObjectAttributesObjectSize,
+			types.ObjectAttributesObjectParts,
+		},
+	})
+	require.NoError(t, err)
+
+	// ETag should be present and non-empty
+	require.NotNil(t, resp.ETag)
+	assert.NotEmpty(t, *resp.ETag)
+	assert.False(t, strings.Contains(*resp.ETag, `"`), "ETag in XML body should not have quotes")
+
+	// ObjectSize should match
+	require.NotNil(t, resp.ObjectSize)
+	assert.Equal(t, int64(len(objectData)), *resp.ObjectSize)
+
+	// StorageClass should be STANDARD (default)
+	assert.Equal(t, "STANDARD", string(resp.StorageClass))
+
+	// ObjectParts should be nil for non-multipart objects
+	assert.Nil(t, resp.ObjectParts)
+
+	// LastModified header should be present
+	assert.NotNil(t, resp.LastModified)
+
+	t.Logf("Basic GetObjectAttributes passed: ETag=%s, Size=%d, StorageClass=%s",
+		*resp.ETag, *resp.ObjectSize, resp.StorageClass)
+}
+
+func testGetObjectAttributesMultipart(t *testing.T, cluster *TestCluster) {
+	bucketName := createTestBucket(t, cluster, "test-goa-mp-")
+	objectKey := "test-multipart.bin"
+
+	// Create a 2-part multipart upload
+	part1Data := bytes.Repeat([]byte("A"), 5*1024*1024) // 5MB (minimum part size)
+	part2Data := bytes.Repeat([]byte("B"), 3*1024*1024) // 3MB
+
+	initResp, err := cluster.s3Client.CreateMultipartUpload(&v1s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	uploadID := initResp.UploadId
+
+	part1Resp, err := cluster.s3Client.UploadPart(&v1s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		PartNumber: aws.Int64(1),
+		UploadId:   uploadID,
+		Body:       bytes.NewReader(part1Data),
+	})
+	require.NoError(t, err)
+
+	part2Resp, err := cluster.s3Client.UploadPart(&v1s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		PartNumber: aws.Int64(2),
+		UploadId:   uploadID,
+		Body:       bytes.NewReader(part2Data),
+	})
+	require.NoError(t, err)
+
+	_, err = cluster.s3Client.CompleteMultipartUpload(&v1s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: uploadID,
+		MultipartUpload: &v1s3.CompletedMultipartUpload{
+			Parts: []*v1s3.CompletedPart{
+				{ETag: part1Resp.ETag, PartNumber: aws.Int64(1)},
+				{ETag: part2Resp.ETag, PartNumber: aws.Int64(2)},
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	// Wait briefly for metadata to settle
+	time.Sleep(200 * time.Millisecond)
+
+	client := newS3V2Client(cluster)
+	resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String(objectKey),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesObjectParts,
+			types.ObjectAttributesObjectSize,
+		},
+	})
+	require.NoError(t, err)
+
+	require.NotNil(t, resp.ObjectSize)
+	assert.Equal(t, int64(len(part1Data)+len(part2Data)), *resp.ObjectSize)
+
+	require.NotNil(t, resp.ObjectParts, "ObjectParts should be present for multipart objects")
+	assert.Equal(t, int32(2), *resp.ObjectParts.TotalPartsCount)
+	require.Len(t, resp.ObjectParts.Parts, 2)
+	assert.Equal(t, int32(1), *resp.ObjectParts.Parts[0].PartNumber)
+	assert.Equal(t, int64(len(part1Data)), *resp.ObjectParts.Parts[0].Size)
+	assert.Equal(t, int32(2), *resp.ObjectParts.Parts[1].PartNumber)
+	assert.Equal(t, int64(len(part2Data)), *resp.ObjectParts.Parts[1].Size)
+
+	// Test pagination: MaxParts=1
+	resp2, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket:  v2aws.String(bucketName),
+		Key:     v2aws.String(objectKey),
+		MaxParts: v2aws.Int32(1),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesObjectParts,
+		},
+	})
+	require.NoError(t, err)
+	require.NotNil(t, resp2.ObjectParts)
+	assert.Len(t, resp2.ObjectParts.Parts, 1)
+	assert.True(t, *resp2.ObjectParts.IsTruncated)
+	assert.Equal(t, int32(2), *resp2.ObjectParts.TotalPartsCount)
+
+	t.Logf("Multipart GetObjectAttributes passed: %d parts, total size %d",
+		*resp.ObjectParts.TotalPartsCount, *resp.ObjectSize)
+}
+
+func testGetObjectAttributesSelective(t *testing.T, cluster *TestCluster) {
+	bucketName := createTestBucket(t, cluster, "test-goa-sel-")
+	objectKey := "test-selective.txt"
+	objectData := "Selective attributes test"
+
+	_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader([]byte(objectData)),
+	})
+	require.NoError(t, err)
+
+	client := newS3V2Client(cluster)
+
+	// Request only ETag
+	resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String(objectKey),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesEtag,
+		},
+	})
+	require.NoError(t, err)
+	require.NotNil(t, resp.ETag)
+	assert.NotEmpty(t, *resp.ETag)
+	assert.Nil(t, resp.ObjectSize, "ObjectSize should not be present when not requested")
+	assert.Empty(t, string(resp.StorageClass), "StorageClass should not be present when not requested")
+	assert.Nil(t, resp.ObjectParts, "ObjectParts should not be present when not requested")
+
+	t.Logf("Selective GetObjectAttributes passed: ETag=%s", *resp.ETag)
+}
+
+func testGetObjectAttributesInvalid(t *testing.T, cluster *TestCluster) {
+	bucketName := createTestBucket(t, cluster, "test-goa-inv-")
+	objectKey := "test-object.txt"
+
+	_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader([]byte("test")),
+	})
+	require.NoError(t, err)
+
+	// Use raw HTTP to send an invalid attribute name since the SDK validates
+	reqURL := fmt.Sprintf("%s/%s/%s?attributes", cluster.s3Endpoint, bucketName, objectKey)
+	req, err := http.NewRequest("GET", reqURL, nil)
+	require.NoError(t, err)
+	req.Header.Set("X-Amz-Object-Attributes", "InvalidAttr")
+
+	signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
+	_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
+	require.NoError(t, err)
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	io.Copy(io.Discard, resp.Body)
+
+	assert.Equal(t, 400, resp.StatusCode)
+	t.Logf("Invalid attribute test passed: got %d", resp.StatusCode)
+}
+
+func testGetObjectAttributesNotFound(t *testing.T, cluster *TestCluster) {
+	bucketName := createTestBucket(t, cluster, "test-goa-nf-")
+
+	client := newS3V2Client(cluster)
+	_, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String("nonexistent-key"),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesEtag,
+		},
+	})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "NoSuchKey")
+
+	t.Logf("NotFound GetObjectAttributes passed")
+}
+
+func testGetObjectAttributesVersioned(t *testing.T, cluster *TestCluster) {
+	client := newS3V2Client(cluster)
+	bucketName := createTestBucket(t, cluster, "test-goa-ver-")
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.Background(), &v2s3.PutBucketVersioningInput{
+		Bucket: v2aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err)
+	time.Sleep(200 * time.Millisecond)
+
+	// Put two versions of the same object
+	v1Data := "version 1 content"
+	putResp1, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String("versioned-key"),
+		Body:   strings.NewReader(v1Data),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, putResp1.VersionId)
+	versionId1 := *putResp1.VersionId
+
+	v2Data := "version 2 content - longer"
+	putResp2, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String("versioned-key"),
+		Body:   strings.NewReader(v2Data),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, putResp2.VersionId)
+	versionId2 := *putResp2.VersionId
+
+	assert.NotEqual(t, versionId1, versionId2, "versions should differ")
+
+	// GetObjectAttributes for latest version (v2)
+	resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String("versioned-key"),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesObjectSize,
+			types.ObjectAttributesEtag,
+		},
+	})
+	require.NoError(t, err)
+	require.NotNil(t, resp.ObjectSize)
+	assert.Equal(t, int64(len(v2Data)), *resp.ObjectSize)
+	require.NotNil(t, resp.VersionId)
+	assert.Equal(t, versionId2, *resp.VersionId)
+
+	// GetObjectAttributes for specific older version (v1)
+	resp1, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
+		Bucket:    v2aws.String(bucketName),
+		Key:       v2aws.String("versioned-key"),
+		VersionId: v2aws.String(versionId1),
+		ObjectAttributes: []types.ObjectAttributes{
+			types.ObjectAttributesObjectSize,
+			types.ObjectAttributesEtag,
+		},
+	})
+	require.NoError(t, err)
+	require.NotNil(t, resp1.ObjectSize)
+	assert.Equal(t, int64(len(v1Data)), *resp1.ObjectSize)
+	require.NotNil(t, resp1.VersionId)
+	assert.Equal(t, versionId1, *resp1.VersionId)
+
+	t.Logf("Versioned GetObjectAttributes passed: v1 size=%d (id=%s), v2 size=%d (id=%s)",
+		*resp1.ObjectSize, versionId1, *resp.ObjectSize, versionId2)
+}
+
+// signedGetObjectAttributes creates a signed GET request for ?attributes with custom headers.
+func signedGetObjectAttributes(t *testing.T, cluster *TestCluster, bucketName, objectKey string, extraHeaders map[string]string) *http.Response {
+	reqURL := fmt.Sprintf("%s/%s/%s?attributes", cluster.s3Endpoint, bucketName, objectKey)
+	req, err := http.NewRequest("GET", reqURL, nil)
+	require.NoError(t, err)
+	req.Header.Set("X-Amz-Object-Attributes", "ETag,ObjectSize")
+	for k, v := range extraHeaders {
+		req.Header.Set(k, v)
+	}
+	signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
+	_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
+	require.NoError(t, err)
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	return resp
+}
+
+func testGetObjectAttributesConditionalHeaders(t *testing.T, cluster *TestCluster) {
+	bucketName := createTestBucket(t, cluster, "test-goa-cond-")
+	objectKey := "cond-test.txt"
+
+	_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader([]byte("conditional headers test")),
+	})
+	require.NoError(t, err)
+
+	// Get the ETag and Last-Modified for the object
+	headResp, err := cluster.s3Client.HeadObject(&v1s3.HeadObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	etag := aws.StringValue(headResp.ETag)
+	lastModified := headResp.LastModified
+	require.NotNil(t, lastModified)
+
+	pastDate := lastModified.Add(-1 * time.Hour).UTC().Format(http.TimeFormat)
+	futureDate := lastModified.Add(1 * time.Hour).UTC().Format(http.TimeFormat)
+
+	// RFC 7232: If-Match true + If-Unmodified-Since false => 200 OK
+	// If-Unmodified-Since is ignored when If-Match is present
+	t.Run("IfMatch_true_IfUnmodifiedSince_false", func(t *testing.T) {
+		resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
+			"If-Match":            etag,
+			"If-Unmodified-Since": pastDate, // object was modified after this => false
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 200, resp.StatusCode,
+			"If-Match=true should return 200 even when If-Unmodified-Since=false (RFC 7232 Section 3.4)")
+	})
+
+	// RFC 7232: If-None-Match false + If-Modified-Since true => 304 Not Modified
+	// If-Modified-Since is ignored when If-None-Match is present
+	t.Run("IfNoneMatch_false_IfModifiedSince_true", func(t *testing.T) {
+		resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
+			"If-None-Match":     etag,
+			"If-Modified-Since": pastDate, // object was modified after this => true
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 304, resp.StatusCode,
+			"If-None-Match=false (ETag match) should return 304 even when If-Modified-Since=true (RFC 7232 Section 3.3)")
+	})
+
+	// If-Match succeeds, If-Unmodified-Since also succeeds => 200
+	t.Run("IfMatch_true_IfUnmodifiedSince_true", func(t *testing.T) {
+		resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
+			"If-Match":            etag,
+			"If-Unmodified-Since": futureDate,
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 200, resp.StatusCode)
+	})
+
+	// If-None-Match passes (ETag differs), If-Modified-Since ignored => 200
+	// Per RFC 7232, If-Modified-Since is ignored when If-None-Match is present
+	t.Run("IfNoneMatch_true_IfModifiedSince_ignored", func(t *testing.T) {
+		resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
+			"If-None-Match":     `"nonexistent-etag"`,
+			"If-Modified-Since": futureDate, // would fail alone, but is ignored
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 200, resp.StatusCode,
+			"If-None-Match=true means If-Modified-Since is ignored, should return 200 (RFC 7232 Section 3.3)")
+	})
+
+	// If-Match fails => 412 regardless of If-Unmodified-Since
+	t.Run("IfMatch_false", func(t *testing.T) {
+		resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
+			"If-Match":            `"wrong-etag"`,
+			"If-Unmodified-Since": futureDate,
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 412, resp.StatusCode)
+	})
+
+	t.Logf("Conditional headers tests passed")
+}
+
+// signedGetObjectAttributesVersioned creates a signed GET request for ?attributes&versionId=... with custom headers.
+func signedGetObjectAttributesVersioned(t *testing.T, cluster *TestCluster, bucketName, objectKey, versionId string, extraHeaders map[string]string) *http.Response {
+	reqURL := fmt.Sprintf("%s/%s/%s?attributes&versionId=%s", cluster.s3Endpoint, bucketName, objectKey, versionId)
+	req, err := http.NewRequest("GET", reqURL, nil)
+	require.NoError(t, err)
+	req.Header.Set("X-Amz-Object-Attributes", "ETag,ObjectSize")
+	for k, v := range extraHeaders {
+		req.Header.Set(k, v)
+	}
+	signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
+	_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
+	require.NoError(t, err)
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	return resp
+}
+
+func testGetObjectAttributesVersionedConditionalHeaders(t *testing.T, cluster *TestCluster) {
+	client := newS3V2Client(cluster)
+	bucketName := createTestBucket(t, cluster, "test-goa-vcond-")
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.Background(), &v2s3.PutBucketVersioningInput{
+		Bucket: v2aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err)
+	time.Sleep(200 * time.Millisecond)
+
+	// Put two versions with different content (different ETags)
+	v1Data := "version 1 - original"
+	putResp1, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String("vcond-key"),
+		Body:   strings.NewReader(v1Data),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, putResp1.VersionId)
+	vid1 := *putResp1.VersionId
+
+	v2Data := "version 2 - updated content"
+	putResp2, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
+		Bucket: v2aws.String(bucketName),
+		Key:    v2aws.String("vcond-key"),
+		Body:   strings.NewReader(v2Data),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, putResp2.VersionId)
+	vid2 := *putResp2.VersionId
+
+	// Get ETags for each version
+	headV1, err := client.HeadObject(context.Background(), &v2s3.HeadObjectInput{
+		Bucket:    v2aws.String(bucketName),
+		Key:       v2aws.String("vcond-key"),
+		VersionId: v2aws.String(vid1),
+	})
+	require.NoError(t, err)
+	etagV1 := *headV1.ETag
+
+	headV2, err := client.HeadObject(context.Background(), &v2s3.HeadObjectInput{
+		Bucket:    v2aws.String(bucketName),
+		Key:       v2aws.String("vcond-key"),
+		VersionId: v2aws.String(vid2),
+	})
+	require.NoError(t, err)
+	etagV2 := *headV2.ETag
+	require.NotEqual(t, etagV1, etagV2, "versions should have different ETags")
+
+	// If-Match with v1's ETag + versionId=v1 => 200
+	// Before the fix, this would fail with 412 because conditional headers
+	// were evaluated against the latest version (v2) whose ETag differs
+	t.Run("IfMatch_v1_etag_versionId_v1", func(t *testing.T) {
+		resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
+			"If-Match": etagV1,
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 200, resp.StatusCode,
+			"If-Match with v1 ETag targeting versionId=v1 should return 200")
+	})
+
+	// If-Match with v2's ETag + versionId=v1 => 412
+	// The ETag doesn't match v1, so this should fail
+	t.Run("IfMatch_v2_etag_versionId_v1", func(t *testing.T) {
+		resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
+			"If-Match": etagV2,
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 412, resp.StatusCode,
+			"If-Match with v2 ETag targeting versionId=v1 should return 412")
+	})
+
+	// If-None-Match with v1's ETag + versionId=v1 => 304
+	t.Run("IfNoneMatch_v1_etag_versionId_v1", func(t *testing.T) {
+		resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
+			"If-None-Match": etagV1,
+		})
+		defer resp.Body.Close()
+		io.Copy(io.Discard, resp.Body)
+		assert.Equal(t, 304, resp.StatusCode,
+			"If-None-Match with v1 ETag targeting versionId=v1 should return 304")
+	})
+
+	t.Logf("Versioned conditional headers tests passed: vid1=%s, vid2=%s", vid1, vid2)
+}

test/s3/policy/policy_test.go

@@ -422,6 +422,218 @@ func TestS3MultipartOperationsInheritPutObjectPermissions(t *testing.T) {
 	require.Equal(t, 0, len(listUploadsOut.Uploads))
 }
 
+// TestS3IAMManagedPolicyLifecycle is an end-to-end integration test covering the
+// user-reported use case in https://github.com/seaweedfs/seaweedfs/issues/8506
+// where managed policy operations (GetPolicy, ListPolicies, DeletePolicy,
+// AttachUserPolicy, DetachUserPolicy) returned 500 errors.
+func TestS3IAMManagedPolicyLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	iamClient := newIAMClient(t, cluster.s3Endpoint)
+
+	// Step 1: Create a user (this already worked per the issue)
+	userName := uniqueName("lifecycle-user")
+	_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+	require.NoError(t, err, "CreateUser should succeed")
+
+	// Step 2: Create a managed policy via IAM API
+	policyName := uniqueName("lifecycle-policy")
+	policyArn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
+	policyDoc := `{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Effect": "Allow",
+			"Action": ["s3:GetObject", "s3:PutObject"],
+			"Resource": "arn:aws:s3:::*"
+		}]
+	}`
+	createOut, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
+		PolicyName:     aws.String(policyName),
+		PolicyDocument: aws.String(policyDoc),
+	})
+	require.NoError(t, err, "CreatePolicy should succeed")
+	require.NotNil(t, createOut.Policy)
+	require.Equal(t, policyName, *createOut.Policy.PolicyName)
+
+	// Step 3: ListPolicies — should include the created policy (was returning 500)
+	listOut, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
+	require.NoError(t, err, "ListPolicies should succeed (was returning 500)")
+	require.True(t, managedPolicyContains(listOut.Policies, policyName),
+		"ListPolicies should contain the newly created policy")
+
+	// Step 4: GetPolicy by ARN — should return the policy (was returning 500)
+	getOut, err := iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
+	require.NoError(t, err, "GetPolicy should succeed (was returning 500)")
+	require.NotNil(t, getOut.Policy)
+	require.Equal(t, policyName, *getOut.Policy.PolicyName)
+	require.Equal(t, policyArn, *getOut.Policy.Arn)
+
+	// Step 5: AttachUserPolicy — should succeed (was returning 500)
+	_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
+		UserName:  aws.String(userName),
+		PolicyArn: aws.String(policyArn),
+	})
+	require.NoError(t, err, "AttachUserPolicy should succeed (was returning 500)")
+
+	// Step 6: ListAttachedUserPolicies — verify the policy is attached
+	attachedOut, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err, "ListAttachedUserPolicies should succeed")
+	require.True(t, attachedPolicyContains(attachedOut.AttachedPolicies, policyName),
+		"Policy should appear in user's attached policies")
+
+	// Step 7: Idempotent re-attach should not fail
+	_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
+		UserName:  aws.String(userName),
+		PolicyArn: aws.String(policyArn),
+	})
+	require.NoError(t, err, "Re-attaching same policy should be idempotent")
+
+	// Step 8: DeletePolicy while attached — should fail with DeleteConflict (AWS behavior)
+	_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
+	require.Error(t, err, "DeletePolicy should fail while policy is attached")
+	var awsErr awserr.Error
+	require.True(t, errors.As(err, &awsErr))
+	require.Equal(t, iam.ErrCodeDeleteConflictException, awsErr.Code(),
+		"Should return DeleteConflict when deleting attached policy")
+
+	// Step 9: DetachUserPolicy
+	_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
+		UserName:  aws.String(userName),
+		PolicyArn: aws.String(policyArn),
+	})
+	require.NoError(t, err, "DetachUserPolicy should succeed")
+
+	// Verify detached
+	attachedOut, err = iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+	require.False(t, attachedPolicyContains(attachedOut.AttachedPolicies, policyName),
+		"Policy should no longer appear in user's attached policies after detach")
+
+	// Step 10: DeletePolicy — should now succeed (was returning XML parsing error)
+	_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
+	require.NoError(t, err, "DeletePolicy should succeed after detach (was returning XML parsing error)")
+
+	// Step 11: Verify the policy is gone
+	listOut, err = iamClient.ListPolicies(&iam.ListPoliciesInput{})
+	require.NoError(t, err)
+	require.False(t, managedPolicyContains(listOut.Policies, policyName),
+		"Deleted policy should not appear in ListPolicies")
+
+	_, err = iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
+	require.Error(t, err, "GetPolicy should fail for deleted policy")
+	require.True(t, errors.As(err, &awsErr))
+	require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+}
+
+// TestS3IAMManagedPolicyErrorCases covers error cases from the user-reported issue:
+// invalid ARNs, missing policies, and missing users.
+func TestS3IAMManagedPolicyErrorCases(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	iamClient := newIAMClient(t, cluster.s3Endpoint)
+
+	t.Run("GetPolicy with nonexistent ARN returns NoSuchEntity", func(t *testing.T) {
+		_, err := iamClient.GetPolicy(&iam.GetPolicyInput{
+			PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
+		})
+		require.Error(t, err)
+		var awsErr awserr.Error
+		require.True(t, errors.As(err, &awsErr))
+		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+
+	t.Run("DeletePolicy with nonexistent ARN returns NoSuchEntity", func(t *testing.T) {
+		_, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{
+			PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
+		})
+		require.Error(t, err)
+		var awsErr awserr.Error
+		require.True(t, errors.As(err, &awsErr))
+		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+
+	t.Run("AttachUserPolicy with nonexistent policy returns NoSuchEntity", func(t *testing.T) {
+		userName := uniqueName("err-user")
+		_, err := iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+		require.NoError(t, err)
+
+		_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
+			UserName:  aws.String(userName),
+			PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
+		})
+		require.Error(t, err)
+		var awsErr awserr.Error
+		require.True(t, errors.As(err, &awsErr))
+		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+
+	t.Run("AttachUserPolicy with nonexistent user returns NoSuchEntity", func(t *testing.T) {
+		policyName := uniqueName("err-policy")
+		_, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`),
+		})
+		require.NoError(t, err)
+
+		_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
+			UserName:  aws.String("nonexistent-user"),
+			PolicyArn: aws.String(fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)),
+		})
+		require.Error(t, err)
+		var awsErr awserr.Error
+		require.True(t, errors.As(err, &awsErr))
+		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+
+	t.Run("DetachUserPolicy that is not attached returns NoSuchEntity", func(t *testing.T) {
+		userName := uniqueName("detach-user")
+		_, err := iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+		require.NoError(t, err)
+
+		policyName := uniqueName("detach-policy")
+		_, err = iamClient.CreatePolicy(&iam.CreatePolicyInput{
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`),
+		})
+		require.NoError(t, err)
+
+		_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
+			UserName:  aws.String(userName),
+			PolicyArn: aws.String(fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)),
+		})
+		require.Error(t, err)
+		var awsErr awserr.Error
+		require.True(t, errors.As(err, &awsErr))
+		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+
+	t.Run("ListAttachedUserPolicies for nonexistent user returns NoSuchEntity", func(t *testing.T) {
+		_, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
+			UserName: aws.String("nonexistent-user"),
+		})
+		require.Error(t, err)
+		var awsErr awserr.Error
+		require.True(t, errors.As(err, &awsErr))
+		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+}
+
 func execShell(t *testing.T, weedCmd, master, filer, shellCmd string) string {
 	// weed shell -master=... -filer=...
 	args := []string{"shell", "-master=" + master, "-filer=" + filer}

test/volume_server/http/admin_test.go

@@ -23,8 +23,6 @@ func TestAdminStatusAndHealthz(t *testing.T) {
 	if err != nil {
 		t.Fatalf("create status request: %v", err)
 	}
-	statusReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-1")
-
 	statusResp := framework.DoRequest(t, client, statusReq)
 	statusBody := framework.ReadAllAndClose(t, statusResp)
 
@@ -34,8 +32,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
 	if got := statusResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
 		t.Fatalf("expected /status Server header to contain SeaweedFS Volume, got %q", got)
 	}
-	if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-1" {
-		t.Fatalf("expected echoed request id, got %q", got)
+	if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
+		t.Fatal("expected server-generated request id in response header")
 	}
 
 	var payload map[string]interface{}
@@ -49,7 +47,6 @@ func TestAdminStatusAndHealthz(t *testing.T) {
 	}
 
 	healthReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz")
-	healthReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-2")
 	healthResp := framework.DoRequest(t, client, healthReq)
 	_ = framework.ReadAllAndClose(t, healthResp)
 	if healthResp.StatusCode != http.StatusOK {
@@ -58,8 +55,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
 	if got := healthResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
 		t.Fatalf("expected /healthz Server header to contain SeaweedFS Volume, got %q", got)
 	}
-	if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-2" {
-		t.Fatalf("expected /healthz echoed request id, got %q", got)
+	if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
+		t.Fatal("expected /healthz server-generated request id in response header")
 	}
 
 	uiResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))

weed/admin/dash/admin_lock_manager.go

@@ -4,12 +4,13 @@ import (
 	"sync"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
 )
 
 const (
-	adminLockName       = "shell"
+	adminLockName       = cluster.AdminShellLockName
 	adminLockClientName = "admin-plugin"
 )
 

weed/admin/dash/admin_presence_lock.go

@@ -0,0 +1,61 @@
+package dash
+
+import (
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
+)
+
+const adminPresenceClientName = "admin-server"
+
+type adminPresenceLock struct {
+	locker *exclusive_locks.ExclusiveLocker
+	stopCh chan struct{}
+}
+
+func newAdminPresenceLock(masterClient *wdclient.MasterClient) *adminPresenceLock {
+	if masterClient == nil {
+		return nil
+	}
+	return &adminPresenceLock{
+		locker: exclusive_locks.NewExclusiveLocker(masterClient, cluster.AdminServerPresenceLockName),
+		stopCh: make(chan struct{}),
+	}
+}
+
+func (l *adminPresenceLock) Start() {
+	if l == nil || l.locker == nil {
+		return
+	}
+	l.locker.SetMessage("admin server connected")
+	go func() {
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		for {
+			if !l.locker.IsLocked() {
+				l.locker.RequestLock(adminPresenceClientName)
+			}
+			select {
+			case <-l.stopCh:
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+}
+
+func (l *adminPresenceLock) Stop() {
+	if l == nil {
+		return
+	}
+	select {
+	case <-l.stopCh:
+	default:
+		close(l.stopCh)
+	}
+	if l.locker != nil {
+		l.locker.ReleaseLock()
+	}
+}

weed/admin/dash/admin_server.go

@@ -11,6 +11,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
 	adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin"
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	clustermaintenance "github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
 	"github.com/seaweedfs/seaweedfs/weed/credential"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -99,6 +100,7 @@ type AdminServer struct {
 	maintenanceManager *maintenance.MaintenanceManager
 	plugin             *adminplugin.Plugin
 	pluginLock         *AdminLockManager
+	adminPresenceLock  *adminPresenceLock
 	expireJobHandler   func(jobID string, reason string) (*adminplugin.TrackedJob, bool, error)
 
 	// Topic retention purger
@@ -137,6 +139,10 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 	go masterClient.KeepConnectedToMaster(ctx)
 
 	lockManager := NewAdminLockManager(masterClient, adminLockClientName)
+	presenceLock := newAdminPresenceLock(masterClient)
+	if presenceLock != nil {
+		presenceLock.Start()
+	}
 
 	server := &AdminServer{
 		masterClient:                  masterClient,
@@ -150,6 +156,7 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 		s3TablesManager:               newS3TablesManager(),
 		icebergPort:                   icebergPort,
 		pluginLock:                    lockManager,
+		adminPresenceLock:             presenceLock,
 	}
 
 	// Initialize topic retention purger
@@ -228,22 +235,19 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 		}()
 	}
 
-	plugin, err := adminplugin.New(adminplugin.Options{
+	pluginOpts := adminplugin.Options{
 		DataDir: dataDir,
 		ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
 			return server.buildDefaultPluginClusterContext(), nil
 		},
-		LockManager: lockManager,
-	})
+		LockManager:            lockManager,
+		ConfigDefaultsProvider: server.enrichConfigDefaults,
+	}
+	plugin, err := adminplugin.New(pluginOpts)
 	if err != nil && dataDir != "" {
 		glog.Warningf("Failed to initialize plugin with dataDir=%q: %v. Falling back to in-memory plugin state.", dataDir, err)
-		plugin, err = adminplugin.New(adminplugin.Options{
-			DataDir: "",
-			ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
-				return server.buildDefaultPluginClusterContext(), nil
-			},
-			LockManager: lockManager,
-		})
+		pluginOpts.DataDir = ""
+		plugin, err = adminplugin.New(pluginOpts)
 	}
 	if err != nil {
 		glog.Errorf("Failed to initialize plugin: %v", err)
@@ -267,6 +271,89 @@ func (s *AdminServer) loadTaskConfigurationsFromPersistence() {
 	configUpdateRegistry.UpdateAllConfigs(s.configPersistence)
 }
 
+// enrichConfigDefaults is called by the plugin when bootstrapping a job type's
+// default config from its descriptor. For admin_script, it fetches maintenance
+// scripts from the master and uses them as the script default.
+//
+// MIGRATION: This exists to help users migrate from master.toml [master.maintenance]
+// to the admin script plugin worker. Remove after March 2027.
+func (s *AdminServer) enrichConfigDefaults(cfg *plugin_pb.PersistedJobTypeConfig) *plugin_pb.PersistedJobTypeConfig {
+	if cfg.JobType != "admin_script" {
+		return cfg
+	}
+
+	var maintenanceScripts string
+	var sleepMinutes uint32
+	err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		maintenanceScripts = resp.MaintenanceScripts
+		sleepMinutes = resp.MaintenanceSleepMinutes
+		return nil
+	})
+	if err != nil {
+		glog.V(1).Infof("Could not fetch master configuration for admin_script defaults: %v", err)
+		return cfg
+	}
+
+	script := cleanMaintenanceScript(maintenanceScripts)
+	if script == "" {
+		return cfg
+	}
+
+	interval := int64(sleepMinutes)
+	if interval <= 0 {
+		interval = clustermaintenance.DefaultMaintenanceSleepMinutes
+	}
+
+	glog.V(0).Infof("Enriching admin_script defaults from master maintenance scripts (interval=%dm)", interval)
+
+	if cfg.AdminConfigValues == nil {
+		cfg.AdminConfigValues = make(map[string]*plugin_pb.ConfigValue)
+	}
+	cfg.AdminConfigValues["script"] = &plugin_pb.ConfigValue{
+		Kind: &plugin_pb.ConfigValue_StringValue{StringValue: script},
+	}
+	cfg.AdminConfigValues["run_interval_minutes"] = &plugin_pb.ConfigValue{
+		Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: interval},
+	}
+	cfg.UpdatedBy = "master_migration"
+
+	return cfg
+}
+
+// cleanMaintenanceScript strips lock/unlock commands and normalizes a
+// maintenance script string for use with the admin script plugin worker.
+//
+// MIGRATION: Used by enrichConfigDefaults. Remove after March 2027.
+func cleanMaintenanceScript(script string) string {
+	script = strings.ReplaceAll(script, "\r\n", "\n")
+	var lines []string
+	for _, line := range strings.Split(script, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" || strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+		// Strip inline comments (e.g., "lock # migration note")
+		if idx := strings.Index(trimmed, "#"); idx >= 0 {
+			trimmed = strings.TrimSpace(trimmed[:idx])
+			if trimmed == "" {
+				continue
+			}
+		}
+		firstToken := strings.ToLower(strings.Fields(trimmed)[0])
+		if firstToken == "lock" || firstToken == "unlock" {
+			continue
+		}
+		lines = append(lines, trimmed)
+	}
+	return strings.Join(lines, "\n")
+}
+
 // GetCredentialManager returns the credential manager
 func (s *AdminServer) GetCredentialManager() *credential.CredentialManager {
 	return s.credentialManager
@@ -340,7 +427,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
 				return err
 			}
 
-			if resp.Entry.IsDirectory {
+			if resp.Entry != nil && resp.Entry.IsDirectory {
 				bucketName := resp.Entry.Name
 				if strings.HasPrefix(bucketName, ".") {
 					// Skip internal/system directories from Object Store bucket listing.
@@ -393,13 +480,18 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
 					}
 				}
 
+				var createdAt, lastModified time.Time
+				if resp.Entry.Attributes != nil {
+					createdAt = time.Unix(resp.Entry.Attributes.Crtime, 0)
+					lastModified = time.Unix(resp.Entry.Attributes.Mtime, 0)
+				}
 				bucket := S3Bucket{
 					Name:               bucketName,
-					CreatedAt:          time.Unix(resp.Entry.Attributes.Crtime, 0),
+					CreatedAt:          createdAt,
 					LogicalSize:        logicalSize,
 					PhysicalSize:       physicalSize,
 					ObjectCount:        objectCount,
-					LastModified:       time.Unix(resp.Entry.Attributes.Mtime, 0),
+					LastModified:       lastModified,
 					Quota:              quota,
 					QuotaEnabled:       quotaEnabled,
 					VersioningStatus:   versioningStatus,
@@ -1286,6 +1378,9 @@ func (s *AdminServer) Shutdown() {
 
 	// Stop maintenance manager
 	s.StopMaintenanceManager()
+	if s.adminPresenceLock != nil {
+		s.adminPresenceLock.Stop()
+	}
 
 	if s.plugin != nil {
 		s.plugin.Shutdown()

weed/admin/dash/admin_server_seed_test.go

@@ -0,0 +1,77 @@
+// MIGRATION: Tests for enrichConfigDefaults helpers. Remove after March 2027.
+package dash
+
+import "testing"
+
+func TestCleanMaintenanceScript(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "empty",
+			input:    "",
+			expected: "",
+		},
+		{
+			name:     "only lock unlock",
+			input:    "  lock\n  unlock\n",
+			expected: "",
+		},
+		{
+			name:     "strips lock and unlock",
+			input:    "  lock\n  ec.balance -apply\n  volume.fix.replication -apply\n  unlock\n",
+			expected: "ec.balance -apply\nvolume.fix.replication -apply",
+		},
+		{
+			name:     "case insensitive lock",
+			input:    "Lock\nec.balance -apply\nUNLOCK",
+			expected: "ec.balance -apply",
+		},
+		{
+			name:     "preserves comments removal",
+			input:    "lock\n# a comment\nec.balance -apply\nunlock",
+			expected: "ec.balance -apply",
+		},
+		{
+			name:     "no lock unlock present",
+			input:    "ec.balance -apply\nvolume.fix.replication -apply",
+			expected: "ec.balance -apply\nvolume.fix.replication -apply",
+		},
+		{
+			name:     "windows line endings",
+			input:    "lock\r\nec.balance -apply\r\nunlock\r\n",
+			expected: "ec.balance -apply",
+		},
+		{
+			name:     "lock with inline comment",
+			input:    "lock # migration\nec.balance -apply\nunlock # done",
+			expected: "ec.balance -apply",
+		},
+		{
+			name:     "command with inline comment preserved",
+			input:    "lock\nec.balance -apply # rebalance shards\nunlock",
+			expected: "ec.balance -apply",
+		},
+		{
+			name:     "only inline comment after stripping",
+			input:    "# full line comment\n  # indented comment\n",
+			expected: "",
+		},
+		{
+			name:     "typical master default",
+			input:    "\n  lock\n  ec.encode -fullPercent=95 -quietFor=1h\n  ec.rebuild -apply\n  ec.balance -apply\n  fs.log.purge -daysAgo=7\n  volume.deleteEmpty -quietFor=24h -apply\n  volume.balance -apply\n  volume.fix.replication -apply\n  s3.clean.uploads -timeAgo=24h\n  unlock\n",
+			expected: "ec.encode -fullPercent=95 -quietFor=1h\nec.rebuild -apply\nec.balance -apply\nfs.log.purge -daysAgo=7\nvolume.deleteEmpty -quietFor=24h -apply\nvolume.balance -apply\nvolume.fix.replication -apply\ns3.clean.uploads -timeAgo=24h",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := cleanMaintenanceScript(tt.input)
+			if got != tt.expected {
+				t.Errorf("cleanMaintenanceScript(%q) = %q, want %q", tt.input, got, tt.expected)
+			}
+		})
+	}
+}

weed/admin/dash/mq_management.go

@@ -324,7 +324,7 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
 			}
 
 			// Only process directories that are versions (start with "v")
-			if versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
+			if versionResp.Entry != nil && versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
 				versionDir := filepath.Join(topicDir, versionResp.Entry.Name)
 
 				// List all partition directories under the version directory (e.g., 0315-0630)
@@ -352,7 +352,7 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
 					}
 
 					// Only process directories that are partitions (format: NNNN-NNNN)
-					if partitionResp.Entry.IsDirectory {
+					if partitionResp.Entry != nil && partitionResp.Entry.IsDirectory {
 						// Parse partition range to get partition start ID (e.g., "0315-0630" -> 315)
 						var partitionStart, partitionStop int32
 						if n, err := fmt.Sscanf(partitionResp.Entry.Name, "%04d-%04d", &partitionStart, &partitionStop); n != 2 || err != nil {
@@ -387,11 +387,11 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
 							}
 
 							// Only process .offset files
-							if !offsetResp.Entry.IsDirectory && strings.HasSuffix(offsetResp.Entry.Name, ".offset") {
+							if offsetResp.Entry != nil && !offsetResp.Entry.IsDirectory && strings.HasSuffix(offsetResp.Entry.Name, ".offset") {
 								consumerGroup := strings.TrimSuffix(offsetResp.Entry.Name, ".offset")
 
 								// Read the offset value from the file
-								offsetData, err := filer.ReadInsideFiler(client, partitionDir, offsetResp.Entry.Name)
+								offsetData, err := filer.ReadInsideFiler(context.Background(), client, partitionDir, offsetResp.Entry.Name)
 								if err != nil {
 									glog.Warningf("Failed to read offset file %s: %v", offsetResp.Entry.Name, err)
 									continue
@@ -401,7 +401,10 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
 									offset := int64(util.BytesToUint64(offsetData))
 
 									// Get the file modification time
-									lastUpdated := time.Unix(offsetResp.Entry.Attributes.Mtime, 0)
+									var lastUpdated time.Time
+									if offsetResp.Entry.Attributes != nil {
+										lastUpdated = time.Unix(offsetResp.Entry.Attributes.Mtime, 0)
+									}
 
 									offsets = append(offsets, ConsumerGroupOffsetInfo{
 										ConsumerGroup: consumerGroup,

weed/admin/dash/plugin_api.go

@@ -235,6 +235,53 @@ func (s *AdminServer) GetPluginSchedulerStatusAPI(w http.ResponseWriter, r *http
 	writeJSON(w, http.StatusOK, response)
 }
 
+// GetPluginSchedulerConfigAPI returns scheduler configuration.
+func (s *AdminServer) GetPluginSchedulerConfigAPI(w http.ResponseWriter, r *http.Request) {
+	pluginSvc := s.GetPlugin()
+	if pluginSvc == nil {
+		writeJSONError(w, http.StatusNotFound, "plugin is not enabled")
+		return
+	}
+
+	writeJSON(w, http.StatusOK, pluginSvc.GetSchedulerConfig())
+}
+
+// UpdatePluginSchedulerConfigAPI updates scheduler configuration.
+func (s *AdminServer) UpdatePluginSchedulerConfigAPI(w http.ResponseWriter, r *http.Request) {
+	pluginSvc := s.GetPlugin()
+	if pluginSvc == nil {
+		writeJSONError(w, http.StatusNotFound, "plugin is not enabled")
+		return
+	}
+
+	var req struct {
+		IdleSleepSeconds *int32 `json:"idle_sleep_seconds"`
+	}
+
+	if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
+		if errors.Is(err, io.EOF) {
+			writeJSONError(w, http.StatusBadRequest, "request body is required")
+			return
+		}
+		writeJSONError(w, http.StatusBadRequest, "invalid request body: "+err.Error())
+		return
+	}
+	if req.IdleSleepSeconds == nil {
+		writeJSONError(w, http.StatusBadRequest, "idle_sleep_seconds is required")
+		return
+	}
+
+	updated, err := pluginSvc.UpdateSchedulerConfig(plugin.SchedulerConfig{
+		IdleSleepSeconds: *req.IdleSleepSeconds,
+	})
+	if err != nil {
+		writeJSONError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	writeJSON(w, http.StatusOK, updated)
+}
+
 // RequestPluginJobTypeSchemaAPI asks a worker for one job type schema.
 func (s *AdminServer) RequestPluginJobTypeSchemaAPI(w http.ResponseWriter, r *http.Request) {
 	jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
@@ -867,6 +914,9 @@ func applyDescriptorDefaultsToPersistedConfig(
 		if runtime.PerWorkerExecutionConcurrency <= 0 {
 			runtime.PerWorkerExecutionConcurrency = defaults.PerWorkerExecutionConcurrency
 		}
+		if runtime.JobTypeMaxRuntimeSeconds <= 0 {
+			runtime.JobTypeMaxRuntimeSeconds = defaults.JobTypeMaxRuntimeSeconds
+		}
 		if runtime.RetryBackoffSeconds <= 0 {
 			runtime.RetryBackoffSeconds = defaults.RetryBackoffSeconds
 		}

weed/admin/dash/topic_retention.go

@@ -151,17 +151,21 @@ func (p *TopicRetentionPurger) purgeTopicData(topicRetention TopicRetentionConfi
 			}
 
 			// Only process directories that are versions (start with "v")
-			if versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
+			if versionResp.Entry != nil && versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
 				versionTime, err := p.parseVersionTime(versionResp.Entry.Name)
 				if err != nil {
 					glog.Warningf("Failed to parse version time from %s: %v", versionResp.Entry.Name, err)
 					continue
 				}
 
+				var modTime time.Time
+				if versionResp.Entry.Attributes != nil {
+					modTime = time.Unix(versionResp.Entry.Attributes.Mtime, 0)
+				}
 				versionDirs = append(versionDirs, VersionDirInfo{
 					Name:        versionResp.Entry.Name,
 					VersionTime: versionTime,
-					ModTime:     time.Unix(versionResp.Entry.Attributes.Mtime, 0),
+					ModTime:     modTime,
 				})
 			}
 		}
@@ -260,6 +264,9 @@ func (p *TopicRetentionPurger) deleteDirectoryRecursively(client filer_pb.Seawee
 			return fmt.Errorf("failed to receive entries: %w", err)
 		}
 
+		if resp.Entry == nil {
+			continue
+		}
 		entryPath := filepath.Join(dirPath, resp.Entry.Name)
 
 		if resp.Entry.IsDirectory {

weed/admin/handlers/admin_handlers.go

@@ -229,6 +229,8 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
 	pluginApi.HandleFunc("/status", h.adminServer.GetPluginStatusAPI).Methods(http.MethodGet)
 	pluginApi.HandleFunc("/workers", h.adminServer.GetPluginWorkersAPI).Methods(http.MethodGet)
 	pluginApi.HandleFunc("/job-types", h.adminServer.GetPluginJobTypesAPI).Methods(http.MethodGet)
+	pluginApi.HandleFunc("/scheduler-config", h.adminServer.GetPluginSchedulerConfigAPI).Methods(http.MethodGet)
+	pluginApi.Handle("/scheduler-config", wrapWrite(h.adminServer.UpdatePluginSchedulerConfigAPI)).Methods(http.MethodPut)
 	pluginApi.HandleFunc("/jobs", h.adminServer.GetPluginJobsAPI).Methods(http.MethodGet)
 	pluginApi.HandleFunc("/jobs/{jobId}", h.adminServer.GetPluginJobAPI).Methods(http.MethodGet)
 	pluginApi.HandleFunc("/jobs/{jobId}/detail", h.adminServer.GetPluginJobDetailAPI).Methods(http.MethodGet)

weed/admin/maintenance/maintenance_integration.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/admin/topology"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
 	"github.com/seaweedfs/seaweedfs/weed/worker/types"
@@ -229,6 +230,12 @@ func (s *MaintenanceIntegration) ScanWithTaskDetectors(volumeMetrics []*types.Vo
 			continue
 		}
 
+		// Cancel stale pending tasks for this type before re-detection
+		maintenanceType := s.taskTypeMap[taskType]
+		if cancelled := s.maintenanceQueue.CancelPendingTasksByType(maintenanceType); cancelled > 0 {
+			glog.Infof("Cancelled %d stale pending %s tasks before re-detection", cancelled, taskType)
+		}
+
 		glog.V(2).Infof("Running detection for task type: %s", taskType)
 
 		results, err := detector.ScanForTasks(filteredMetrics, clusterInfo)
@@ -528,10 +535,15 @@ func (s *MaintenanceIntegration) SyncTask(task *MaintenanceTask) {
 		// Volume size is not currently used for Balance/Vacuum impact and is not stored in MaintenanceTask
 		sourceImpact, targetImpact := topology.CalculateTaskStorageImpact(topology.TaskType(string(taskType)), 0)
 
-		// Use unified sources and targets from TaskParams
+		// Use unified sources and targets from TaskParams.
+		// Task protos store ServerAddresses (with gRPC port, e.g., "host:port.grpcPort")
+		// but the topology indexes disks by NodeId (e.g., "host:port").
+		// Strip the gRPC port suffix via ToHttpAddress() to match the topology key.
 		for _, src := range task.TypedParams.Sources {
+			resolvedSrc := pb.ServerAddress(src.Node).ToHttpAddress()
+			glog.V(2).Infof("SyncTask %s: source proto Node=%q resolved to %q, diskId=%d", task.ID, src.Node, resolvedSrc, src.DiskId)
 			sources = append(sources, topology.TaskSource{
-				SourceServer:  src.Node,
+				SourceServer:  resolvedSrc,
 				SourceDisk:    src.DiskId,
 				StorageChange: sourceImpact,
 			})
@@ -539,8 +551,10 @@ func (s *MaintenanceIntegration) SyncTask(task *MaintenanceTask) {
 			estimatedSize += int64(src.EstimatedSize)
 		}
 		for _, target := range task.TypedParams.Targets {
+			resolvedTarget := pb.ServerAddress(target.Node).ToHttpAddress()
+			glog.V(2).Infof("SyncTask %s: target proto Node=%q resolved to %q, diskId=%d", task.ID, target.Node, resolvedTarget, target.DiskId)
 			destinations = append(destinations, topology.TaskDestination{
-				TargetServer:  target.Node,
+				TargetServer:  resolvedTarget,
 				TargetDisk:    target.DiskId,
 				StorageChange: targetImpact,
 			})

weed/admin/maintenance/maintenance_queue.go

@@ -4,6 +4,7 @@ import (
 	"crypto/rand"
 	"fmt"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -230,6 +231,46 @@ func (mq *MaintenanceQueue) hasDuplicateTask(newTask *MaintenanceTask) bool {
 	return false
 }
 
+// CancelPendingTasksByType cancels all pending tasks of a given type.
+// This is called before each detection cycle to ensure stale proposals
+// from previous cycles are cleaned up before creating new ones.
+func (mq *MaintenanceQueue) CancelPendingTasksByType(taskType MaintenanceTaskType) int {
+	mq.mutex.Lock()
+
+	var remaining []*MaintenanceTask
+	var cancelledSnapshots []*MaintenanceTask
+	cancelled := 0
+	for _, task := range mq.pendingTasks {
+		if task.Type == taskType {
+			task.Status = TaskStatusCancelled
+			now := time.Now()
+			task.CompletedAt = &now
+			cancelled++
+			cancelledSnapshots = append(cancelledSnapshots, snapshotTask(task))
+			glog.V(1).Infof("Cancelled stale pending task %s (%s) for volume %d before re-detection",
+				task.ID, task.Type, task.VolumeID)
+
+			// Release capacity in ActiveTopology and remove pending operation
+			if mq.integration != nil {
+				if at := mq.integration.GetActiveTopology(); at != nil {
+					_ = at.CompleteTask(task.ID)
+				}
+			}
+			mq.removePendingOperation(task.ID)
+		} else {
+			remaining = append(remaining, task)
+		}
+	}
+	mq.pendingTasks = remaining
+	mq.mutex.Unlock()
+
+	// Persist cancelled state outside the lock to avoid blocking
+	for _, snapshot := range cancelledSnapshots {
+		mq.saveTaskState(snapshot)
+	}
+	return cancelled
+}
+
 // AddTasksFromResults converts detection results to tasks and adds them to the queue
 func (mq *MaintenanceQueue) AddTasksFromResults(results []*TaskDetectionResult) {
 	for _, result := range results {
@@ -455,8 +496,8 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 		task.Status = TaskStatusFailed
 		task.Error = error
 
-		// Check if task should be retried
-		if task.RetryCount < task.MaxRetries {
+		// Check if task should be retried (skip retry for permanent errors)
+		if task.RetryCount < task.MaxRetries && !isNonRetriableError(error) {
 			// Record unassignment due to failure/retry
 			if task.WorkerID != "" && len(task.AssignmentHistory) > 0 {
 				lastAssignment := task.AssignmentHistory[len(task.AssignmentHistory)-1]
@@ -559,6 +600,12 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 	}
 }
 
+// isNonRetriableError returns true for errors that will never succeed on retry,
+// such as when the volume doesn't exist on the source server.
+func isNonRetriableError(errMsg string) bool {
+	return strings.Contains(errMsg, "not found")
+}
+
 // UpdateTaskProgress updates the progress of a running task
 func (mq *MaintenanceQueue) UpdateTaskProgress(taskID string, progress float64) {
 	mq.mutex.Lock()

weed/admin/plugin/config_store.go

@@ -30,6 +30,7 @@ const (
 	runsJSONFileName        = "runs.json"
 	trackedJobsJSONFileName = "tracked_jobs.json"
 	activitiesJSONFileName  = "activities.json"
+	schedulerJSONFileName   = "scheduler.json"
 	defaultDirPerm          = 0o755
 	defaultFilePerm         = 0o644
 )
@@ -53,6 +54,7 @@ type ConfigStore struct {
 	memTrackedJobs []TrackedJob
 	memActivities  []JobActivity
 	memJobDetails  map[string]TrackedJob
+	memScheduler   *SchedulerConfig
 }
 
 func NewConfigStore(adminDataDir string) (*ConfigStore, error) {
@@ -93,6 +95,60 @@ func (s *ConfigStore) BaseDir() string {
 	return s.baseDir
 }
 
+func (s *ConfigStore) LoadSchedulerConfig() (*SchedulerConfig, error) {
+	s.mu.RLock()
+	if !s.configured {
+		cfg := s.memScheduler
+		s.mu.RUnlock()
+		if cfg == nil {
+			return nil, nil
+		}
+		clone := *cfg
+		return &clone, nil
+	}
+	s.mu.RUnlock()
+
+	path := filepath.Join(s.baseDir, schedulerJSONFileName)
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("read scheduler config: %w", err)
+	}
+
+	var cfg SchedulerConfig
+	if err := json.Unmarshal(data, &cfg); err != nil {
+		return nil, fmt.Errorf("unmarshal scheduler config: %w", err)
+	}
+	return &cfg, nil
+}
+
+func (s *ConfigStore) SaveSchedulerConfig(config *SchedulerConfig) error {
+	if config == nil {
+		return fmt.Errorf("scheduler config is nil")
+	}
+	normalized := normalizeSchedulerConfig(*config)
+
+	s.mu.Lock()
+	if !s.configured {
+		s.memScheduler = &normalized
+		s.mu.Unlock()
+		return nil
+	}
+	s.mu.Unlock()
+
+	payload, err := json.MarshalIndent(normalized, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal scheduler config: %w", err)
+	}
+	path := filepath.Join(s.baseDir, schedulerJSONFileName)
+	if err := os.WriteFile(path, payload, defaultFilePerm); err != nil {
+		return fmt.Errorf("save scheduler config: %w", err)
+	}
+	return nil
+}
+
 func (s *ConfigStore) SaveDescriptor(jobType string, descriptor *plugin_pb.JobTypeDescriptor) error {
 	if descriptor == nil {
 		return fmt.Errorf("descriptor is nil")
@@ -200,6 +256,53 @@ func (s *ConfigStore) SaveJobTypeConfig(config *plugin_pb.PersistedJobTypeConfig
 	return nil
 }
 
+// SaveJobTypeConfigIfNotExists atomically checks whether a config for the
+// given job type already exists and only persists config when none is found.
+// Returns true if the config was saved, false if a config already existed.
+func (s *ConfigStore) SaveJobTypeConfigIfNotExists(config *plugin_pb.PersistedJobTypeConfig) (bool, error) {
+	if config == nil {
+		return false, fmt.Errorf("job type config is nil")
+	}
+	if config.JobType == "" {
+		return false, fmt.Errorf("job type config has empty job_type")
+	}
+	sanitizedJobType, err := sanitizeJobType(config.JobType)
+	if err != nil {
+		return false, err
+	}
+	config.JobType = sanitizedJobType
+
+	clone := proto.Clone(config).(*plugin_pb.PersistedJobTypeConfig)
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if !s.configured {
+		if _, exists := s.memConfigs[config.JobType]; exists {
+			return false, nil
+		}
+		s.memConfigs[config.JobType] = clone
+		return true, nil
+	}
+
+	pbPath := filepath.Join(s.baseDir, jobTypesDirName, config.JobType, configPBFileName)
+	if _, statErr := os.Stat(pbPath); statErr == nil {
+		return false, nil
+	}
+
+	jobTypeDir, err := s.ensureJobTypeDir(config.JobType)
+	if err != nil {
+		return false, err
+	}
+
+	jsonPath := filepath.Join(jobTypeDir, configJSONFileName)
+	if err := writeProtoFiles(clone, filepath.Join(jobTypeDir, configPBFileName), jsonPath); err != nil {
+		return false, fmt.Errorf("save job type config for %s: %w", config.JobType, err)
+	}
+
+	return true, nil
+}
+
 func (s *ConfigStore) LoadJobTypeConfig(jobType string) (*plugin_pb.PersistedJobTypeConfig, error) {
 	if _, err := sanitizeJobType(jobType); err != nil {
 		return nil, err

weed/admin/plugin/config_store_test.go

@@ -208,6 +208,81 @@ func TestConfigStoreMonitorStateRoundTrip(t *testing.T) {
 	}
 }
 
+func TestConfigStoreSaveJobTypeConfigIfNotExists(t *testing.T) {
+	t.Parallel()
+
+	t.Run("in-memory", func(t *testing.T) {
+		t.Parallel()
+		store, err := NewConfigStore("")
+		if err != nil {
+			t.Fatalf("NewConfigStore: %v", err)
+		}
+		testSaveJobTypeConfigIfNotExists(t, store)
+	})
+
+	t.Run("on-disk", func(t *testing.T) {
+		t.Parallel()
+		store, err := NewConfigStore(t.TempDir())
+		if err != nil {
+			t.Fatalf("NewConfigStore: %v", err)
+		}
+		testSaveJobTypeConfigIfNotExists(t, store)
+	})
+}
+
+func testSaveJobTypeConfigIfNotExists(t *testing.T, store *ConfigStore) {
+	t.Helper()
+
+	cfg := &plugin_pb.PersistedJobTypeConfig{
+		JobType:      "admin_script",
+		AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true},
+	}
+
+	// First call should save.
+	saved, err := store.SaveJobTypeConfigIfNotExists(cfg)
+	if err != nil {
+		t.Fatalf("first SaveJobTypeConfigIfNotExists: %v", err)
+	}
+	if !saved {
+		t.Fatal("expected first call to save the config")
+	}
+
+	// Second call with same job type should not save.
+	saved, err = store.SaveJobTypeConfigIfNotExists(&plugin_pb.PersistedJobTypeConfig{
+		JobType:      "admin_script",
+		AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: false},
+	})
+	if err != nil {
+		t.Fatalf("second SaveJobTypeConfigIfNotExists: %v", err)
+	}
+	if saved {
+		t.Fatal("expected second call to be a no-op")
+	}
+
+	// Verify the original config was preserved.
+	loaded, err := store.LoadJobTypeConfig("admin_script")
+	if err != nil {
+		t.Fatalf("LoadJobTypeConfig: %v", err)
+	}
+	if loaded == nil {
+		t.Fatal("expected config to exist")
+	}
+	if !loaded.AdminRuntime.Enabled {
+		t.Fatal("expected original config (Enabled=true) to be preserved")
+	}
+
+	// Different job type should still save.
+	saved, err = store.SaveJobTypeConfigIfNotExists(&plugin_pb.PersistedJobTypeConfig{
+		JobType: "vacuum",
+	})
+	if err != nil {
+		t.Fatalf("SaveJobTypeConfigIfNotExists for different type: %v", err)
+	}
+	if !saved {
+		t.Fatal("expected save for a different job type")
+	}
+}
+
 func TestConfigStoreJobDetailRoundTrip(t *testing.T) {
 	t.Parallel()
 

weed/admin/plugin/plugin.go

@@ -34,6 +34,11 @@ type Options struct {
 	SchedulerTick          time.Duration
 	ClusterContextProvider func(context.Context) (*plugin_pb.ClusterContext, error)
 	LockManager            LockManager
+	// ConfigDefaultsProvider is an optional callback invoked when a job type's
+	// config is being bootstrapped from its descriptor defaults. It can enrich
+	// or replace the default config before it is persisted. If nil, descriptor
+	// defaults are used as-is.
+	ConfigDefaultsProvider func(config *plugin_pb.PersistedJobTypeConfig) *plugin_pb.PersistedJobTypeConfig
 }
 
 // JobTypeInfo contains metadata about a plugin job type.
@@ -54,6 +59,7 @@ type Plugin struct {
 
 	schedulerTick          time.Duration
 	clusterContextProvider func(context.Context) (*plugin_pb.ClusterContext, error)
+	configDefaultsProvider func(config *plugin_pb.PersistedJobTypeConfig) *plugin_pb.PersistedJobTypeConfig
 	lockManager            LockManager
 
 	schedulerMu       sync.Mutex
@@ -68,6 +74,13 @@ type Plugin struct {
 	adminScriptRunMu          sync.RWMutex
 	schedulerDetectionMu      sync.Mutex
 	schedulerDetection        map[string]*schedulerDetectionInfo
+	schedulerRunMu            sync.Mutex
+	schedulerRun              map[string]*schedulerRunInfo
+	schedulerLoopMu           sync.Mutex
+	schedulerLoopState        schedulerLoopState
+	schedulerConfigMu         sync.RWMutex
+	schedulerConfig           SchedulerConfig
+	schedulerWakeCh           chan struct{}
 
 	dedupeMu           sync.Mutex
 	recentDedupeByType map[string]map[string]time.Time
@@ -154,6 +167,7 @@ func New(options Options) (*Plugin, error) {
 		sendTimeout:               sendTimeout,
 		schedulerTick:             schedulerTick,
 		clusterContextProvider:    options.ClusterContextProvider,
+		configDefaultsProvider:    options.ConfigDefaultsProvider,
 		lockManager:               options.LockManager,
 		sessions:                  make(map[string]*streamSession),
 		pendingSchema:             make(map[string]chan *plugin_pb.ConfigSchemaResponse),
@@ -164,14 +178,31 @@ func New(options Options) (*Plugin, error) {
 		detectorLeases:            make(map[string]string),
 		schedulerExecReservations: make(map[string]int),
 		schedulerDetection:        make(map[string]*schedulerDetectionInfo),
+		schedulerRun:              make(map[string]*schedulerRunInfo),
 		recentDedupeByType:        make(map[string]map[string]time.Time),
 		jobs:                      make(map[string]*TrackedJob),
 		activities:                make([]JobActivity, 0, 256),
 		persistTicker:             time.NewTicker(2 * time.Second),
+		schedulerWakeCh:           make(chan struct{}, 1),
 		shutdownCh:                make(chan struct{}),
 	}
 	plugin.ctx, plugin.ctxCancel = context.WithCancel(context.Background())
 
+	if cfg, err := plugin.store.LoadSchedulerConfig(); err != nil {
+		glog.Warningf("Plugin failed to load scheduler config: %v", err)
+		plugin.schedulerConfig = DefaultSchedulerConfig()
+	} else if cfg == nil {
+		defaults := DefaultSchedulerConfig()
+		plugin.schedulerConfig = defaults
+		if plugin.store.IsConfigured() {
+			if err := plugin.store.SaveSchedulerConfig(&defaults); err != nil {
+				glog.Warningf("Plugin failed to persist scheduler defaults: %v", err)
+			}
+		}
+	} else {
+		plugin.schedulerConfig = normalizeSchedulerConfig(*cfg)
+	}
+
 	if err := plugin.loadPersistedMonitorState(); err != nil {
 		glog.Warningf("Plugin failed to load persisted monitoring state: %v", err)
 	}
@@ -371,9 +402,14 @@ func (r *Plugin) LoadJobTypeConfig(jobType string) (*plugin_pb.PersistedJobTypeC
 }
 
 func (r *Plugin) SaveJobTypeConfig(config *plugin_pb.PersistedJobTypeConfig) error {
-	return r.store.SaveJobTypeConfig(config)
+	if err := r.store.SaveJobTypeConfig(config); err != nil {
+		return err
+	}
+	r.wakeScheduler()
+	return nil
 }
 
+
 func (r *Plugin) LoadDescriptor(jobType string) (*plugin_pb.JobTypeDescriptor, error) {
 	return r.store.LoadDescriptor(jobType)
 }
@@ -390,6 +426,31 @@ func (r *Plugin) BaseDir() string {
 	return r.store.BaseDir()
 }
 
+func (r *Plugin) GetSchedulerConfig() SchedulerConfig {
+	if r == nil {
+		return DefaultSchedulerConfig()
+	}
+	r.schedulerConfigMu.RLock()
+	cfg := r.schedulerConfig
+	r.schedulerConfigMu.RUnlock()
+	return normalizeSchedulerConfig(cfg)
+}
+
+func (r *Plugin) UpdateSchedulerConfig(cfg SchedulerConfig) (SchedulerConfig, error) {
+	if r == nil {
+		return DefaultSchedulerConfig(), fmt.Errorf("plugin is not initialized")
+	}
+	normalized := normalizeSchedulerConfig(cfg)
+	if err := r.store.SaveSchedulerConfig(&normalized); err != nil {
+		return SchedulerConfig{}, err
+	}
+	r.schedulerConfigMu.Lock()
+	r.schedulerConfig = normalized
+	r.schedulerConfigMu.Unlock()
+	r.wakeScheduler()
+	return normalized, nil
+}
+
 func (r *Plugin) acquireAdminLock(reason string) (func(), error) {
 	if r == nil || r.lockManager == nil {
 		return func() {}, nil
@@ -912,6 +973,7 @@ func (r *Plugin) handleWorkerMessage(workerID string, message *plugin_pb.WorkerT
 	switch body := message.Body.(type) {
 	case *plugin_pb.WorkerToAdminMessage_Hello:
 		r.registry.UpsertFromHello(body.Hello)
+		r.wakeScheduler()
 	case *plugin_pb.WorkerToAdminMessage_Heartbeat:
 		r.registry.UpdateHeartbeat(workerID, body.Heartbeat)
 	case *plugin_pb.WorkerToAdminMessage_ConfigSchemaResponse:
@@ -981,14 +1043,6 @@ func (r *Plugin) ensureJobTypeConfigFromDescriptor(jobType string, descriptor *p
 		return nil
 	}
 
-	existing, err := r.store.LoadJobTypeConfig(jobType)
-	if err != nil {
-		return err
-	}
-	if existing != nil {
-		return nil
-	}
-
 	workerDefaults := CloneConfigValueMap(descriptor.WorkerDefaultValues)
 	if len(workerDefaults) == 0 && descriptor.WorkerConfigForm != nil {
 		workerDefaults = CloneConfigValueMap(descriptor.WorkerConfigForm.DefaultValues)
@@ -1011,6 +1065,7 @@ func (r *Plugin) ensureJobTypeConfigFromDescriptor(jobType string, descriptor *p
 			PerWorkerExecutionConcurrency: defaults.PerWorkerExecutionConcurrency,
 			RetryLimit:                    defaults.RetryLimit,
 			RetryBackoffSeconds:           defaults.RetryBackoffSeconds,
+			JobTypeMaxRuntimeSeconds:      defaults.JobTypeMaxRuntimeSeconds,
 		}
 	}
 
@@ -1024,7 +1079,22 @@ func (r *Plugin) ensureJobTypeConfigFromDescriptor(jobType string, descriptor *p
 		UpdatedBy:          "plugin",
 	}
 
-	return r.store.SaveJobTypeConfig(cfg)
+	// Check existence first to avoid calling configDefaultsProvider unnecessarily
+	// (e.g., it may make a blocking gRPC call to fetch master config).
+	existing, err := r.store.LoadJobTypeConfig(jobType)
+	if err != nil {
+		return err
+	}
+	if existing != nil {
+		return nil
+	}
+
+	if r.configDefaultsProvider != nil {
+		cfg = r.configDefaultsProvider(cfg)
+	}
+
+	_, err = r.store.SaveJobTypeConfigIfNotExists(cfg)
+	return err
 }
 
 func (r *Plugin) handleDetectionProposals(workerID string, message *plugin_pb.DetectionProposals) {

weed/admin/plugin/plugin_monitor.go

@@ -861,6 +861,79 @@ func (r *Plugin) trackExecutionQueued(job *plugin_pb.JobSpec) {
 	})
 }
 
+func (r *Plugin) cancelQueuedJob(job *plugin_pb.JobSpec, cause error) {
+	reason := "job canceled"
+	if cause != nil {
+		reason = cause.Error()
+	}
+	r.markJobCanceled(job, reason)
+}
+
+func (r *Plugin) markJobCanceled(job *plugin_pb.JobSpec, reason string) {
+	if job == nil || strings.TrimSpace(job.JobId) == "" {
+		return
+	}
+
+	now := time.Now().UTC()
+	if strings.TrimSpace(reason) == "" {
+		reason = "job canceled"
+	}
+
+	r.jobsMu.Lock()
+	tracked := r.jobs[job.JobId]
+	if tracked == nil {
+		tracked = &TrackedJob{
+			JobID:     job.JobId,
+			CreatedAt: timeToPtr(now),
+		}
+		r.jobs[job.JobId] = tracked
+	}
+
+	if job.JobType != "" {
+		tracked.JobType = job.JobType
+	}
+	tracked.State = StateCanceled
+	tracked.Stage = "canceled"
+	tracked.Message = reason
+	tracked.ErrorMessage = reason
+	tracked.Progress = 0
+	if tracked.CreatedAt == nil || tracked.CreatedAt.IsZero() {
+		tracked.CreatedAt = timeToPtr(now)
+	}
+	tracked.UpdatedAt = timeToPtr(now)
+	tracked.CompletedAt = timeToPtr(now)
+	trackedSnapshot := cloneTrackedJob(*tracked)
+	r.pruneTrackedJobsLocked()
+	r.dirtyJobs = true
+	r.jobsMu.Unlock()
+
+	r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
+		detail.JobID = job.JobId
+		if job.JobType != "" {
+			detail.JobType = job.JobType
+		}
+		detail.State = trackedSnapshot.State
+		detail.Stage = trackedSnapshot.Stage
+		detail.Message = trackedSnapshot.Message
+		detail.ErrorMessage = trackedSnapshot.ErrorMessage
+		detail.Progress = trackedSnapshot.Progress
+		if detail.CreatedAt == nil || detail.CreatedAt.IsZero() {
+			detail.CreatedAt = trackedSnapshot.CreatedAt
+		}
+		detail.UpdatedAt = trackedSnapshot.UpdatedAt
+		detail.CompletedAt = trackedSnapshot.CompletedAt
+	})
+
+	r.appendActivity(JobActivity{
+		JobID:      job.JobId,
+		JobType:    job.JobType,
+		Source:     "admin_scheduler",
+		Message:    reason,
+		Stage:      "canceled",
+		OccurredAt: timeToPtr(now),
+	})
+}
+
 func (r *Plugin) trackExecutionCompletion(completed *plugin_pb.JobCompleted) *TrackedJob {
 	if completed == nil || strings.TrimSpace(completed.JobId) == "" {
 		return nil

weed/admin/plugin/plugin_scheduler.go

@@ -13,13 +13,17 @@ import (
 	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
-var errExecutorAtCapacity = errors.New("executor is at capacity")
+var (
+	errExecutorAtCapacity = errors.New("executor is at capacity")
+	errSchedulerShutdown  = errors.New("scheduler shutdown")
+)
 
 const (
 	defaultSchedulerTick                       = 5 * time.Second
 	defaultScheduledDetectionInterval          = 300 * time.Second
 	defaultScheduledDetectionTimeout           = 45 * time.Second
 	defaultScheduledExecutionTimeout           = 90 * time.Second
+	defaultScheduledJobTypeMaxRuntime          = 30 * time.Minute
 	defaultScheduledMaxResults           int32 = 1000
 	defaultScheduledExecutionConcurrency       = 1
 	defaultScheduledPerWorkerConcurrency       = 1
@@ -34,6 +38,7 @@ type schedulerPolicy struct {
 	DetectionInterval      time.Duration
 	DetectionTimeout       time.Duration
 	ExecutionTimeout       time.Duration
+	JobTypeMaxRuntime      time.Duration
 	RetryBackoff           time.Duration
 	MaxResults             int32
 	ExecutionConcurrency   int
@@ -44,31 +49,71 @@ type schedulerPolicy struct {
 
 func (r *Plugin) schedulerLoop() {
 	defer r.wg.Done()
-	ticker := time.NewTicker(r.schedulerTick)
-	defer ticker.Stop()
-
-	// Try once immediately on startup.
-	r.runSchedulerTick()
-
 	for {
 		select {
 		case <-r.shutdownCh:
 			return
-		case <-ticker.C:
-			r.runSchedulerTick()
+		default:
+		}
+
+		hadJobs := r.runSchedulerIteration()
+		r.recordSchedulerIterationComplete(hadJobs)
+
+		if hadJobs {
+			continue
+		}
+
+		r.setSchedulerLoopState("", "sleeping")
+		idleSleep := r.GetSchedulerConfig().IdleSleepDuration()
+		if nextRun := r.earliestNextDetectionAt(); !nextRun.IsZero() {
+			if until := time.Until(nextRun); until <= 0 {
+				idleSleep = 0
+			} else if until < idleSleep {
+				idleSleep = until
+			}
+		}
+		if idleSleep <= 0 {
+			continue
+		}
+
+		timer := time.NewTimer(idleSleep)
+		select {
+		case <-r.shutdownCh:
+			timer.Stop()
+			return
+		case <-r.schedulerWakeCh:
+			if !timer.Stop() {
+				<-timer.C
+			}
+			continue
+		case <-timer.C:
 		}
 	}
 }
 
-func (r *Plugin) runSchedulerTick() {
+func (r *Plugin) runSchedulerIteration() bool {
 	r.expireStaleJobs(time.Now().UTC())
 
 	jobTypes := r.registry.DetectableJobTypes()
 	if len(jobTypes) == 0 {
-		return
+		r.setSchedulerLoopState("", "idle")
+		return false
+	}
+
+	r.setSchedulerLoopState("", "waiting_for_lock")
+	releaseLock, err := r.acquireAdminLock("plugin scheduler iteration")
+	if err != nil {
+		glog.Warningf("Plugin scheduler failed to acquire lock: %v", err)
+		r.setSchedulerLoopState("", "idle")
+		return false
+	}
+	if releaseLock != nil {
+		defer releaseLock()
 	}
 
 	active := make(map[string]struct{}, len(jobTypes))
+	hadJobs := false
+
 	for _, jobType := range jobTypes {
 		active[jobType] = struct{}{}
 
@@ -81,20 +126,212 @@ func (r *Plugin) runSchedulerTick() {
 			r.clearSchedulerJobType(jobType)
 			continue
 		}
-
-		if !r.markDetectionDue(jobType, policy.DetectionInterval) {
+		initialDelay := time.Duration(0)
+		if runInfo := r.snapshotSchedulerRun(jobType); runInfo.lastRunStartedAt.IsZero() {
+			initialDelay = 5 * time.Second
+		}
+		if !r.markDetectionDue(jobType, policy.DetectionInterval, initialDelay) {
 			continue
 		}
 
-		r.wg.Add(1)
-		go func(jt string, p schedulerPolicy) {
-			defer r.wg.Done()
-			r.runScheduledDetection(jt, p)
-		}(jobType, policy)
+		detected := r.runJobTypeIteration(jobType, policy)
+		if detected {
+			hadJobs = true
+		}
 	}
 
 	r.pruneSchedulerState(active)
 	r.pruneDetectorLeases(active)
+	r.setSchedulerLoopState("", "idle")
+	return hadJobs
+}
+
+func (r *Plugin) wakeScheduler() {
+	if r == nil {
+		return
+	}
+	select {
+	case r.schedulerWakeCh <- struct{}{}:
+	default:
+	}
+}
+
+func (r *Plugin) runJobTypeIteration(jobType string, policy schedulerPolicy) bool {
+	r.recordSchedulerRunStart(jobType)
+	r.clearWaitingJobQueue(jobType)
+	r.setSchedulerLoopState(jobType, "detecting")
+	r.markJobTypeInFlight(jobType)
+	defer r.finishDetection(jobType)
+
+	start := time.Now().UTC()
+	r.appendActivity(JobActivity{
+		JobType:    jobType,
+		Source:     "admin_scheduler",
+		Message:    "scheduled detection started",
+		Stage:      "detecting",
+		OccurredAt: timeToPtr(start),
+	})
+
+	if skip, waitingCount, waitingThreshold := r.shouldSkipDetectionForWaitingJobs(jobType, policy); skip {
+		r.recordSchedulerDetectionSkip(jobType, fmt.Sprintf("waiting backlog %d reached threshold %d", waitingCount, waitingThreshold))
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduled detection skipped: waiting backlog %d reached threshold %d", waitingCount, waitingThreshold),
+			Stage:      "skipped_waiting_backlog",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		r.recordSchedulerRunComplete(jobType, "skipped")
+		return false
+	}
+
+	maxRuntime := policy.JobTypeMaxRuntime
+	if maxRuntime <= 0 {
+		maxRuntime = defaultScheduledJobTypeMaxRuntime
+	}
+	jobCtx, cancel := context.WithTimeout(context.Background(), maxRuntime)
+	defer cancel()
+
+	clusterContext, err := r.loadSchedulerClusterContext(jobCtx)
+	if err != nil {
+		r.recordSchedulerDetectionError(jobType, err)
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduled detection aborted: %v", err),
+			Stage:      "failed",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		r.recordSchedulerRunComplete(jobType, "error")
+		return false
+	}
+
+	detectionTimeout := policy.DetectionTimeout
+	remaining := time.Until(start.Add(maxRuntime))
+	if remaining <= 0 {
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    "scheduled run timed out before detection",
+			Stage:      "timeout",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		r.recordSchedulerRunComplete(jobType, "timeout")
+		return false
+	}
+	if detectionTimeout <= 0 {
+		detectionTimeout = defaultScheduledDetectionTimeout
+	}
+	if detectionTimeout > remaining {
+		detectionTimeout = remaining
+	}
+
+	detectCtx, cancelDetect := context.WithTimeout(jobCtx, detectionTimeout)
+	proposals, err := r.RunDetection(detectCtx, jobType, clusterContext, policy.MaxResults)
+	cancelDetect()
+	if err != nil {
+		r.recordSchedulerDetectionError(jobType, err)
+		stage := "failed"
+		status := "error"
+		if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+			stage = "timeout"
+			status = "timeout"
+		}
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduled detection failed: %v", err),
+			Stage:      stage,
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		r.recordSchedulerRunComplete(jobType, status)
+		return false
+	}
+
+	r.appendActivity(JobActivity{
+		JobType:    jobType,
+		Source:     "admin_scheduler",
+		Message:    fmt.Sprintf("scheduled detection completed: %d proposal(s)", len(proposals)),
+		Stage:      "detected",
+		OccurredAt: timeToPtr(time.Now().UTC()),
+	})
+	r.recordSchedulerDetectionSuccess(jobType, len(proposals))
+
+	detected := len(proposals) > 0
+
+	filteredByActive, skippedActive := r.filterProposalsWithActiveJobs(jobType, proposals)
+	if skippedActive > 0 {
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduled detection skipped %d proposal(s) due to active assigned/running jobs", skippedActive),
+			Stage:      "deduped_active_jobs",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+	}
+
+	if len(filteredByActive) == 0 {
+		r.recordSchedulerRunComplete(jobType, "success")
+		return detected
+	}
+
+	filtered := r.filterScheduledProposals(filteredByActive)
+	if len(filtered) != len(filteredByActive) {
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduled detection deduped %d proposal(s) within this run", len(filteredByActive)-len(filtered)),
+			Stage:      "deduped",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+	}
+
+	if len(filtered) == 0 {
+		r.recordSchedulerRunComplete(jobType, "success")
+		return detected
+	}
+
+	r.setSchedulerLoopState(jobType, "executing")
+
+	remaining = time.Until(start.Add(maxRuntime))
+	if remaining <= 0 {
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    "scheduled execution skipped: job type max runtime reached",
+			Stage:      "timeout",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		r.recordSchedulerRunComplete(jobType, "timeout")
+		return detected
+	}
+
+	execPolicy := policy
+	if execPolicy.ExecutionTimeout <= 0 {
+		execPolicy.ExecutionTimeout = defaultScheduledExecutionTimeout
+	}
+	if execPolicy.ExecutionTimeout > remaining {
+		execPolicy.ExecutionTimeout = remaining
+	}
+
+	successCount, errorCount, canceledCount := r.dispatchScheduledProposals(jobCtx, jobType, filtered, clusterContext, execPolicy)
+
+	status := "success"
+	if jobCtx.Err() != nil {
+		status = "timeout"
+	} else if errorCount > 0 || canceledCount > 0 {
+		status = "error"
+	}
+
+	r.appendActivity(JobActivity{
+		JobType:    jobType,
+		Source:     "admin_scheduler",
+		Message:    fmt.Sprintf("scheduled execution finished: success=%d error=%d canceled=%d", successCount, errorCount, canceledCount),
+		Stage:      "executed",
+		OccurredAt: timeToPtr(time.Now().UTC()),
+	})
+	r.recordSchedulerRunComplete(jobType, status)
+	return detected
 }
 
 func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, error) {
@@ -119,6 +356,7 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
 		DetectionInterval:      durationFromSeconds(adminRuntime.DetectionIntervalSeconds, defaultScheduledDetectionInterval),
 		DetectionTimeout:       durationFromSeconds(adminRuntime.DetectionTimeoutSeconds, defaultScheduledDetectionTimeout),
 		ExecutionTimeout:       defaultScheduledExecutionTimeout,
+		JobTypeMaxRuntime:      durationFromSeconds(adminRuntime.JobTypeMaxRuntimeSeconds, defaultScheduledJobTypeMaxRuntime),
 		RetryBackoff:           durationFromSeconds(adminRuntime.RetryBackoffSeconds, defaultScheduledRetryBackoff),
 		MaxResults:             adminRuntime.MaxJobsPerDetection,
 		ExecutionConcurrency:   int(adminRuntime.GlobalExecutionConcurrency),
@@ -148,6 +386,9 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
 	if policy.RetryLimit < 0 {
 		policy.RetryLimit = 0
 	}
+	if policy.JobTypeMaxRuntime <= 0 {
+		policy.JobTypeMaxRuntime = defaultScheduledJobTypeMaxRuntime
+	}
 
 	// Plugin protocol currently has only detection timeout in admin settings.
 	execTimeout := time.Duration(adminRuntime.DetectionTimeoutSeconds*2) * time.Second
@@ -199,6 +440,7 @@ func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
 				state.DetectionIntervalSeconds = secondsFromDuration(policy.DetectionInterval)
 				state.DetectionTimeoutSeconds = secondsFromDuration(policy.DetectionTimeout)
 				state.ExecutionTimeoutSeconds = secondsFromDuration(policy.ExecutionTimeout)
+				state.JobTypeMaxRuntimeSeconds = secondsFromDuration(policy.JobTypeMaxRuntime)
 				state.MaxJobsPerDetection = policy.MaxResults
 				state.GlobalExecutionConcurrency = policy.ExecutionConcurrency
 				state.PerWorkerExecutionConcurrency = policy.PerWorkerConcurrency
@@ -207,6 +449,19 @@ func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
 			}
 		}
 
+		runInfo := r.snapshotSchedulerRun(jobType)
+		if !runInfo.lastRunStartedAt.IsZero() {
+			at := runInfo.lastRunStartedAt
+			state.LastRunStartedAt = &at
+		}
+		if !runInfo.lastRunCompletedAt.IsZero() {
+			at := runInfo.lastRunCompletedAt
+			state.LastRunCompletedAt = &at
+		}
+		if runInfo.lastRunStatus != "" {
+			state.LastRunStatus = runInfo.lastRunStatus
+		}
+
 		leasedWorkerID := r.getDetectorLease(jobType)
 		if leasedWorkerID != "" {
 			state.DetectorWorkerID = leasedWorkerID
@@ -258,10 +513,11 @@ func deriveSchedulerAdminRuntime(
 		PerWorkerExecutionConcurrency: defaults.PerWorkerExecutionConcurrency,
 		RetryLimit:                    defaults.RetryLimit,
 		RetryBackoffSeconds:           defaults.RetryBackoffSeconds,
+		JobTypeMaxRuntimeSeconds:      defaults.JobTypeMaxRuntimeSeconds,
 	}
 }
 
-func (r *Plugin) markDetectionDue(jobType string, interval time.Duration) bool {
+func (r *Plugin) markDetectionDue(jobType string, interval, initialDelay time.Duration) bool {
 	now := time.Now().UTC()
 
 	r.schedulerMu.Lock()
@@ -275,12 +531,43 @@ func (r *Plugin) markDetectionDue(jobType string, interval time.Duration) bool {
 	if exists && now.Before(nextRun) {
 		return false
 	}
+	if !exists && initialDelay > 0 {
+		r.nextDetectionAt[jobType] = now.Add(initialDelay)
+		return false
+	}
 
 	r.nextDetectionAt[jobType] = now.Add(interval)
 	r.detectionInFlight[jobType] = true
 	return true
 }
 
+func (r *Plugin) earliestNextDetectionAt() time.Time {
+	if r == nil {
+		return time.Time{}
+	}
+
+	r.schedulerMu.Lock()
+	defer r.schedulerMu.Unlock()
+
+	var earliest time.Time
+	for _, nextRun := range r.nextDetectionAt {
+		if nextRun.IsZero() {
+			continue
+		}
+		if earliest.IsZero() || nextRun.Before(earliest) {
+			earliest = nextRun
+		}
+	}
+
+	return earliest
+}
+
+func (r *Plugin) markJobTypeInFlight(jobType string) {
+	r.schedulerMu.Lock()
+	r.detectionInFlight[jobType] = true
+	r.schedulerMu.Unlock()
+}
+
 func (r *Plugin) finishDetection(jobType string) {
 	r.schedulerMu.Lock()
 	delete(r.detectionInFlight, jobType)
@@ -318,125 +605,18 @@ func (r *Plugin) pruneDetectorLeases(activeJobTypes map[string]struct{}) {
 	}
 }
 
-func (r *Plugin) runScheduledDetection(jobType string, policy schedulerPolicy) {
-	defer r.finishDetection(jobType)
-
-	releaseLock, lockErr := r.acquireAdminLock(fmt.Sprintf("plugin scheduled detection %s", jobType))
-	if lockErr != nil {
-		r.recordSchedulerDetectionError(jobType, lockErr)
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection aborted: failed to acquire lock: %v", lockErr),
-			Stage:      "failed",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-		return
-	}
-	if releaseLock != nil {
-		defer releaseLock()
-	}
-
-	start := time.Now().UTC()
-	r.appendActivity(JobActivity{
-		JobType:    jobType,
-		Source:     "admin_scheduler",
-		Message:    "scheduled detection started",
-		Stage:      "detecting",
-		OccurredAt: timeToPtr(start),
-	})
-
-	if skip, waitingCount, waitingThreshold := r.shouldSkipDetectionForWaitingJobs(jobType, policy); skip {
-		r.recordSchedulerDetectionSkip(jobType, fmt.Sprintf("waiting backlog %d reached threshold %d", waitingCount, waitingThreshold))
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection skipped: waiting backlog %d reached threshold %d", waitingCount, waitingThreshold),
-			Stage:      "skipped_waiting_backlog",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-		return
-	}
-
-	clusterContext, err := r.loadSchedulerClusterContext()
-	if err != nil {
-		r.recordSchedulerDetectionError(jobType, err)
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection aborted: %v", err),
-			Stage:      "failed",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-		return
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), policy.DetectionTimeout)
-	proposals, err := r.RunDetection(ctx, jobType, clusterContext, policy.MaxResults)
-	cancel()
-	if err != nil {
-		r.recordSchedulerDetectionError(jobType, err)
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection failed: %v", err),
-			Stage:      "failed",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-		return
-	}
-
-	r.appendActivity(JobActivity{
-		JobType:    jobType,
-		Source:     "admin_scheduler",
-		Message:    fmt.Sprintf("scheduled detection completed: %d proposal(s)", len(proposals)),
-		Stage:      "detected",
-		OccurredAt: timeToPtr(time.Now().UTC()),
-	})
-	r.recordSchedulerDetectionSuccess(jobType, len(proposals))
-
-	filteredByActive, skippedActive := r.filterProposalsWithActiveJobs(jobType, proposals)
-	if skippedActive > 0 {
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection skipped %d proposal(s) due to active assigned/running jobs", skippedActive),
-			Stage:      "deduped_active_jobs",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-	}
-
-	if len(filteredByActive) == 0 {
-		return
-	}
-
-	filtered := r.filterScheduledProposals(filteredByActive)
-	if len(filtered) != len(filteredByActive) {
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection deduped %d proposal(s) within this run", len(filteredByActive)-len(filtered)),
-			Stage:      "deduped",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-	}
-
-	if len(filtered) == 0 {
-		return
-	}
-
-	r.dispatchScheduledProposals(jobType, filtered, clusterContext, policy)
-}
-
-func (r *Plugin) loadSchedulerClusterContext() (*plugin_pb.ClusterContext, error) {
+func (r *Plugin) loadSchedulerClusterContext(ctx context.Context) (*plugin_pb.ClusterContext, error) {
 	if r.clusterContextProvider == nil {
 		return nil, fmt.Errorf("cluster context provider is not configured")
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultClusterContextTimeout)
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	clusterCtx, cancel := context.WithTimeout(ctx, defaultClusterContextTimeout)
 	defer cancel()
 
-	clusterContext, err := r.clusterContextProvider(ctx)
+	clusterContext, err := r.clusterContextProvider(clusterCtx)
 	if err != nil {
 		return nil, err
 	}
@@ -447,11 +627,16 @@ func (r *Plugin) loadSchedulerClusterContext() (*plugin_pb.ClusterContext, error
 }
 
 func (r *Plugin) dispatchScheduledProposals(
+	ctx context.Context,
 	jobType string,
 	proposals []*plugin_pb.JobProposal,
 	clusterContext *plugin_pb.ClusterContext,
 	policy schedulerPolicy,
-) {
+) (int, int, int) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
 	jobQueue := make(chan *plugin_pb.JobSpec, len(proposals))
 	for index, proposal := range proposals {
 		job := buildScheduledJobSpec(jobType, proposal, index)
@@ -459,7 +644,7 @@ func (r *Plugin) dispatchScheduledProposals(
 		select {
 		case <-r.shutdownCh:
 			close(jobQueue)
-			return
+			return 0, 0, 0
 		default:
 			jobQueue <- job
 		}
@@ -470,6 +655,7 @@ func (r *Plugin) dispatchScheduledProposals(
 	var statsMu sync.Mutex
 	successCount := 0
 	errorCount := 0
+	canceledCount := 0
 
 	workerCount := policy.ExecutionConcurrency
 	if workerCount < 1 {
@@ -481,6 +667,7 @@ func (r *Plugin) dispatchScheduledProposals(
 		go func() {
 			defer wg.Done()
 
+		jobLoop:
 			for job := range jobQueue {
 				select {
 				case <-r.shutdownCh:
@@ -488,19 +675,36 @@ func (r *Plugin) dispatchScheduledProposals(
 				default:
 				}
 
+				if ctx.Err() != nil {
+					r.cancelQueuedJob(job, ctx.Err())
+					statsMu.Lock()
+					canceledCount++
+					statsMu.Unlock()
+					continue
+				}
+
 				for {
 					select {
 					case <-r.shutdownCh:
 						return
 					default:
 					}
+					if ctx.Err() != nil {
+						r.cancelQueuedJob(job, ctx.Err())
+						statsMu.Lock()
+						canceledCount++
+						statsMu.Unlock()
+						continue jobLoop
+					}
 
-					executor, release, reserveErr := r.reserveScheduledExecutor(jobType, policy)
+					executor, release, reserveErr := r.reserveScheduledExecutor(ctx, jobType, policy)
 					if reserveErr != nil {
-						select {
-						case <-r.shutdownCh:
-							return
-						default:
+						if ctx.Err() != nil {
+							r.cancelQueuedJob(job, ctx.Err())
+							statsMu.Lock()
+							canceledCount++
+							statsMu.Unlock()
+							continue jobLoop
 						}
 						statsMu.Lock()
 						errorCount++
@@ -515,16 +719,23 @@ func (r *Plugin) dispatchScheduledProposals(
 						break
 					}
 
-					err := r.executeScheduledJobWithExecutor(executor, job, clusterContext, policy)
+					err := r.executeScheduledJobWithExecutor(ctx, executor, job, clusterContext, policy)
 					release()
 					if errors.Is(err, errExecutorAtCapacity) {
 						r.trackExecutionQueued(job)
-						if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
+						if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
 							return
 						}
 						continue
 					}
 					if err != nil {
+						if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+							r.cancelQueuedJob(job, err)
+							statsMu.Lock()
+							canceledCount++
+							statsMu.Unlock()
+							continue jobLoop
+						}
 						statsMu.Lock()
 						errorCount++
 						statsMu.Unlock()
@@ -550,23 +761,34 @@ func (r *Plugin) dispatchScheduledProposals(
 
 	wg.Wait()
 
-	r.appendActivity(JobActivity{
-		JobType:    jobType,
-		Source:     "admin_scheduler",
-		Message:    fmt.Sprintf("scheduled execution finished: success=%d error=%d", successCount, errorCount),
-		Stage:      "executed",
-		OccurredAt: timeToPtr(time.Now().UTC()),
-	})
+	drainErr := ctx.Err()
+	if drainErr == nil {
+		drainErr = errSchedulerShutdown
+	}
+	for job := range jobQueue {
+		r.cancelQueuedJob(job, drainErr)
+		canceledCount++
+	}
+
+	return successCount, errorCount, canceledCount
 }
 
 func (r *Plugin) reserveScheduledExecutor(
+	ctx context.Context,
 	jobType string,
 	policy schedulerPolicy,
 ) (*WorkerSession, func(), error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
 	deadline := time.Now().Add(policy.ExecutionTimeout)
 	if policy.ExecutionTimeout <= 0 {
 		deadline = time.Now().Add(10 * time.Minute) // Default cap
 	}
+	if ctxDeadline, ok := ctx.Deadline(); ok && ctxDeadline.Before(deadline) {
+		deadline = ctxDeadline
+	}
 
 	for {
 		select {
@@ -574,6 +796,9 @@ func (r *Plugin) reserveScheduledExecutor(
 			return nil, nil, fmt.Errorf("plugin is shutting down")
 		default:
 		}
+		if ctx.Err() != nil {
+			return nil, nil, ctx.Err()
+		}
 
 		if time.Now().After(deadline) {
 			return nil, nil, fmt.Errorf("timed out waiting for executor capacity for %s", jobType)
@@ -581,7 +806,10 @@ func (r *Plugin) reserveScheduledExecutor(
 
 		executors, err := r.registry.ListExecutors(jobType)
 		if err != nil {
-			if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
+			if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
+				if ctx.Err() != nil {
+					return nil, nil, ctx.Err()
+				}
 				return nil, nil, fmt.Errorf("plugin is shutting down")
 			}
 			continue
@@ -595,7 +823,10 @@ func (r *Plugin) reserveScheduledExecutor(
 			return executor, release, nil
 		}
 
-		if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
+		if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
+			if ctx.Err() != nil {
+				return nil, nil, ctx.Err()
+			}
 			return nil, nil, fmt.Errorf("plugin is shutting down")
 		}
 	}
@@ -680,6 +911,7 @@ func schedulerWorkerExecutionLimit(executor *WorkerSession, jobType string, poli
 }
 
 func (r *Plugin) executeScheduledJobWithExecutor(
+	ctx context.Context,
 	executor *WorkerSession,
 	job *plugin_pb.JobSpec,
 	clusterContext *plugin_pb.ClusterContext,
@@ -697,8 +929,15 @@ func (r *Plugin) executeScheduledJobWithExecutor(
 			return fmt.Errorf("plugin is shutting down")
 		default:
 		}
+		if ctx != nil && ctx.Err() != nil {
+			return ctx.Err()
+		}
 
-		execCtx, cancel := context.WithTimeout(context.Background(), policy.ExecutionTimeout)
+		parent := ctx
+		if parent == nil {
+			parent = context.Background()
+		}
+		execCtx, cancel := context.WithTimeout(parent, policy.ExecutionTimeout)
 		_, err := r.executeJobWithExecutor(execCtx, executor, job, clusterContext, int32(attempt))
 		cancel()
 		if err == nil {
@@ -718,7 +957,10 @@ func (r *Plugin) executeScheduledJobWithExecutor(
 				Stage:      "retry",
 				OccurredAt: timeToPtr(time.Now().UTC()),
 			})
-			if !waitForShutdownOrTimer(r.shutdownCh, policy.RetryBackoff) {
+			if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.RetryBackoff) {
+				if ctx != nil && ctx.Err() != nil {
+					return ctx.Err()
+				}
 				return fmt.Errorf("plugin is shutting down")
 			}
 		}
@@ -764,6 +1006,53 @@ func (r *Plugin) countWaitingTrackedJobs(jobType string) int {
 	return waiting
 }
 
+func (r *Plugin) clearWaitingJobQueue(jobType string) int {
+	normalizedJobType := strings.TrimSpace(jobType)
+	if normalizedJobType == "" {
+		return 0
+	}
+
+	jobIDs := make([]string, 0)
+	seen := make(map[string]struct{})
+
+	r.jobsMu.RLock()
+	for _, job := range r.jobs {
+		if job == nil {
+			continue
+		}
+		if strings.TrimSpace(job.JobType) != normalizedJobType {
+			continue
+		}
+		if !isWaitingTrackedJobState(job.State) {
+			continue
+		}
+		jobID := strings.TrimSpace(job.JobID)
+		if jobID == "" {
+			continue
+		}
+		if _, ok := seen[jobID]; ok {
+			continue
+		}
+		seen[jobID] = struct{}{}
+		jobIDs = append(jobIDs, jobID)
+	}
+	r.jobsMu.RUnlock()
+
+	if len(jobIDs) == 0 {
+		return 0
+	}
+
+	reason := fmt.Sprintf("cleared queued job before %s run", normalizedJobType)
+	for _, jobID := range jobIDs {
+		r.markJobCanceled(&plugin_pb.JobSpec{
+			JobId:   jobID,
+			JobType: normalizedJobType,
+		}, reason)
+	}
+
+	return len(jobIDs)
+}
+
 func waitingBacklogThreshold(policy schedulerPolicy) int {
 	concurrency := policy.ExecutionConcurrency
 	if concurrency <= 0 {
@@ -861,6 +1150,27 @@ func waitForShutdownOrTimer(shutdown <-chan struct{}, duration time.Duration) bo
 	}
 }
 
+func waitForShutdownOrTimerWithContext(shutdown <-chan struct{}, ctx context.Context, duration time.Duration) bool {
+	if duration <= 0 {
+		return true
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	timer := time.NewTimer(duration)
+	defer timer.Stop()
+
+	select {
+	case <-shutdown:
+		return false
+	case <-ctx.Done():
+		return false
+	case <-timer.C:
+		return true
+	}
+}
+
 // filterProposalsWithActiveJobs removes proposals whose dedupe keys already have active jobs.
 // It first expires stale tracked jobs via expireStaleJobs, which can mutate scheduler state,
 // so callers should treat this method as a stateful operation.

weed/admin/plugin/plugin_scheduler_test.go

@@ -1,6 +1,7 @@
 package plugin
 
 import (
+	"context"
 	"fmt"
 	"testing"
 	"time"
@@ -28,6 +29,7 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
 			PerWorkerExecutionConcurrency: 2,
 			RetryLimit:                    4,
 			RetryBackoffSeconds:           7,
+			JobTypeMaxRuntimeSeconds:      1800,
 		},
 	})
 	if err != nil {
@@ -53,6 +55,9 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
 	if policy.RetryLimit != 4 {
 		t.Fatalf("unexpected retry limit: got=%d", policy.RetryLimit)
 	}
+	if policy.JobTypeMaxRuntime != 30*time.Minute {
+		t.Fatalf("unexpected max runtime: got=%v", policy.JobTypeMaxRuntime)
+	}
 }
 
 func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T) {
@@ -75,6 +80,7 @@ func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T
 			PerWorkerExecutionConcurrency: 2,
 			RetryLimit:                    3,
 			RetryBackoffSeconds:           6,
+			JobTypeMaxRuntimeSeconds:      1200,
 		},
 	})
 	if err != nil {
@@ -97,6 +103,9 @@ func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T
 	if policy.PerWorkerConcurrency != 2 {
 		t.Fatalf("unexpected per-worker concurrency: got=%d", policy.PerWorkerConcurrency)
 	}
+	if policy.JobTypeMaxRuntime != 20*time.Minute {
+		t.Fatalf("unexpected max runtime: got=%v", policy.JobTypeMaxRuntime)
+	}
 }
 
 func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) {
@@ -126,13 +135,13 @@ func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) {
 		ExecutorReserveBackoff: time.Millisecond,
 	}
 
-	executor1, release1, err := pluginSvc.reserveScheduledExecutor("balance", policy)
+	executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
 	if err != nil {
 		t.Fatalf("reserve executor 1: %v", err)
 	}
 	defer release1()
 
-	executor2, release2, err := pluginSvc.reserveScheduledExecutor("balance", policy)
+	executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
 	if err != nil {
 		t.Fatalf("reserve executor 2: %v", err)
 	}
@@ -254,7 +263,7 @@ func TestReserveScheduledExecutorTimesOutWhenNoExecutor(t *testing.T) {
 
 	start := time.Now()
 	pluginSvc.Shutdown()
-	_, _, err = pluginSvc.reserveScheduledExecutor("missing-job-type", policy)
+	_, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy)
 	if err == nil {
 		t.Fatalf("expected reservation shutdown error")
 	}
@@ -285,7 +294,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
 		ExecutorReserveBackoff: 5 * time.Millisecond,
 	}
 
-	_, release1, err := pluginSvc.reserveScheduledExecutor("balance", policy)
+	_, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
 	if err != nil {
 		t.Fatalf("reserve executor 1: %v", err)
 	}
@@ -296,7 +305,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
 	}
 	secondReserveCh := make(chan reserveResult, 1)
 	go func() {
-		_, release2, reserveErr := pluginSvc.reserveScheduledExecutor("balance", policy)
+		_, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
 		if release2 != nil {
 			release2()
 		}
@@ -394,6 +403,7 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
 			PerWorkerExecutionConcurrency: 2,
 			RetryLimit:                    1,
 			RetryBackoffSeconds:           9,
+			JobTypeMaxRuntimeSeconds:      900,
 		},
 	})
 	if err != nil {
@@ -446,6 +456,9 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
 	if state.ExecutionTimeoutSeconds != 90 {
 		t.Fatalf("unexpected execution timeout: got=%d", state.ExecutionTimeoutSeconds)
 	}
+	if state.JobTypeMaxRuntimeSeconds != 900 {
+		t.Fatalf("unexpected job type max runtime: got=%d", state.JobTypeMaxRuntimeSeconds)
+	}
 	if state.MaxJobsPerDetection != 80 {
 		t.Fatalf("unexpected max jobs per detection: got=%d", state.MaxJobsPerDetection)
 	}

weed/admin/plugin/scheduler_config.go

@@ -0,0 +1,31 @@
+package plugin
+
+import "time"
+
+const (
+	defaultSchedulerIdleSleep = 613 * time.Second
+)
+
+type SchedulerConfig struct {
+	IdleSleepSeconds int32 `json:"idle_sleep_seconds"`
+}
+
+func DefaultSchedulerConfig() SchedulerConfig {
+	return SchedulerConfig{
+		IdleSleepSeconds: int32(defaultSchedulerIdleSleep / time.Second),
+	}
+}
+
+func normalizeSchedulerConfig(cfg SchedulerConfig) SchedulerConfig {
+	if cfg.IdleSleepSeconds <= 0 {
+		return DefaultSchedulerConfig()
+	}
+	return cfg
+}
+
+func (c SchedulerConfig) IdleSleepDuration() time.Duration {
+	if c.IdleSleepSeconds <= 0 {
+		return defaultSchedulerIdleSleep
+	}
+	return time.Duration(c.IdleSleepSeconds) * time.Second
+}

weed/admin/plugin/scheduler_status.go

@@ -9,6 +9,12 @@ import (
 type SchedulerStatus struct {
 	Now                  time.Time                `json:"now"`
 	SchedulerTickSeconds int                      `json:"scheduler_tick_seconds"`
+	IdleSleepSeconds     int                      `json:"idle_sleep_seconds,omitempty"`
+	NextDetectionAt      *time.Time               `json:"next_detection_at,omitempty"`
+	CurrentJobType       string                   `json:"current_job_type,omitempty"`
+	CurrentPhase         string                   `json:"current_phase,omitempty"`
+	LastIterationHadJobs bool                     `json:"last_iteration_had_jobs,omitempty"`
+	LastIterationDoneAt  *time.Time               `json:"last_iteration_done_at,omitempty"`
 	Waiting              []SchedulerWaitingStatus `json:"waiting,omitempty"`
 	InProcessJobs        []SchedulerJobStatus     `json:"in_process_jobs,omitempty"`
 	JobTypes             []SchedulerJobTypeStatus `json:"job_types,omitempty"`
@@ -56,6 +62,19 @@ type schedulerDetectionInfo struct {
 	lastSkippedReason string
 }
 
+type schedulerRunInfo struct {
+	lastRunStartedAt   time.Time
+	lastRunCompletedAt time.Time
+	lastRunStatus      string
+}
+
+type schedulerLoopState struct {
+	currentJobType         string
+	currentPhase           string
+	lastIterationHadJobs   bool
+	lastIterationCompleted time.Time
+}
+
 func (r *Plugin) recordSchedulerDetectionSuccess(jobType string, count int) {
 	if r == nil {
 		return
@@ -122,12 +141,105 @@ func (r *Plugin) snapshotSchedulerDetection(jobType string) schedulerDetectionIn
 	return *info
 }
 
+func (r *Plugin) recordSchedulerRunStart(jobType string) {
+	if r == nil {
+		return
+	}
+	r.schedulerRunMu.Lock()
+	defer r.schedulerRunMu.Unlock()
+	info := r.schedulerRun[jobType]
+	if info == nil {
+		info = &schedulerRunInfo{}
+		r.schedulerRun[jobType] = info
+	}
+	info.lastRunStartedAt = time.Now().UTC()
+	info.lastRunStatus = ""
+}
+
+func (r *Plugin) recordSchedulerRunComplete(jobType, status string) {
+	if r == nil {
+		return
+	}
+	r.schedulerRunMu.Lock()
+	defer r.schedulerRunMu.Unlock()
+	info := r.schedulerRun[jobType]
+	if info == nil {
+		info = &schedulerRunInfo{}
+		r.schedulerRun[jobType] = info
+	}
+	info.lastRunCompletedAt = time.Now().UTC()
+	info.lastRunStatus = status
+}
+
+func (r *Plugin) snapshotSchedulerRun(jobType string) schedulerRunInfo {
+	if r == nil {
+		return schedulerRunInfo{}
+	}
+	r.schedulerRunMu.Lock()
+	defer r.schedulerRunMu.Unlock()
+	info := r.schedulerRun[jobType]
+	if info == nil {
+		return schedulerRunInfo{}
+	}
+	return *info
+}
+
+func (r *Plugin) setSchedulerLoopState(jobType, phase string) {
+	if r == nil {
+		return
+	}
+	r.schedulerLoopMu.Lock()
+	r.schedulerLoopState.currentJobType = jobType
+	r.schedulerLoopState.currentPhase = phase
+	r.schedulerLoopMu.Unlock()
+}
+
+func (r *Plugin) recordSchedulerIterationComplete(hadJobs bool) {
+	if r == nil {
+		return
+	}
+	r.schedulerLoopMu.Lock()
+	r.schedulerLoopState.lastIterationHadJobs = hadJobs
+	r.schedulerLoopState.lastIterationCompleted = time.Now().UTC()
+	r.schedulerLoopMu.Unlock()
+}
+
+func (r *Plugin) snapshotSchedulerLoopState() schedulerLoopState {
+	if r == nil {
+		return schedulerLoopState{}
+	}
+	r.schedulerLoopMu.Lock()
+	defer r.schedulerLoopMu.Unlock()
+	return r.schedulerLoopState
+}
+
 func (r *Plugin) GetSchedulerStatus() SchedulerStatus {
 	now := time.Now().UTC()
+	loopState := r.snapshotSchedulerLoopState()
+	schedulerConfig := r.GetSchedulerConfig()
 	status := SchedulerStatus{
 		Now:                  now,
 		SchedulerTickSeconds: int(secondsFromDuration(r.schedulerTick)),
 		InProcessJobs:        r.listInProcessJobs(now),
+		IdleSleepSeconds:     int(schedulerConfig.IdleSleepSeconds),
+		CurrentJobType:       loopState.currentJobType,
+		CurrentPhase:         loopState.currentPhase,
+		LastIterationHadJobs: loopState.lastIterationHadJobs,
+	}
+	nextDetectionAt := r.earliestNextDetectionAt()
+	if nextDetectionAt.IsZero() && loopState.currentPhase == "sleeping" && !loopState.lastIterationCompleted.IsZero() {
+		idleSleep := schedulerConfig.IdleSleepDuration()
+		if idleSleep > 0 {
+			nextDetectionAt = loopState.lastIterationCompleted.Add(idleSleep)
+		}
+	}
+	if !nextDetectionAt.IsZero() {
+		at := nextDetectionAt
+		status.NextDetectionAt = &at
+	}
+	if !loopState.lastIterationCompleted.IsZero() {
+		at := loopState.lastIterationCompleted
+		status.LastIterationDoneAt = &at
 	}
 
 	states, err := r.ListSchedulerStates()

weed/admin/plugin/types.go

@@ -90,6 +90,7 @@ type SchedulerJobTypeState struct {
 	DetectionIntervalSeconds      int32      `json:"detection_interval_seconds,omitempty"`
 	DetectionTimeoutSeconds       int32      `json:"detection_timeout_seconds,omitempty"`
 	ExecutionTimeoutSeconds       int32      `json:"execution_timeout_seconds,omitempty"`
+	JobTypeMaxRuntimeSeconds      int32      `json:"job_type_max_runtime_seconds,omitempty"`
 	MaxJobsPerDetection           int32      `json:"max_jobs_per_detection,omitempty"`
 	GlobalExecutionConcurrency    int        `json:"global_execution_concurrency,omitempty"`
 	PerWorkerExecutionConcurrency int        `json:"per_worker_execution_concurrency,omitempty"`
@@ -98,6 +99,9 @@ type SchedulerJobTypeState struct {
 	DetectorAvailable             bool       `json:"detector_available"`
 	DetectorWorkerID              string     `json:"detector_worker_id,omitempty"`
 	ExecutorWorkerCount           int        `json:"executor_worker_count"`
+	LastRunStartedAt              *time.Time `json:"last_run_started_at,omitempty"`
+	LastRunCompletedAt            *time.Time `json:"last_run_completed_at,omitempty"`
+	LastRunStatus                 string     `json:"last_run_status,omitempty"`
 }
 
 func timeToPtr(t time.Time) *time.Time {

weed/admin/static/css/admin.css

@@ -1,5 +1,9 @@
 /* SeaweedFS Dashboard Custom Styles */
 
+:root {
+    --navbar-height: 56px;
+}
+
 /* Link colors - muted */
 a {
     color: #5b7c99;
@@ -12,7 +16,7 @@ a:hover {
 /* Sidebar Styles */
 .sidebar {
     position: fixed;
-    top: 56px;
+    top: var(--navbar-height);
     bottom: 0;
     left: 0;
     z-index: 100;
@@ -51,13 +55,32 @@ main {
 
 @media (max-width: 767.98px) {
     .sidebar {
-        top: 5rem;
+        top: var(--navbar-height);
+        padding-top: 0;
+        width: 240px;
+        background-color: #f8f9fa !important;
+        z-index: 1050;
+    }
+    .sidebar.show ~ .sidebar-backdrop {
+        display: block;
     }
     main {
         margin-left: 0;
     }
 }
 
+/* Sidebar backdrop for mobile overlay */
+.sidebar-backdrop {
+    display: none;
+    position: fixed;
+    top: var(--navbar-height);
+    left: 0;
+    right: 0;
+    bottom: 0;
+    background-color: rgba(0, 0, 0, 0.5);
+    z-index: 1040;
+}
+
 /* Custom card styles */
 .border-left-primary {
     border-left: 0.25rem solid #6b8caf !important;
@@ -262,6 +285,11 @@ main {
     box-shadow: 0 0.15rem 1.75rem 0 rgba(58, 59, 69, 0.15) !important;
 }
 
+/* Navbar user icon color */
+.navbar-toggler .fa-user {
+    color: rgba(255, 255, 255, 0.75);
+}
+
 /* Collapsible menu styles */
 .nav-link[data-bs-toggle="collapse"] {
     position: relative;

weed/admin/static/js/admin.js

@@ -31,6 +31,9 @@ function initializeDashboard() {
 
     // Set up submenu behavior
     setupSubmenuBehavior();
+
+    // Set up mobile sidebar behavior
+    setupMobileSidebar();
 }
 
 // HTMX event listeners
@@ -194,6 +197,33 @@ function setupSubmenuBehavior() {
 
 }
 
+// Mobile sidebar toggle and backdrop behavior
+function setupMobileSidebar() {
+    const sidebar = document.getElementById('sidebarMenu');
+    const backdrop = document.getElementById('sidebarBackdrop');
+    if (!sidebar || !backdrop) return;
+
+    const hideSidebar = () => {
+        const bsCollapse = bootstrap.Collapse.getInstance(sidebar);
+        if (bsCollapse) {
+            bsCollapse.hide();
+        }
+    };
+
+    // Close sidebar when backdrop is clicked
+    backdrop.addEventListener('click', hideSidebar);
+
+    // Close sidebar when a nav link is clicked (on mobile)
+    const sidebarToggler = document.querySelector("button[data-bs-target='#sidebarMenu']");
+    sidebar.querySelectorAll('a.nav-link:not([data-bs-toggle="collapse"])').forEach(function (link) {
+        link.addEventListener('click', function () {
+            if (sidebarToggler && getComputedStyle(sidebarToggler).display !== 'none') {
+                hideSidebar();
+            }
+        });
+    });
+}
+
 // Loading indicator functions
 function showLoadingIndicator() {
     const indicator = document.getElementById('loading-indicator');

weed/admin/topology/task_management.go

@@ -17,20 +17,30 @@ func (at *ActiveTopology) AssignTask(taskID string) error {
 		return fmt.Errorf("pending task %s not found", taskID)
 	}
 
-	// Check if all destination disks have sufficient capacity to reserve
-	for _, dest := range task.Destinations {
-		targetKey := fmt.Sprintf("%s:%d", dest.TargetServer, dest.TargetDisk)
-		if targetDisk, exists := at.disks[targetKey]; exists {
-			availableCapacity := at.getEffectiveAvailableCapacityUnsafe(targetDisk)
+	// Skip capacity check if topology hasn't been populated yet
+	if len(at.disks) == 0 {
+		glog.Warningf("AssignTask %s: topology has no disks yet, skipping capacity check", taskID)
+	} else {
+		// Check if all destination disks have sufficient capacity to reserve
+		for _, dest := range task.Destinations {
+			targetKey := fmt.Sprintf("%s:%d", dest.TargetServer, dest.TargetDisk)
+			if targetDisk, exists := at.disks[targetKey]; exists {
+				availableCapacity := at.getEffectiveAvailableCapacityUnsafe(targetDisk)
 
-			// Check if we have enough total capacity using the improved unified comparison
-			if !availableCapacity.CanAccommodate(dest.StorageChange) {
-				return fmt.Errorf("insufficient capacity on target disk %s:%d. Available: %+v, Required: %+v",
-					dest.TargetServer, dest.TargetDisk, availableCapacity, dest.StorageChange)
+				// Check if we have enough total capacity using the improved unified comparison
+				if !availableCapacity.CanAccommodate(dest.StorageChange) {
+					return fmt.Errorf("insufficient capacity on target disk %s:%d. Available: %+v, Required: %+v",
+						dest.TargetServer, dest.TargetDisk, availableCapacity, dest.StorageChange)
+				}
+			} else if dest.TargetServer != "" {
+				// Fail fast if destination disk is not found in topology
+				var existingKeys []string
+				for k := range at.disks {
+					existingKeys = append(existingKeys, k)
+				}
+				glog.Warningf("destination disk %s not found in topology. Existing disk keys: %v", targetKey, existingKeys)
+				return fmt.Errorf("destination disk %s not found in topology", targetKey)
 			}
-		} else if dest.TargetServer != "" {
-			// Fail fast if destination disk is not found in topology
-			return fmt.Errorf("destination disk %s not found in topology", targetKey)
 		}
 	}
 

weed/admin/topology/topology_management.go

@@ -87,6 +87,8 @@ func (at *ActiveTopology) UpdateTopology(topologyInfo *master_pb.TopologyInfo) e
 					}
 
 					diskKey := fmt.Sprintf("%s:%d", nodeInfo.Id, diskInfo.DiskId)
+					glog.V(2).Infof("UpdateTopology: adding disk key=%q nodeId=%q diskId=%d diskType=%q address=%q grpcPort=%d volumes=%d maxVolumes=%d",
+						diskKey, nodeInfo.Id, diskInfo.DiskId, diskType, nodeInfo.Address, nodeInfo.GrpcPort, diskInfo.VolumeCount, diskInfo.MaxVolumeCount)
 					node.disks[diskInfo.DiskId] = disk
 					at.disks[diskKey] = disk
 				}

weed/admin/view/app/plugin.templ

@@ -65,7 +65,7 @@ templ Plugin(page string) {
 
         <div class="plugin-section plugin-section-overview">
             <div class="row mb-4">
-                <div class="col-md-4 mb-3">
+                <div class="col-md-3 mb-3">
                     <div class="card border-info h-100">
                         <div class="card-body text-center">
                             <div class="text-uppercase text-muted small mb-1">Workers</div>
@@ -73,7 +73,7 @@ templ Plugin(page string) {
                         </div>
                     </div>
                 </div>
-                <div class="col-md-4 mb-3">
+                <div class="col-md-3 mb-3">
                     <div class="card border-warning h-100">
                         <div class="card-body text-center">
                             <div class="text-uppercase text-muted small mb-1">Active Jobs</div>
@@ -81,7 +81,7 @@ templ Plugin(page string) {
                         </div>
                     </div>
                 </div>
-                <div class="col-md-4 mb-3">
+                <div class="col-md-3 mb-3">
                     <div class="card border-success h-100">
                         <div class="card-body text-center">
                             <div class="text-uppercase text-muted small mb-1">Activities (recent)</div>
@@ -89,6 +89,15 @@ templ Plugin(page string) {
                         </div>
                     </div>
                 </div>
+                <div class="col-md-3 mb-3">
+                    <div class="card border-secondary h-100">
+                        <div class="card-body text-center">
+                            <div class="text-uppercase text-muted small mb-1">Next Run</div>
+                            <h4 class="mb-0 plugin-scheduler-next-run">-</h4>
+                            <div class="text-muted small plugin-scheduler-next-run-meta"></div>
+                        </div>
+                    </div>
+                </div>
             </div>
             <div class="row mb-4">
                 <div class="col-12">
@@ -120,7 +129,7 @@ templ Plugin(page string) {
                     <div class="card shadow-sm">
                         <div class="card-header d-flex justify-content-between align-items-center flex-wrap gap-2">
                             <h5 class="mb-0"><i class="fas fa-clock me-2"></i>Scheduler State</h5>
-                            <small class="text-muted">Per job type detection schedule and execution limits</small>
+                            <small class="text-muted">Sequential scheduler with per-job runtime limits</small>
                         </div>
                         <div class="card-body p-0">
                             <div class="table-responsive">
@@ -131,12 +140,12 @@ templ Plugin(page string) {
                                             <th>Enabled</th>
                                             <th>Detector</th>
                                             <th>In Flight</th>
-                                            <th>Next Detection</th>
-                                            <th>Interval</th>
+                                            <th>Max Runtime</th>
                                             <th>Exec Global</th>
                                             <th>Exec/Worker</th>
                                             <th>Executor Workers</th>
                                             <th>Effective Exec</th>
+                                            <th>Last Run</th>
                                         </tr>
                                     </thead>
                                     <tbody id="plugin-scheduler-table-body">
@@ -245,11 +254,16 @@ templ Plugin(page string) {
                                 <div class="col-12">
                                     <label class="form-label" for="plugin-admin-detection-interval">Detection Interval (s)</label>
                                     <input type="number" class="form-control" id="plugin-admin-detection-interval" min="0"/>
+                                    <div class="form-text">How often to check for new work.</div>
                                 </div>
                                 <div class="col-12">
                                     <label class="form-label" for="plugin-admin-detection-timeout">Detection Timeout (s)</label>
                                     <input type="number" class="form-control" id="plugin-admin-detection-timeout" min="0"/>
                                 </div>
+                                <div class="col-12">
+                                    <label class="form-label" for="plugin-admin-max-runtime">Job Type Max Runtime (s)</label>
+                                    <input type="number" class="form-control" id="plugin-admin-max-runtime" min="0"/>
+                                </div>
                                 <div class="col-12">
                                     <label class="form-label" for="plugin-admin-max-results">Max Jobs / Detection</label>
                                     <input type="number" class="form-control" id="plugin-admin-max-results" min="0"/>
@@ -273,6 +287,17 @@ templ Plugin(page string) {
                             </div>
                         </div>
                     </div>
+
+                    <div class="card shadow-sm mt-3">
+                        <div class="card-header d-flex justify-content-between align-items-center">
+                            <h5 class="mb-0"><i class="fas fa-hourglass-half me-2"></i>Next Run</h5>
+                            <small class="text-muted">Scheduler</small>
+                        </div>
+                        <div class="card-body">
+                            <div class="h5 mb-1 plugin-scheduler-next-run">-</div>
+                            <div class="text-muted small plugin-scheduler-next-run-meta">Not scheduled</div>
+                        </div>
+                    </div>
                 </div>
             </div>
 
@@ -572,6 +597,7 @@ templ Plugin(page string) {
                 jobs: [],
                 activities: [],
                 schedulerStates: [],
+                schedulerStatus: null,
                 allJobs: [],
                 allActivities: [],
                 loadedJobType: '',
@@ -1442,8 +1468,8 @@ templ Plugin(page string) {
                     var enabled = !!item.enabled;
                     var inFlight = !!item.detection_in_flight;
                     var detector = item.detector_available ? textOrDash(item.detector_worker_id) : 'No detector';
-                    var intervalSeconds = Number(item.detection_interval_seconds || 0);
-                    var intervalText = intervalSeconds > 0 ? (String(intervalSeconds) + 's') : '-';
+                    var maxRuntimeSeconds = Number(item.job_type_max_runtime_seconds || 0);
+                    var maxRuntimeText = maxRuntimeSeconds > 0 ? (String(maxRuntimeSeconds) + 's') : '-';
                     var globalExec = Number(item.global_execution_concurrency || 0);
                     var perWorkerExec = Number(item.per_worker_execution_concurrency || 0);
                     var executorWorkers = Number(item.executor_worker_count || 0);
@@ -1452,6 +1478,13 @@ templ Plugin(page string) {
                     var perWorkerExecText = enabled ? String(perWorkerExec) : '-';
                     var executorWorkersText = enabled ? String(executorWorkers) : '-';
                     var effectiveExecText = enabled ? String(effectiveExec) : '-';
+                    var lastRunStatus = textOrDash(item.last_run_status);
+                    var lastRunTime = parseTime(item.last_run_completed_at);
+                    var lastRunText = '-';
+                    if (lastRunStatus !== '-' || lastRunTime) {
+                        var statusLabel = lastRunStatus !== '-' ? lastRunStatus : 'run';
+                        lastRunText = lastRunTime ? (statusLabel + ' @ ' + lastRunTime) : statusLabel;
+                    }
 
                     var enabledBadge = enabled ? '<span class="badge bg-success">Enabled</span>' : '<span class="badge bg-secondary">Disabled</span>';
                     var inFlightBadge = inFlight ? '<span class="badge bg-warning text-dark">Yes</span>' : '<span class="badge bg-light text-dark">No</span>';
@@ -1465,18 +1498,38 @@ templ Plugin(page string) {
                         '<td>' + enabledBadge + '</td>' +
                         '<td><small>' + escapeHtml(detector) + '</small></td>' +
                         '<td>' + inFlightBadge + '</td>' +
-                        '<td><small>' + escapeHtml(parseTime(item.next_detection_at) || '-') + '</small></td>' +
-                        '<td><small>' + escapeHtml(intervalText) + '</small></td>' +
+                        '<td><small>' + escapeHtml(maxRuntimeText) + '</small></td>' +
                         '<td><small>' + escapeHtml(globalExecText) + '</small></td>' +
                         '<td><small>' + escapeHtml(perWorkerExecText) + '</small></td>' +
                         '<td><small>' + escapeHtml(executorWorkersText) + '</small></td>' +
                         '<td><small>' + escapeHtml(effectiveExecText) + '</small></td>' +
+                        '<td><small>' + escapeHtml(lastRunText) + '</small></td>' +
                         '</tr>';
                 }
 
                 tbody.innerHTML = rows;
             }
 
+            function renderSchedulerStatus() {
+                var valueNodes = document.querySelectorAll('.plugin-scheduler-next-run');
+                if (!valueNodes.length) {
+                    return;
+                }
+                var metaNodes = document.querySelectorAll('.plugin-scheduler-next-run-meta');
+                var status = state.schedulerStatus || {};
+                var nextRun = parseTime(status.next_detection_at);
+                var display = nextRun || '-';
+                valueNodes.forEach(function(node) {
+                    node.textContent = display;
+                });
+                var metaText = nextRun ? 'Local time' : 'Not scheduled';
+                if (metaNodes.length) {
+                    metaNodes.forEach(function(node) {
+                        node.textContent = metaText;
+                    });
+                }
+            }
+
             function renderWorkers() {
                 var tbody = document.getElementById('plugin-workers-table-body');
                 if (!state.workers.length) {
@@ -2374,6 +2427,7 @@ templ Plugin(page string) {
                 document.getElementById('plugin-admin-enabled').checked = pickBool('enabled');
                 document.getElementById('plugin-admin-detection-interval').value = String(pickNumber('detection_interval_seconds'));
                 document.getElementById('plugin-admin-detection-timeout').value = String(pickNumber('detection_timeout_seconds'));
+                document.getElementById('plugin-admin-max-runtime').value = String(pickNumber('job_type_max_runtime_seconds'));
                 document.getElementById('plugin-admin-max-results').value = String(pickNumber('max_jobs_per_detection'));
                 document.getElementById('plugin-admin-global-exec').value = String(pickNumber('global_execution_concurrency'));
                 document.getElementById('plugin-admin-per-worker-exec').value = String(pickNumber('per_worker_execution_concurrency'));
@@ -2398,6 +2452,7 @@ templ Plugin(page string) {
                     enabled: !!document.getElementById('plugin-admin-enabled').checked,
                     detection_interval_seconds: getInt('plugin-admin-detection-interval'),
                     detection_timeout_seconds: getInt('plugin-admin-detection-timeout'),
+                    job_type_max_runtime_seconds: getInt('plugin-admin-max-runtime'),
                     max_jobs_per_detection: getInt('plugin-admin-max-results'),
                     global_execution_concurrency: getInt('plugin-admin-global-exec'),
                     per_worker_execution_concurrency: getInt('plugin-admin-per-worker-exec'),
@@ -2788,21 +2843,30 @@ templ Plugin(page string) {
                 var allJobsPromise = pluginRequest('GET', '/api/plugin/jobs?limit=500');
                 var allActivitiesPromise = pluginRequest('GET', '/api/plugin/activities?limit=500');
                 var schedulerPromise = pluginRequest('GET', '/api/plugin/scheduler-states');
+                var schedulerStatusPromise = pluginRequest('GET', '/api/plugin/scheduler-status');
 
                 var allJobs = await allJobsPromise;
                 var allActivities = await allActivitiesPromise;
                 var schedulerStates = await schedulerPromise;
+                var schedulerStatus = null;
+                try {
+                    schedulerStatus = await schedulerStatusPromise;
+                } catch (e) {
+                    schedulerStatus = null;
+                }
 
                 state.jobs = Array.isArray(allJobs) ? allJobs : [];
                 state.activities = Array.isArray(allActivities) ? allActivities : [];
                 state.allJobs = state.jobs;
                 state.allActivities = state.activities;
                 state.schedulerStates = Array.isArray(schedulerStates) ? schedulerStates : [];
+                state.schedulerStatus = schedulerStatus && schedulerStatus.scheduler ? schedulerStatus.scheduler : null;
                 renderQueueJobs();
                 renderDetectionJobs();
                 renderExecutionJobs();
                 renderExecutionActivities();
                 renderSchedulerStates();
+                renderSchedulerStatus();
                 renderStatus();
                 renderJobTypeSummary();
             }

weed/admin/view/layout/layout.templ

@@ -51,19 +51,21 @@ templ Layout(view ViewContext, content templ.Component) {
     <link rel="stylesheet" href="/static/css/admin.css">
 </head>
 <body>
-    <div class="container-fluid">
+    <div class="container-fluid p-0">
         <!-- Header -->
         <header class="navbar navbar-expand-lg navbar-dark bg-primary sticky-top">
             <div class="container-fluid">
                 <a class="navbar-brand fw-bold" href="/admin">
                     <i class="fas fa-server me-2"></i>
                     SeaweedFS Admin
-                    <span class="badge bg-warning text-dark ms-2">ALPHA</span>
                 </a>
                 
-                <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarNav">
+                <button class="navbar-toggler d-md-none me-2" type="button" data-bs-toggle="collapse" data-bs-target="#sidebarMenu" aria-controls="sidebarMenu" aria-expanded="false" aria-label="Toggle navigation menu">
                     <span class="navbar-toggler-icon"></span>
                 </button>
+                <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarNav" aria-controls="navbarNav" aria-expanded="false" aria-label="User menu">
+                    <i class="fas fa-user"></i>
+                </button>
                 
                 <div class="collapse navbar-collapse" id="navbarNav">
                     <ul class="navbar-nav ms-auto">
@@ -84,7 +86,7 @@ templ Layout(view ViewContext, content templ.Component) {
 
         <div class="row g-0">
             <!-- Sidebar -->
-            <div class="col-md-3 col-lg-2 d-md-block bg-light sidebar collapse">
+            <div class="col-md-3 col-lg-2 d-md-block bg-light sidebar collapse" id="sidebarMenu">
                 <div class="position-sticky pt-3">
                     <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted">
                         <span>MAIN</span>
@@ -245,19 +247,6 @@ templ Layout(view ViewContext, content templ.Component) {
                                 </div>
                             }
                         </li>
-                        <!-- Commented out for later -->
-                        <!--
-                        <li class="nav-item">
-                            <a class="nav-link" href="/metrics">
-                                <i class="fas fa-chart-line me-2"></i>Metrics
-                            </a>
-                        </li>
-                        <li class="nav-item">
-                            <a class="nav-link" href="/logs">
-                                <i class="fas fa-file-alt me-2"></i>Logs
-                            </a>
-                        </li>
-                        -->
                     </ul>
 
                     <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted">
@@ -323,8 +312,11 @@ templ Layout(view ViewContext, content templ.Component) {
                 </div>
             </div>
 
+            <!-- Sidebar backdrop for mobile -->
+            <div class="sidebar-backdrop" id="sidebarBackdrop"></div>
+
             <!-- Main content -->
-            <main class="col-md-9 ms-sm-auto col-lg-10 px-md-4">
+            <main class="col-md-9 ms-sm-auto col-lg-10 px-3 px-md-4">
                 <div class="pt-3">
                     @content
                 </div>

weed/admin/view/layout/layout_templ.go

@@ -71,20 +71,20 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\"><link rel=\"icon\" href=\"/static/favicon.ico\" type=\"image/x-icon\"><!-- Bootstrap CSS --><link href=\"/static/css/bootstrap.min.css\" rel=\"stylesheet\"><!-- Font Awesome CSS --><link href=\"/static/css/fontawesome.min.css\" rel=\"stylesheet\"><!-- HTMX --><script src=\"/static/js/htmx.min.js\"></script><!-- Custom CSS --><link rel=\"stylesheet\" href=\"/static/css/admin.css\"></head><body><div class=\"container-fluid\"><!-- Header --><header class=\"navbar navbar-expand-lg navbar-dark bg-primary sticky-top\"><div class=\"container-fluid\"><a class=\"navbar-brand fw-bold\" href=\"/admin\"><i class=\"fas fa-server me-2\"></i> SeaweedFS Admin <span class=\"badge bg-warning text-dark ms-2\">ALPHA</span></a> <button class=\"navbar-toggler\" type=\"button\" data-bs-toggle=\"collapse\" data-bs-target=\"#navbarNav\"><span class=\"navbar-toggler-icon\"></span></button><div class=\"collapse navbar-collapse\" id=\"navbarNav\"><ul class=\"navbar-nav ms-auto\"><li class=\"nav-item dropdown\"><a class=\"nav-link dropdown-toggle\" href=\"#\" role=\"button\" data-bs-toggle=\"dropdown\"><i class=\"fas fa-user me-1\"></i>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\"><link rel=\"icon\" href=\"/static/favicon.ico\" type=\"image/x-icon\"><!-- Bootstrap CSS --><link href=\"/static/css/bootstrap.min.css\" rel=\"stylesheet\"><!-- Font Awesome CSS --><link href=\"/static/css/fontawesome.min.css\" rel=\"stylesheet\"><!-- HTMX --><script src=\"/static/js/htmx.min.js\"></script><!-- Custom CSS --><link rel=\"stylesheet\" href=\"/static/css/admin.css\"></head><body><div class=\"container-fluid p-0\"><!-- Header --><header class=\"navbar navbar-expand-lg navbar-dark bg-primary sticky-top\"><div class=\"container-fluid\"><a class=\"navbar-brand fw-bold\" href=\"/admin\"><i class=\"fas fa-server me-2\"></i> SeaweedFS Admin</a> <button class=\"navbar-toggler d-md-none me-2\" type=\"button\" data-bs-toggle=\"collapse\" data-bs-target=\"#sidebarMenu\" aria-controls=\"sidebarMenu\" aria-expanded=\"false\" aria-label=\"Toggle navigation menu\"><span class=\"navbar-toggler-icon\"></span></button> <button class=\"navbar-toggler\" type=\"button\" data-bs-toggle=\"collapse\" data-bs-target=\"#navbarNav\" aria-controls=\"navbarNav\" aria-expanded=\"false\" aria-label=\"User menu\"><i class=\"fas fa-user\"></i></button><div class=\"collapse navbar-collapse\" id=\"navbarNav\"><ul class=\"navbar-nav ms-auto\"><li class=\"nav-item dropdown\"><a class=\"nav-link dropdown-toggle\" href=\"#\" role=\"button\" data-bs-toggle=\"dropdown\"><i class=\"fas fa-user me-1\"></i>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var3 string
 		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(username)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 72, Col: 73}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 74, Col: 73}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "</a><ul class=\"dropdown-menu\"><li><a class=\"dropdown-item\" href=\"/logout\"><i class=\"fas fa-sign-out-alt me-2\"></i>Logout</a></li></ul></li></ul></div></div></header><div class=\"row g-0\"><!-- Sidebar --><div class=\"col-md-3 col-lg-2 d-md-block bg-light sidebar collapse\"><div class=\"position-sticky pt-3\"><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MAIN</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"/admin\"><i class=\"fas fa-tachometer-alt me-2\"></i>Dashboard</a></li><li class=\"nav-item\">")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "</a><ul class=\"dropdown-menu\"><li><a class=\"dropdown-item\" href=\"/logout\"><i class=\"fas fa-sign-out-alt me-2\"></i>Logout</a></li></ul></li></ul></div></div></header><div class=\"row g-0\"><!-- Sidebar --><div class=\"col-md-3 col-lg-2 d-md-block bg-light sidebar collapse\" id=\"sidebarMenu\"><div class=\"position-sticky pt-3\"><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MAIN</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"/admin\"><i class=\"fas fa-tachometer-alt me-2\"></i>Dashboard</a></li><li class=\"nav-item\">")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
@@ -113,7 +113,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 		var templ_7745c5c3_Var6 string
 		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", isClusterPage))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 99, Col: 207}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 101, Col: 207}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {
@@ -170,7 +170,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 		var templ_7745c5c3_Var11 string
 		templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", isStoragePage))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 124, Col: 207}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 126, Col: 207}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
 		if templ_7745c5c3_Err != nil {
@@ -254,7 +254,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "</li><!-- Commented out for later --><!--\n                        <li class=\"nav-item\">\n                            <a class=\"nav-link\" href=\"/metrics\">\n                                <i class=\"fas fa-chart-line me-2\"></i>Metrics\n                            </a>\n                        </li>\n                        <li class=\"nav-item\">\n                            <a class=\"nav-link\" href=\"/logs\">\n                                <i class=\"fas fa-file-alt me-2\"></i>Logs\n                            </a>\n                        </li>\n                        --></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>WORKERS</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\">")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "</li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>WORKERS</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\">")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
@@ -329,7 +329,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "</li></ul></div></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-md-4\"><div class=\"pt-3\">")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "</li></ul></div></div><!-- Sidebar backdrop for mobile --><div class=\"sidebar-backdrop\" id=\"sidebarBackdrop\"></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-3 px-md-4\"><div class=\"pt-3\">")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
@@ -344,7 +344,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 		var templ_7745c5c3_Var14 string
 		templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 339, Col: 60}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 331, Col: 60}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
 		if templ_7745c5c3_Err != nil {
@@ -357,7 +357,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
 		var templ_7745c5c3_Var15 string
 		templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 339, Col: 102}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 331, Col: 102}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
 		if templ_7745c5c3_Err != nil {
@@ -409,7 +409,7 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
 		var templ_7745c5c3_Var17 string
 		templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(title)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 367, Col: 17}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 359, Col: 17}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 		if templ_7745c5c3_Err != nil {
@@ -422,7 +422,7 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
 		var templ_7745c5c3_Var18 string
 		templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(title)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 381, Col: 57}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 373, Col: 57}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 		if templ_7745c5c3_Err != nil {
@@ -440,7 +440,7 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
 			var templ_7745c5c3_Var19 string
 			templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 388, Col: 45}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 380, Col: 45}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 			if templ_7745c5c3_Err != nil {
@@ -458,7 +458,7 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
 		var templ_7745c5c3_Var20 string
 		templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(csrfToken)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 393, Col: 84}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 385, Col: 84}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
 		if templ_7745c5c3_Err != nil {

weed/cluster/admin_locks.go

@@ -0,0 +1,6 @@
+package cluster
+
+const (
+	AdminShellLockName          = "shell"
+	AdminServerPresenceLockName = "admin-server"
+)

weed/cluster/maintenance/maintenance_config.go

@@ -1,5 +1,7 @@
 package maintenance
 
+const DefaultMaintenanceSleepMinutes = 17
+
 const DefaultMasterMaintenanceScripts = `
   lock
   ec.encode -fullPercent=95 -quietFor=1h

weed/command/mini_plugin_test.go

@@ -1,13 +1,21 @@
 package command
 
-import "testing"
+import (
+	"testing"
+
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
 
 func TestMiniDefaultPluginJobTypes(t *testing.T) {
-	jobTypes, err := parsePluginWorkerJobTypes(defaultMiniPluginJobTypes)
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	// defaultMiniPluginJobTypes is an explicit list: "vacuum,volume_balance,erasure_coding,admin_script"
+	handlers, err := buildPluginWorkerHandlers(defaultMiniPluginJobTypes, dialOption, int(pluginworker.DefaultMaxExecutionConcurrency), "")
 	if err != nil {
-		t.Fatalf("parsePluginWorkerJobTypes(mini default) err = %v", err)
+		t.Fatalf("buildPluginWorkerHandlers(mini default) err = %v", err)
 	}
-	if len(jobTypes) != 4 {
-		t.Fatalf("expected mini default job types to include 4 handlers, got %v", jobTypes)
+	if len(handlers) != 4 {
+		t.Fatalf("expected mini default job types to include 4 handlers, got %d", len(handlers))
 	}
 }

weed/command/plugin_worker_test.go

@@ -14,68 +14,46 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 )
 
-func TestBuildPluginWorkerHandler(t *testing.T) {
+func TestBuildPluginWorkerHandlerExplicitTypes(t *testing.T) {
 	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
-
 	testMaxConcurrency := int(pluginworker.DefaultMaxExecutionConcurrency)
 
-	handler, err := buildPluginWorkerHandler("vacuum", dialOption, testMaxConcurrency, "")
-	if err != nil {
-		t.Fatalf("buildPluginWorkerHandler(vacuum) err = %v", err)
-	}
-	if handler == nil {
-		t.Fatalf("expected non-nil handler")
+	for _, jobType := range []string{"vacuum", "volume_balance", "erasure_coding", "admin_script", "iceberg_maintenance"} {
+		handlers, err := buildPluginWorkerHandlers(jobType, dialOption, testMaxConcurrency, "")
+		if err != nil {
+			t.Fatalf("buildPluginWorkerHandlers(%s) err = %v", jobType, err)
+		}
+		if len(handlers) != 1 {
+			t.Fatalf("expected 1 handler for %s, got %d", jobType, len(handlers))
+		}
 	}
+}
 
-	handler, err = buildPluginWorkerHandler("", dialOption, testMaxConcurrency, "")
-	if err != nil {
-		t.Fatalf("buildPluginWorkerHandler(default) err = %v", err)
-	}
-	if handler == nil {
-		t.Fatalf("expected non-nil default handler")
-	}
+func TestBuildPluginWorkerHandlerAliases(t *testing.T) {
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	testMaxConcurrency := int(pluginworker.DefaultMaxExecutionConcurrency)
 
-	handler, err = buildPluginWorkerHandler("volume_balance", dialOption, testMaxConcurrency, "")
-	if err != nil {
-		t.Fatalf("buildPluginWorkerHandler(volume_balance) err = %v", err)
-	}
-	if handler == nil {
-		t.Fatalf("expected non-nil volume_balance handler")
+	for _, alias := range []string{"balance", "ec", "iceberg", "admin", "script"} {
+		handlers, err := buildPluginWorkerHandlers(alias, dialOption, testMaxConcurrency, "")
+		if err != nil {
+			t.Fatalf("buildPluginWorkerHandlers(%s) err = %v", alias, err)
+		}
+		if len(handlers) != 1 {
+			t.Fatalf("expected 1 handler for alias %s, got %d", alias, len(handlers))
+		}
 	}
+}
 
-	handler, err = buildPluginWorkerHandler("balance", dialOption, testMaxConcurrency, "")
-	if err != nil {
-		t.Fatalf("buildPluginWorkerHandler(balance alias) err = %v", err)
-	}
-	if handler == nil {
-		t.Fatalf("expected non-nil balance alias handler")
-	}
-
-	handler, err = buildPluginWorkerHandler("erasure_coding", dialOption, testMaxConcurrency, "")
-	if err != nil {
-		t.Fatalf("buildPluginWorkerHandler(erasure_coding) err = %v", err)
-	}
-	if handler == nil {
-		t.Fatalf("expected non-nil erasure_coding handler")
-	}
-
-	handler, err = buildPluginWorkerHandler("ec", dialOption, testMaxConcurrency, "")
-	if err != nil {
-		t.Fatalf("buildPluginWorkerHandler(ec alias) err = %v", err)
-	}
-	if handler == nil {
-		t.Fatalf("expected non-nil ec alias handler")
-	}
-
-	_, err = buildPluginWorkerHandler("unknown", dialOption, testMaxConcurrency, "")
+func TestBuildPluginWorkerHandlerUnknown(t *testing.T) {
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	_, err := buildPluginWorkerHandlers("unknown", dialOption, 1, "")
 	if err == nil {
-		t.Fatalf("expected unsupported job type error")
+		t.Fatalf("expected error for unknown job type")
 	}
 }
 
 func TestBuildPluginWorkerHandlers(t *testing.T) {
 	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
-
 	testMaxConcurrency := int(pluginworker.DefaultMaxExecutionConcurrency)
 
 	handlers, err := buildPluginWorkerHandlers("vacuum,volume_balance,erasure_coding", dialOption, testMaxConcurrency, "")
@@ -100,47 +78,97 @@ func TestBuildPluginWorkerHandlers(t *testing.T) {
 	}
 }
 
-func TestParsePluginWorkerJobTypes(t *testing.T) {
-	jobTypes, err := parsePluginWorkerJobTypes("")
+func TestBuildPluginWorkerHandlersCategories(t *testing.T) {
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	testMaxConcurrency := int(pluginworker.DefaultMaxExecutionConcurrency)
+
+	allHandlers, err := buildPluginWorkerHandlers("all", dialOption, testMaxConcurrency, "")
 	if err != nil {
-		t.Fatalf("parsePluginWorkerJobTypes(default) err = %v", err)
+		t.Fatalf("buildPluginWorkerHandlers(all) err = %v", err)
 	}
-	if len(jobTypes) != 1 || jobTypes[0] != "vacuum" {
-		t.Fatalf("expected default [vacuum], got %v", jobTypes)
+	// "all" must include at least vacuum and erasure_coding (one default, one heavy)
+	allNames := handlerJobTypes(allHandlers)
+	for _, required := range []string{"vacuum", "erasure_coding", "iceberg_maintenance"} {
+		if !allNames[required] {
+			t.Fatalf("'all' missing expected job type %q, got %v", required, allNames)
+		}
 	}
 
-	jobTypes, err = parsePluginWorkerJobTypes(" volume_balance , ec , vacuum , volume_balance ")
+	defaultHandlers, err := buildPluginWorkerHandlers("default", dialOption, testMaxConcurrency, "")
 	if err != nil {
-		t.Fatalf("parsePluginWorkerJobTypes(list) err = %v", err)
+		t.Fatalf("buildPluginWorkerHandlers(default) err = %v", err)
 	}
-	if len(jobTypes) != 3 {
-		t.Fatalf("expected 3 deduped job types, got %d (%v)", len(jobTypes), jobTypes)
+	defaultNames := handlerJobTypes(defaultHandlers)
+
+	heavyHandlers, err := buildPluginWorkerHandlers("heavy", dialOption, testMaxConcurrency, "")
+	if err != nil {
+		t.Fatalf("buildPluginWorkerHandlers(heavy) err = %v", err)
 	}
-	if jobTypes[0] != "volume_balance" || jobTypes[1] != "erasure_coding" || jobTypes[2] != "vacuum" {
-		t.Fatalf("unexpected parsed order %v", jobTypes)
+	heavyNames := handlerJobTypes(heavyHandlers)
+
+	// default and heavy must both be non-empty subsets of all
+	if len(defaultNames) == 0 {
+		t.Fatalf("'default' resolved no handlers")
+	}
+	if len(heavyNames) == 0 {
+		t.Fatalf("'heavy' resolved no handlers")
+	}
+	for name := range defaultNames {
+		if !allNames[name] {
+			t.Fatalf("default handler %q not in 'all'", name)
+		}
+	}
+	for name := range heavyNames {
+		if !allNames[name] {
+			t.Fatalf("heavy handler %q not in 'all'", name)
+		}
 	}
 
-	if _, err = parsePluginWorkerJobTypes(" , "); err != nil {
-		t.Fatalf("expected empty list to resolve to default vacuum: %v", err)
+	// default and heavy must be disjoint and their union must equal all
+	for name := range defaultNames {
+		if heavyNames[name] {
+			t.Fatalf("handler %q appears in both default and heavy", name)
+		}
+	}
+	if len(defaultNames)+len(heavyNames) != len(allNames) {
+		t.Fatalf("union(default=%d, heavy=%d) != all(%d)", len(defaultNames), len(heavyNames), len(allNames))
 	}
 
-	jobTypes, err = parsePluginWorkerJobTypes("admin-script,script,admin_script")
+	// mix category + explicit: "default,iceberg" adds one heavy to default set
+	mixedHandlers, err := buildPluginWorkerHandlers("default,iceberg", dialOption, testMaxConcurrency, "")
 	if err != nil {
-		t.Fatalf("parsePluginWorkerJobTypes(admin script aliases) err = %v", err)
+		t.Fatalf("buildPluginWorkerHandlers(default,iceberg) err = %v", err)
 	}
-	if len(jobTypes) != 1 || jobTypes[0] != "admin_script" {
-		t.Fatalf("expected admin_script alias to resolve, got %v", jobTypes)
+	if len(mixedHandlers) != len(defaultHandlers)+1 {
+		t.Fatalf("expected default+1 handlers for 'default,iceberg', got %d (default=%d)", len(mixedHandlers), len(defaultHandlers))
 	}
 }
 
 func TestPluginWorkerDefaultJobTypes(t *testing.T) {
-	jobTypes, err := parsePluginWorkerJobTypes(defaultPluginWorkerJobTypes)
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	testMaxConcurrency := int(pluginworker.DefaultMaxExecutionConcurrency)
+
+	// defaultPluginWorkerJobTypes is "all", so it should match the "all" category exactly
+	defaultHandlers, err := buildPluginWorkerHandlers(defaultPluginWorkerJobTypes, dialOption, testMaxConcurrency, "")
 	if err != nil {
-		t.Fatalf("parsePluginWorkerJobTypes(default setting) err = %v", err)
+		t.Fatalf("buildPluginWorkerHandlers(default setting) err = %v", err)
 	}
-	if len(jobTypes) != 4 {
-		t.Fatalf("expected default job types to include 4 handlers, got %v", jobTypes)
+	allHandlers, err := buildPluginWorkerHandlers("all", dialOption, testMaxConcurrency, "")
+	if err != nil {
+		t.Fatalf("buildPluginWorkerHandlers(all) err = %v", err)
 	}
+	if len(defaultHandlers) != len(allHandlers) {
+		t.Fatalf("default setting resolved %d handlers, 'all' resolved %d", len(defaultHandlers), len(allHandlers))
+	}
+}
+
+// handlerJobTypes returns the set of job type names from a slice of handlers.
+func handlerJobTypes(handlers []pluginworker.JobHandler) map[string]bool {
+	m := make(map[string]bool, len(handlers))
+	for _, h := range handlers {
+		m[h.Capability().JobType] = true
+	}
+	return m
 }
 
 func TestResolvePluginWorkerID(t *testing.T) {

weed/command/scaffold/master.toml

@@ -6,6 +6,7 @@
 
 [master.maintenance]
 # periodically run these scripts are the same as running them from 'weed shell'
+# Scripts are skipped while an admin server is connected.
 scripts = """{{DEFAULT_MAINTENANCE_SCRIPTS}}"""
 sleep_minutes = 17          # sleep minutes between each script execution
 

weed/command/worker.go

@@ -7,25 +7,30 @@ import (
 )
 
 var cmdWorker = &Command{
-	UsageLine: "worker -admin=<admin_server> [-id=<worker_id>] [-jobType=vacuum,volume_balance,erasure_coding,admin_script] [-workingDir=<path>] [-heartbeat=15s] [-reconnect=5s] [-maxDetect=1] [-maxExecute=4] [-metricsPort=<port>] [-metricsIp=<ip>] [-debug]",
+	UsageLine: "worker -admin=<admin_server> [-id=<worker_id>] [-jobType=all] [-workingDir=<path>] [-heartbeat=15s] [-reconnect=5s] [-maxDetect=1] [-maxExecute=4] [-metricsPort=<port>] [-metricsIp=<ip>] [-debug]",
 	Short:     "start a plugin.proto worker process",
 	Long: `Start an external plugin worker using weed/pb/plugin.proto over gRPC.
 
-This command provides vacuum, volume_balance, erasure_coding, and admin_script job type
-contracts with the plugin stream runtime, including descriptor delivery,
-heartbeat/load reporting, detection, and execution.
+This command provides plugin job type handlers for cluster maintenance,
+including descriptor delivery, heartbeat/load reporting, detection, and execution.
 
 Behavior:
-  - Use -jobType to choose one or more plugin job handlers (comma-separated list)
+  - Use -jobType to choose handlers by category or explicit name (comma-separated)
+  - Categories: "all" (every registered handler), "default" (lightweight jobs),
+    "heavy" (resource-intensive jobs like erasure coding)
+  - Explicit job type names and aliases are still supported (e.g. "vacuum", "ec")
+  - Categories and explicit names can be mixed (e.g. "default,iceberg")
   - Use -workingDir to persist worker.id for stable worker identity across restarts
   - Use -metricsPort/-metricsIp to expose /health, /ready, and /metrics
 
 Examples:
   weed worker -admin=localhost:23646
-  weed worker -admin=localhost:23646 -jobType=volume_balance
+  weed worker -admin=localhost:23646 -jobType=all
+  weed worker -admin=localhost:23646 -jobType=default
+  weed worker -admin=localhost:23646 -jobType=heavy
+  weed worker -admin=localhost:23646 -jobType=default,iceberg
   weed worker -admin=localhost:23646 -jobType=vacuum,volume_balance
   weed worker -admin=localhost:23646 -jobType=erasure_coding
-  weed worker -admin=localhost:23646 -jobType=admin_script
   weed worker -admin=admin.example.com:23646 -id=plugin-vacuum-a -heartbeat=10s
   weed worker -admin=localhost:23646 -workingDir=/var/lib/seaweedfs-plugin
   weed worker -admin=localhost:23646 -metricsPort=9327 -metricsIp=0.0.0.0
@@ -36,7 +41,7 @@ var (
 	workerAdminServer = cmdWorker.Flag.String("admin", "localhost:23646", "admin server address")
 	workerID          = cmdWorker.Flag.String("id", "", "worker ID (auto-generated when empty)")
 	workerWorkingDir  = cmdWorker.Flag.String("workingDir", "", "working directory for persistent worker state")
-	workerJobType     = cmdWorker.Flag.String("jobType", defaultPluginWorkerJobTypes, "job types to serve (comma-separated list)")
+	workerJobType     = cmdWorker.Flag.String("jobType", defaultPluginWorkerJobTypes, "job types or categories to serve: all, default, heavy, or explicit names/aliases such as ec, balance, iceberg (comma-separated)")
 	workerHeartbeat   = cmdWorker.Flag.Duration("heartbeat", 15*time.Second, "heartbeat interval")
 	workerReconnect   = cmdWorker.Flag.Duration("reconnect", 5*time.Second, "reconnect delay")
 	workerMaxDetect   = cmdWorker.Flag.Int("maxDetect", 1, "max concurrent detection requests")

weed/command/worker_runtime.go

@@ -15,6 +15,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	_ "github.com/seaweedfs/seaweedfs/weed/plugin/worker/handlers" // register all handler subpackages
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	statsCollect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -23,7 +24,7 @@ import (
 	"google.golang.org/grpc"
 )
 
-const defaultPluginWorkerJobTypes = "vacuum,volume_balance,erasure_coding,admin_script"
+const defaultPluginWorkerJobTypes = "all"
 
 type pluginWorkerRunOptions struct {
 	AdminServer string
@@ -137,98 +138,37 @@ func resolvePluginWorkerID(explicitID string, workingDir string) (string, error)
 	return worker.GenerateOrLoadWorkerID(workingDir)
 }
 
-// buildPluginWorkerHandler constructs the JobHandler for the given job type.
-// maxExecute is forwarded to handlers that use it to report their own
-// MaxExecutionConcurrency in Capability for consistency and future-proofing.
-// The scheduler's effective per-worker MaxExecutionConcurrency is derived from
-// the worker-level configuration (e.g. WorkerOptions.MaxExecutionConcurrency),
-// not directly from the handler's Capability.
-func buildPluginWorkerHandler(jobType string, dialOption grpc.DialOption, maxExecute int, workingDir string) (pluginworker.JobHandler, error) {
-	canonicalJobType, err := canonicalPluginWorkerJobType(jobType)
-	if err != nil {
-		return nil, err
-	}
-
-	switch canonicalJobType {
-	case "vacuum":
-		return pluginworker.NewVacuumHandler(dialOption, int32(maxExecute)), nil
-	case "volume_balance":
-		return pluginworker.NewVolumeBalanceHandler(dialOption), nil
-	case "erasure_coding":
-		return pluginworker.NewErasureCodingHandler(dialOption, workingDir), nil
-	case "admin_script":
-		return pluginworker.NewAdminScriptHandler(dialOption), nil
-	default:
-		return nil, fmt.Errorf("unsupported plugin job type %q", canonicalJobType)
-	}
-}
-
-// buildPluginWorkerHandlers constructs a deduplicated slice of JobHandlers for
-// the comma-separated jobTypes string, forwarding maxExecute to each handler.
+// buildPluginWorkerHandlers resolves the comma-separated jobTypes string
+// (which may contain category names like "all", "default", "heavy" and/or
+// explicit job type names/aliases) into a deduplicated slice of JobHandlers.
 func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string) ([]pluginworker.JobHandler, error) {
-	parsedJobTypes, err := parsePluginWorkerJobTypes(jobTypes)
+	jobTypes = strings.TrimSpace(jobTypes)
+	if jobTypes == "" {
+		jobTypes = defaultPluginWorkerJobTypes
+	}
+
+	factories, err := pluginworker.ResolveHandlerFactories(jobTypes)
 	if err != nil {
 		return nil, err
 	}
 
-	handlers := make([]pluginworker.JobHandler, 0, len(parsedJobTypes))
-	for _, jobType := range parsedJobTypes {
-		handler, buildErr := buildPluginWorkerHandler(jobType, dialOption, maxExecute, workingDir)
+	opts := pluginworker.HandlerBuildOptions{
+		GrpcDialOption: dialOption,
+		MaxExecute:     maxExecute,
+		WorkingDir:     workingDir,
+	}
+
+	handlers := make([]pluginworker.JobHandler, 0, len(factories))
+	for _, f := range factories {
+		handler, buildErr := f.Build(opts)
 		if buildErr != nil {
-			return nil, buildErr
+			return nil, fmt.Errorf("building handler for %q: %w", f.JobType, buildErr)
 		}
 		handlers = append(handlers, handler)
 	}
 	return handlers, nil
 }
 
-func parsePluginWorkerJobTypes(jobTypes string) ([]string, error) {
-	jobTypes = strings.TrimSpace(jobTypes)
-	if jobTypes == "" {
-		return []string{"vacuum"}, nil
-	}
-
-	parts := strings.Split(jobTypes, ",")
-	parsed := make([]string, 0, len(parts))
-	seen := make(map[string]struct{}, len(parts))
-
-	for _, part := range parts {
-		part = strings.TrimSpace(part)
-		if part == "" {
-			continue
-		}
-		canonical, err := canonicalPluginWorkerJobType(part)
-		if err != nil {
-			return nil, err
-		}
-		if _, found := seen[canonical]; found {
-			continue
-		}
-		seen[canonical] = struct{}{}
-		parsed = append(parsed, canonical)
-	}
-
-	if len(parsed) == 0 {
-		return []string{"vacuum"}, nil
-	}
-	return parsed, nil
-}
-
-func canonicalPluginWorkerJobType(jobType string) (string, error) {
-	switch strings.ToLower(strings.TrimSpace(jobType)) {
-	case "", "vacuum":
-		return "vacuum", nil
-	case "volume_balance", "balance", "volume.balance", "volume-balance":
-		return "volume_balance", nil
-	case "erasure_coding", "erasure-coding", "erasure.coding", "ec":
-		return "erasure_coding", nil
-	case "admin_script", "admin-script", "admin.script", "script", "admin":
-		return "admin_script", nil
-	default:
-		return "", fmt.Errorf("unsupported plugin job type %q", jobType)
-	}
-}
-
 func resolvePluginWorkerAdminServer(adminServer string) string {
 	adminServer = strings.TrimSpace(adminServer)
 	host, httpPort, hasExplicitGrpcPort, err := parsePluginWorkerAdminAddress(adminServer)

weed/command/worker_test.go

@@ -1,13 +1,21 @@
 package command
 
-import "testing"
+import (
+	"testing"
+
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
 
 func TestWorkerDefaultJobTypes(t *testing.T) {
-	jobTypes, err := parsePluginWorkerJobTypes(*workerJobType)
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	handlers, err := buildPluginWorkerHandlers(*workerJobType, dialOption, int(pluginworker.DefaultMaxExecutionConcurrency), "")
 	if err != nil {
-		t.Fatalf("parsePluginWorkerJobTypes(default worker flag) err = %v", err)
+		t.Fatalf("buildPluginWorkerHandlers(default worker flag) err = %v", err)
 	}
-	if len(jobTypes) != 4 {
-		t.Fatalf("expected default worker job types to include 4 handlers, got %v", jobTypes)
+	// Expected: vacuum, volume_balance, admin_script, erasure_coding, iceberg_maintenance
+	if len(handlers) != 5 {
+		t.Fatalf("expected default worker job types to include 5 handlers, got %d", len(handlers))
 	}
 }

weed/credential/filer_etc/filer_etc_identity.go

@@ -24,7 +24,7 @@ func (store *FilerEtcStore) LoadConfiguration(ctx context.Context) (*iam_pb.S3Ap
 	s3cfg := &iam_pb.S3ApiConfiguration{}
 
 	// 1. Load from legacy single file (low priority)
-	content, foundLegacy, err := store.readInsideFiler(filer.IamConfigDirectory, IamLegacyIdentityFile)
+	content, foundLegacy, err := store.readInsideFiler(ctx, filer.IamConfigDirectory, IamLegacyIdentityFile)
 	if err != nil {
 		return s3cfg, err
 	}
@@ -93,7 +93,7 @@ func (store *FilerEtcStore) loadFromMultiFile(ctx context.Context, s3cfg *iam_pb
 			if len(entry.Content) > 0 {
 				content = entry.Content
 			} else {
-				c, err := filer.ReadInsideFiler(client, dir, entry.Name)
+				c, err := filer.ReadInsideFiler(ctx, client, dir, entry.Name)
 				if err != nil {
 					glog.Warningf("Failed to read identity file %s: %v", entry.Name, err)
 					continue
@@ -249,7 +249,7 @@ func (store *FilerEtcStore) CreateUser(ctx context.Context, identity *iam_pb.Ide
 func (store *FilerEtcStore) GetUser(ctx context.Context, username string) (*iam_pb.Identity, error) {
 	var identity *iam_pb.Identity
 	err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		data, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory+"/"+IamIdentitiesDirectory, username+".json")
+		data, err := filer.ReadInsideFiler(ctx, client, filer.IamConfigDirectory+"/"+IamIdentitiesDirectory, username+".json")
 		if err != nil {
 			if err == filer_pb.ErrNotFound {
 				return credential.ErrUserNotFound
@@ -350,7 +350,7 @@ func (store *FilerEtcStore) GetUserByAccessKey(ctx context.Context, accessKey st
 			if len(entry.Content) > 0 {
 				content = entry.Content
 			} else {
-				c, err := filer.ReadInsideFiler(client, dir, entry.Name)
+				c, err := filer.ReadInsideFiler(ctx, client, dir, entry.Name)
 				if err != nil {
 					continue
 				}
@@ -435,11 +435,11 @@ func (store *FilerEtcStore) saveIdentity(ctx context.Context, identity *iam_pb.I
 	})
 }
 
-func (store *FilerEtcStore) readInsideFiler(dir string, name string) ([]byte, bool, error) {
+func (store *FilerEtcStore) readInsideFiler(ctx context.Context, dir string, name string) ([]byte, bool, error) {
 	var content []byte
 	found := false
 	err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		c, err := filer.ReadInsideFiler(client, dir, name)
+		c, err := filer.ReadInsideFiler(ctx, client, dir, name)
 		if err != nil {
 			if err == filer_pb.ErrNotFound {
 				return nil

weed/credential/filer_etc/filer_etc_policy.go

@@ -9,6 +9,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
 )
 
@@ -18,13 +19,113 @@ const (
 )
 
 type PoliciesCollection struct {
-	Policies map[string]policy_engine.PolicyDocument `json:"policies"`
+	Policies       map[string]policy_engine.PolicyDocument            `json:"policies"`
+	InlinePolicies map[string]map[string]policy_engine.PolicyDocument `json:"inlinePolicies"`
 }
 
 func validatePolicyName(name string) error {
 	return credential.ValidatePolicyName(name)
 }
 
+func newPoliciesCollection() *PoliciesCollection {
+	return &PoliciesCollection{
+		Policies:       make(map[string]policy_engine.PolicyDocument),
+		InlinePolicies: make(map[string]map[string]policy_engine.PolicyDocument),
+	}
+}
+
+func (store *FilerEtcStore) loadLegacyPoliciesCollection(ctx context.Context) (*PoliciesCollection, bool, error) {
+	policiesCollection := newPoliciesCollection()
+
+	content, foundLegacy, err := store.readInsideFiler(ctx, filer.IamConfigDirectory, filer.IamPoliciesFile)
+	if err != nil {
+		return nil, false, err
+	}
+	if !foundLegacy || len(content) == 0 {
+		return policiesCollection, foundLegacy, nil
+	}
+
+	if err := json.Unmarshal(content, policiesCollection); err != nil {
+		return nil, false, err
+	}
+	if policiesCollection.Policies == nil {
+		policiesCollection.Policies = make(map[string]policy_engine.PolicyDocument)
+	}
+	if policiesCollection.InlinePolicies == nil {
+		policiesCollection.InlinePolicies = make(map[string]map[string]policy_engine.PolicyDocument)
+	}
+
+	return policiesCollection, true, nil
+}
+
+func (store *FilerEtcStore) saveLegacyPoliciesCollection(ctx context.Context, policiesCollection *PoliciesCollection) error {
+	return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		content, err := json.MarshalIndent(policiesCollection, "", "  ")
+		if err != nil {
+			return err
+		}
+		return filer.SaveInsideFiler(client, filer.IamConfigDirectory, filer.IamPoliciesFile, content)
+	})
+}
+
+func policyDocumentToPbPolicy(name string, policy policy_engine.PolicyDocument) (*iam_pb.Policy, error) {
+	content, err := json.Marshal(policy)
+	if err != nil {
+		return nil, err
+	}
+	return &iam_pb.Policy{Name: name, Content: string(content)}, nil
+}
+
+// LoadManagedPolicies loads managed policies for the S3 runtime without
+// triggering legacy-to-multifile migration. This lets the runtime hydrate
+// policies while preserving any legacy inline policy data stored alongside
+// managed policies.
+func (store *FilerEtcStore) LoadManagedPolicies(ctx context.Context) ([]*iam_pb.Policy, error) {
+	policiesCollection, _, err := store.loadLegacyPoliciesCollection(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	policies := make(map[string]policy_engine.PolicyDocument, len(policiesCollection.Policies))
+	for name, policy := range policiesCollection.Policies {
+		policies[name] = policy
+	}
+
+	if err := store.loadPoliciesFromMultiFile(ctx, policies); err != nil {
+		return nil, err
+	}
+
+	managedPolicies := make([]*iam_pb.Policy, 0, len(policies))
+	for name, policy := range policies {
+		pbPolicy, err := policyDocumentToPbPolicy(name, policy)
+		if err != nil {
+			return nil, err
+		}
+		managedPolicies = append(managedPolicies, pbPolicy)
+	}
+
+	return managedPolicies, nil
+}
+
+// LoadInlinePolicies loads legacy inline policies keyed by user name. Inline
+// policies are still stored in the legacy shared policies file.
+func (store *FilerEtcStore) LoadInlinePolicies(ctx context.Context) (map[string]map[string]policy_engine.PolicyDocument, error) {
+	policiesCollection, _, err := store.loadLegacyPoliciesCollection(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	inlinePolicies := make(map[string]map[string]policy_engine.PolicyDocument, len(policiesCollection.InlinePolicies))
+	for userName, userPolicies := range policiesCollection.InlinePolicies {
+		inlinePolicies[userName] = make(map[string]policy_engine.PolicyDocument, len(userPolicies))
+		for policyName, policy := range userPolicies {
+			inlinePolicies[userName][policyName] = policy
+		}
+	}
+
+	return inlinePolicies, nil
+}
+
 // GetPolicies retrieves all IAM policies from the filer
 func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_engine.PolicyDocument, error) {
 	policies := make(map[string]policy_engine.PolicyDocument)
@@ -43,23 +144,12 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_
 		filer.IamConfigDirectory, filer.IamPoliciesFile)
 
 	// 1. Load from legacy single file (low priority)
-	content, foundLegacy, err := store.readInsideFiler(filer.IamConfigDirectory, filer.IamPoliciesFile)
+	policiesCollection, _, err := store.loadLegacyPoliciesCollection(ctx)
 	if err != nil {
 		return nil, err
 	}
-
-	if foundLegacy && len(content) > 0 {
-		policiesCollection := &PoliciesCollection{
-			Policies: make(map[string]policy_engine.PolicyDocument),
-		}
-		if err := json.Unmarshal(content, policiesCollection); err != nil {
-			glog.Errorf("Failed to parse legacy IAM policies from %s/%s: %v",
-				filer.IamConfigDirectory, filer.IamPoliciesFile, err)
-		} else {
-			for name, policy := range policiesCollection.Policies {
-				policies[name] = policy
-			}
-		}
+	for name, policy := range policiesCollection.Policies {
+		policies[name] = policy
 	}
 
 	// 2. Load from multi-file structure (high priority, overrides legacy)
@@ -67,14 +157,6 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_
 		return nil, err
 	}
 
-	// 3. Perform migration if we loaded legacy config
-	if foundLegacy {
-		if err := store.migratePoliciesToMultiFile(ctx, policies); err != nil {
-			glog.Errorf("Failed to migrate IAM policies to multi-file layout: %v", err)
-			return policies, err
-		}
-	}
-
 	return policies, nil
 }
 
@@ -98,7 +180,7 @@ func (store *FilerEtcStore) loadPoliciesFromMultiFile(ctx context.Context, polic
 			if len(entry.Content) > 0 {
 				content = entry.Content
 			} else {
-				c, err := filer.ReadInsideFiler(client, dir, entry.Name)
+				c, err := filer.ReadInsideFiler(ctx, client, dir, entry.Name)
 				if err != nil {
 					glog.Warningf("Failed to read policy file %s: %v", entry.Name, err)
 					continue
@@ -115,7 +197,7 @@ func (store *FilerEtcStore) loadPoliciesFromMultiFile(ctx context.Context, polic
 
 				// The file name is "policyName.json"
 				policyName := entry.Name
-				if len(policyName) > 5 && policyName[len(policyName)-5:] == ".json" {
+				if strings.HasSuffix(policyName, ".json") {
 					policyName = policyName[:len(policyName)-5]
 					policies[policyName] = policy
 				}
@@ -184,7 +266,23 @@ func (store *FilerEtcStore) DeletePolicy(ctx context.Context, name string) error
 	if err := validatePolicyName(name); err != nil {
 		return err
 	}
-	return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	store.policyMu.Lock()
+	defer store.policyMu.Unlock()
+
+	policiesCollection, foundLegacy, err := store.loadLegacyPoliciesCollection(ctx)
+	if err != nil {
+		return err
+	}
+
+	deleteLegacyPolicy := false
+	if foundLegacy {
+		if _, exists := policiesCollection.Policies[name]; exists {
+			delete(policiesCollection.Policies, name)
+			deleteLegacyPolicy = true
+		}
+	}
+
+	if err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 		_, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{
 			Directory: filer.IamConfigDirectory + "/" + IamPoliciesDirectory,
 			Name:      name + ".json",
@@ -193,7 +291,15 @@ func (store *FilerEtcStore) DeletePolicy(ctx context.Context, name string) error
 			return err
 		}
 		return nil
-	})
+	}); err != nil {
+		return err
+	}
+
+	if deleteLegacyPolicy {
+		return store.saveLegacyPoliciesCollection(ctx, policiesCollection)
+	}
+
+	return nil
 }
 
 // GetPolicy retrieves a specific IAM policy by name from the filer
@@ -204,7 +310,7 @@ func (store *FilerEtcStore) GetPolicy(ctx context.Context, name string) (*policy
 
 	var policy *policy_engine.PolicyDocument
 	err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		data, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory+"/"+IamPoliciesDirectory, name+".json")
+		data, err := filer.ReadInsideFiler(ctx, client, filer.IamConfigDirectory+"/"+IamPoliciesDirectory, name+".json")
 		if err != nil {
 			if err == filer_pb.ErrNotFound {
 				return nil
@@ -239,6 +345,7 @@ func (store *FilerEtcStore) GetPolicy(ctx context.Context, name string) (*policy
 // ListPolicyNames returns all managed policy names stored in the filer.
 func (store *FilerEtcStore) ListPolicyNames(ctx context.Context) ([]string, error) {
 	names := make([]string, 0)
+	seenNames := make(map[string]struct{})
 
 	store.mu.RLock()
 	configured := store.filerAddressFunc != nil
@@ -248,7 +355,19 @@ func (store *FilerEtcStore) ListPolicyNames(ctx context.Context) ([]string, erro
 		return names, nil
 	}
 
-	err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	policiesCollection, _, err := store.loadLegacyPoliciesCollection(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for name := range policiesCollection.Policies {
+		if _, found := seenNames[name]; found {
+			continue
+		}
+		names = append(names, name)
+		seenNames[name] = struct{}{}
+	}
+
+	err = store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 		dir := filer.IamConfigDirectory + "/" + IamPoliciesDirectory
 		entries, err := listEntries(ctx, client, dir)
 		if err != nil {
@@ -266,7 +385,11 @@ func (store *FilerEtcStore) ListPolicyNames(ctx context.Context) ([]string, erro
 			if strings.HasSuffix(name, ".json") {
 				name = name[:len(name)-5]
 			}
+			if _, found := seenNames[name]; found {
+				continue
+			}
 			names = append(names, name)
+			seenNames[name] = struct{}{}
 		}
 
 		return nil

weed/credential/filer_etc/filer_etc_policy_test.go

@@ -0,0 +1,369 @@
+package filer_etc
+
+import (
+	"context"
+	"net"
+	"sort"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+)
+
+type policyTestFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	mu                   sync.RWMutex
+	entries              map[string]*filer_pb.Entry
+	contentlessListEntry map[string]struct{}
+	beforeLookup         func(context.Context, string, string) error
+	afterListEntry       func(string, string)
+	beforeDelete         func(string, string) error
+	beforeUpdate         func(string, string) error
+}
+
+func newPolicyTestFilerServer() *policyTestFilerServer {
+	return &policyTestFilerServer{
+		entries:              make(map[string]*filer_pb.Entry),
+		contentlessListEntry: make(map[string]struct{}),
+	}
+}
+
+func (s *policyTestFilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	s.mu.RLock()
+	beforeLookup := s.beforeLookup
+	s.mu.RUnlock()
+	if beforeLookup != nil {
+		if err := beforeLookup(ctx, req.Directory, req.Name); err != nil {
+			return nil, err
+		}
+	}
+
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	entry, found := s.entries[filerEntryKey(req.Directory, req.Name)]
+	if !found {
+		return nil, status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())
+	}
+
+	return &filer_pb.LookupDirectoryEntryResponse{Entry: cloneEntry(entry)}, nil
+}
+
+func (s *policyTestFilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream grpc.ServerStreamingServer[filer_pb.ListEntriesResponse]) error {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	names := make([]string, 0)
+	for key := range s.entries {
+		dir, name := splitFilerEntryKey(key)
+		if dir != req.Directory {
+			continue
+		}
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	for _, name := range names {
+		entry := cloneEntry(s.entries[filerEntryKey(req.Directory, name)])
+		if _, found := s.contentlessListEntry[filerEntryKey(req.Directory, name)]; found {
+			entry.Content = nil
+		}
+		if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: entry}); err != nil {
+			return err
+		}
+		if s.afterListEntry != nil {
+			s.afterListEntry(req.Directory, name)
+		}
+	}
+
+	return nil
+}
+
+func (s *policyTestFilerServer) CreateEntry(_ context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.entries[filerEntryKey(req.Directory, req.Entry.Name)] = cloneEntry(req.Entry)
+	return &filer_pb.CreateEntryResponse{}, nil
+}
+
+func (s *policyTestFilerServer) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
+	s.mu.RLock()
+	beforeUpdate := s.beforeUpdate
+	s.mu.RUnlock()
+	if beforeUpdate != nil {
+		if err := beforeUpdate(req.Directory, req.Entry.Name); err != nil {
+			return nil, err
+		}
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.entries[filerEntryKey(req.Directory, req.Entry.Name)] = cloneEntry(req.Entry)
+	return &filer_pb.UpdateEntryResponse{}, nil
+}
+
+func (s *policyTestFilerServer) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
+	s.mu.RLock()
+	beforeDelete := s.beforeDelete
+	s.mu.RUnlock()
+	if beforeDelete != nil {
+		if err := beforeDelete(req.Directory, req.Name); err != nil {
+			return nil, err
+		}
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	key := filerEntryKey(req.Directory, req.Name)
+	if _, found := s.entries[key]; !found {
+		return nil, status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())
+	}
+
+	delete(s.entries, key)
+	return &filer_pb.DeleteEntryResponse{}, nil
+}
+
+func newPolicyTestStore(t *testing.T) *FilerEtcStore {
+	store, _ := newPolicyTestStoreWithServer(t)
+	return store
+}
+
+func newPolicyTestStoreWithServer(t *testing.T) (*FilerEtcStore, *policyTestFilerServer) {
+	t.Helper()
+
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+
+	server := newPolicyTestFilerServer()
+	grpcServer := pb.NewGrpcServer()
+	filer_pb.RegisterSeaweedFilerServer(grpcServer, server)
+	go func() {
+		_ = grpcServer.Serve(lis)
+	}()
+
+	t.Cleanup(func() {
+		grpcServer.Stop()
+		_ = lis.Close()
+	})
+
+	store := &FilerEtcStore{}
+	host, portString, err := net.SplitHostPort(lis.Addr().String())
+	require.NoError(t, err)
+	grpcPort, err := strconv.Atoi(portString)
+	require.NoError(t, err)
+	store.SetFilerAddressFunc(func() pb.ServerAddress {
+		return pb.NewServerAddress(host, 1, grpcPort)
+	}, grpc.WithTransportCredentials(insecure.NewCredentials()))
+
+	return store, server
+}
+
+func TestFilerEtcStoreListPolicyNamesIncludesLegacyPolicies(t *testing.T) {
+	ctx := context.Background()
+	store := newPolicyTestStore(t)
+
+	legacyPolicies := newPoliciesCollection()
+	legacyPolicies.Policies["legacy-only"] = testPolicyDocument("s3:GetObject", "arn:aws:s3:::legacy-only/*")
+	legacyPolicies.Policies["shared"] = testPolicyDocument("s3:GetObject", "arn:aws:s3:::shared/*")
+	require.NoError(t, store.saveLegacyPoliciesCollection(ctx, legacyPolicies))
+
+	require.NoError(t, store.savePolicy(ctx, "multi-file-only", testPolicyDocument("s3:PutObject", "arn:aws:s3:::multi-file-only/*")))
+	require.NoError(t, store.savePolicy(ctx, "shared", testPolicyDocument("s3:DeleteObject", "arn:aws:s3:::shared/*")))
+
+	names, err := store.ListPolicyNames(ctx)
+	require.NoError(t, err)
+
+	assert.ElementsMatch(t, []string{"legacy-only", "multi-file-only", "shared"}, names)
+}
+
+func TestFilerEtcStoreDeletePolicyRemovesLegacyManagedCopy(t *testing.T) {
+	ctx := context.Background()
+	store := newPolicyTestStore(t)
+
+	inlinePolicy := testPolicyDocument("s3:PutObject", "arn:aws:s3:::inline-user/*")
+	legacyPolicies := newPoliciesCollection()
+	legacyPolicies.Policies["legacy-only"] = testPolicyDocument("s3:GetObject", "arn:aws:s3:::legacy-only/*")
+	legacyPolicies.InlinePolicies["inline-user"] = map[string]policy_engine.PolicyDocument{
+		"PutOnly": inlinePolicy,
+	}
+	require.NoError(t, store.saveLegacyPoliciesCollection(ctx, legacyPolicies))
+
+	managedPolicies, err := store.LoadManagedPolicies(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"legacy-only"}, managedPolicyNames(managedPolicies))
+
+	require.NoError(t, store.DeletePolicy(ctx, "legacy-only"))
+
+	managedPolicies, err = store.LoadManagedPolicies(ctx)
+	require.NoError(t, err)
+	assert.Empty(t, managedPolicies)
+
+	inlinePolicies, err := store.LoadInlinePolicies(ctx)
+	require.NoError(t, err)
+	assertInlinePolicyPreserved(t, inlinePolicies, "inline-user", "PutOnly")
+
+	loadedLegacyPolicies, foundLegacy, err := store.loadLegacyPoliciesCollection(ctx)
+	require.NoError(t, err)
+	require.True(t, foundLegacy)
+	assert.Empty(t, loadedLegacyPolicies.Policies)
+	assertInlinePolicyPreserved(t, loadedLegacyPolicies.InlinePolicies, "inline-user", "PutOnly")
+}
+
+func TestFilerEtcStoreDeletePolicySerializesLegacyUpdates(t *testing.T) {
+	ctx := context.Background()
+	store, server := newPolicyTestStoreWithServer(t)
+
+	legacyPolicies := newPoliciesCollection()
+	legacyPolicies.Policies["first"] = testPolicyDocument("s3:GetObject", "arn:aws:s3:::first/*")
+	legacyPolicies.Policies["second"] = testPolicyDocument("s3:GetObject", "arn:aws:s3:::second/*")
+	require.NoError(t, store.saveLegacyPoliciesCollection(ctx, legacyPolicies))
+	require.NoError(t, store.savePolicy(ctx, "first", testPolicyDocument("s3:GetObject", "arn:aws:s3:::first/*")))
+	require.NoError(t, store.savePolicy(ctx, "second", testPolicyDocument("s3:GetObject", "arn:aws:s3:::second/*")))
+
+	firstSaveStarted := make(chan struct{})
+	releaseFirstSave := make(chan struct{})
+	secondReachedDelete := make(chan struct{}, 1)
+	var blockOnce sync.Once
+
+	server.mu.Lock()
+	server.beforeUpdate = func(dir string, name string) error {
+		if dir == filer.IamConfigDirectory && name == filer.IamPoliciesFile {
+			blockOnce.Do(func() {
+				close(firstSaveStarted)
+				<-releaseFirstSave
+			})
+		}
+		return nil
+	}
+	server.beforeDelete = func(dir string, name string) error {
+		if dir == filer.IamConfigDirectory+"/"+IamPoliciesDirectory && name == "second.json" {
+			select {
+			case secondReachedDelete <- struct{}{}:
+			default:
+			}
+		}
+		return nil
+	}
+	server.mu.Unlock()
+
+	firstDeleteErr := make(chan error, 1)
+	go func() {
+		firstDeleteErr <- store.DeletePolicy(ctx, "first")
+	}()
+
+	<-firstSaveStarted
+
+	secondDeleteErr := make(chan error, 1)
+	go func() {
+		secondDeleteErr <- store.DeletePolicy(ctx, "second")
+	}()
+
+	select {
+	case <-secondReachedDelete:
+		t.Fatal("second delete reached filer mutation while first delete was still blocked")
+	case <-time.After(300 * time.Millisecond):
+	}
+
+	close(releaseFirstSave)
+
+	require.NoError(t, <-firstDeleteErr)
+	require.NoError(t, <-secondDeleteErr)
+
+	loadedLegacyPolicies, foundLegacy, err := store.loadLegacyPoliciesCollection(ctx)
+	require.NoError(t, err)
+	require.True(t, foundLegacy)
+	assert.Empty(t, loadedLegacyPolicies.Policies)
+}
+
+func TestFilerEtcStoreLoadManagedPoliciesRespectsReadContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	store, server := newPolicyTestStoreWithServer(t)
+
+	require.NoError(t, store.savePolicy(context.Background(), "cancel-me", testPolicyDocument("s3:GetObject", "arn:aws:s3:::cancel-me/*")))
+
+	server.mu.Lock()
+	server.contentlessListEntry[filerEntryKey(filer.IamConfigDirectory+"/"+IamPoliciesDirectory, "cancel-me.json")] = struct{}{}
+	server.beforeLookup = func(ctx context.Context, dir string, name string) error {
+		if dir == filer.IamConfigDirectory+"/"+IamPoliciesDirectory && name == "cancel-me.json" {
+			cancel()
+			return status.Error(codes.Canceled, context.Canceled.Error())
+		}
+		return nil
+	}
+	server.mu.Unlock()
+
+	managedPolicies, err := store.LoadManagedPolicies(ctx)
+	require.NoError(t, err)
+	assert.Empty(t, managedPolicies)
+}
+
+func testPolicyDocument(action string, resource string) policy_engine.PolicyDocument {
+	return policy_engine.PolicyDocument{
+		Version: policy_engine.PolicyVersion2012_10_17,
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice(action),
+				Resource: policy_engine.NewStringOrStringSlice(resource),
+			},
+		},
+	}
+}
+
+func managedPolicyNames(policies []*iam_pb.Policy) []string {
+	names := make([]string, 0, len(policies))
+	for _, policy := range policies {
+		names = append(names, policy.Name)
+	}
+	sort.Strings(names)
+	return names
+}
+
+func assertInlinePolicyPreserved(t *testing.T, inlinePolicies map[string]map[string]policy_engine.PolicyDocument, userName string, policyName string) {
+	t.Helper()
+
+	userPolicies, found := inlinePolicies[userName]
+	require.True(t, found)
+
+	policy, found := userPolicies[policyName]
+	require.True(t, found)
+	assert.Equal(t, policy_engine.PolicyVersion2012_10_17, policy.Version)
+	require.Len(t, policy.Statement, 1)
+	assert.Equal(t, policy_engine.PolicyEffectAllow, policy.Statement[0].Effect)
+}
+
+func cloneEntry(entry *filer_pb.Entry) *filer_pb.Entry {
+	if entry == nil {
+		return nil
+	}
+	return proto.Clone(entry).(*filer_pb.Entry)
+}
+
+func filerEntryKey(dir string, name string) string {
+	return dir + "\x00" + name
+}
+
+func splitFilerEntryKey(key string) (dir string, name string) {
+	for idx := 0; idx < len(key); idx++ {
+		if key[idx] == '\x00' {
+			return key[:idx], key[idx+1:]
+		}
+	}
+	return key, ""
+}

weed/credential/filer_etc/filer_etc_service_account.go

@@ -38,7 +38,7 @@ func (store *FilerEtcStore) loadServiceAccountsFromMultiFile(ctx context.Context
 			if len(entry.Content) > 0 {
 				content = entry.Content
 			} else {
-				c, err := filer.ReadInsideFiler(client, dir, entry.Name)
+				c, err := filer.ReadInsideFiler(ctx, client, dir, entry.Name)
 				if err != nil {
 					glog.Warningf("Failed to read service account file %s: %v", entry.Name, err)
 					continue
@@ -133,7 +133,7 @@ func (store *FilerEtcStore) GetServiceAccount(ctx context.Context, id string) (*
 	}
 	var sa *iam_pb.ServiceAccount
 	err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		data, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory+"/"+IamServiceAccountsDirectory, id+".json")
+		data, err := filer.ReadInsideFiler(ctx, client, filer.IamConfigDirectory+"/"+IamServiceAccountsDirectory, id+".json")
 		if err != nil {
 			if err == filer_pb.ErrNotFound {
 				return credential.ErrServiceAccountNotFound
@@ -170,7 +170,7 @@ func (store *FilerEtcStore) ListServiceAccounts(ctx context.Context) ([]*iam_pb.
 			if len(entry.Content) > 0 {
 				content = entry.Content
 			} else {
-				c, err := filer.ReadInsideFiler(client, dir, entry.Name)
+				c, err := filer.ReadInsideFiler(ctx, client, dir, entry.Name)
 				if err != nil {
 					glog.Warningf("Failed to read service account file %s: %v", entry.Name, err)
 					continue

weed/credential/filer_etc/filer_etc_store.go

@@ -20,6 +20,7 @@ type FilerEtcStore struct {
 	filerAddressFunc func() pb.ServerAddress // Function to get current active filer
 	grpcDialOption   grpc.DialOption
 	mu               sync.RWMutex // Protects filerAddressFunc and grpcDialOption
+	policyMu         sync.Mutex   // Serializes legacy managed-policy mutations
 }
 
 func (store *FilerEtcStore) GetName() credential.CredentialStoreTypeName {

weed/credential/propagating_store.go

@@ -20,6 +20,14 @@ import (
 var _ CredentialStore = &PropagatingCredentialStore{}
 var _ PolicyManager = &PropagatingCredentialStore{}
 
+type propagatingManagedPolicyLoader interface {
+	LoadManagedPolicies(ctx context.Context) ([]*iam_pb.Policy, error)
+}
+
+type propagatingInlinePolicyLoader interface {
+	LoadInlinePolicies(ctx context.Context) (map[string]map[string]policy_engine.PolicyDocument, error)
+}
+
 type PropagatingCredentialStore struct {
 	CredentialStore
 	masterClient   *wdclient.MasterClient
@@ -240,6 +248,38 @@ func (s *PropagatingCredentialStore) ListPolicyNames(ctx context.Context) ([]str
 	return s.CredentialStore.ListPolicyNames(ctx)
 }
 
+func (s *PropagatingCredentialStore) LoadManagedPolicies(ctx context.Context) ([]*iam_pb.Policy, error) {
+	if loader, ok := s.CredentialStore.(propagatingManagedPolicyLoader); ok {
+		return loader.LoadManagedPolicies(ctx)
+	}
+
+	policies, err := s.CredentialStore.GetPolicies(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	managedPolicies := make([]*iam_pb.Policy, 0, len(policies))
+	for name, policyDocument := range policies {
+		content, err := json.Marshal(policyDocument)
+		if err != nil {
+			return nil, err
+		}
+		managedPolicies = append(managedPolicies, &iam_pb.Policy{
+			Name:    name,
+			Content: string(content),
+		})
+	}
+
+	return managedPolicies, nil
+}
+
+func (s *PropagatingCredentialStore) LoadInlinePolicies(ctx context.Context) (map[string]map[string]policy_engine.PolicyDocument, error) {
+	if loader, ok := s.CredentialStore.(propagatingInlinePolicyLoader); ok {
+		return loader.LoadInlinePolicies(ctx)
+	}
+	return nil, nil
+}
+
 func (s *PropagatingCredentialStore) CreatePolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error {
 	if pm, ok := s.CredentialStore.(PolicyManager); ok {
 		if err := pm.CreatePolicy(ctx, name, document); err != nil {

weed/filer/filer_conf.go

@@ -47,7 +47,7 @@ func ReadFilerConfFromFilers(filerGrpcAddresses []pb.ServerAddress, grpcDialOpti
 			data = buf.Bytes()
 			return nil
 		}
-		content, err := ReadInsideFiler(client, DirectoryEtcSeaweedFS, FilerConfName)
+		content, err := ReadInsideFiler(context.Background(), client, DirectoryEtcSeaweedFS, FilerConfName)
 		if err != nil {
 			return err
 		}

weed/filer/filer_notify.go

@@ -19,19 +19,23 @@ import (
 )
 
 func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) {
+	f.notifyUpdateEvent(ctx, oldEntry, newEntry, deleteChunks, isFromOtherCluster, signatures)
+}
+
+func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) *filer_pb.SubscribeMetadataResponse {
 	var fullpath string
 	if oldEntry != nil {
 		fullpath = string(oldEntry.FullPath)
 	} else if newEntry != nil {
 		fullpath = string(newEntry.FullPath)
 	} else {
-		return
+		return nil
 	}
 
 	// println("fullpath:", fullpath)
 
 	if strings.HasPrefix(fullpath, SystemLogDir) {
-		return
+		return nil
 	}
 	foundSelf := false
 	for _, sig := range signatures {
@@ -43,18 +47,8 @@ func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry
 		signatures = append(signatures, f.Signature)
 	}
 
-	newParentPath := ""
-	if newEntry != nil {
-		newParentPath, _ = newEntry.FullPath.DirAndName()
-	}
-	eventNotification := &filer_pb.EventNotification{
-		OldEntry:           oldEntry.ToProtoEntry(),
-		NewEntry:           newEntry.ToProtoEntry(),
-		DeleteChunks:       deleteChunks,
-		NewParentPath:      newParentPath,
-		IsFromOtherCluster: isFromOtherCluster,
-		Signatures:         signatures,
-	}
+	event := f.newMetadataEvent(oldEntry, newEntry, deleteChunks, isFromOtherCluster, signatures)
+	eventNotification := event.EventNotification
 
 	if notification.Queue != nil {
 		glog.V(3).Infof("notifying entry update %v", fullpath)
@@ -64,31 +58,57 @@ func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry
 		}
 	}
 
-	f.logMetaEvent(ctx, fullpath, eventNotification)
+	f.logMetaEvent(ctx, event)
+	if sink := metadataEventSinkFromContext(ctx); sink != nil {
+		sink.Record(event)
+	}
 
 	// Trigger empty folder cleanup for local events
 	// Remote events are handled via MetaAggregator.onMetadataChangeEvent
 	f.triggerLocalEmptyFolderCleanup(oldEntry, newEntry)
 
+	return event
 }
 
-func (f *Filer) logMetaEvent(ctx context.Context, fullpath string, eventNotification *filer_pb.EventNotification) {
-
-	dir, _ := util.FullPath(fullpath).DirAndName()
-
-	event := &filer_pb.SubscribeMetadataResponse{
-		Directory:         dir,
-		EventNotification: eventNotification,
-		TsNs:              time.Now().UnixNano(),
+func (f *Filer) newMetadataEvent(oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) *filer_pb.SubscribeMetadataResponse {
+	if oldEntry == nil && newEntry == nil {
+		return nil
 	}
+	var fullpath util.FullPath
+	if oldEntry != nil {
+		fullpath = oldEntry.FullPath
+	}
+	if fullpath == "" && newEntry != nil {
+		fullpath = newEntry.FullPath
+	}
+	dir, _ := fullpath.DirAndName()
+	newParentPath := ""
+	if newEntry != nil {
+		newParentPath, _ = newEntry.FullPath.DirAndName()
+	}
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory: dir,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:           oldEntry.ToProtoEntry(),
+			NewEntry:           newEntry.ToProtoEntry(),
+			DeleteChunks:       deleteChunks,
+			NewParentPath:      newParentPath,
+			IsFromOtherCluster: isFromOtherCluster,
+			Signatures:         signatures,
+		},
+		TsNs: time.Now().UnixNano(),
+	}
+}
+
+func (f *Filer) logMetaEvent(ctx context.Context, event *filer_pb.SubscribeMetadataResponse) {
 	data, err := proto.Marshal(event)
 	if err != nil {
 		glog.Errorf("failed to marshal filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
 		return
 	}
 
-	if err := f.LocalMetaLogBuffer.AddDataToBuffer([]byte(dir), data, event.TsNs); err != nil {
-		glog.Errorf("failed to add data to log buffer for %s: %v", dir, err)
+	if err := f.LocalMetaLogBuffer.AddDataToBuffer([]byte(event.Directory), data, event.TsNs); err != nil {
+		glog.Errorf("failed to add data to log buffer for %s: %v", event.Directory, err)
 	}
 
 }

weed/filer/metadata_event_sink.go

@@ -0,0 +1,47 @@
+package filer
+
+import (
+	"context"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+type metadataEventSinkKey struct{}
+
+// MetadataEventSink captures the last metadata event emitted while serving a
+// request. It is request-scoped and accessed only by the goroutine handling
+// the gRPC call, so no mutex is needed.
+type MetadataEventSink struct {
+	last *filer_pb.SubscribeMetadataResponse
+}
+
+func WithMetadataEventSink(ctx context.Context) (context.Context, *MetadataEventSink) {
+	sink := &MetadataEventSink{}
+	return context.WithValue(ctx, metadataEventSinkKey{}, sink), sink
+}
+
+func metadataEventSinkFromContext(ctx context.Context) *MetadataEventSink {
+	if ctx == nil {
+		return nil
+	}
+	sink, _ := ctx.Value(metadataEventSinkKey{}).(*MetadataEventSink)
+	return sink
+}
+
+// Record stores the event, replacing any previously recorded one.
+// Each filer RPC emits at most one NotifyUpdateEvent, so only the last
+// event is retained. If an RPC were to emit multiple events, only the
+// final one would be returned to the caller.
+func (s *MetadataEventSink) Record(event *filer_pb.SubscribeMetadataResponse) {
+	if s == nil || event == nil {
+		return
+	}
+	s.last = event
+}
+
+func (s *MetadataEventSink) Last() *filer_pb.SubscribeMetadataResponse {
+	if s == nil {
+		return nil
+	}
+	return s.last
+}

weed/filer/metadata_event_sink_test.go

@@ -0,0 +1,43 @@
+package filer
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+)
+
+func TestNotifyUpdateEventRecordsRequestMetadataEvent(t *testing.T) {
+	f := &Filer{
+		Signature: 42,
+		LocalMetaLogBuffer: log_buffer.NewLogBuffer(
+			"test",
+			time.Hour,
+			func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {},
+			nil,
+			nil,
+		),
+	}
+
+	ctx, sink := WithMetadataEventSink(context.Background())
+	f.NotifyUpdateEvent(ctx, &Entry{FullPath: util.FullPath("/dir/file.txt")}, nil, true, false, []int32{7})
+
+	event := sink.Last()
+	if event == nil {
+		t.Fatal("expected metadata event to be recorded")
+	}
+	if event.Directory != "/dir" {
+		t.Fatalf("directory = %q, want /dir", event.Directory)
+	}
+	if event.EventNotification.OldEntry == nil || event.EventNotification.OldEntry.Name != "file.txt" {
+		t.Fatalf("old entry = %+v, want file.txt", event.EventNotification.OldEntry)
+	}
+	if got := event.EventNotification.Signatures; len(got) != 2 || got[0] != 7 || got[1] != 42 {
+		t.Fatalf("signatures = %v, want [7 42]", got)
+	}
+	if event.TsNs == 0 {
+		t.Fatal("expected event timestamp to be set")
+	}
+}

weed/filer/read_write.go

@@ -28,12 +28,12 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed
 
 }
 
-func ReadInsideFiler(filerClient filer_pb.SeaweedFilerClient, dir, name string) (content []byte, err error) {
+func ReadInsideFiler(ctx context.Context, filerClient filer_pb.SeaweedFilerClient, dir, name string) (content []byte, err error) {
 	request := &filer_pb.LookupDirectoryEntryRequest{
 		Directory: dir,
 		Name:      name,
 	}
-	respLookupEntry, err := filer_pb.LookupEntry(context.Background(), filerClient, request)
+	respLookupEntry, err := filer_pb.LookupEntry(ctx, filerClient, request)
 	if err != nil {
 		return
 	}

weed/filer/remote_mapping.go

@@ -1,6 +1,7 @@
 package filer
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -13,7 +14,7 @@ import (
 func ReadMountMappings(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress) (mappings *remote_pb.RemoteStorageMapping, readErr error) {
 	var oldContent []byte
 	if readErr = pb.WithFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
+		oldContent, readErr = ReadInsideFiler(context.Background(), client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
 		return readErr
 	}); readErr != nil {
 		if readErr != filer_pb.ErrNotFound {
@@ -34,7 +35,7 @@ func InsertMountMapping(filerClient filer_pb.FilerClient, dir string, remoteStor
 	// read current mapping
 	var oldContent, newContent []byte
 	err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		oldContent, err = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
+		oldContent, err = ReadInsideFiler(context.Background(), client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
 		return err
 	})
 	if err != nil {
@@ -65,7 +66,7 @@ func DeleteMountMapping(filerClient filer_pb.FilerClient, dir string) (err error
 	// read current mapping
 	var oldContent, newContent []byte
 	err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		oldContent, err = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
+		oldContent, err = ReadInsideFiler(context.Background(), client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
 		return err
 	})
 	if err != nil {

weed/filer/remote_storage.go

@@ -138,7 +138,7 @@ func UnmarshalRemoteStorageMappings(oldContent []byte) (mappings *remote_pb.Remo
 func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, storageName string) (conf *remote_pb.RemoteConf, readErr error) {
 	var oldContent []byte
 	if readErr = pb.WithFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, storageName+REMOTE_STORAGE_CONF_SUFFIX)
+		oldContent, readErr = ReadInsideFiler(context.Background(), client, DirectoryEtcRemote, storageName+REMOTE_STORAGE_CONF_SUFFIX)
 		return readErr
 	}); readErr != nil {
 		return nil, readErr

weed/iam/integration/iam_manager.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"sync"
 
 	"github.com/golang-jwt/jwt/v5"
 	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
@@ -28,6 +29,8 @@ type IAMManager struct {
 	userStore            UserStore
 	filerAddressProvider func() string // Function to get current filer address
 	initialized          bool
+	runtimePolicyMu      sync.Mutex
+	runtimePolicyNames   map[string]struct{}
 }
 
 // IAMConfig holds configuration for all IAM components
@@ -105,6 +108,57 @@ func (m *IAMManager) SetUserStore(store UserStore) {
 	m.userStore = store
 }
 
+// SyncRuntimePolicies keeps zero-config runtime policies available to the
+// in-memory policy engine used by the advanced IAM authorizer.
+func (m *IAMManager) SyncRuntimePolicies(ctx context.Context, policies []*iam_pb.Policy) error {
+	if !m.initialized || m.policyEngine == nil {
+		return nil
+	}
+	if m.policyEngine.StoreType() != sts.StoreTypeMemory {
+		return nil
+	}
+
+	desiredPolicies := make(map[string]*policy.PolicyDocument, len(policies))
+	for _, runtimePolicy := range policies {
+		if runtimePolicy == nil || runtimePolicy.Name == "" {
+			continue
+		}
+
+		var document policy.PolicyDocument
+		if err := json.Unmarshal([]byte(runtimePolicy.Content), &document); err != nil {
+			return fmt.Errorf("failed to parse runtime policy %q: %w", runtimePolicy.Name, err)
+		}
+
+		desiredPolicies[runtimePolicy.Name] = &document
+	}
+
+	m.runtimePolicyMu.Lock()
+	defer m.runtimePolicyMu.Unlock()
+
+	filerAddress := m.getFilerAddress()
+	for policyName := range m.runtimePolicyNames {
+		if _, keep := desiredPolicies[policyName]; keep {
+			continue
+		}
+		if err := m.policyEngine.DeletePolicy(ctx, filerAddress, policyName); err != nil {
+			return fmt.Errorf("failed to delete runtime policy %q: %w", policyName, err)
+		}
+	}
+
+	for policyName, document := range desiredPolicies {
+		if err := m.policyEngine.AddPolicy(filerAddress, policyName, document); err != nil {
+			return fmt.Errorf("failed to sync runtime policy %q: %w", policyName, err)
+		}
+	}
+
+	m.runtimePolicyNames = make(map[string]struct{}, len(desiredPolicies))
+	for policyName := range desiredPolicies {
+		m.runtimePolicyNames[policyName] = struct{}{}
+	}
+
+	return nil
+}
+
 // Initialize initializes the IAM manager with all components
 func (m *IAMManager) Initialize(config *IAMConfig, filerAddressProvider func() string) error {
 	if config == nil {
@@ -422,6 +476,7 @@ func (m *IAMManager) IsActionAllowed(ctx context.Context, request *ActionRequest
 
 	var baseResult *policy.EvaluationResult
 	var err error
+	subjectPolicyCount := 0
 
 	if isAdmin {
 		// Admin always has base access allowed
@@ -454,6 +509,7 @@ func (m *IAMManager) IsActionAllowed(ctx context.Context, request *ActionRequest
 				policies = roleDef.AttachedPolicies
 			}
 		}
+		subjectPolicyCount = len(policies)
 
 		if bucketPolicyName != "" {
 			// Enforce an upper bound on the number of policies to avoid excessive allocations
@@ -477,6 +533,14 @@ func (m *IAMManager) IsActionAllowed(ctx context.Context, request *ActionRequest
 		return false, nil
 	}
 
+	// Zero-config IAM uses DefaultEffect=Allow to preserve open-by-default behavior
+	// for requests without any subject policies. Once a user or role has attached
+	// policies, "no matching statement" must fall back to deny so the attachment
+	// actually scopes access.
+	if subjectPolicyCount > 0 && len(baseResult.MatchingStatements) == 0 {
+		return false, nil
+	}
+
 	// If there's a session policy, it must also allow the action
 	if sessionInfo != nil && sessionInfo.SessionPolicy != "" {
 		var sessionPolicy policy.PolicyDocument

weed/iam/policy/policy_engine.go

@@ -353,6 +353,27 @@ func (e *PolicyEngine) AddPolicy(filerAddress string, name string, policy *Polic
 	return e.store.StorePolicy(context.Background(), filerAddress, name, policy)
 }
 
+// DeletePolicy removes a policy from the configured store.
+func (e *PolicyEngine) DeletePolicy(ctx context.Context, filerAddress string, name string) error {
+	if !e.initialized {
+		return fmt.Errorf("policy engine not initialized")
+	}
+
+	if name == "" {
+		return fmt.Errorf("policy name cannot be empty")
+	}
+
+	return e.store.DeletePolicy(ctx, filerAddress, name)
+}
+
+// StoreType returns the configured backend type for the policy store.
+func (e *PolicyEngine) StoreType() string {
+	if e.config == nil {
+		return ""
+	}
+	return e.config.StoreType
+}
+
 // Evaluate evaluates policies against a request context (filerAddress ignored for memory stores)
 func (e *PolicyEngine) Evaluate(ctx context.Context, filerAddress string, evalCtx *EvaluationContext, policyNames []string) (*EvaluationResult, error) {
 	if !e.initialized {

weed/iam/policy/policy_store.go

@@ -3,15 +3,19 @@ package policy
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/credential"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 // MemoryPolicyStore implements PolicyStore using in-memory storage
@@ -134,11 +138,15 @@ func copyPolicyDocument(original *PolicyDocument) *PolicyDocument {
 			copy(copied.Statement[i].NotResource, stmt.NotResource)
 		}
 
-		// Copy condition map (shallow copy for now)
+		// Copy condition map
 		if stmt.Condition != nil {
 			copied.Statement[i].Condition = make(map[string]map[string]interface{})
-			for k, v := range stmt.Condition {
-				copied.Statement[i].Condition[k] = v
+			for conditionType, conditionValues := range stmt.Condition {
+				copiedConditionValues := make(map[string]interface{}, len(conditionValues))
+				for conditionKey, conditionValue := range conditionValues {
+					copiedConditionValues[conditionKey] = copyPolicyConditionValue(conditionValue)
+				}
+				copied.Statement[i].Condition[conditionType] = copiedConditionValues
 			}
 		}
 	}
@@ -146,6 +154,29 @@ func copyPolicyDocument(original *PolicyDocument) *PolicyDocument {
 	return copied
 }
 
+func copyPolicyConditionValue(value interface{}) interface{} {
+	switch v := value.(type) {
+	case []string:
+		copied := make([]string, len(v))
+		copy(copied, v)
+		return copied
+	case []interface{}:
+		copied := make([]interface{}, len(v))
+		for i := range v {
+			copied[i] = copyPolicyConditionValue(v[i])
+		}
+		return copied
+	case map[string]interface{}:
+		copied := make(map[string]interface{}, len(v))
+		for key, nestedValue := range v {
+			copied[key] = copyPolicyConditionValue(nestedValue)
+		}
+		return copied
+	default:
+		return v
+	}
+}
+
 // FilerPolicyStore implements PolicyStore using SeaweedFS filer
 type FilerPolicyStore struct {
 	grpcDialOption       grpc.DialOption
@@ -198,27 +229,13 @@ func (s *FilerPolicyStore) StorePolicy(ctx context.Context, filerAddress string,
 
 	// Store in filer
 	return s.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
-		request := &filer_pb.CreateEntryRequest{
-			Directory: s.basePath,
-			Entry: &filer_pb.Entry{
-				Name:        s.getPolicyFileName(name),
-				IsDirectory: false,
-				Attributes: &filer_pb.FuseAttributes{
-					Mtime:    time.Now().Unix(),
-					Crtime:   time.Now().Unix(),
-					FileMode: uint32(0600), // Read/write for owner only
-					Uid:      uint32(0),
-					Gid:      uint32(0),
-				},
-				Content: policyData,
-			},
-		}
-
 		glog.V(3).Infof("Storing policy %s at %s", name, policyPath)
-		_, err := client.CreateEntry(ctx, request)
-		if err != nil {
+		if err := s.savePolicyFile(ctx, client, s.getPolicyFileName(name), policyData); err != nil {
 			return fmt.Errorf("failed to store policy %s: %v", name, err)
 		}
+		if err := s.deleteLegacyPolicyFileIfPresent(ctx, client, name); err != nil {
+			return err
+		}
 
 		return nil
 	})
@@ -239,23 +256,30 @@ func (s *FilerPolicyStore) GetPolicy(ctx context.Context, filerAddress string, n
 
 	var policyData []byte
 	err := s.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
-		request := &filer_pb.LookupDirectoryEntryRequest{
-			Directory: s.basePath,
-			Name:      s.getPolicyFileName(name),
+		for _, fileName := range s.getPolicyLookupFileNames(name) {
+			request := &filer_pb.LookupDirectoryEntryRequest{
+				Directory: s.basePath,
+				Name:      fileName,
+			}
+
+			glog.V(3).Infof("Looking up policy %s as %s", name, fileName)
+			response, err := client.LookupDirectoryEntry(ctx, request)
+			if err != nil {
+				if isNotFoundPolicyStoreError(err) {
+					continue
+				}
+				return fmt.Errorf("policy lookup failed: %v", err)
+			}
+
+			if response.Entry == nil {
+				continue
+			}
+
+			policyData = response.Entry.Content
+			return nil
 		}
 
-		glog.V(3).Infof("Looking up policy %s", name)
-		response, err := client.LookupDirectoryEntry(ctx, request)
-		if err != nil {
-			return fmt.Errorf("policy not found: %v", err)
-		}
-
-		if response.Entry == nil {
-			return fmt.Errorf("policy not found")
-		}
-
-		policyData = response.Entry.Content
-		return nil
+		return fmt.Errorf("policy not found")
 	})
 
 	if err != nil {
@@ -285,31 +309,27 @@ func (s *FilerPolicyStore) DeletePolicy(ctx context.Context, filerAddress string
 	}
 
 	return s.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
-		request := &filer_pb.DeleteEntryRequest{
-			Directory:            s.basePath,
-			Name:                 s.getPolicyFileName(name),
-			IsDeleteData:         true,
-			IsRecursive:          false,
-			IgnoreRecursiveError: false,
-		}
-
-		glog.V(3).Infof("Deleting policy %s", name)
-		resp, err := client.DeleteEntry(ctx, request)
-		if err != nil {
-			// Ignore "not found" errors - policy may already be deleted
-			if strings.Contains(err.Error(), "not found") {
-				return nil
+		for _, fileName := range s.getPolicyLookupFileNames(name) {
+			request := &filer_pb.DeleteEntryRequest{
+				Directory:            s.basePath,
+				Name:                 fileName,
+				IsDeleteData:         true,
+				IsRecursive:          false,
+				IgnoreRecursiveError: false,
 			}
-			return fmt.Errorf("failed to delete policy %s: %v", name, err)
-		}
 
-		// Check response error
-		if resp.Error != "" {
-			// Ignore "not found" errors - policy may already be deleted
-			if strings.Contains(resp.Error, "not found") {
-				return nil
+			glog.V(3).Infof("Deleting policy %s as %s", name, fileName)
+			resp, err := client.DeleteEntry(ctx, request)
+			if err != nil {
+				if isNotFoundPolicyStoreError(err) {
+					continue
+				}
+				return fmt.Errorf("failed to delete policy %s: %v", name, err)
+			}
+
+			if resp.Error != "" {
+				return fmt.Errorf("failed to delete policy %s: %s", name, resp.Error)
 			}
-			return fmt.Errorf("failed to delete policy %s: %s", name, resp.Error)
 		}
 
 		return nil
@@ -332,7 +352,7 @@ func (s *FilerPolicyStore) ListPolicies(ctx context.Context, filerAddress string
 		// List all entries in the policy directory
 		request := &filer_pb.ListEntriesRequest{
 			Directory:          s.basePath,
-			Prefix:             "policy_",
+			Prefix:             "",
 			StartFromFileName:  "",
 			InclusiveStartFrom: false,
 			Limit:              1000, // Process in batches of 1000
@@ -353,11 +373,7 @@ func (s *FilerPolicyStore) ListPolicies(ctx context.Context, filerAddress string
 				continue
 			}
 
-			// Extract policy name from filename
-			filename := resp.Entry.Name
-			if strings.HasPrefix(filename, "policy_") && strings.HasSuffix(filename, ".json") {
-				// Remove "policy_" prefix and ".json" suffix
-				policyName := strings.TrimSuffix(strings.TrimPrefix(filename, "policy_"), ".json")
+			if policyName, ok := s.policyNameFromFileName(resp.Entry.Name); ok {
 				policyNames = append(policyNames, policyName)
 			}
 		}
@@ -369,7 +385,17 @@ func (s *FilerPolicyStore) ListPolicies(ctx context.Context, filerAddress string
 		return nil, err
 	}
 
-	return policyNames, nil
+	uniquePolicyNames := make([]string, 0, len(policyNames))
+	seen := make(map[string]struct{}, len(policyNames))
+	for _, policyName := range policyNames {
+		if _, found := seen[policyName]; found {
+			continue
+		}
+		seen[policyName] = struct{}{}
+		uniquePolicyNames = append(uniquePolicyNames, policyName)
+	}
+
+	return uniquePolicyNames, nil
 }
 
 // Helper methods
@@ -391,5 +417,115 @@ func (s *FilerPolicyStore) getPolicyPath(policyName string) string {
 
 // getPolicyFileName returns the filename for a policy
 func (s *FilerPolicyStore) getPolicyFileName(policyName string) string {
+	return s.getCanonicalPolicyFileName(policyName)
+}
+
+func (s *FilerPolicyStore) getLegacyPolicyFileName(policyName string) string {
 	return "policy_" + policyName + ".json"
 }
+
+func (s *FilerPolicyStore) getCanonicalPolicyFileName(policyName string) string {
+	return policyName + ".json"
+}
+
+func (s *FilerPolicyStore) getPolicyLookupFileNames(policyName string) []string {
+	return []string{
+		s.getCanonicalPolicyFileName(policyName),
+		s.getLegacyPolicyFileName(policyName),
+	}
+}
+
+func (s *FilerPolicyStore) policyNameFromFileName(fileName string) (string, bool) {
+	if !strings.HasSuffix(fileName, ".json") {
+		return "", false
+	}
+	policyName := strings.TrimSuffix(fileName, ".json")
+	if strings.HasPrefix(fileName, "policy_") {
+		policyName = strings.TrimPrefix(policyName, "policy_")
+	}
+	if s.isSupportedPolicyName(policyName) {
+		return policyName, true
+	}
+	return "", false
+}
+
+func (s *FilerPolicyStore) isSupportedPolicyName(policyName string) bool {
+	if policyName == "" {
+		return false
+	}
+	// Bucket policies are stored alongside IAM policies but use the internal
+	// "bucket-policy:<bucket>" naming scheme, which is intentionally outside the
+	// public IAM policy-name validator.
+	if strings.HasPrefix(policyName, "bucket-policy:") {
+		return len(policyName) > len("bucket-policy:")
+	}
+	return credential.ValidatePolicyName(policyName) == nil
+}
+
+func (s *FilerPolicyStore) deleteLegacyPolicyFileIfPresent(ctx context.Context, client filer_pb.SeaweedFilerClient, policyName string) error {
+	legacyFileName := s.getLegacyPolicyFileName(policyName)
+	response, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{
+		Directory:            s.basePath,
+		Name:                 legacyFileName,
+		IsDeleteData:         true,
+		IsRecursive:          false,
+		IgnoreRecursiveError: false,
+	})
+	if err != nil {
+		if isNotFoundPolicyStoreError(err) {
+			return nil
+		}
+		return fmt.Errorf("failed to delete legacy policy %s: %v", policyName, err)
+	}
+	if response.Error != "" {
+		return fmt.Errorf("failed to delete legacy policy %s: %s", policyName, response.Error)
+	}
+	return nil
+}
+
+func (s *FilerPolicyStore) savePolicyFile(ctx context.Context, client filer_pb.SeaweedFilerClient, fileName string, content []byte) error {
+	now := time.Now().Unix()
+	entry := &filer_pb.Entry{
+		Name:        fileName,
+		IsDirectory: false,
+		Attributes: &filer_pb.FuseAttributes{
+			Mtime:    now,
+			Crtime:   now,
+			FileMode: uint32(0600),
+			Uid:      uint32(0),
+			Gid:      uint32(0),
+			FileSize: uint64(len(content)),
+		},
+		Content: content,
+	}
+
+	createRequest := &filer_pb.CreateEntryRequest{
+		Directory: s.basePath,
+		Entry:     entry,
+	}
+
+	if err := filer_pb.CreateEntry(ctx, client, createRequest); err == nil {
+		return nil
+	} else if !isAlreadyExistsPolicyStoreError(err) {
+		return err
+	}
+
+	return filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{
+		Directory: s.basePath,
+		Entry:     entry,
+	})
+}
+
+func isNotFoundPolicyStoreError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return errors.Is(err, filer_pb.ErrNotFound) || status.Code(err) == codes.NotFound
+}
+
+func isAlreadyExistsPolicyStoreError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return status.Code(err) == codes.AlreadyExists
+}

weed/iam/policy/policy_store_test.go

@@ -0,0 +1,314 @@
+package policy
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"sort"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+)
+
+type policyStoreTestFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	mu      sync.RWMutex
+	entries map[string]*filer_pb.Entry
+}
+
+func newPolicyStoreTestFilerServer() *policyStoreTestFilerServer {
+	return &policyStoreTestFilerServer{
+		entries: make(map[string]*filer_pb.Entry),
+	}
+}
+
+func (s *policyStoreTestFilerServer) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	entry, found := s.entries[policyStoreTestEntryKey(req.Directory, req.Name)]
+	if !found {
+		return nil, status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())
+	}
+
+	return &filer_pb.LookupDirectoryEntryResponse{Entry: clonePolicyStoreEntry(entry)}, nil
+}
+
+func (s *policyStoreTestFilerServer) CreateEntry(_ context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	key := policyStoreTestEntryKey(req.Directory, req.Entry.Name)
+	if _, found := s.entries[key]; found {
+		return nil, status.Error(codes.AlreadyExists, "entry already exists")
+	}
+
+	s.entries[key] = clonePolicyStoreEntry(req.Entry)
+	return &filer_pb.CreateEntryResponse{}, nil
+}
+
+func (s *policyStoreTestFilerServer) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	key := policyStoreTestEntryKey(req.Directory, req.Entry.Name)
+	if _, found := s.entries[key]; !found {
+		return nil, status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())
+	}
+
+	s.entries[key] = clonePolicyStoreEntry(req.Entry)
+	return &filer_pb.UpdateEntryResponse{}, nil
+}
+
+func (s *policyStoreTestFilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream grpc.ServerStreamingServer[filer_pb.ListEntriesResponse]) error {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	names := make([]string, 0)
+	for key := range s.entries {
+		dir, name := splitPolicyStoreEntryKey(key)
+		if dir != req.Directory {
+			continue
+		}
+		if req.Prefix != "" && len(name) >= len(req.Prefix) && name[:len(req.Prefix)] != req.Prefix {
+			continue
+		}
+		if req.Prefix != "" && len(name) < len(req.Prefix) {
+			continue
+		}
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	for _, name := range names {
+		if err := stream.Send(&filer_pb.ListEntriesResponse{
+			Entry: clonePolicyStoreEntry(s.entries[policyStoreTestEntryKey(req.Directory, name)]),
+		}); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *policyStoreTestFilerServer) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	key := policyStoreTestEntryKey(req.Directory, req.Name)
+	if _, found := s.entries[key]; !found {
+		return nil, status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())
+	}
+
+	delete(s.entries, key)
+	return &filer_pb.DeleteEntryResponse{}, nil
+}
+
+func (s *policyStoreTestFilerServer) putPolicyFile(t *testing.T, dir string, name string, document *PolicyDocument) {
+	t.Helper()
+
+	content, err := json.Marshal(document)
+	require.NoError(t, err)
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.entries[policyStoreTestEntryKey(dir, name)] = &filer_pb.Entry{
+		Name:    name,
+		Content: content,
+	}
+}
+
+func (s *policyStoreTestFilerServer) hasEntry(dir string, name string) bool {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	_, found := s.entries[policyStoreTestEntryKey(dir, name)]
+	return found
+}
+
+func newTestFilerPolicyStore(t *testing.T) (*FilerPolicyStore, *policyStoreTestFilerServer) {
+	t.Helper()
+
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+
+	server := newPolicyStoreTestFilerServer()
+	grpcServer := pb.NewGrpcServer()
+	filer_pb.RegisterSeaweedFilerServer(grpcServer, server)
+	go func() {
+		_ = grpcServer.Serve(lis)
+	}()
+
+	t.Cleanup(func() {
+		grpcServer.Stop()
+		_ = lis.Close()
+	})
+
+	host, portString, err := net.SplitHostPort(lis.Addr().String())
+	require.NoError(t, err)
+	grpcPort, err := strconv.Atoi(portString)
+	require.NoError(t, err)
+
+	store, err := NewFilerPolicyStore(nil, func() string {
+		return string(pb.NewServerAddress(host, 1, grpcPort))
+	})
+	require.NoError(t, err)
+	store.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
+
+	return store, server
+}
+
+func TestFilerPolicyStoreGetPolicyPrefersCanonicalFiles(t *testing.T) {
+	ctx := context.Background()
+	store, server := newTestFilerPolicyStore(t)
+
+	server.putPolicyFile(t, store.basePath, "cli-bucket-access-policy.json", testPolicyDocument("s3:ListBucket", "arn:aws:s3:::cli-allowed-bucket"))
+	server.putPolicyFile(t, store.basePath, "policy_cli-bucket-access-policy.json", testPolicyDocument("s3:PutObject", "arn:aws:s3:::cli-forbidden-bucket/*"))
+
+	document, err := store.GetPolicy(ctx, "", "cli-bucket-access-policy")
+	require.NoError(t, err)
+	require.Len(t, document.Statement, 1)
+	assert.Equal(t, "s3:ListBucket", document.Statement[0].Action[0])
+	assert.Equal(t, "arn:aws:s3:::cli-allowed-bucket", document.Statement[0].Resource[0])
+}
+
+func TestFilerPolicyStoreListPoliciesIncludesCanonicalAndLegacyFiles(t *testing.T) {
+	ctx := context.Background()
+	store, server := newTestFilerPolicyStore(t)
+
+	server.putPolicyFile(t, store.basePath, "canonical-only.json", testPolicyDocument("s3:GetObject", "arn:aws:s3:::canonical-only/*"))
+	server.putPolicyFile(t, store.basePath, "policy_legacy-only.json", testPolicyDocument("s3:PutObject", "arn:aws:s3:::legacy-only/*"))
+	server.putPolicyFile(t, store.basePath, "shared.json", testPolicyDocument("s3:DeleteObject", "arn:aws:s3:::shared/*"))
+	server.putPolicyFile(t, store.basePath, "policy_shared.json", testPolicyDocument("s3:ListBucket", "arn:aws:s3:::shared"))
+	server.putPolicyFile(t, store.basePath, "policy_invalid:name.json", testPolicyDocument("s3:GetObject", "arn:aws:s3:::ignored/*"))
+	server.putPolicyFile(t, store.basePath, "bucket-policy:bucket-a.json", testPolicyDocument("s3:ListBucket", "arn:aws:s3:::bucket-a"))
+
+	names, err := store.ListPolicies(ctx, "")
+	require.NoError(t, err)
+
+	assert.ElementsMatch(t, []string{"canonical-only", "legacy-only", "shared", "bucket-policy:bucket-a"}, names)
+}
+
+func TestFilerPolicyStoreDeletePolicyRemovesCanonicalAndLegacyFiles(t *testing.T) {
+	ctx := context.Background()
+	store, server := newTestFilerPolicyStore(t)
+
+	server.putPolicyFile(t, store.basePath, "dual-format.json", testPolicyDocument("s3:GetObject", "arn:aws:s3:::dual-format/*"))
+	server.putPolicyFile(t, store.basePath, "policy_dual-format.json", testPolicyDocument("s3:PutObject", "arn:aws:s3:::dual-format/*"))
+
+	require.NoError(t, store.DeletePolicy(ctx, "", "dual-format"))
+	assert.False(t, server.hasEntry(store.basePath, "dual-format.json"))
+	assert.False(t, server.hasEntry(store.basePath, "policy_dual-format.json"))
+}
+
+func TestFilerPolicyStoreStorePolicyWritesCanonicalFileAndRemovesLegacyTwin(t *testing.T) {
+	ctx := context.Background()
+	store, server := newTestFilerPolicyStore(t)
+
+	server.putPolicyFile(t, store.basePath, "policy_dual-format.json", testPolicyDocument("s3:PutObject", "arn:aws:s3:::dual-format/*"))
+
+	require.NoError(t, store.StorePolicy(ctx, "", "dual-format", testPolicyDocument("s3:GetObject", "arn:aws:s3:::dual-format/*")))
+
+	assert.True(t, server.hasEntry(store.basePath, "dual-format.json"))
+	assert.False(t, server.hasEntry(store.basePath, "policy_dual-format.json"))
+
+	document, err := store.GetPolicy(ctx, "", "dual-format")
+	require.NoError(t, err)
+	require.Len(t, document.Statement, 1)
+	assert.Equal(t, "s3:GetObject", document.Statement[0].Action[0])
+}
+
+func TestFilerPolicyStoreStorePolicyUpdatesExistingCanonicalFile(t *testing.T) {
+	ctx := context.Background()
+	store, server := newTestFilerPolicyStore(t)
+
+	server.putPolicyFile(t, store.basePath, "existing.json", testPolicyDocument("s3:PutObject", "arn:aws:s3:::existing/*"))
+
+	require.NoError(t, store.StorePolicy(ctx, "", "existing", testPolicyDocument("s3:GetObject", "arn:aws:s3:::existing/*")))
+
+	document, err := store.GetPolicy(ctx, "", "existing")
+	require.NoError(t, err)
+	require.Len(t, document.Statement, 1)
+	assert.Equal(t, "s3:GetObject", document.Statement[0].Action[0])
+	assert.Equal(t, "arn:aws:s3:::existing/*", document.Statement[0].Resource[0])
+}
+
+func TestCopyPolicyDocumentClonesConditionState(t *testing.T) {
+	original := &PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []Statement{
+			{
+				Effect: "Allow",
+				Action: []string{"s3:GetObject"},
+				Resource: []string{
+					"arn:aws:s3:::test-bucket/*",
+				},
+				Condition: map[string]map[string]interface{}{
+					"StringEquals": {
+						"s3:prefix": []string{"public/", "private/"},
+					},
+					"Null": {
+						"aws:PrincipalArn": "false",
+					},
+				},
+			},
+		},
+	}
+
+	copied := copyPolicyDocument(original)
+	require.NotNil(t, copied)
+
+	original.Statement[0].Condition["StringEquals"]["s3:prefix"] = []string{"mutated/"}
+	original.Statement[0].Condition["Null"]["aws:PrincipalArn"] = "true"
+
+	assert.Equal(t, []string{"public/", "private/"}, copied.Statement[0].Condition["StringEquals"]["s3:prefix"])
+	assert.Equal(t, "false", copied.Statement[0].Condition["Null"]["aws:PrincipalArn"])
+}
+
+func TestIsAlreadyExistsPolicyStoreErrorUsesStatusCode(t *testing.T) {
+	assert.True(t, isAlreadyExistsPolicyStoreError(status.Error(codes.AlreadyExists, "entry already exists")))
+	assert.False(t, isAlreadyExistsPolicyStoreError(fmt.Errorf("entry already exists")))
+}
+
+func testPolicyDocument(action string, resource string) *PolicyDocument {
+	return &PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []Statement{
+			{
+				Effect:   "Allow",
+				Action:   []string{action},
+				Resource: []string{resource},
+			},
+		},
+	}
+}
+
+func clonePolicyStoreEntry(entry *filer_pb.Entry) *filer_pb.Entry {
+	if entry == nil {
+		return nil
+	}
+	return proto.Clone(entry).(*filer_pb.Entry)
+}
+
+func policyStoreTestEntryKey(dir string, name string) string {
+	return dir + "\x00" + name
+}
+
+func splitPolicyStoreEntryKey(key string) (string, string) {
+	for i := 0; i < len(key); i++ {
+		if key[i] == '\x00' {
+			return key[:i], key[i+1:]
+		}
+	}
+	return key, ""
+}

weed/iam/responses.go

@@ -2,198 +2,207 @@ package iam
 
 import (
 	"encoding/xml"
-	"fmt"
-	"time"
 
 	"github.com/aws/aws-sdk-go/service/iam"
 )
 
-// CommonResponse is embedded in all IAM response types to provide RequestId.
+// CommonResponse is embedded in IAM success response types to provide RequestId.
 type CommonResponse struct {
 	ResponseMetadata struct {
 		RequestId string `xml:"RequestId"`
 	} `xml:"ResponseMetadata"`
 }
 
-// SetRequestId sets a unique request ID based on current timestamp.
-func (r *CommonResponse) SetRequestId() {
-	r.ResponseMetadata.RequestId = fmt.Sprintf("%d", time.Now().UnixNano())
+// SetRequestId stores the request ID generated for the current HTTP request.
+func (r *CommonResponse) SetRequestId(requestID string) {
+	r.ResponseMetadata.RequestId = requestID
+}
+
+// RequestIDSetter is implemented by IAM responses that can carry a RequestId.
+type RequestIDSetter interface {
+	SetRequestId(string)
 }
 
 // ListUsersResponse is the response for ListUsers action.
 type ListUsersResponse struct {
-	CommonResponse
 	XMLName         xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListUsersResponse"`
 	ListUsersResult struct {
 		Users       []*iam.User `xml:"Users>member"`
 		IsTruncated bool        `xml:"IsTruncated"`
 	} `xml:"ListUsersResult"`
+	CommonResponse
 }
 
 // ListAccessKeysResponse is the response for ListAccessKeys action.
 type ListAccessKeysResponse struct {
-	CommonResponse
 	XMLName              xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListAccessKeysResponse"`
 	ListAccessKeysResult struct {
 		AccessKeyMetadata []*iam.AccessKeyMetadata `xml:"AccessKeyMetadata>member"`
 		IsTruncated       bool                     `xml:"IsTruncated"`
 	} `xml:"ListAccessKeysResult"`
+	CommonResponse
 }
 
 // DeleteAccessKeyResponse is the response for DeleteAccessKey action.
 type DeleteAccessKeyResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteAccessKeyResponse"`
+	CommonResponse
 }
 
 // CreatePolicyResponse is the response for CreatePolicy action.
 type CreatePolicyResponse struct {
-	CommonResponse
 	XMLName            xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreatePolicyResponse"`
 	CreatePolicyResult struct {
 		Policy iam.Policy `xml:"Policy"`
 	} `xml:"CreatePolicyResult"`
+	CommonResponse
 }
 
 // DeletePolicyResponse is the response for DeletePolicy action.
 type DeletePolicyResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeletePolicyResponse"`
+	CommonResponse
 }
 
 // ListPoliciesResponse is the response for ListPolicies action.
 type ListPoliciesResponse struct {
-	CommonResponse
 	XMLName            xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListPoliciesResponse"`
 	ListPoliciesResult struct {
 		Policies    []*iam.Policy `xml:"Policies>member"`
 		IsTruncated bool          `xml:"IsTruncated"`
 		Marker      string        `xml:"Marker,omitempty"`
 	} `xml:"ListPoliciesResult"`
+	CommonResponse
 }
 
 // GetPolicyResponse is the response for GetPolicy action.
 type GetPolicyResponse struct {
-	CommonResponse
 	XMLName         xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetPolicyResponse"`
 	GetPolicyResult struct {
 		Policy iam.Policy `xml:"Policy"`
 	} `xml:"GetPolicyResult"`
+	CommonResponse
 }
 
 // ListPolicyVersionsResponse is the response for ListPolicyVersions action.
 type ListPolicyVersionsResponse struct {
-	CommonResponse
 	XMLName                  xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListPolicyVersionsResponse"`
 	ListPolicyVersionsResult struct {
 		Versions    []*iam.PolicyVersion `xml:"Versions>member"`
 		IsTruncated bool                 `xml:"IsTruncated"`
 		Marker      string               `xml:"Marker,omitempty"`
 	} `xml:"ListPolicyVersionsResult"`
+	CommonResponse
 }
 
 // GetPolicyVersionResponse is the response for GetPolicyVersion action.
 type GetPolicyVersionResponse struct {
-	CommonResponse
 	XMLName                xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetPolicyVersionResponse"`
 	GetPolicyVersionResult struct {
 		PolicyVersion iam.PolicyVersion `xml:"PolicyVersion"`
 	} `xml:"GetPolicyVersionResult"`
+	CommonResponse
 }
 
 // CreateUserResponse is the response for CreateUser action.
 type CreateUserResponse struct {
-	CommonResponse
 	XMLName          xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateUserResponse"`
 	CreateUserResult struct {
 		User iam.User `xml:"User"`
 	} `xml:"CreateUserResult"`
+	CommonResponse
 }
 
 // DeleteUserResponse is the response for DeleteUser action.
 type DeleteUserResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteUserResponse"`
+	CommonResponse
 }
 
 // GetUserResponse is the response for GetUser action.
 type GetUserResponse struct {
-	CommonResponse
 	XMLName       xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetUserResponse"`
 	GetUserResult struct {
 		User iam.User `xml:"User"`
 	} `xml:"GetUserResult"`
+	CommonResponse
 }
 
 // UpdateUserResponse is the response for UpdateUser action.
 type UpdateUserResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ UpdateUserResponse"`
+	CommonResponse
 }
 
 // CreateAccessKeyResponse is the response for CreateAccessKey action.
 type CreateAccessKeyResponse struct {
-	CommonResponse
 	XMLName               xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateAccessKeyResponse"`
 	CreateAccessKeyResult struct {
 		AccessKey iam.AccessKey `xml:"AccessKey"`
 	} `xml:"CreateAccessKeyResult"`
+	CommonResponse
 }
 
 // PutUserPolicyResponse is the response for PutUserPolicy action.
 type PutUserPolicyResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ PutUserPolicyResponse"`
+	CommonResponse
 }
 
 // DeleteUserPolicyResponse is the response for DeleteUserPolicy action.
 type DeleteUserPolicyResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteUserPolicyResponse"`
+	CommonResponse
 }
 
 // AttachUserPolicyResponse is the response for AttachUserPolicy action.
 type AttachUserPolicyResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ AttachUserPolicyResponse"`
+	CommonResponse
 }
 
 // DetachUserPolicyResponse is the response for DetachUserPolicy action.
 type DetachUserPolicyResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DetachUserPolicyResponse"`
+	CommonResponse
 }
 
 // ListAttachedUserPoliciesResponse is the response for ListAttachedUserPolicies action.
 type ListAttachedUserPoliciesResponse struct {
-	CommonResponse
 	XMLName                        xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListAttachedUserPoliciesResponse"`
 	ListAttachedUserPoliciesResult struct {
 		AttachedPolicies []*iam.AttachedPolicy `xml:"AttachedPolicies>member"`
 		IsTruncated      bool                  `xml:"IsTruncated"`
 		Marker           string                `xml:"Marker,omitempty"`
 	} `xml:"ListAttachedUserPoliciesResult"`
+	CommonResponse
 }
 
 // GetUserPolicyResponse is the response for GetUserPolicy action.
 type GetUserPolicyResponse struct {
-	CommonResponse
 	XMLName             xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetUserPolicyResponse"`
 	GetUserPolicyResult struct {
 		UserName       string `xml:"UserName"`
 		PolicyName     string `xml:"PolicyName"`
 		PolicyDocument string `xml:"PolicyDocument"`
 	} `xml:"GetUserPolicyResult"`
+	CommonResponse
 }
 
 // ErrorResponse is the IAM error response format.
+// AWS IAM uses a bare <RequestId> at root level for errors, not <ResponseMetadata>.
 type ErrorResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ErrorResponse"`
 	Error   struct {
 		iam.ErrorDetails
 		Type string `xml:"Type"`
 	} `xml:"Error"`
+	RequestId string `xml:"RequestId"`
+}
+
+// SetRequestId stores the request ID generated for the current HTTP request.
+func (r *ErrorResponse) SetRequestId(requestID string) {
+	r.RequestId = requestID
 }
 
 // Error represents an IAM API error with code and underlying error.
@@ -210,14 +219,14 @@ type Policies struct {
 // SetUserStatusResponse is the response for SetUserStatus action.
 // This is a SeaweedFS extension to enable/disable users without deleting them.
 type SetUserStatusResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ SetUserStatusResponse"`
+	CommonResponse
 }
 
 // UpdateAccessKeyResponse is the response for UpdateAccessKey action.
 type UpdateAccessKeyResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ UpdateAccessKeyResponse"`
+	CommonResponse
 }
 
 // ServiceAccountInfo contains service account details for API responses.
@@ -234,40 +243,40 @@ type ServiceAccountInfo struct {
 
 // CreateServiceAccountResponse is the response for CreateServiceAccount action.
 type CreateServiceAccountResponse struct {
-	CommonResponse
 	XMLName                    xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateServiceAccountResponse"`
 	CreateServiceAccountResult struct {
 		ServiceAccount ServiceAccountInfo `xml:"ServiceAccount"`
 	} `xml:"CreateServiceAccountResult"`
+	CommonResponse
 }
 
 // DeleteServiceAccountResponse is the response for DeleteServiceAccount action.
 type DeleteServiceAccountResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteServiceAccountResponse"`
+	CommonResponse
 }
 
 // ListServiceAccountsResponse is the response for ListServiceAccounts action.
 type ListServiceAccountsResponse struct {
-	CommonResponse
 	XMLName                   xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListServiceAccountsResponse"`
 	ListServiceAccountsResult struct {
 		ServiceAccounts []*ServiceAccountInfo `xml:"ServiceAccounts>member"`
 		IsTruncated     bool                  `xml:"IsTruncated"`
 	} `xml:"ListServiceAccountsResult"`
+	CommonResponse
 }
 
 // GetServiceAccountResponse is the response for GetServiceAccount action.
 type GetServiceAccountResponse struct {
-	CommonResponse
 	XMLName                 xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetServiceAccountResponse"`
 	GetServiceAccountResult struct {
 		ServiceAccount ServiceAccountInfo `xml:"ServiceAccount"`
 	} `xml:"GetServiceAccountResult"`
+	CommonResponse
 }
 
 // UpdateServiceAccountResponse is the response for UpdateServiceAccount action.
 type UpdateServiceAccountResponse struct {
-	CommonResponse
 	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ UpdateServiceAccountResponse"`
+	CommonResponse
 }

weed/iam/responses_test.go

@@ -0,0 +1,52 @@
+package iam
+
+import (
+	"encoding/xml"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestListUsersResponseXMLOrdering(t *testing.T) {
+	resp := ListUsersResponse{}
+	resp.SetRequestId("test-req-id")
+
+	output, err := xml.Marshal(resp)
+	require.NoError(t, err)
+
+	xmlString := string(output)
+	listUsersResultIndex := strings.Index(xmlString, "<ListUsersResult>")
+	responseMetadataIndex := strings.Index(xmlString, "<ResponseMetadata>")
+
+	assert.NotEqual(t, -1, listUsersResultIndex, "ListUsersResult should be present")
+	assert.NotEqual(t, -1, responseMetadataIndex, "ResponseMetadata should be present")
+	assert.Less(t, listUsersResultIndex, responseMetadataIndex,
+		"ListUsersResult should appear before ResponseMetadata")
+}
+
+func TestErrorResponseXMLUsesTopLevelRequestId(t *testing.T) {
+	errCode := "NoSuchEntity"
+	errMsg := "the requested IAM entity does not exist"
+
+	resp := ErrorResponse{}
+	resp.Error.Type = "Sender"
+	resp.Error.Code = &errCode
+	resp.Error.Message = &errMsg
+	resp.SetRequestId("request-123")
+
+	output, err := xml.Marshal(resp)
+	require.NoError(t, err)
+
+	xmlString := string(output)
+	errorIndex := strings.Index(xmlString, "<Error>")
+	requestIDIndex := strings.Index(xmlString, "<RequestId>request-123</RequestId>")
+
+	assert.NotEqual(t, -1, errorIndex, "Error should be present")
+	assert.NotEqual(t, -1, requestIDIndex, "RequestId should be present")
+	assert.NotContains(t, xmlString, "<ResponseMetadata>",
+		"ErrorResponse should use bare RequestId, not ResponseMetadata wrapper")
+	assert.Less(t, errorIndex, requestIDIndex,
+		"RequestId should appear after Error at the root level")
+}

weed/iamapi/iamapi_handlers.go

@@ -8,19 +8,20 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
 
-func newErrorResponse(errCode string, errMsg string) ErrorResponse {
+func newErrorResponse(errCode string, errMsg string, requestID string) ErrorResponse {
 	errorResp := ErrorResponse{}
 	errorResp.Error.Type = "Sender"
 	errorResp.Error.Code = &errCode
 	errorResp.Error.Message = &errMsg
+	errorResp.SetRequestId(requestID)
 	return errorResp
 }
 
-func writeIamErrorResponse(w http.ResponseWriter, r *http.Request, iamError *IamError) {
-
+func writeIamErrorResponse(w http.ResponseWriter, r *http.Request, reqID string, iamError *IamError) {
 	if iamError == nil {
-		// Do nothing if there is no error
-		glog.Errorf("No error found")
+		glog.Errorf("writeIamErrorResponse called with nil error")
+		internalResp := newErrorResponse(iam.ErrCodeServiceFailureException, "Internal server error", reqID)
+		s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, internalResp)
 		return
 	}
 
@@ -28,14 +29,16 @@ func writeIamErrorResponse(w http.ResponseWriter, r *http.Request, iamError *Iam
 	errMsg := iamError.Error.Error()
 	glog.Errorf("Response %+v", errMsg)
 
-	errorResp := newErrorResponse(errCode, errMsg)
-	internalErrorResponse := newErrorResponse(iam.ErrCodeServiceFailureException, "Internal server error")
+	errorResp := newErrorResponse(errCode, errMsg, reqID)
+	internalErrorResponse := newErrorResponse(iam.ErrCodeServiceFailureException, "Internal server error", reqID)
 
 	switch errCode {
 	case iam.ErrCodeNoSuchEntityException:
 		s3err.WriteXMLResponse(w, r, http.StatusNotFound, errorResp)
 	case iam.ErrCodeMalformedPolicyDocumentException, iam.ErrCodeInvalidInputException:
 		s3err.WriteXMLResponse(w, r, http.StatusBadRequest, errorResp)
+	case iam.ErrCodeDeleteConflictException:
+		s3err.WriteXMLResponse(w, r, http.StatusConflict, errorResp)
 	case iam.ErrCodeServiceFailureException:
 		// We do not want to expose internal server error to the client
 		s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, internalErrorResponse)

weed/iamapi/iamapi_management_handlers.go

@@ -19,6 +19,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
 )
 
 // Constants from shared package
@@ -41,6 +42,20 @@ const (
 
 var policyLock = sync.RWMutex{}
 
+const policyArnPrefix = "arn:aws:iam:::policy/"
+
+// parsePolicyArn validates an IAM policy ARN and extracts the policy name.
+func parsePolicyArn(policyArn string) (string, *IamError) {
+	if !strings.HasPrefix(policyArn, policyArnPrefix) {
+		return "", &IamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("invalid policy ARN: %s", policyArn)}
+	}
+	policyName := strings.TrimPrefix(policyArn, policyArnPrefix)
+	if policyName == "" {
+		return "", &IamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("invalid policy ARN: %s", policyArn)}
+	}
+	return policyName, nil
+}
+
 // userPolicyKey returns a namespaced key for inline user policies to prevent collision with managed policies.
 // getOrCreateUserPolicies returns the policy map for a user, creating it if needed.
 // Returns a pointer to the user's policy map from Policies.InlinePolicies.
@@ -148,14 +163,16 @@ func validateAccessKeyStatus(status string) error {
 	}
 }
 
-func (iama *IamApiServer) ListUsers(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp ListUsersResponse) {
+func (iama *IamApiServer) ListUsers(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *ListUsersResponse) {
+	resp = &ListUsersResponse{}
 	for _, ident := range s3cfg.Identities {
 		resp.ListUsersResult.Users = append(resp.ListUsersResult.Users, &iam.User{UserName: &ident.Name})
 	}
 	return resp
 }
 
-func (iama *IamApiServer) ListAccessKeys(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp ListAccessKeysResponse) {
+func (iama *IamApiServer) ListAccessKeys(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *ListAccessKeysResponse) {
+	resp = &ListAccessKeysResponse{}
 	userName := values.Get("UserName")
 	for _, ident := range s3cfg.Identities {
 		if userName != "" && userName != ident.Name {
@@ -176,16 +193,31 @@ func (iama *IamApiServer) ListAccessKeys(s3cfg *iam_pb.S3ApiConfiguration, value
 	return resp
 }
 
-func (iama *IamApiServer) CreateUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp CreateUserResponse) {
+func (iama *IamApiServer) CreateUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *CreateUserResponse) {
+	resp = &CreateUserResponse{}
 	userName := values.Get("UserName")
 	resp.CreateUserResult.User.UserName = &userName
 	s3cfg.Identities = append(s3cfg.Identities, &iam_pb.Identity{Name: userName})
 	return resp
 }
 
-func (iama *IamApiServer) DeleteUser(s3cfg *iam_pb.S3ApiConfiguration, userName string) (resp DeleteUserResponse, err *IamError) {
+func (iama *IamApiServer) DeleteUser(s3cfg *iam_pb.S3ApiConfiguration, userName string) (resp *DeleteUserResponse, err *IamError) {
+	resp = &DeleteUserResponse{}
 	for i, ident := range s3cfg.Identities {
 		if userName == ident.Name {
+			// Clean up any inline policies stored for this user
+			policies := Policies{}
+			if pErr := iama.s3ApiConfig.GetPolicies(&policies); pErr != nil && !errors.Is(pErr, filer_pb.ErrNotFound) {
+				return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: pErr}
+			}
+			if policies.InlinePolicies != nil {
+				if _, exists := policies.InlinePolicies[userName]; exists {
+					delete(policies.InlinePolicies, userName)
+					if pErr := iama.s3ApiConfig.PutPolicies(&policies); pErr != nil {
+						return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: pErr}
+					}
+				}
+			}
 			s3cfg.Identities = append(s3cfg.Identities[:i], s3cfg.Identities[i+1:]...)
 			return resp, nil
 		}
@@ -193,7 +225,8 @@ func (iama *IamApiServer) DeleteUser(s3cfg *iam_pb.S3ApiConfiguration, userName
 	return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(USER_DOES_NOT_EXIST, userName)}
 }
 
-func (iama *IamApiServer) GetUser(s3cfg *iam_pb.S3ApiConfiguration, userName string) (resp GetUserResponse, err *IamError) {
+func (iama *IamApiServer) GetUser(s3cfg *iam_pb.S3ApiConfiguration, userName string) (resp *GetUserResponse, err *IamError) {
+	resp = &GetUserResponse{}
 	for _, ident := range s3cfg.Identities {
 		if userName == ident.Name {
 			resp.GetUserResult.User = iam.User{UserName: &ident.Name}
@@ -203,13 +236,28 @@ func (iama *IamApiServer) GetUser(s3cfg *iam_pb.S3ApiConfiguration, userName str
 	return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(USER_DOES_NOT_EXIST, userName)}
 }
 
-func (iama *IamApiServer) UpdateUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp UpdateUserResponse, err *IamError) {
+func (iama *IamApiServer) UpdateUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *UpdateUserResponse, err *IamError) {
+	resp = &UpdateUserResponse{}
 	userName := values.Get("UserName")
 	newUserName := values.Get("NewUserName")
 	if newUserName != "" {
 		for _, ident := range s3cfg.Identities {
 			if userName == ident.Name {
 				ident.Name = newUserName
+				// Move any inline policies from old username to new username
+				policies := Policies{}
+				if pErr := iama.s3ApiConfig.GetPolicies(&policies); pErr != nil && !errors.Is(pErr, filer_pb.ErrNotFound) {
+					return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: pErr}
+				}
+				if policies.InlinePolicies != nil {
+					if userPolicies, exists := policies.InlinePolicies[userName]; exists {
+						delete(policies.InlinePolicies, userName)
+						policies.InlinePolicies[newUserName] = userPolicies
+						if pErr := iama.s3ApiConfig.PutPolicies(&policies); pErr != nil {
+							return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: pErr}
+						}
+					}
+				}
 				return resp, nil
 			}
 		}
@@ -227,23 +275,27 @@ func GetPolicyDocument(policy *string) (policy_engine.PolicyDocument, error) {
 	return policyDocument, nil
 }
 
-func (iama *IamApiServer) CreatePolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp CreatePolicyResponse, iamError *IamError) {
+func (iama *IamApiServer) CreatePolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *CreatePolicyResponse, iamError *IamError) {
+	resp = &CreatePolicyResponse{}
 	policyName := values.Get("PolicyName")
 	policyDocumentString := values.Get("PolicyDocument")
 	policyDocument, err := GetPolicyDocument(&policyDocumentString)
 	if err != nil {
-		return CreatePolicyResponse{}, &IamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err}
+		return resp, &IamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err}
 	}
-	policyId := Hash(&policyDocumentString)
+	policyId := Hash(&policyName)
 	arn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
 	resp.CreatePolicyResult.Policy.PolicyName = &policyName
 	resp.CreatePolicyResult.Policy.Arn = &arn
 	resp.CreatePolicyResult.Policy.PolicyId = &policyId
 	policies := Policies{}
 	// Note: Lock is already held by DoActions, no need to acquire here
-	if err = iama.s3ApiConfig.GetPolicies(&policies); err != nil {
+	if err = iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
 		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
 	}
+	if policies.Policies == nil {
+		policies.Policies = make(map[string]policy_engine.PolicyDocument)
+	}
 	policies.Policies[policyName] = policyDocument
 	if err = iama.s3ApiConfig.PutPolicies(&policies); err != nil {
 		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
@@ -257,54 +309,56 @@ type IamError struct {
 }
 
 // https://docs.aws.amazon.com/IAM/latest/APIReference/API_PutUserPolicy.html
-func (iama *IamApiServer) PutUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp PutUserPolicyResponse, iamError *IamError) {
+func (iama *IamApiServer) PutUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *PutUserPolicyResponse, iamError *IamError) {
+	resp = &PutUserPolicyResponse{}
 	userName := values.Get("UserName")
 	policyName := values.Get("PolicyName")
 	policyDocumentString := values.Get("PolicyDocument")
 	policyDocument, err := GetPolicyDocument(&policyDocumentString)
 	if err != nil {
-		return PutUserPolicyResponse{}, &IamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err}
+		return resp, &IamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err}
 	}
-	actions, err := GetActions(&policyDocument)
-	if err != nil {
-		return PutUserPolicyResponse{}, &IamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err}
+	if _, err := GetActions(&policyDocument); err != nil {
+		return resp, &IamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err}
+	}
+
+	// Verify the user exists before persisting the policy
+	var targetIdent *iam_pb.Identity
+	for _, ident := range s3cfg.Identities {
+		if ident.Name == userName {
+			targetIdent = ident
+			break
+		}
+	}
+	if targetIdent == nil {
+		return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("the user with name %s cannot be found", userName)}
 	}
 
 	// Persist inline policy to storage using per-user indexed structure
 	policies := Policies{}
 	if err = iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
-		return PutUserPolicyResponse{}, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
 	}
 
-	// Get or create user's policy map
 	userPolicies := policies.getOrCreateUserPolicies(userName)
 	userPolicies[policyName] = policyDocument
-	// policies.InlinePolicies[userName] now contains the updated map
 
 	if err = iama.s3ApiConfig.PutPolicies(&policies); err != nil {
-		return PutUserPolicyResponse{}, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
 	}
 
-	// Compute aggregated actions from all user's inline policies, passing the local policies
-	// to avoid redundant I/O (reuses the just-written Policies map)
-	aggregatedActions, computeErr := computeAggregatedActionsForUser(iama, userName, &policies)
+	// Recompute aggregated actions (inline + managed)
+	aggregatedActions, computeErr := computeAllActionsForUser(iama, userName, &policies, targetIdent)
 	if computeErr != nil {
-		glog.Warningf("Failed to compute aggregated actions for user %s: %v", userName, computeErr)
-		aggregatedActions = actions // Fall back to current policy's actions
+		glog.Warningf("Failed to compute aggregated actions for user %s: %v; keeping existing actions", userName, computeErr)
+	} else {
+		targetIdent.Actions = aggregatedActions
 	}
-
-	glog.V(3).Infof("PutUserPolicy: aggregated actions=%v", aggregatedActions)
-	for _, ident := range s3cfg.Identities {
-		if userName != ident.Name {
-			continue
-		}
-		ident.Actions = aggregatedActions
-		return resp, nil
-	}
-	return PutUserPolicyResponse{}, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("the user with name %s cannot be found", userName)}
+	return resp, nil
 }
 
-func (iama *IamApiServer) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp GetUserPolicyResponse, err *IamError) {
+func (iama *IamApiServer) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *GetUserPolicyResponse, err *IamError) {
+	resp = &GetUserPolicyResponse{}
 	userName := values.Get("UserName")
 	policyName := values.Get("PolicyName")
 	for _, ident := range s3cfg.Identities {
@@ -387,7 +441,8 @@ func (iama *IamApiServer) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values
 }
 
 // DeleteUserPolicy removes the inline policy from a user (clears their actions).
-func (iama *IamApiServer) DeleteUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp DeleteUserPolicyResponse, err *IamError) {
+func (iama *IamApiServer) DeleteUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *DeleteUserPolicyResponse, err *IamError) {
+	resp = &DeleteUserPolicyResponse{}
 	userName := values.Get("UserName")
 	policyName := values.Get("PolicyName")
 
@@ -419,17 +474,253 @@ func (iama *IamApiServer) DeleteUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, val
 		}
 	}
 
-	// Recompute aggregated actions from remaining inline policies (passing policies to avoid redundant GetPolicies)
-	aggregatedActions, computeErr := computeAggregatedActionsForUser(iama, userName, &policies)
+	// Recompute aggregated actions from remaining inline + managed policies
+	aggregatedActions, computeErr := computeAllActionsForUser(iama, userName, &policies, targetIdent)
 	if computeErr != nil {
-		glog.Warningf("Failed to recompute aggregated actions for user %s: %v", userName, computeErr)
+		glog.Warningf("Failed to recompute aggregated actions for user %s: %v; keeping existing actions", userName, computeErr)
+	} else {
+		targetIdent.Actions = aggregatedActions
+	}
+	return resp, nil
+}
+
+// GetPolicy retrieves a managed policy by ARN.
+func (iama *IamApiServer) GetPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *GetPolicyResponse, iamError *IamError) {
+	resp = &GetPolicyResponse{}
+	policyArn := values.Get("PolicyArn")
+	policyName, iamError := parsePolicyArn(policyArn)
+	if iamError != nil {
+		return resp, iamError
 	}
 
-	// Update the found identity's actions
-	targetIdent.Actions = aggregatedActions
+	policies := Policies{}
+	if err := iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+
+	if _, exists := policies.Policies[policyName]; !exists {
+		return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("policy %s not found", policyName)}
+	}
+
+	policyId := Hash(&policyName)
+	resp.GetPolicyResult.Policy.PolicyName = &policyName
+	resp.GetPolicyResult.Policy.Arn = &policyArn
+	resp.GetPolicyResult.Policy.PolicyId = &policyId
 	return resp, nil
 }
 
+// DeletePolicy removes a managed policy. Rejects deletion if the policy is still attached to any user
+// (matching AWS IAM behavior: must detach before deleting).
+func (iama *IamApiServer) DeletePolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *DeletePolicyResponse, iamError *IamError) {
+	resp = &DeletePolicyResponse{}
+	policyArn := values.Get("PolicyArn")
+	policyName, iamError := parsePolicyArn(policyArn)
+	if iamError != nil {
+		return resp, iamError
+	}
+
+	policies := Policies{}
+	if err := iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+
+	if _, exists := policies.Policies[policyName]; !exists {
+		return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("policy %s not found", policyName)}
+	}
+
+	// Reject deletion if the policy is still attached to any user
+	for _, ident := range s3cfg.Identities {
+		for _, name := range ident.PolicyNames {
+			if name == policyName {
+				return resp, &IamError{
+					Code:  iam.ErrCodeDeleteConflictException,
+					Error: fmt.Errorf("policy %s is still attached to user %s", policyName, ident.Name),
+				}
+			}
+		}
+	}
+
+	delete(policies.Policies, policyName)
+	if err := iama.s3ApiConfig.PutPolicies(&policies); err != nil {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+
+	return resp, nil
+}
+
+// ListPolicies lists all managed policies.
+func (iama *IamApiServer) ListPolicies(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *ListPoliciesResponse, iamError *IamError) {
+	resp = &ListPoliciesResponse{}
+	policies := Policies{}
+	if err := iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+
+	for policyName := range policies.Policies {
+		name := policyName
+		arn := fmt.Sprintf("arn:aws:iam:::policy/%s", name)
+		policyId := Hash(&name)
+		resp.ListPoliciesResult.Policies = append(resp.ListPoliciesResult.Policies, &iam.Policy{
+			PolicyName: &name,
+			Arn:        &arn,
+			PolicyId:   &policyId,
+		})
+	}
+	return resp, nil
+}
+
+// AttachUserPolicy attaches a managed policy to a user.
+func (iama *IamApiServer) AttachUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *AttachUserPolicyResponse, iamError *IamError) {
+	resp = &AttachUserPolicyResponse{}
+	userName := values.Get("UserName")
+	policyArn := values.Get("PolicyArn")
+	policyName, iamError := parsePolicyArn(policyArn)
+	if iamError != nil {
+		return resp, iamError
+	}
+
+	// Verify managed policy exists
+	policies := Policies{}
+	if err := iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+	if _, exists := policies.Policies[policyName]; !exists {
+		return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("policy %s not found", policyName)}
+	}
+
+	// Find user and attach policy
+	for _, ident := range s3cfg.Identities {
+		if ident.Name != userName {
+			continue
+		}
+		// Check if already attached
+		for _, name := range ident.PolicyNames {
+			if name == policyName {
+				return resp, nil // Already attached, idempotent
+			}
+		}
+		prevPolicyNames := ident.PolicyNames
+		ident.PolicyNames = append(ident.PolicyNames, policyName)
+
+		// Recompute aggregated actions (inline + managed)
+		aggregatedActions, err := computeAllActionsForUser(iama, userName, &policies, ident)
+		if err != nil {
+			// Roll back PolicyNames to keep identity consistent
+			ident.PolicyNames = prevPolicyNames
+			return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: fmt.Errorf("failed to compute actions after attaching policy: %w", err)}
+		}
+		ident.Actions = aggregatedActions
+		return resp, nil
+	}
+	return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(USER_DOES_NOT_EXIST, userName)}
+}
+
+// DetachUserPolicy detaches a managed policy from a user.
+func (iama *IamApiServer) DetachUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *DetachUserPolicyResponse, iamError *IamError) {
+	resp = &DetachUserPolicyResponse{}
+	userName := values.Get("UserName")
+	policyArn := values.Get("PolicyArn")
+	policyName, iamError := parsePolicyArn(policyArn)
+	if iamError != nil {
+		return resp, iamError
+	}
+
+	for _, ident := range s3cfg.Identities {
+		if ident.Name != userName {
+			continue
+		}
+		// Find and remove policy name from the list
+		prevPolicyNames := make([]string, len(ident.PolicyNames))
+		copy(prevPolicyNames, ident.PolicyNames)
+
+		found := false
+		for i, name := range ident.PolicyNames {
+			if name == policyName {
+				ident.PolicyNames = append(ident.PolicyNames[:i], ident.PolicyNames[i+1:]...)
+				found = true
+				break
+			}
+		}
+		if !found {
+			return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("policy %s is not attached to user %s", policyName, userName)}
+		}
+
+		// Recompute aggregated actions (inline + managed)
+		policies := Policies{}
+		if err := iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+			// Roll back PolicyNames on storage error
+			ident.PolicyNames = prevPolicyNames
+			return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+		}
+		aggregatedActions, err := computeAllActionsForUser(iama, userName, &policies, ident)
+		if err != nil {
+			// Roll back PolicyNames to keep identity consistent
+			ident.PolicyNames = prevPolicyNames
+			return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: fmt.Errorf("failed to compute actions after detaching policy: %w", err)}
+		}
+		ident.Actions = aggregatedActions
+		return resp, nil
+	}
+	return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(USER_DOES_NOT_EXIST, userName)}
+}
+
+// ListAttachedUserPolicies lists the managed policies attached to a user.
+func (iama *IamApiServer) ListAttachedUserPolicies(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *ListAttachedUserPoliciesResponse, iamError *IamError) {
+	resp = &ListAttachedUserPoliciesResponse{}
+	userName := values.Get("UserName")
+	for _, ident := range s3cfg.Identities {
+		if ident.Name != userName {
+			continue
+		}
+		for _, policyName := range ident.PolicyNames {
+			name := policyName
+			arn := fmt.Sprintf("arn:aws:iam:::policy/%s", name)
+			resp.ListAttachedUserPoliciesResult.AttachedPolicies = append(
+				resp.ListAttachedUserPoliciesResult.AttachedPolicies,
+				&iam.AttachedPolicy{PolicyName: &name, PolicyArn: &arn},
+			)
+		}
+		return resp, nil
+	}
+	return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(USER_DOES_NOT_EXIST, userName)}
+}
+
+// computeAllActionsForUser computes the union of actions from both inline and managed policies.
+func computeAllActionsForUser(iama *IamApiServer, userName string, policies *Policies, ident *iam_pb.Identity) ([]string, error) {
+	actionSet := make(map[string]bool)
+	var aggregatedActions []string
+
+	addUniqueActions := func(actions []string) {
+		for _, action := range actions {
+			if !actionSet[action] {
+				actionSet[action] = true
+				aggregatedActions = append(aggregatedActions, action)
+			}
+		}
+	}
+
+	// Include inline policy actions
+	inlineActions, err := computeAggregatedActionsForUser(iama, userName, policies)
+	if err != nil {
+		return nil, err
+	}
+	addUniqueActions(inlineActions)
+
+	// Include managed policy actions
+	for _, policyName := range ident.PolicyNames {
+		if policyDoc, exists := policies.Policies[policyName]; exists {
+			actions, err := GetActions(&policyDoc)
+			if err != nil {
+				glog.Warningf("Failed to get actions from managed policy '%s' for user %s: %v", policyName, userName, err)
+				continue
+			}
+			addUniqueActions(actions)
+		}
+	}
+
+	return aggregatedActions, nil
+}
+
 func GetActions(policy *policy_engine.PolicyDocument) ([]string, error) {
 	var actions []string
 
@@ -467,7 +758,8 @@ func GetActions(policy *policy_engine.PolicyDocument) ([]string, error) {
 	return actions, nil
 }
 
-func (iama *IamApiServer) CreateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp CreateAccessKeyResponse, iamErr *IamError) {
+func (iama *IamApiServer) CreateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *CreateAccessKeyResponse, iamErr *IamError) {
+	resp = &CreateAccessKeyResponse{}
 	userName := values.Get("UserName")
 	status := iam.StatusTypeActive
 
@@ -511,7 +803,8 @@ func (iama *IamApiServer) CreateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu
 }
 
 // UpdateAccessKey updates the status of an access key (Active or Inactive).
-func (iama *IamApiServer) UpdateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp UpdateAccessKeyResponse, err *IamError) {
+func (iama *IamApiServer) UpdateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *UpdateAccessKeyResponse, err *IamError) {
+	resp = &UpdateAccessKeyResponse{}
 	userName := values.Get("UserName")
 	accessKeyId := values.Get("AccessKeyId")
 	status := values.Get("Status")
@@ -541,7 +834,8 @@ func (iama *IamApiServer) UpdateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu
 	return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(USER_DOES_NOT_EXIST, userName)}
 }
 
-func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp DeleteAccessKeyResponse) {
+func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *DeleteAccessKeyResponse) {
+	resp = &DeleteAccessKeyResponse{}
 	userName := values.Get("UserName")
 	accessKeyId := values.Get("AccessKeyId")
 	for _, ident := range s3cfg.Identities {
@@ -612,6 +906,8 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 	policyLock.Lock()
 	defer policyLock.Unlock()
 
+	r, reqID := request_id.Ensure(r)
+
 	if err := r.ParseForm(); err != nil {
 		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
 		return
@@ -624,8 +920,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 	}
 
 	glog.V(4).Infof("DoActions: %+v", values)
-	var response interface{}
-	var iamError *IamError
+	var response iamlib.RequestIDSetter
 	changed := true
 	switch r.Form.Get("Action") {
 	case "ListUsers":
@@ -639,32 +934,35 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		response = iama.CreateUser(s3cfg, values)
 	case "GetUser":
 		userName := values.Get("UserName")
-		response, iamError = iama.GetUser(s3cfg, userName)
-		if iamError != nil {
-			writeIamErrorResponse(w, r, iamError)
+		var err *IamError
+		response, err = iama.GetUser(s3cfg, userName)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
 		changed = false
 	case "UpdateUser":
-		response, iamError = iama.UpdateUser(s3cfg, values)
-		if iamError != nil {
-			glog.Errorf("UpdateUser: %+v", iamError.Error)
-			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		var err *IamError
+		response, err = iama.UpdateUser(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
 	case "DeleteUser":
 		userName := values.Get("UserName")
-		response, iamError = iama.DeleteUser(s3cfg, userName)
-		if iamError != nil {
-			writeIamErrorResponse(w, r, iamError)
+		var err *IamError
+		response, err = iama.DeleteUser(s3cfg, userName)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
 	case "CreateAccessKey":
 		iama.handleImplicitUsername(r, values)
-		response, iamError = iama.CreateAccessKey(s3cfg, values)
-		if iamError != nil {
-			glog.Errorf("CreateAccessKey: %+v", iamError.Error)
-			writeIamErrorResponse(w, r, iamError)
+		var err *IamError
+		response, err = iama.CreateAccessKey(s3cfg, values)
+		if err != nil {
+			glog.Errorf("CreateAccessKey: %+v", err.Error)
+			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
 	case "DeleteAccessKey":
@@ -672,47 +970,94 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		response = iama.DeleteAccessKey(s3cfg, values)
 	case "UpdateAccessKey":
 		iama.handleImplicitUsername(r, values)
-		response, iamError = iama.UpdateAccessKey(s3cfg, values)
-		if iamError != nil {
-			writeIamErrorResponse(w, r, iamError)
+		var err *IamError
+		response, err = iama.UpdateAccessKey(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
 	case "CreatePolicy":
-		response, iamError = iama.CreatePolicy(s3cfg, values)
-		if iamError != nil {
-			glog.Errorf("CreatePolicy:  %+v", iamError.Error)
-			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		var err *IamError
+		response, err = iama.CreatePolicy(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
 		// CreatePolicy persists the policy document via iama.s3ApiConfig.PutPolicies().
 		// The `changed` flag is false because this does not modify the main s3cfg.Identities configuration.
 		changed = false
 	case "PutUserPolicy":
-		var iamError *IamError
-		response, iamError = iama.PutUserPolicy(s3cfg, values)
-		if iamError != nil {
-			glog.Errorf("PutUserPolicy:  %+v", iamError.Error)
-
-			writeIamErrorResponse(w, r, iamError)
+		var err *IamError
+		response, err = iama.PutUserPolicy(s3cfg, values)
+		if err != nil {
+			glog.Errorf("PutUserPolicy:  %+v", err.Error)
+			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
 	case "GetUserPolicy":
-		response, iamError = iama.GetUserPolicy(s3cfg, values)
-		if iamError != nil {
-			writeIamErrorResponse(w, r, iamError)
+		var err *IamError
+		response, err = iama.GetUserPolicy(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
 		changed = false
 	case "DeleteUserPolicy":
-		if response, iamError = iama.DeleteUserPolicy(s3cfg, values); iamError != nil {
-			writeIamErrorResponse(w, r, iamError)
+		var err *IamError
+		response, err = iama.DeleteUserPolicy(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
+	case "GetPolicy":
+		var err *IamError
+		response, err = iama.GetPolicy(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
+			return
+		}
+		changed = false
+	case "DeletePolicy":
+		var err *IamError
+		response, err = iama.DeletePolicy(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
+			return
+		}
+		changed = false
+	case "ListPolicies":
+		var err *IamError
+		response, err = iama.ListPolicies(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
+			return
+		}
+		changed = false
+	case "AttachUserPolicy":
+		var err *IamError
+		response, err = iama.AttachUserPolicy(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
+			return
+		}
+	case "DetachUserPolicy":
+		var err *IamError
+		response, err = iama.DetachUserPolicy(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
+			return
+		}
+	case "ListAttachedUserPolicies":
+		var err *IamError
+		response, err = iama.ListAttachedUserPolicies(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
+			return
+		}
+		changed = false
 	default:
 		errNotImplemented := s3err.GetAPIError(s3err.ErrNotImplemented)
-		errorResponse := ErrorResponse{}
-		errorResponse.Error.Code = &errNotImplemented.Code
-		errorResponse.Error.Message = &errNotImplemented.Description
+		errorResponse := newErrorResponse(errNotImplemented.Code, errNotImplemented.Description, reqID)
 		s3err.WriteXMLResponse(w, r, errNotImplemented.HTTPStatusCode, errorResponse)
 		return
 	}
@@ -720,7 +1065,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		err := iama.s3ApiConfig.PutS3ApiConfiguration(s3cfg)
 		if err != nil {
 			var iamError = IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
-			writeIamErrorResponse(w, r, &iamError)
+			writeIamErrorResponse(w, r, reqID, &iamError)
 			return
 		}
 		// Reload in-memory identity maps so subsequent LookupByAccessKey calls
@@ -732,5 +1077,6 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 	}
+	response.SetRequestId(reqID)
 	s3err.WriteXMLResponse(w, r, http.StatusOK, response)
 }

weed/iamapi/iamapi_management_handlers_test.go

@@ -5,6 +5,7 @@ import (
 	"net/url"
 	"testing"
 
+	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
@@ -261,3 +262,299 @@ func TestMultipleInlinePoliciesAggregateActions(t *testing.T) {
 		assert.True(t, actionSet[expectedAction], "Expected action '%s' not found in aggregated actions. Got: %v", expectedAction, aliceIdent.Actions)
 	}
 }
+
+// newTestIamApiServer creates a minimal IamApiServer for unit testing with only s3ApiConfig set.
+// Other fields (iam, masterClient, etc.) are left nil — tests must not call code paths that use them.
+func newTestIamApiServer(policies Policies) *IamApiServer {
+	return &IamApiServer{
+		s3ApiConfig: &mockIamS3ApiConfig{policies: policies},
+	}
+}
+
+func TestGetPolicy(t *testing.T) {
+	policyDoc := policy_engine.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+				Resource: policy_engine.NewStringOrStringSlice("arn:aws:s3:::*"),
+			},
+		},
+	}
+	iama := newTestIamApiServer(Policies{
+		Policies: map[string]policy_engine.PolicyDocument{"my-policy": policyDoc},
+	})
+	s3cfg := &iam_pb.S3ApiConfiguration{}
+
+	// Success case
+	values := url.Values{"PolicyArn": []string{"arn:aws:iam:::policy/my-policy"}}
+	resp, iamErr := iama.GetPolicy(s3cfg, values)
+	assert.Nil(t, iamErr)
+	assert.Equal(t, "my-policy", *resp.GetPolicyResult.Policy.PolicyName)
+	assert.Equal(t, "arn:aws:iam:::policy/my-policy", *resp.GetPolicyResult.Policy.Arn)
+	policyName := "my-policy"
+	expectedId := Hash(&policyName)
+	assert.Equal(t, expectedId, *resp.GetPolicyResult.Policy.PolicyId)
+
+	// Not found case
+	values = url.Values{"PolicyArn": []string{"arn:aws:iam:::policy/nonexistent"}}
+	_, iamErr = iama.GetPolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code)
+
+	// Invalid ARN
+	values = url.Values{"PolicyArn": []string{"invalid-arn"}}
+	_, iamErr = iama.GetPolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeInvalidInputException, iamErr.Code)
+
+	// Empty ARN
+	values = url.Values{"PolicyArn": []string{""}}
+	_, iamErr = iama.GetPolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeInvalidInputException, iamErr.Code)
+}
+
+func TestDeletePolicy(t *testing.T) {
+	policyDoc := policy_engine.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+				Resource: policy_engine.NewStringOrStringSlice("arn:aws:s3:::*"),
+			},
+		},
+	}
+	mock := &mockIamS3ApiConfig{policies: Policies{
+		Policies: map[string]policy_engine.PolicyDocument{"my-policy": policyDoc},
+	}}
+	iama := &IamApiServer{s3ApiConfig: mock}
+
+	// Reject deletion when policy is attached to a user (AWS-compatible behavior)
+	s3cfg := &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{
+			Name:        "alice",
+			PolicyNames: []string{"my-policy"},
+		}},
+	}
+	values := url.Values{"PolicyArn": []string{"arn:aws:iam:::policy/my-policy"}}
+	_, iamErr := iama.DeletePolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeDeleteConflictException, iamErr.Code)
+
+	// Succeed when no users are attached
+	s3cfgEmpty := &iam_pb.S3ApiConfiguration{}
+	_, iamErr = iama.DeletePolicy(s3cfgEmpty, values)
+	assert.Nil(t, iamErr)
+
+	// Verify deleted
+	_, iamErr = iama.GetPolicy(s3cfgEmpty, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code)
+}
+
+func TestListPolicies(t *testing.T) {
+	iama := newTestIamApiServer(Policies{})
+	s3cfg := &iam_pb.S3ApiConfiguration{}
+
+	// Empty case
+	resp, iamErr := iama.ListPolicies(s3cfg, url.Values{})
+	assert.Nil(t, iamErr)
+	assert.Empty(t, resp.ListPoliciesResult.Policies)
+
+	// Populated case
+	policyDoc := policy_engine.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+				Resource: policy_engine.NewStringOrStringSlice("arn:aws:s3:::*"),
+			},
+		},
+	}
+	iama = newTestIamApiServer(Policies{
+		Policies: map[string]policy_engine.PolicyDocument{
+			"policy-a": policyDoc,
+			"policy-b": policyDoc,
+		},
+	})
+
+	resp, iamErr = iama.ListPolicies(s3cfg, url.Values{})
+	assert.Nil(t, iamErr)
+	assert.Equal(t, 2, len(resp.ListPoliciesResult.Policies))
+	for _, p := range resp.ListPoliciesResult.Policies {
+		name := *p.PolicyName
+		expectedId := Hash(&name)
+		assert.Equal(t, expectedId, *p.PolicyId, "PolicyId should be Hash(policyName) for %s", name)
+	}
+}
+
+func TestAttachUserPolicy(t *testing.T) {
+	policyDoc := policy_engine.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+				Resource: policy_engine.NewStringOrStringSlice("arn:aws:s3:::*"),
+			},
+		},
+	}
+	iama := newTestIamApiServer(Policies{
+		Policies: map[string]policy_engine.PolicyDocument{"my-policy": policyDoc},
+	})
+	s3cfg := &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice"}},
+	}
+
+	// Success case
+	values := url.Values{
+		"UserName":  []string{"alice"},
+		"PolicyArn": []string{"arn:aws:iam:::policy/my-policy"},
+	}
+	_, iamErr := iama.AttachUserPolicy(s3cfg, values)
+	assert.Nil(t, iamErr)
+	assert.Contains(t, s3cfg.Identities[0].PolicyNames, "my-policy")
+	// Verify actions were computed from the managed policy
+	assert.Greater(t, len(s3cfg.Identities[0].Actions), 0)
+
+	// Idempotent re-attach
+	_, iamErr = iama.AttachUserPolicy(s3cfg, values)
+	assert.Nil(t, iamErr)
+	// Should still have exactly one entry
+	count := 0
+	for _, name := range s3cfg.Identities[0].PolicyNames {
+		if name == "my-policy" {
+			count++
+		}
+	}
+	assert.Equal(t, 1, count)
+
+	// Policy not found
+	values = url.Values{
+		"UserName":  []string{"alice"},
+		"PolicyArn": []string{"arn:aws:iam:::policy/nonexistent"},
+	}
+	_, iamErr = iama.AttachUserPolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code)
+
+	// User not found
+	values = url.Values{
+		"UserName":  []string{"bob"},
+		"PolicyArn": []string{"arn:aws:iam:::policy/my-policy"},
+	}
+	_, iamErr = iama.AttachUserPolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code)
+}
+
+func TestManagedPolicyActionsPreservedAcrossInlineMutations(t *testing.T) {
+	managedPolicyDoc := policy_engine.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+				Resource: policy_engine.NewStringOrStringSlice("arn:aws:s3:::*"),
+			},
+		},
+	}
+	iama := newTestIamApiServer(Policies{
+		Policies: map[string]policy_engine.PolicyDocument{"my-policy": managedPolicyDoc},
+	})
+	s3cfg := &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice"}},
+	}
+
+	// Attach managed policy
+	_, iamErr := iama.AttachUserPolicy(s3cfg, url.Values{
+		"UserName":  []string{"alice"},
+		"PolicyArn": []string{"arn:aws:iam:::policy/my-policy"},
+	})
+	assert.Nil(t, iamErr)
+	assert.Contains(t, s3cfg.Identities[0].Actions, "Read", "Managed policy should grant Read action")
+
+	// Add an inline policy
+	inlinePolicyJSON := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::bucket-x/*"}]}`
+	_, iamErr = iama.PutUserPolicy(s3cfg, url.Values{
+		"UserName":       []string{"alice"},
+		"PolicyName":     []string{"inline-write"},
+		"PolicyDocument": []string{inlinePolicyJSON},
+	})
+	assert.Nil(t, iamErr)
+
+	// Should have both managed (Read) and inline (Write:bucket-x/*) actions
+	actionSet := make(map[string]bool)
+	for _, a := range s3cfg.Identities[0].Actions {
+		actionSet[a] = true
+	}
+	assert.True(t, actionSet["Read"], "Managed policy Read action should persist after PutUserPolicy")
+	assert.True(t, actionSet["Write:bucket-x/*"], "Inline policy Write action should be present")
+
+	// Delete the inline policy
+	_, iamErr = iama.DeleteUserPolicy(s3cfg, url.Values{
+		"UserName":   []string{"alice"},
+		"PolicyName": []string{"inline-write"},
+	})
+	assert.Nil(t, iamErr)
+
+	// Managed policy actions should still be present
+	assert.Contains(t, s3cfg.Identities[0].PolicyNames, "my-policy", "Managed policy should still be attached")
+	assert.Contains(t, s3cfg.Identities[0].Actions, "Read", "Managed policy Read action should persist after DeleteUserPolicy")
+}
+
+func TestDetachUserPolicy(t *testing.T) {
+	policyDoc := policy_engine.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy_engine.PolicyStatement{
+			{
+				Effect:   policy_engine.PolicyEffectAllow,
+				Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+				Resource: policy_engine.NewStringOrStringSlice("arn:aws:s3:::*"),
+			},
+		},
+	}
+	iama := newTestIamApiServer(Policies{
+		Policies: map[string]policy_engine.PolicyDocument{"my-policy": policyDoc},
+	})
+	s3cfg := &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice", PolicyNames: []string{"my-policy"}}},
+	}
+
+	values := url.Values{
+		"UserName":  []string{"alice"},
+		"PolicyArn": []string{"arn:aws:iam:::policy/my-policy"},
+	}
+	_, iamErr := iama.DetachUserPolicy(s3cfg, values)
+	assert.Nil(t, iamErr)
+	assert.Empty(t, s3cfg.Identities[0].PolicyNames)
+
+	// Detach again should fail (not attached)
+	_, iamErr = iama.DetachUserPolicy(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code)
+}
+
+func TestListAttachedUserPolicies(t *testing.T) {
+	iama := newTestIamApiServer(Policies{})
+	s3cfg := &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice", PolicyNames: []string{"policy-a", "policy-b"}}},
+	}
+
+	values := url.Values{"UserName": []string{"alice"}}
+	resp, iamErr := iama.ListAttachedUserPolicies(s3cfg, values)
+	assert.Nil(t, iamErr)
+	assert.Equal(t, 2, len(resp.ListAttachedUserPoliciesResult.AttachedPolicies))
+	assert.Equal(t, "policy-a", *resp.ListAttachedUserPoliciesResult.AttachedPolicies[0].PolicyName)
+	assert.Equal(t, "arn:aws:iam:::policy/policy-a", *resp.ListAttachedUserPoliciesResult.AttachedPolicies[0].PolicyArn)
+
+	// User not found
+	values = url.Values{"UserName": []string{"bob"}}
+	_, iamErr = iama.ListAttachedUserPolicies(s3cfg, values)
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code)
+}

weed/iamapi/iamapi_response.go

@@ -22,8 +22,14 @@ type (
 	UpdateAccessKeyResponse      = iamlib.UpdateAccessKeyResponse
 	PutUserPolicyResponse        = iamlib.PutUserPolicyResponse
 	DeleteUserPolicyResponse     = iamlib.DeleteUserPolicyResponse
-	GetUserPolicyResponse        = iamlib.GetUserPolicyResponse
-	ErrorResponse                = iamlib.ErrorResponse
+	GetUserPolicyResponse            = iamlib.GetUserPolicyResponse
+	GetPolicyResponse                = iamlib.GetPolicyResponse
+	DeletePolicyResponse             = iamlib.DeletePolicyResponse
+	ListPoliciesResponse             = iamlib.ListPoliciesResponse
+	AttachUserPolicyResponse         = iamlib.AttachUserPolicyResponse
+	DetachUserPolicyResponse         = iamlib.DetachUserPolicyResponse
+	ListAttachedUserPoliciesResponse = iamlib.ListAttachedUserPoliciesResponse
+	ErrorResponse                    = iamlib.ErrorResponse
 	ServiceAccountInfo           = iamlib.ServiceAccountInfo
 	CreateServiceAccountResponse = iamlib.CreateServiceAccountResponse
 	DeleteServiceAccountResponse = iamlib.DeleteServiceAccountResponse

weed/iamapi/iamapi_server.go

@@ -21,6 +21,7 @@ import (
 	. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/proto"
@@ -117,6 +118,7 @@ func NewIamApiServerWithStore(router *mux.Router, option *IamServerOption, expli
 func (iama *IamApiServer) registerRouter(router *mux.Router) {
 	// API Router
 	apiRouter := router.PathPrefix("/").Subrouter()
+	apiRouter.Use(request_id.Middleware)
 	// ListBuckets
 
 	// apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_ADMIN), "LIST"))

weed/iamapi/iamapi_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
+	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -73,6 +74,21 @@ func TestListUsers(t *testing.T) {
 	assert.Equal(t, http.StatusOK, response.Code)
 }
 
+func TestListUsersRequestIdMatchesResponseHeader(t *testing.T) {
+	params := &iam.ListUsersInput{}
+	req, _ := iam.New(session.New()).ListUsersRequest(params)
+	_ = req.Build()
+
+	out := ListUsersResponse{}
+	response, err := executeRequest(req.HTTPRequest, out)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, http.StatusOK, response.Code)
+
+	headerRequestID := response.Header().Get(request_id.AmzRequestIDHeader)
+	assert.NotEmpty(t, headerRequestID)
+	assert.Equal(t, headerRequestID, extractRequestID(response))
+}
+
 func TestListAccessKeys(t *testing.T) {
 	svc := iam.New(session.New())
 	params := &iam.ListAccessKeysInput{}
@@ -244,6 +260,9 @@ func TestPutUserPolicyError(t *testing.T) {
 
 	assert.Equal(t, expectedMessage, message)
 	assert.Equal(t, expectedCode, code)
+	assert.Contains(t, response.Body.String(), "<RequestId>")
+	assert.NotContains(t, response.Body.String(), "<ResponseMetadata>")
+	assert.Equal(t, response.Header().Get(request_id.AmzRequestIDHeader), extractRequestID(response))
 }
 
 func extractErrorCodeAndMessage(response *httptest.ResponseRecorder) (string, string) {
@@ -255,6 +274,15 @@ func extractErrorCodeAndMessage(response *httptest.ResponseRecorder) (string, st
 	return code, message
 }
 
+func extractRequestID(response *httptest.ResponseRecorder) string {
+	re := regexp.MustCompile(`<RequestId>([^<]+)</RequestId>`)
+	matches := re.FindStringSubmatch(response.Body.String())
+	if len(matches) < 2 {
+		return ""
+	}
+	return matches[1]
+}
+
 func TestGetUserPolicy(t *testing.T) {
 	userName := aws.String("Test")
 	params := &iam.GetUserPolicyInput{UserName: userName, PolicyName: aws.String("S3-read-only-example-bucket")}

weed/mount/filehandle_read.go

@@ -9,7 +9,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 func (fh *FileHandle) lockForRead(startOffset int64, size int) {
@@ -163,11 +162,12 @@ func (fh *FileHandle) downloadRemoteEntry(entry *LockedEntry) error {
 
 		fh.SetEntry(resp.Entry)
 
-		// Only update cache if the parent directory is cached
-		if fh.wfs.metaCache.IsDirectoryCached(util.FullPath(dir)) {
-			if err := fh.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, resp.Entry)); err != nil {
-				return fmt.Errorf("update meta cache for %s: %w", fileFullPath, err)
-			}
+		event := resp.GetMetadataEvent()
+		if event == nil {
+			event = metadataUpdateEvent(request.Directory, resp.Entry)
+		}
+		if applyErr := fh.wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil {
+			glog.Warningf("CacheRemoteObject %s: best-effort metadata apply failed: %v", fileFullPath, applyErr)
 		}
 
 		return nil

weed/mount/filer_conf.go

@@ -1,6 +1,7 @@
 package mount
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"path/filepath"
@@ -20,7 +21,7 @@ func (wfs *WFS) subscribeFilerConfEvents() (*meta_cache.MetadataFollower, error)
 
 	// read current conf
 	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		content, err := filer.ReadInsideFiler(client, confDir, confName)
+		content, err := filer.ReadInsideFiler(context.Background(), client, confDir, confName)
 		if err != nil {
 			return err
 		}

weed/mount/inode_to_path.go

@@ -21,18 +21,18 @@ type InodeEntry struct {
 	nlookup           uint64
 	isDirectory       bool
 	isChildrenCached  bool
+	readDirDirect     bool
 	cachedExpiresTime time.Time
 	lastAccess        time.Time
 	lastRefresh       time.Time
 	updateWindowStart time.Time
 	updateCount       int
-	needsRefresh      bool
 }
 
 func (ie *InodeEntry) resetCacheState() {
 	ie.isChildrenCached = false
+	ie.readDirDirect = false
 	ie.cachedExpiresTime = time.Time{}
-	ie.needsRefresh = false
 	ie.updateCount = 0
 	ie.updateWindowStart = time.Time{}
 }
@@ -188,11 +188,11 @@ func (i *InodeToPath) MarkChildrenCached(fullpath util.FullPath) {
 		return
 	}
 	path.isChildrenCached = true
+	path.readDirDirect = false
 	now := time.Now()
 	path.lastAccess = now
 	path.lastRefresh = now
 	path.updateCount = 0
-	path.needsRefresh = false
 	path.updateWindowStart = time.Time{}
 	if i.cacheMetaTtlSec > 0 {
 		path.cachedExpiresTime = now.Add(i.cacheMetaTtlSec)
@@ -264,6 +264,27 @@ func (i *InodeToPath) TouchDirectory(fullpath util.FullPath) {
 	entry.lastAccess = time.Now()
 }
 
+func (i *InodeToPath) MarkDirectoryReadThrough(fullpath util.FullPath, now time.Time) bool {
+	i.Lock()
+	defer i.Unlock()
+	inode, found := i.path2inode[fullpath]
+	if !found {
+		return false
+	}
+	entry, found := i.inode2path[inode]
+	if !found || !entry.isDirectory {
+		return false
+	}
+	entry.isChildrenCached = false
+	entry.readDirDirect = true
+	entry.cachedExpiresTime = time.Time{}
+	entry.lastAccess = now
+	entry.lastRefresh = time.Time{}
+	entry.updateCount = 0
+	entry.updateWindowStart = time.Time{}
+	return true
+}
+
 func (i *InodeToPath) RecordDirectoryUpdate(fullpath util.FullPath, now time.Time, window time.Duration, threshold int) bool {
 	if threshold <= 0 || window <= 0 {
 		return false
@@ -284,13 +305,19 @@ func (i *InodeToPath) RecordDirectoryUpdate(fullpath util.FullPath, now time.Tim
 	}
 	entry.updateCount++
 	if entry.updateCount >= threshold {
-		entry.needsRefresh = true
+		entry.isChildrenCached = false
+		entry.readDirDirect = true
+		entry.cachedExpiresTime = time.Time{}
+		entry.lastAccess = now
+		entry.lastRefresh = time.Time{}
+		entry.updateCount = 0
+		entry.updateWindowStart = time.Time{}
 		return true
 	}
 	return false
 }
 
-func (i *InodeToPath) NeedsRefresh(fullpath util.FullPath) bool {
+func (i *InodeToPath) ShouldReadDirectoryDirect(fullpath util.FullPath) bool {
 	i.RLock()
 	defer i.RUnlock()
 	inode, found := i.path2inode[fullpath]
@@ -301,7 +328,7 @@ func (i *InodeToPath) NeedsRefresh(fullpath util.FullPath) bool {
 	if !found || !entry.isDirectory {
 		return false
 	}
-	return entry.isChildrenCached && entry.needsRefresh
+	return entry.readDirDirect
 }
 
 func (i *InodeToPath) MarkDirectoryRefreshed(fullpath util.FullPath, now time.Time) {
@@ -317,8 +344,8 @@ func (i *InodeToPath) MarkDirectoryRefreshed(fullpath util.FullPath, now time.Ti
 	}
 	entry.lastRefresh = now
 	entry.lastAccess = now
+	entry.readDirDirect = false
 	entry.updateCount = 0
-	entry.needsRefresh = false
 	entry.updateWindowStart = time.Time{}
 	if i.cacheMetaTtlSec > 0 {
 		entry.cachedExpiresTime = now.Add(i.cacheMetaTtlSec)

weed/mount/inode_to_path_test.go

@@ -2,6 +2,7 @@ package mount
 
 import (
 	"testing"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
@@ -92,3 +93,43 @@ func TestInodeEntry_removeOnePath(t *testing.T) {
 		})
 	}
 }
+
+func TestRecordDirectoryUpdateSwitchesDirectoryToReadThrough(t *testing.T) {
+	root := util.FullPath("/")
+	dir := util.FullPath("/data")
+
+	inodeToPath := NewInodeToPath(root, 60)
+	inodeToPath.Lookup(dir, time.Now().Unix(), true, false, 0, true)
+	inodeToPath.MarkChildrenCached(dir)
+
+	now := time.Now()
+	if !inodeToPath.RecordDirectoryUpdate(dir, now, time.Second, 1) {
+		t.Fatal("expected directory to switch to read-through mode")
+	}
+	if inodeToPath.IsChildrenCached(dir) {
+		t.Fatal("directory should no longer be marked cached")
+	}
+	if !inodeToPath.ShouldReadDirectoryDirect(dir) {
+		t.Fatal("directory should be served via direct reads after hot invalidation")
+	}
+}
+
+func TestMarkChildrenCachedClearsReadThroughMode(t *testing.T) {
+	root := util.FullPath("/")
+	dir := util.FullPath("/data")
+
+	inodeToPath := NewInodeToPath(root, 60)
+	inodeToPath.Lookup(dir, time.Now().Unix(), true, false, 0, true)
+
+	if !inodeToPath.MarkDirectoryReadThrough(dir, time.Now()) {
+		t.Fatal("expected read-through flag to be set")
+	}
+	inodeToPath.MarkChildrenCached(dir)
+
+	if !inodeToPath.IsChildrenCached(dir) {
+		t.Fatal("directory should be cached after MarkChildrenCached")
+	}
+	if inodeToPath.ShouldReadDirectoryDirect(dir) {
+		t.Fatal("directory should leave read-through mode after caching")
+	}
+}

weed/mount/meta_cache/meta_cache.go

@@ -2,17 +2,21 @@ package meta_cache
 
 import (
 	"context"
+	"errors"
 	"os"
 	"sync"
 	"time"
 
 	"golang.org/x/sync/singleflight"
 
+	"fmt"
+
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/filer/leveldb"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
 )
 
 // need to have logic similar to FilerStoreWrapper
@@ -29,12 +33,61 @@ type MetaCache struct {
 	invalidateFunc    func(fullpath util.FullPath, entry *filer_pb.Entry)
 	onDirectoryUpdate func(dir util.FullPath)
 	visitGroup        singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path
+	applyCh           chan metadataApplyRequest
+	applyDone         chan struct{}
+	applyStateMu      sync.Mutex
+	applyClosed       bool
+	buildingDirs map[util.FullPath]*directoryBuildState
+	dedupRing    dedupRingBuffer
+}
+
+var errMetaCacheClosed = errors.New("metadata cache is shut down")
+
+type MetadataResponseApplyOptions struct {
+	NotifyDirectories bool
+	InvalidateEntries bool
+}
+
+var (
+	LocalMetadataResponseApplyOptions = MetadataResponseApplyOptions{
+		NotifyDirectories: true,
+	}
+	SubscriberMetadataResponseApplyOptions = MetadataResponseApplyOptions{
+		NotifyDirectories: true,
+		InvalidateEntries: true,
+	}
+)
+
+type directoryBuildState struct {
+	bufferedEvents []*filer_pb.SubscribeMetadataResponse
+}
+
+const recentEventDedupWindow = 4096
+
+type metadataApplyRequestKind int
+
+const (
+	metadataApplyEvent metadataApplyRequestKind = iota
+	metadataBeginBuild
+	metadataCompleteBuild
+	metadataAbortBuild
+	metadataShutdown
+)
+
+type metadataApplyRequest struct {
+	ctx          context.Context
+	kind         metadataApplyRequestKind
+	resp         *filer_pb.SubscribeMetadataResponse
+	options      MetadataResponseApplyOptions
+	buildPath    util.FullPath
+	snapshotTsNs int64
+	done         chan error
 }
 
 func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath,
 	markCachedFn func(path util.FullPath), isCachedFn func(path util.FullPath) bool, invalidateFunc func(util.FullPath, *filer_pb.Entry), onDirectoryUpdate func(dir util.FullPath)) *MetaCache {
 	leveldbStore, virtualStore := openMetaStore(dbFolder)
-	return &MetaCache{
+	mc := &MetaCache{
 		root:              root,
 		localStore:        virtualStore,
 		leveldbStore:      leveldbStore,
@@ -45,7 +98,13 @@ func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPat
 		invalidateFunc: func(fullpath util.FullPath, entry *filer_pb.Entry) {
 			invalidateFunc(fullpath, entry)
 		},
+		applyCh:        make(chan metadataApplyRequest, 128),
+		applyDone:      make(chan struct{}),
+		buildingDirs: make(map[util.FullPath]*directoryBuildState),
+		dedupRing:    newDedupRingBuffer(),
 	}
+	go mc.runApplyLoop()
+	return mc
 }
 
 func openMetaStore(dbFolder string) (*leveldb.LevelDBStore, filer.VirtualFilerStore) {
@@ -85,7 +144,10 @@ func (mc *MetaCache) doBatchInsertEntries(ctx context.Context, entries []*filer.
 func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry) error {
 	mc.Lock()
 	defer mc.Unlock()
+	return mc.atomicUpdateEntryFromFilerLocked(ctx, oldPath, newEntry, false)
+}
 
+func (mc *MetaCache) atomicUpdateEntryFromFilerLocked(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry, allowUncachedInsert bool) error {
 	entry, err := mc.localStore.FindEntry(ctx, oldPath)
 	if err != nil && err != filer_pb.ErrNotFound {
 		glog.Errorf("Metacache: find entry error: %v", err)
@@ -110,7 +172,7 @@ func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath uti
 
 	if newEntry != nil {
 		newDir, _ := newEntry.DirAndName()
-		if mc.isCachedFn(util.FullPath(newDir)) {
+		if allowUncachedInsert || mc.isCachedFn(util.FullPath(newDir)) {
 			glog.V(3).Infof("InsertEntry %s/%s", newDir, newEntry.Name())
 			if err := mc.localStore.InsertEntry(ctx, newEntry); err != nil {
 				return err
@@ -120,6 +182,71 @@ func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath uti
 	return nil
 }
 
+func (mc *MetaCache) ApplyMetadataResponse(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions) error {
+	if resp == nil || resp.EventNotification == nil {
+		return nil
+	}
+	clonedResp := proto.Clone(resp).(*filer_pb.SubscribeMetadataResponse)
+	return mc.applyMetadataResponseEnqueue(ctx, clonedResp, options)
+}
+
+// ApplyMetadataResponseOwned is like ApplyMetadataResponse but takes ownership
+// of resp without cloning. The caller must not use resp after this call.
+func (mc *MetaCache) ApplyMetadataResponseOwned(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions) error {
+	if resp == nil || resp.EventNotification == nil {
+		return nil
+	}
+	return mc.applyMetadataResponseEnqueue(ctx, resp, options)
+}
+
+func (mc *MetaCache) applyMetadataResponseEnqueue(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	req := metadataApplyRequest{
+		// Use a non-cancellable context for the queued mutation so a
+		// cancelled caller doesn't abort the apply loop mid-write.
+		ctx:     context.Background(),
+		kind:    metadataApplyEvent,
+		resp:    resp,
+		options: options,
+		done:    make(chan error, 1),
+	}
+
+	if err := mc.enqueueApplyRequest(req); err != nil {
+		return err
+	}
+
+	select {
+	case err := <-req.done:
+		return err
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (mc *MetaCache) BeginDirectoryBuild(ctx context.Context, dirPath util.FullPath) error {
+	return mc.enqueueAndWait(ctx, metadataApplyRequest{
+		kind:      metadataBeginBuild,
+		buildPath: dirPath,
+	})
+}
+
+func (mc *MetaCache) CompleteDirectoryBuild(ctx context.Context, dirPath util.FullPath, snapshotTsNs int64) error {
+	return mc.enqueueAndWait(ctx, metadataApplyRequest{
+		kind:         metadataCompleteBuild,
+		buildPath:    dirPath,
+		snapshotTsNs: snapshotTsNs,
+	})
+}
+
+func (mc *MetaCache) AbortDirectoryBuild(ctx context.Context, dirPath util.FullPath) error {
+	return mc.enqueueAndWait(ctx, metadataApplyRequest{
+		kind:      metadataAbortBuild,
+		buildPath: dirPath,
+	})
+}
+
 func (mc *MetaCache) UpdateEntry(ctx context.Context, entry *filer.Entry) error {
 	mc.Lock()
 	defer mc.Unlock()
@@ -174,6 +301,25 @@ func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.Full
 }
 
 func (mc *MetaCache) Shutdown() {
+	done := make(chan error, 1)
+
+	mc.applyStateMu.Lock()
+	if !mc.applyClosed {
+		mc.applyClosed = true
+		mc.applyCh <- metadataApplyRequest{
+			kind: metadataShutdown,
+			done: done,
+		}
+	}
+	mc.applyStateMu.Unlock()
+
+	select {
+	case <-done:
+	case <-mc.applyDone:
+	}
+
+	<-mc.applyDone
+
 	mc.Lock()
 	defer mc.Unlock()
 	mc.localStore.Shutdown()
@@ -201,3 +347,494 @@ func (mc *MetaCache) noteDirectoryUpdate(dirPath util.FullPath) {
 		mc.onDirectoryUpdate(dirPath)
 	}
 }
+
+func (mc *MetaCache) enqueueAndWait(ctx context.Context, req metadataApplyRequest) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	// Use a non-cancellable context for the queued operation so a
+	// cancelled caller doesn't abort a build/complete mid-way.
+	req.ctx = context.Background()
+	req.done = make(chan error, 1)
+	if err := mc.enqueueApplyRequest(req); err != nil {
+		return err
+	}
+	select {
+	case err := <-req.done:
+		return err
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (mc *MetaCache) enqueueApplyRequest(req metadataApplyRequest) error {
+	mc.applyStateMu.Lock()
+	if mc.applyClosed {
+		mc.applyStateMu.Unlock()
+		return errMetaCacheClosed
+	}
+	// Release the mutex before the potentially-blocking channel send so that
+	// Shutdown can still acquire it to set applyClosed when the channel is full.
+	mc.applyStateMu.Unlock()
+	select {
+	case mc.applyCh <- req:
+		return nil
+	case <-mc.applyDone:
+		return errMetaCacheClosed
+	}
+}
+
+func (mc *MetaCache) runApplyLoop() {
+	defer close(mc.applyDone)
+
+	for req := range mc.applyCh {
+		req.done <- mc.handleApplyRequest(req)
+		close(req.done)
+		if req.kind == metadataShutdown {
+			mc.drainApplyCh()
+			return
+		}
+	}
+}
+
+// drainApplyCh non-blockingly drains any remaining requests from applyCh
+// after a shutdown sentinel, signalling each caller so they don't block.
+func (mc *MetaCache) drainApplyCh() {
+	for {
+		select {
+		case req := <-mc.applyCh:
+			req.done <- errMetaCacheClosed
+			close(req.done)
+		default:
+			return
+		}
+	}
+}
+
+func (mc *MetaCache) handleApplyRequest(req metadataApplyRequest) error {
+	switch req.kind {
+	case metadataApplyEvent:
+		return mc.applyMetadataResponseNow(req.ctx, req.resp, req.options)
+	case metadataBeginBuild:
+		return mc.beginDirectoryBuildNow(req.buildPath)
+	case metadataCompleteBuild:
+		return mc.completeDirectoryBuildNow(req.ctx, req.buildPath, req.snapshotTsNs)
+	case metadataAbortBuild:
+		return mc.abortDirectoryBuildNow(req.buildPath)
+	case metadataShutdown:
+		return nil
+	default:
+		return nil
+	}
+}
+
+type metadataInvalidation struct {
+	path  util.FullPath
+	entry *filer_pb.Entry
+}
+
+type metadataResponseSideEffects struct {
+	dirsToNotify  []util.FullPath
+	invalidations []metadataInvalidation
+}
+
+func (mc *MetaCache) applyMetadataResponseNow(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions) error {
+	if mc.shouldSkipDuplicateEvent(resp) {
+		return nil
+	}
+
+	immediateEvents, bufferedEvents := mc.routeMetadataResponse(resp)
+	if len(bufferedEvents) == 0 {
+		return mc.applyMetadataResponseDirect(ctx, resp, options, false)
+	}
+
+	// Apply side effects but skip directory notifications for dirs that are
+	// currently being built. Notifying a building dir can trigger
+	// markDirectoryReadThrough → DeleteFolderChildren, wiping entries that
+	// EnsureVisited already inserted, leaving an incomplete cache.
+	mc.applyMetadataSideEffectsSkippingBuildingDirs(resp, options)
+	for buildDir, events := range bufferedEvents {
+		state := mc.buildingDirs[buildDir]
+		if state == nil {
+			continue
+		}
+		state.bufferedEvents = append(state.bufferedEvents, events...)
+	}
+	for _, immediateEvent := range immediateEvents {
+		if err := mc.applyMetadataResponseDirect(ctx, immediateEvent, MetadataResponseApplyOptions{}, false); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (mc *MetaCache) applyMetadataResponseDirect(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions, allowUncachedInsert bool) error {
+	if _, err := mc.applyMetadataResponseLocked(ctx, resp, options, allowUncachedInsert); err != nil {
+		return err
+	}
+	mc.applyMetadataSideEffects(resp, options)
+	return nil
+}
+
+func (mc *MetaCache) applyMetadataSideEffects(resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions) {
+	sideEffects := metadataResponseSideEffects{}
+	if options.NotifyDirectories {
+		sideEffects.dirsToNotify = collectDirectoryNotifications(resp)
+	}
+	if options.InvalidateEntries {
+		sideEffects.invalidations = collectEntryInvalidations(resp)
+	}
+	for _, dirPath := range sideEffects.dirsToNotify {
+		mc.noteDirectoryUpdate(dirPath)
+	}
+	for _, invalidation := range sideEffects.invalidations {
+		mc.invalidateFunc(invalidation.path, invalidation.entry)
+	}
+}
+
+// applyMetadataSideEffectsSkippingBuildingDirs is like applyMetadataSideEffects
+// but suppresses directory notifications for dirs currently in buildingDirs.
+// This prevents markDirectoryReadThrough from wiping entries mid-build.
+func (mc *MetaCache) applyMetadataSideEffectsSkippingBuildingDirs(resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions) {
+	sideEffects := metadataResponseSideEffects{}
+	if options.NotifyDirectories {
+		sideEffects.dirsToNotify = collectDirectoryNotifications(resp)
+	}
+	if options.InvalidateEntries {
+		sideEffects.invalidations = collectEntryInvalidations(resp)
+	}
+	for _, dirPath := range sideEffects.dirsToNotify {
+		if _, building := mc.buildingDirs[dirPath]; !building {
+			mc.noteDirectoryUpdate(dirPath)
+		}
+	}
+	for _, invalidation := range sideEffects.invalidations {
+		mc.invalidateFunc(invalidation.path, invalidation.entry)
+	}
+}
+
+func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, _ MetadataResponseApplyOptions, allowUncachedInsert bool) (metadataResponseSideEffects, error) {
+	message := resp.GetEventNotification()
+	if message == nil {
+		return metadataResponseSideEffects{}, nil
+	}
+
+	var oldPath util.FullPath
+	var newEntry *filer.Entry
+	if message.OldEntry != nil {
+		oldPath = util.NewFullPath(resp.Directory, message.OldEntry.Name)
+	}
+
+	if message.NewEntry != nil {
+		dir := resp.Directory
+		if message.NewParentPath != "" {
+			dir = message.NewParentPath
+		}
+		newEntry = filer.FromPbEntry(dir, message.NewEntry)
+	}
+
+	mc.Lock()
+	err := mc.atomicUpdateEntryFromFilerLocked(ctx, oldPath, newEntry, allowUncachedInsert)
+	// When a directory is deleted or moved, remove its cached descendants
+	// so stale children cannot be served from the local cache.
+	if err == nil && oldPath != "" && message.OldEntry != nil && message.OldEntry.IsDirectory {
+		isDelete := message.NewEntry == nil
+		isMove := message.NewEntry != nil && (message.NewParentPath != resp.Directory || message.NewEntry.Name != message.OldEntry.Name)
+		if isDelete || isMove {
+			if deleteErr := mc.localStore.DeleteFolderChildren(ctx, oldPath); deleteErr != nil {
+				glog.V(2).Infof("delete descendants of %s: %v", oldPath, deleteErr)
+			}
+		}
+	}
+	mc.Unlock()
+	if err != nil {
+		return metadataResponseSideEffects{}, err
+	}
+	return metadataResponseSideEffects{}, nil
+}
+
+func (mc *MetaCache) beginDirectoryBuildNow(dirPath util.FullPath) error {
+	if _, found := mc.buildingDirs[dirPath]; found {
+		return nil
+	}
+	mc.buildingDirs[dirPath] = &directoryBuildState{}
+	return nil
+}
+
+func (mc *MetaCache) abortDirectoryBuildNow(dirPath util.FullPath) error {
+	delete(mc.buildingDirs, dirPath)
+	return nil
+}
+
+func (mc *MetaCache) completeDirectoryBuildNow(ctx context.Context, dirPath util.FullPath, snapshotTsNs int64) error {
+	state := mc.buildingDirs[dirPath]
+	delete(mc.buildingDirs, dirPath)
+
+	if state == nil {
+		return nil
+	}
+
+	for _, event := range state.bufferedEvents {
+		// When the server provided a snapshot timestamp, skip events that
+		// the listing already included. When snapshotTsNs == 0 (empty
+		// directory — server returned no entries and no snapshot), replay
+		// ALL buffered events to avoid dropping mutations due to
+		// client/server clock skew.
+		if snapshotTsNs != 0 && event.TsNs != 0 && event.TsNs <= snapshotTsNs {
+			continue
+		}
+		if err := mc.applyMetadataResponseDirect(ctx, event, MetadataResponseApplyOptions{}, true); err != nil {
+			return err
+		}
+	}
+
+	mc.markCachedFn(dirPath)
+	return nil
+}
+
+func (mc *MetaCache) routeMetadataResponse(resp *filer_pb.SubscribeMetadataResponse) ([]*filer_pb.SubscribeMetadataResponse, map[util.FullPath][]*filer_pb.SubscribeMetadataResponse) {
+	message := resp.GetEventNotification()
+	if message == nil {
+		return []*filer_pb.SubscribeMetadataResponse{resp}, nil
+	}
+
+	oldDir, hasOld := metadataOldParentDir(resp)
+	newDir, hasNew := metadataNewParentDir(resp)
+	oldBuilding := hasOld && mc.isBuildingDir(oldDir)
+	newBuilding := hasNew && mc.isBuildingDir(newDir)
+	if !oldBuilding && !newBuilding {
+		return []*filer_pb.SubscribeMetadataResponse{resp}, nil
+	}
+
+	bufferedEvents := make(map[util.FullPath][]*filer_pb.SubscribeMetadataResponse)
+	var immediateEvents []*filer_pb.SubscribeMetadataResponse
+
+	if hasOld && hasNew && oldDir != newDir {
+		deleteEvent := metadataDeleteFragment(resp)
+		createEvent := metadataCreateFragment(resp)
+		if oldBuilding {
+			bufferedEvents[oldDir] = append(bufferedEvents[oldDir], deleteEvent)
+		} else {
+			immediateEvents = append(immediateEvents, deleteEvent)
+		}
+		if newBuilding {
+			bufferedEvents[newDir] = append(bufferedEvents[newDir], createEvent)
+		} else {
+			immediateEvents = append(immediateEvents, createEvent)
+		}
+		return immediateEvents, bufferedEvents
+	}
+
+	targetDir := newDir
+	if hasOld {
+		targetDir = oldDir
+	}
+	if mc.isBuildingDir(targetDir) {
+		bufferedEvents[targetDir] = append(bufferedEvents[targetDir], resp)
+		return nil, bufferedEvents
+	}
+	return []*filer_pb.SubscribeMetadataResponse{resp}, nil
+}
+
+func (mc *MetaCache) isBuildingDir(dirPath util.FullPath) bool {
+	_, found := mc.buildingDirs[dirPath]
+	return found
+}
+
+func metadataOldParentDir(resp *filer_pb.SubscribeMetadataResponse) (util.FullPath, bool) {
+	if resp.GetEventNotification() == nil || resp.EventNotification.OldEntry == nil {
+		return "", false
+	}
+	return util.FullPath(resp.Directory), true
+}
+
+func metadataNewParentDir(resp *filer_pb.SubscribeMetadataResponse) (util.FullPath, bool) {
+	if resp.GetEventNotification() == nil || resp.EventNotification.NewEntry == nil {
+		return "", false
+	}
+	newDir := resp.Directory
+	if resp.EventNotification.NewParentPath != "" {
+		newDir = resp.EventNotification.NewParentPath
+	}
+	return util.FullPath(newDir), true
+}
+
+func metadataDeleteFragment(resp *filer_pb.SubscribeMetadataResponse) *filer_pb.SubscribeMetadataResponse {
+	if resp.GetEventNotification() == nil || resp.EventNotification.OldEntry == nil {
+		return nil
+	}
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory: resp.Directory,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: proto.Clone(resp.EventNotification.OldEntry).(*filer_pb.Entry),
+		},
+		TsNs: resp.TsNs,
+	}
+}
+
+func metadataCreateFragment(resp *filer_pb.SubscribeMetadataResponse) *filer_pb.SubscribeMetadataResponse {
+	if resp.GetEventNotification() == nil || resp.EventNotification.NewEntry == nil {
+		return nil
+	}
+	newDir := resp.Directory
+	if resp.EventNotification.NewParentPath != "" {
+		newDir = resp.EventNotification.NewParentPath
+	}
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory: newDir,
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry:      proto.Clone(resp.EventNotification.NewEntry).(*filer_pb.Entry),
+			NewParentPath: newDir,
+		},
+		TsNs: resp.TsNs,
+	}
+}
+
+func metadataEventDedupKey(resp *filer_pb.SubscribeMetadataResponse) string {
+	var oldName, newName, newParent string
+	hasOld, hasNew := false, false
+	if msg := resp.GetEventNotification(); msg != nil {
+		if msg.OldEntry != nil {
+			oldName = msg.OldEntry.Name
+			hasOld = true
+		}
+		if msg.NewEntry != nil {
+			newName = msg.NewEntry.Name
+			hasNew = true
+			newParent = msg.NewParentPath
+		}
+	}
+	// Encode event shape (create/delete/update/rename) so structurally
+	// different events with the same names are not collapsed.
+	var shape byte
+	switch {
+	case hasOld && hasNew:
+		if resp.Directory != newParent && newParent != "" {
+			shape = 'R' // rename across directories
+		} else {
+			shape = 'U' // update in place
+		}
+	case hasOld:
+		shape = 'D' // delete
+	case hasNew:
+		shape = 'C' // create
+	}
+	return fmt.Sprintf("%d|%c|%s|%s|%s|%s", resp.TsNs, shape, resp.Directory, oldName, newParent, newName)
+}
+
+func (mc *MetaCache) shouldSkipDuplicateEvent(resp *filer_pb.SubscribeMetadataResponse) bool {
+	if resp == nil || resp.TsNs == 0 {
+		return false
+	}
+	key := metadataEventDedupKey(resp)
+	return !mc.dedupRing.Add(key)
+}
+
+type dedupRingBuffer struct {
+	keys [recentEventDedupWindow]string
+	head int
+	size int
+	set  map[string]struct{}
+}
+
+func newDedupRingBuffer() dedupRingBuffer {
+	return dedupRingBuffer{
+		set: make(map[string]struct{}, recentEventDedupWindow),
+	}
+}
+
+func (r *dedupRingBuffer) Add(key string) bool {
+	if _, found := r.set[key]; found {
+		return false // duplicate
+	}
+	if r.size == recentEventDedupWindow {
+		evicted := r.keys[r.head]
+		delete(r.set, evicted)
+	} else {
+		r.size++
+	}
+	r.keys[r.head] = key
+	r.set[key] = struct{}{}
+	r.head = (r.head + 1) % recentEventDedupWindow
+	return true // new entry
+}
+
+func collectDirectoryNotifications(resp *filer_pb.SubscribeMetadataResponse) []util.FullPath {
+	message := resp.GetEventNotification()
+	if message == nil {
+		return nil
+	}
+
+	// At most 3 dirs: old parent, new parent, new child (if directory).
+	// Use a fixed slice with linear dedup to avoid map allocation.
+	var dirs [3]util.FullPath
+	n := 0
+	addUnique := func(p util.FullPath) {
+		for i := 0; i < n; i++ {
+			if dirs[i] == p {
+				return
+			}
+		}
+		dirs[n] = p
+		n++
+	}
+
+	if message.OldEntry != nil {
+		oldPath := util.NewFullPath(resp.Directory, message.OldEntry.Name)
+		parent, _ := oldPath.DirAndName()
+		addUnique(util.FullPath(parent))
+	}
+	if message.NewEntry != nil {
+		newDir := resp.Directory
+		if message.NewParentPath != "" {
+			newDir = message.NewParentPath
+		}
+		newPath := util.NewFullPath(newDir, message.NewEntry.Name)
+		parent, _ := newPath.DirAndName()
+		addUnique(util.FullPath(parent))
+		if message.NewEntry.IsDirectory {
+			addUnique(newPath)
+		}
+	}
+
+	return dirs[:n]
+}
+
+func collectEntryInvalidations(resp *filer_pb.SubscribeMetadataResponse) []metadataInvalidation {
+	message := resp.GetEventNotification()
+	if message == nil {
+		return nil
+	}
+
+	var invalidations []metadataInvalidation
+	if message.OldEntry != nil && message.NewEntry != nil {
+		oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name)
+		invalidations = append(invalidations, metadataInvalidation{path: oldKey, entry: message.OldEntry})
+		// Normalize NewParentPath: empty means same directory as resp.Directory
+		newDir := resp.Directory
+		if message.NewParentPath != "" {
+			newDir = message.NewParentPath
+		}
+		if message.OldEntry.Name != message.NewEntry.Name || resp.Directory != newDir {
+			newKey := util.NewFullPath(newDir, message.NewEntry.Name)
+			invalidations = append(invalidations, metadataInvalidation{path: newKey, entry: message.NewEntry})
+		}
+		return invalidations
+	}
+
+	if filer_pb.IsCreate(resp) && message.NewEntry != nil {
+		newDir := resp.Directory
+		if message.NewParentPath != "" {
+			newDir = message.NewParentPath
+		}
+		newKey := util.NewFullPath(newDir, message.NewEntry.Name)
+		invalidations = append(invalidations, metadataInvalidation{path: newKey, entry: message.NewEntry})
+	}
+
+	if filer_pb.IsDelete(resp) && message.OldEntry != nil {
+		oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name)
+		invalidations = append(invalidations, metadataInvalidation{path: oldKey, entry: message.OldEntry})
+	}
+
+	return invalidations
+}

weed/mount/meta_cache/meta_cache_apply_test.go

@@ -0,0 +1,361 @@
+package meta_cache
+
+import (
+	"context"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestApplyMetadataResponseAppliesEventsInOrder(t *testing.T) {
+	mc, _, notifications, invalidations := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	createResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    1,
+					FileMode: 0100644,
+					FileSize: 11,
+				},
+			},
+		},
+	}
+	updateResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name: "file.txt",
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    2,
+					FileMode: 0100644,
+					FileSize: 29,
+				},
+			},
+			NewParentPath: "/dir",
+		},
+	}
+	deleteResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name: "file.txt",
+			},
+		},
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), createResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply create: %v", err)
+	}
+
+	entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
+	if err != nil {
+		t.Fatalf("find created entry: %v", err)
+	}
+	if entry.FileSize != 11 {
+		t.Fatalf("created file size = %d, want 11", entry.FileSize)
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), updateResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply update: %v", err)
+	}
+
+	entry, err = mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
+	if err != nil {
+		t.Fatalf("find updated entry: %v", err)
+	}
+	if entry.FileSize != 29 {
+		t.Fatalf("updated file size = %d, want 29", entry.FileSize)
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), deleteResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply delete: %v", err)
+	}
+
+	entry, err = mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("find deleted entry error = %v, want %v", err, filer_pb.ErrNotFound)
+	}
+	if entry != nil {
+		t.Fatalf("deleted entry still cached: %+v", entry)
+	}
+
+	if got := countPath(notifications.paths(), util.FullPath("/dir")); got != 3 {
+		t.Fatalf("directory notifications for /dir = %d, want 3", got)
+	}
+	if got := countPath(invalidations.paths(), util.FullPath("/dir/file.txt")); got != 3 {
+		t.Fatalf("invalidations for /dir/file.txt = %d, want 3 (create + update + delete)", got)
+	}
+}
+
+func TestApplyMetadataResponseRenamesAcrossCachedDirectories(t *testing.T) {
+	mc, _, notifications, invalidations := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/src": true,
+		"/dst": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/src/file.tmp",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 7,
+		},
+	}); err != nil {
+		t.Fatalf("insert source entry: %v", err)
+	}
+
+	renameResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/src",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name: "file.tmp",
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    2,
+					FileMode: 0100644,
+					FileSize: 41,
+				},
+			},
+			NewParentPath: "/dst",
+		},
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply rename: %v", err)
+	}
+
+	oldEntry, err := mc.FindEntry(context.Background(), util.FullPath("/src/file.tmp"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("find old path error = %v, want %v", err, filer_pb.ErrNotFound)
+	}
+	if oldEntry != nil {
+		t.Fatalf("old path still cached: %+v", oldEntry)
+	}
+
+	newEntry, err := mc.FindEntry(context.Background(), util.FullPath("/dst/file.txt"))
+	if err != nil {
+		t.Fatalf("find new path: %v", err)
+	}
+	if newEntry.FileSize != 41 {
+		t.Fatalf("renamed file size = %d, want 41", newEntry.FileSize)
+	}
+
+	if got := countPath(notifications.paths(), util.FullPath("/src")); got != 1 {
+		t.Fatalf("directory notifications for /src = %d, want 1", got)
+	}
+	if got := countPath(notifications.paths(), util.FullPath("/dst")); got != 1 {
+		t.Fatalf("directory notifications for /dst = %d, want 1", got)
+	}
+	if got := countPath(invalidations.paths(), util.FullPath("/src/file.tmp")); got != 1 {
+		t.Fatalf("invalidations for /src/file.tmp = %d, want 1", got)
+	}
+	if got := countPath(invalidations.paths(), util.FullPath("/dst/file.txt")); got != 1 {
+		t.Fatalf("invalidations for /dst/file.txt = %d, want 1", got)
+	}
+}
+
+func TestApplyMetadataResponseLocalOptionsSkipInvalidations(t *testing.T) {
+	mc, _, notifications, invalidations := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/dir/file.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 7,
+		},
+	}); err != nil {
+		t.Fatalf("insert source entry: %v", err)
+	}
+
+	updateResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name: "file.txt",
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    2,
+					FileMode: 0100644,
+					FileSize: 17,
+				},
+			},
+			NewParentPath: "/dir",
+		},
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), updateResp, LocalMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply local update: %v", err)
+	}
+
+	entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
+	if err != nil {
+		t.Fatalf("find updated entry: %v", err)
+	}
+	if entry.FileSize != 17 {
+		t.Fatalf("updated file size = %d, want 17", entry.FileSize)
+	}
+	if got := countPath(notifications.paths(), util.FullPath("/dir")); got != 1 {
+		t.Fatalf("directory notifications for /dir = %d, want 1", got)
+	}
+	if got := len(invalidations.paths()); got != 0 {
+		t.Fatalf("invalidations = %d, want 0", got)
+	}
+}
+
+func TestApplyMetadataResponseDeduplicatesRepeatedFilerEvent(t *testing.T) {
+	mc, _, notifications, invalidations := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/dir/file.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 5,
+		},
+	}); err != nil {
+		t.Fatalf("insert source entry: %v", err)
+	}
+
+	updateResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name: "file.txt",
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    2,
+					FileMode: 0100644,
+					FileSize: 15,
+				},
+			},
+			NewParentPath: "/dir",
+			Signatures:    []int32{7},
+		},
+		TsNs: 99,
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), updateResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("first apply: %v", err)
+	}
+	if err := mc.ApplyMetadataResponse(context.Background(), updateResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("second apply: %v", err)
+	}
+
+	entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
+	if err != nil {
+		t.Fatalf("find updated entry: %v", err)
+	}
+	if entry.FileSize != 15 {
+		t.Fatalf("updated file size = %d, want 15", entry.FileSize)
+	}
+	if got := countPath(notifications.paths(), util.FullPath("/dir")); got != 1 {
+		t.Fatalf("directory notifications for /dir = %d, want 1", got)
+	}
+	if got := countPath(invalidations.paths(), util.FullPath("/dir/file.txt")); got != 1 {
+		t.Fatalf("invalidations for /dir/file.txt = %d, want 1", got)
+	}
+}
+
+func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache, map[util.FullPath]bool, *recordedPaths, *recordedPaths) {
+	t.Helper()
+
+	mapper, err := NewUidGidMapper("", "")
+	if err != nil {
+		t.Fatalf("uid/gid mapper: %v", err)
+	}
+
+	var cachedMu sync.Mutex
+	notifications := &recordedPaths{}
+	invalidations := &recordedPaths{}
+
+	mc := NewMetaCache(
+		filepath.Join(t.TempDir(), "meta"),
+		mapper,
+		util.FullPath("/"),
+		func(path util.FullPath) {
+			cachedMu.Lock()
+			defer cachedMu.Unlock()
+			cached[path] = true
+		},
+		func(path util.FullPath) bool {
+			cachedMu.Lock()
+			defer cachedMu.Unlock()
+			return cached[path]
+		},
+		func(path util.FullPath, entry *filer_pb.Entry) {
+			invalidations.record(path)
+		},
+		func(dir util.FullPath) {
+			notifications.record(dir)
+		},
+	)
+
+	return mc, cached, notifications, invalidations
+}
+
+type recordedPaths struct {
+	mu    sync.Mutex
+	items []util.FullPath
+}
+
+func (r *recordedPaths) record(path util.FullPath) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.items = append(r.items, path)
+}
+
+func (r *recordedPaths) paths() []util.FullPath {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return append([]util.FullPath(nil), r.items...)
+}
+
+func countPath(paths []util.FullPath, target util.FullPath) int {
+	count := 0
+	for _, path := range paths {
+		if path == target {
+			count++
+		}
+	}
+	return count
+}

weed/mount/meta_cache/meta_cache_build_test.go

@@ -0,0 +1,459 @@
+package meta_cache
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+type buildListStream struct {
+	responses   []*filer_pb.ListEntriesResponse
+	onFirstRecv func()
+	once        sync.Once
+	index       int
+}
+
+func (s *buildListStream) Recv() (*filer_pb.ListEntriesResponse, error) {
+	s.once.Do(func() {
+		if s.onFirstRecv != nil {
+			s.onFirstRecv()
+		}
+	})
+	if s.index >= len(s.responses) {
+		return nil, io.EOF
+	}
+	resp := s.responses[s.index]
+	s.index++
+	return resp, nil
+}
+
+func (s *buildListStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
+func (s *buildListStream) Trailer() metadata.MD         { return metadata.MD{} }
+func (s *buildListStream) CloseSend() error             { return nil }
+func (s *buildListStream) Context() context.Context     { return context.Background() }
+func (s *buildListStream) SendMsg(any) error            { return nil }
+func (s *buildListStream) RecvMsg(any) error            { return nil }
+
+type buildListClient struct {
+	filer_pb.SeaweedFilerClient
+	responses   []*filer_pb.ListEntriesResponse
+	onFirstRecv func()
+}
+
+func (c *buildListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
+	return &buildListStream{
+		responses:   c.responses,
+		onFirstRecv: c.onFirstRecv,
+	}, nil
+}
+
+type buildFilerAccessor struct {
+	client filer_pb.SeaweedFilerClient
+}
+
+func (a *buildFilerAccessor) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return fn(a.client)
+}
+
+func (a *buildFilerAccessor) AdjustedUrl(*filer_pb.Location) string { return "" }
+func (a *buildFilerAccessor) GetDataCenter() string                 { return "" }
+
+func TestEnsureVisitedReplaysBufferedEventsAfterSnapshot(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/": true,
+	})
+	defer mc.Shutdown()
+
+	var applyErr error
+	accessor := &buildFilerAccessor{
+		client: &buildListClient{
+			responses: []*filer_pb.ListEntriesResponse{
+				{
+					Entry: &filer_pb.Entry{
+						Name: "base.txt",
+						Attributes: &filer_pb.FuseAttributes{
+							Crtime:   1,
+							Mtime:    1,
+							FileMode: 0100644,
+							FileSize: 3,
+						},
+					},
+					SnapshotTsNs: 100,
+				},
+			},
+			onFirstRecv: func() {
+				applyErr = mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
+					Directory: "/dir",
+					EventNotification: &filer_pb.EventNotification{
+						NewEntry: &filer_pb.Entry{
+							Name: "after.txt",
+							Attributes: &filer_pb.FuseAttributes{
+								Crtime:   2,
+								Mtime:    2,
+								FileMode: 0100644,
+								FileSize: 9,
+							},
+						},
+					},
+					TsNs: 101,
+				}, SubscriberMetadataResponseApplyOptions)
+			},
+		},
+	}
+
+	if err := EnsureVisited(mc, accessor, util.FullPath("/dir")); err != nil {
+		t.Fatalf("ensure visited: %v", err)
+	}
+	if applyErr != nil {
+		t.Fatalf("apply buffered event: %v", applyErr)
+	}
+	if !mc.IsDirectoryCached(util.FullPath("/dir")) {
+		t.Fatal("directory /dir should be cached after build completes")
+	}
+
+	baseEntry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/base.txt"))
+	if err != nil {
+		t.Fatalf("find base entry: %v", err)
+	}
+	if baseEntry.FileSize != 3 {
+		t.Fatalf("base entry size = %d, want 3", baseEntry.FileSize)
+	}
+
+	afterEntry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/after.txt"))
+	if err != nil {
+		t.Fatalf("find replayed entry: %v", err)
+	}
+	if afterEntry.FileSize != 9 {
+		t.Fatalf("replayed entry size = %d, want 9", afterEntry.FileSize)
+	}
+}
+
+// TestDirectoryNotificationsSuppressedDuringBuild verifies that metadata events
+// targeting a directory under active build do NOT fire onDirectoryUpdate for
+// that directory. In production, onDirectoryUpdate can trigger
+// markDirectoryReadThrough → DeleteFolderChildren, which would wipe entries
+// that EnsureVisited already inserted mid-build.
+func TestDirectoryNotificationsSuppressedDuringBuild(t *testing.T) {
+	mc, _, notifications, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/": true,
+	})
+	defer mc.Shutdown()
+
+	// Start building /dir (simulates the beginning of EnsureVisited)
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+
+	// Insert an entry as EnsureVisited would during the filer listing
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/dir/existing.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 100,
+		},
+	}); err != nil {
+		t.Fatalf("insert entry during build: %v", err)
+	}
+
+	// Simulate multiple metadata events arriving for /dir while the build
+	// is in progress. Each event would normally call noteDirectoryUpdate,
+	// which in production can trigger markDirectoryReadThrough and wipe entries.
+	for i := 0; i < 5; i++ {
+		resp := &filer_pb.SubscribeMetadataResponse{
+			Directory: "/dir",
+			EventNotification: &filer_pb.EventNotification{
+				NewEntry: &filer_pb.Entry{
+					Name: fmt.Sprintf("new-%d.txt", i),
+					Attributes: &filer_pb.FuseAttributes{
+						Crtime:   int64(10 + i),
+						Mtime:    int64(10 + i),
+						FileMode: 0100644,
+						FileSize: uint64(i + 1),
+					},
+				},
+			},
+			TsNs: int64(200 + i),
+		}
+		if err := mc.ApplyMetadataResponse(context.Background(), resp, SubscriberMetadataResponseApplyOptions); err != nil {
+			t.Fatalf("apply event %d: %v", i, err)
+		}
+	}
+
+	// The building directory /dir must NOT have received any notifications.
+	// If it did, markDirectoryReadThrough would wipe the cache mid-build.
+	for _, p := range notifications.paths() {
+		if p == util.FullPath("/dir") {
+			t.Fatal("onDirectoryUpdate was called for /dir during build; this would cause markDirectoryReadThrough to wipe entries mid-build")
+		}
+	}
+
+	// The entry inserted during the build must still be present
+	entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/existing.txt"))
+	if err != nil {
+		t.Fatalf("entry wiped during build: %v", err)
+	}
+	if entry.FileSize != 100 {
+		t.Fatalf("entry size = %d, want 100", entry.FileSize)
+	}
+
+	// Complete the build — buffered events should be replayed
+	if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 150); err != nil {
+		t.Fatalf("complete build: %v", err)
+	}
+
+	// After build completes, the entry from the listing should still exist
+	entry, err = mc.FindEntry(context.Background(), util.FullPath("/dir/existing.txt"))
+	if err != nil {
+		t.Fatalf("entry lost after build completion: %v", err)
+	}
+	if entry.FileSize != 100 {
+		t.Fatalf("entry size after build = %d, want 100", entry.FileSize)
+	}
+
+	// Buffered events with TsNs > snapshotTsNs (150) should have been replayed
+	for i := 0; i < 5; i++ {
+		name := fmt.Sprintf("new-%d.txt", i)
+		e, err := mc.FindEntry(context.Background(), util.FullPath("/dir/"+name))
+		if err != nil {
+			t.Fatalf("replayed entry %s not found: %v", name, err)
+		}
+		if e.FileSize != uint64(i+1) {
+			t.Fatalf("replayed entry %s size = %d, want %d", name, e.FileSize, i+1)
+		}
+	}
+}
+
+// TestEmptyDirectoryBuildReplaysAllBufferedEvents verifies that when a
+// directory build completes with snapshotTsNs=0 (empty directory — server
+// returned no entries and no snapshot), ALL buffered events are replayed
+// without any TsNs filtering. This prevents clock-skew between client and
+// filer from dropping legitimate mutations.
+func TestEmptyDirectoryBuildReplaysAllBufferedEvents(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/empty")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+
+	// Buffer events with a range of TsNs values — some very old, some recent.
+	// With a client-synthesized snapshot, old events could be incorrectly filtered.
+	tsValues := []int64{1, 50, 500, 5000, 50000}
+	for i, ts := range tsValues {
+		resp := &filer_pb.SubscribeMetadataResponse{
+			Directory: "/empty",
+			EventNotification: &filer_pb.EventNotification{
+				NewEntry: &filer_pb.Entry{
+					Name: fmt.Sprintf("file-%d.txt", i),
+					Attributes: &filer_pb.FuseAttributes{
+						Crtime:   ts,
+						Mtime:    ts,
+						FileMode: 0100644,
+						FileSize: uint64(i + 10),
+					},
+				},
+			},
+			TsNs: ts,
+		}
+		if err := mc.ApplyMetadataResponse(context.Background(), resp, SubscriberMetadataResponseApplyOptions); err != nil {
+			t.Fatalf("apply event %d: %v", i, err)
+		}
+	}
+
+	// Complete with snapshotTsNs=0 — simulates empty directory listing
+	if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/empty"), 0); err != nil {
+		t.Fatalf("complete build: %v", err)
+	}
+
+	// Every buffered event must have been replayed, regardless of TsNs
+	for i := range tsValues {
+		name := fmt.Sprintf("file-%d.txt", i)
+		e, err := mc.FindEntry(context.Background(), util.FullPath("/empty/"+name))
+		if err != nil {
+			t.Fatalf("replayed entry %s not found: %v", name, err)
+		}
+		if e.FileSize != uint64(i+10) {
+			t.Fatalf("replayed entry %s size = %d, want %d", name, e.FileSize, i+10)
+		}
+	}
+
+	if !mc.IsDirectoryCached(util.FullPath("/empty")) {
+		t.Fatal("/empty should be marked cached after build completes")
+	}
+}
+
+// TestBuildCompletionSurvivesCallerCancellation verifies that once
+// CompleteDirectoryBuild is enqueued, a cancelled caller context does not
+// prevent the build from completing. The apply loop uses context.Background()
+// internally, so the operation finishes even if the caller gives up waiting.
+func TestBuildCompletionSurvivesCallerCancellation(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+
+	// Insert an entry during the build (as EnsureVisited would)
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/dir/kept.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 42,
+		},
+	}); err != nil {
+		t.Fatalf("insert entry: %v", err)
+	}
+
+	// Buffer an event that should be replayed
+	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name: "buffered.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   5,
+					Mtime:    5,
+					FileMode: 0100644,
+					FileSize: 77,
+				},
+			},
+		},
+		TsNs: 200,
+	}, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply event: %v", err)
+	}
+
+	// Complete with an already-cancelled context. The operation should still
+	// succeed because enqueueAndWait sets req.ctx = context.Background().
+	cancelledCtx, cancel := context.WithCancel(context.Background())
+	cancel() // cancel immediately
+
+	// CompleteDirectoryBuild may return ctx.Err() if the select picks
+	// ctx.Done() first, but the operation itself still completes in the
+	// apply loop. Poll for the observable side effect instead of using
+	// a fixed sleep.
+	_ = mc.CompleteDirectoryBuild(cancelledCtx, util.FullPath("/dir"), 100)
+
+	// Poll until the build completes or a deadline elapses.
+	deadline := time.After(2 * time.Second)
+	for !mc.IsDirectoryCached(util.FullPath("/dir")) {
+		select {
+		case <-deadline:
+			t.Fatal("/dir should be cached — CompleteDirectoryBuild must have executed despite cancelled context")
+		default:
+			time.Sleep(5 * time.Millisecond)
+		}
+	}
+
+	// The pre-existing entry must survive
+	entry, findErr := mc.FindEntry(context.Background(), util.FullPath("/dir/kept.txt"))
+	if findErr != nil {
+		t.Fatalf("find kept entry: %v", findErr)
+	}
+	if entry.FileSize != 42 {
+		t.Fatalf("kept entry size = %d, want 42", entry.FileSize)
+	}
+
+	// The buffered event (TsNs 200 > snapshot 100) must have been replayed
+	buffered, findErr := mc.FindEntry(context.Background(), util.FullPath("/dir/buffered.txt"))
+	if findErr != nil {
+		t.Fatalf("find buffered entry: %v", findErr)
+	}
+	if buffered.FileSize != 77 {
+		t.Fatalf("buffered entry size = %d, want 77", buffered.FileSize)
+	}
+}
+
+func TestBufferedRenameUpdatesOtherDirectoryBeforeBuildCompletes(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/src": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/src/from.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 7,
+		},
+	}); err != nil {
+		t.Fatalf("insert source entry: %v", err)
+	}
+
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dst")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+
+	renameResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/src",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name: "from.txt",
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "to.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   2,
+					Mtime:    2,
+					FileMode: 0100644,
+					FileSize: 12,
+				},
+			},
+			NewParentPath: "/dst",
+		},
+		TsNs: 101,
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply rename: %v", err)
+	}
+
+	oldEntry, err := mc.FindEntry(context.Background(), util.FullPath("/src/from.txt"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("find old path error = %v, want %v", err, filer_pb.ErrNotFound)
+	}
+	if oldEntry != nil {
+		t.Fatalf("old path should be removed before build completes: %+v", oldEntry)
+	}
+
+	newEntry, err := mc.FindEntry(context.Background(), util.FullPath("/dst/to.txt"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("find buffered new path error = %v, want %v", err, filer_pb.ErrNotFound)
+	}
+	if newEntry != nil {
+		t.Fatalf("new path should stay hidden until build completes: %+v", newEntry)
+	}
+
+	if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dst"), 100); err != nil {
+		t.Fatalf("complete build: %v", err)
+	}
+
+	newEntry, err = mc.FindEntry(context.Background(), util.FullPath("/dst/to.txt"))
+	if err != nil {
+		t.Fatalf("find replayed new path: %v", err)
+	}
+	if newEntry.FileSize != 12 {
+		t.Fatalf("replayed new path size = %d, want 12", newEntry.FileSize)
+	}
+}

weed/mount/meta_cache/meta_cache_init.go

@@ -69,12 +69,43 @@ func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerCl
 
 		glog.V(4).Infof("ReadDirAllEntries %s ...", path)
 
+		// Use context.Background() for build lifecycle calls so that
+		// errgroup cancellation of ctx doesn't cause enqueueAndWait to
+		// return early, which would trigger cleanupBuild while the
+		// operation is still queued.
+		if err := mc.BeginDirectoryBuild(context.Background(), path); err != nil {
+			return nil, fmt.Errorf("begin build %s: %w", path, err)
+		}
+		cleanupDone := false
+		cleanupBuild := func(reason string) {
+			if cleanupDone {
+				return
+			}
+			cleanupDone = true
+			if deleteErr := mc.DeleteFolderChildren(context.Background(), path); deleteErr != nil {
+				glog.V(2).Infof("clear %s build %s: %v", reason, path, deleteErr)
+			}
+			if abortErr := mc.AbortDirectoryBuild(context.Background(), path); abortErr != nil {
+				glog.V(2).Infof("abort %s build %s: %v", reason, path, abortErr)
+			}
+		}
+		defer func() {
+			if !cleanupDone && ctx.Err() != nil {
+				cleanupBuild("canceled")
+			}
+		}()
+
 		// Collect entries in batches for efficient LevelDB writes
 		var batch []*filer.Entry
+		var snapshotTsNs int64
 
 		fetchErr := util.Retry("ReadDirAllEntries", func() error {
 			batch = nil // Reset batch on retry, allow GC of previous entries
-			return filer_pb.ReadDirAllEntries(ctx, client, path, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
+			if err := mc.DeleteFolderChildren(ctx, path); err != nil {
+				return fmt.Errorf("clear existing entries for %s: %w", path, err)
+			}
+			var err error
+			snapshotTsNs, err = filer_pb.ReadDirAllEntriesWithSnapshot(ctx, client, path, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
 				entry := filer.FromPbEntry(string(path), pbEntry)
 				if IsHiddenSystemEntry(string(path), entry.Name()) {
 					return nil
@@ -94,19 +125,26 @@ func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerCl
 				}
 				return nil
 			})
+			return err
 		})
 
 		if fetchErr != nil {
+			cleanupBuild("failed")
 			return nil, fmt.Errorf("list %s: %w", path, fetchErr)
 		}
 
 		// Flush any remaining entries in the batch
 		if len(batch) > 0 {
 			if err := mc.doBatchInsertEntries(ctx, batch); err != nil {
+				cleanupBuild("incomplete")
 				return nil, fmt.Errorf("batch insert remaining for %s: %w", path, err)
 			}
 		}
-		mc.markCachedFn(path)
+		if err := mc.CompleteDirectoryBuild(context.Background(), path, snapshotTsNs); err != nil {
+			cleanupBuild("unreplayed")
+			return nil, fmt.Errorf("complete build for %s: %w", path, err)
+		}
+		cleanupDone = true // Prevent deferred cleanup after successful publish
 		return nil, nil
 	})
 	return err

weed/mount/meta_cache/meta_cache_subscribe.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"strings"
 
-	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -51,67 +50,12 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
 	}
 
 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
-		message := resp.EventNotification
-
-		for _, sig := range message.Signatures {
-			if sig == selfSignature && selfSignature != 0 {
-				return nil
-			}
-		}
-
-		dir := resp.Directory
-		var oldPath util.FullPath
-		var newEntry *filer.Entry
-		if message.OldEntry != nil {
-			oldPath = util.NewFullPath(dir, message.OldEntry.Name)
-			glog.V(4).Infof("deleting %v", oldPath)
-		}
-
-		if message.NewEntry != nil {
-			if message.NewParentPath != "" {
-				dir = message.NewParentPath
-			}
-			key := util.NewFullPath(dir, message.NewEntry.Name)
-			glog.V(4).Infof("creating %v", key)
-			newEntry = filer.FromPbEntry(dir, message.NewEntry)
-		}
-		err := mc.AtomicUpdateEntryFromFiler(context.Background(), oldPath, newEntry)
-		if err == nil {
-			if message.NewEntry != nil || message.OldEntry != nil {
-				dirsToNotify := make(map[util.FullPath]struct{})
-				if oldPath != "" {
-					parent, _ := oldPath.DirAndName()
-					dirsToNotify[util.FullPath(parent)] = struct{}{}
-				}
-				if newEntry != nil {
-					newParent, _ := newEntry.DirAndName()
-					dirsToNotify[util.FullPath(newParent)] = struct{}{}
-				}
-				if message.NewEntry != nil && message.NewEntry.IsDirectory {
-					childPath := util.NewFullPath(dir, message.NewEntry.Name)
-					dirsToNotify[childPath] = struct{}{}
-				}
-				for dirPath := range dirsToNotify {
-					mc.noteDirectoryUpdate(dirPath)
-				}
-			}
-			if message.OldEntry != nil && message.NewEntry != nil {
-				oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name)
-				mc.invalidateFunc(oldKey, message.OldEntry)
-				if message.OldEntry.Name != message.NewEntry.Name {
-					newKey := util.NewFullPath(dir, message.NewEntry.Name)
-					mc.invalidateFunc(newKey, message.NewEntry)
-				}
-			} else if filer_pb.IsCreate(resp) {
-				// no need to invalidate
-			} else if filer_pb.IsDelete(resp) {
-				oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name)
-				mc.invalidateFunc(oldKey, message.OldEntry)
-			}
-		}
-
-		return err
-
+		// Let all events (including self-originated ones) flow through the
+		// applier so that the directory-build buffering and dedup logic
+		// can handle them consistently. The dedupRing in
+		// applyMetadataResponseNow catches duplicates that were already
+		// applied locally via applyLocalMetadataEvent.
+		return mc.ApplyMetadataResponse(context.Background(), resp, SubscriberMetadataResponseApplyOptions)
 	}
 
 	prefix := dir

weed/mount/metadata_events.go

@@ -0,0 +1,66 @@
+package mount
+
+import (
+	"context"
+
+	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"google.golang.org/protobuf/proto"
+)
+
+func (wfs *WFS) applyLocalMetadataEvent(ctx context.Context, event *filer_pb.SubscribeMetadataResponse) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	return wfs.metaCache.ApplyMetadataResponseOwned(ctx, event, meta_cache.LocalMetadataResponseApplyOptions)
+}
+
+func metadataDeleteEvent(directory, name string, isDirectory bool) *filer_pb.SubscribeMetadataResponse {
+	if name == "" {
+		return nil
+	}
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory: directory,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: name, IsDirectory: isDirectory},
+		},
+	}
+}
+
+func metadataCreateEvent(directory string, entry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse {
+	if entry == nil {
+		return nil
+	}
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory: directory,
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry:      proto.Clone(entry).(*filer_pb.Entry),
+			NewParentPath: directory,
+		},
+	}
+}
+
+func metadataUpdateEvent(directory string, entry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse {
+	if entry == nil {
+		return nil
+	}
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory: directory,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: entry.Name},
+			NewEntry:      proto.Clone(entry).(*filer_pb.Entry),
+			NewParentPath: directory,
+		},
+	}
+}
+
+func metadataEventFromRenameResponse(resp *filer_pb.StreamRenameEntryResponse) *filer_pb.SubscribeMetadataResponse {
+	if resp == nil || resp.EventNotification == nil {
+		return nil
+	}
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory:         resp.Directory,
+		EventNotification: proto.Clone(resp.EventNotification).(*filer_pb.EventNotification),
+		TsNs:              resp.TsNs,
+	}
+}

weed/mount/weedfs.go

@@ -200,7 +200,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 			}
 		}, func(dirPath util.FullPath) {
 			if wfs.inodeToPath.RecordDirectoryUpdate(dirPath, time.Now(), wfs.dirHotWindow, wfs.dirHotThreshold) {
-				wfs.maybeRefreshDirectory(dirPath)
+				wfs.markDirectoryReadThrough(dirPath)
 			}
 		})
 	grace.OnInterrupt(func() {
@@ -313,36 +313,42 @@ func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, fuse.St
 }
 
 // lookupEntry looks up an entry by path, checking the local cache first.
-// If the directory is cached, it trusts the cache. Otherwise, it fetches
-// directly from the filer without caching the entire directory.
-// This avoids the performance issue of listing millions of files just to open one.
+// Cached metadata is only authoritative when the parent directory itself is cached.
+// For uncached/read-through directories, always consult the filer directly so stale
+// local entries do not leak back into lookup results.
 func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, fuse.Status) {
 	dir, _ := fullpath.DirAndName()
+	dirPath := util.FullPath(dir)
 
-	// Try to find the entry in the local cache first.
-	cachedEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath)
-	if cacheErr != nil && cacheErr != filer_pb.ErrNotFound {
-		glog.Errorf("lookupEntry: cache lookup for %s failed: %v", fullpath, cacheErr)
-		return nil, fuse.EIO
-	}
-	if cachedEntry != nil {
-		glog.V(4).Infof("lookupEntry cache hit %s", fullpath)
-		return cachedEntry, fuse.OK
-	}
-
-	// If the directory is cached but entry not found, file doesn't exist.
-	// No need to query the filer again.
-	if wfs.metaCache.IsDirectoryCached(util.FullPath(dir)) {
-		glog.V(4).Infof("lookupEntry cache miss (dir cached) %s", fullpath)
-		return nil, fuse.ENOENT
+	if wfs.metaCache.IsDirectoryCached(dirPath) {
+		cachedEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath)
+		if cacheErr != nil && cacheErr != filer_pb.ErrNotFound {
+			glog.Errorf("lookupEntry: cache lookup for %s failed: %v", fullpath, cacheErr)
+			return nil, fuse.EIO
+		}
+		if cachedEntry != nil {
+			glog.V(4).Infof("lookupEntry cache hit %s", fullpath)
+			return cachedEntry, fuse.OK
+		}
+		// Re-check: the directory may have been evicted from cache between
+		// our IsDirectoryCached check and FindEntry (e.g. markDirectoryReadThrough).
+		// If it's no longer cached, fall through to the filer lookup below.
+		if wfs.metaCache.IsDirectoryCached(dirPath) {
+			glog.V(4).Infof("lookupEntry cache miss (dir cached) %s", fullpath)
+			return nil, fuse.ENOENT
+		}
 	}
 
 	// Directory not cached - fetch directly from filer without caching the entire directory.
 	glog.V(4).Infof("lookupEntry fetching from filer %s", fullpath)
 	entry, err := filer_pb.GetEntry(context.Background(), wfs, fullpath)
 	if err != nil {
-		glog.V(1).Infof("lookupEntry GetEntry %s: %v", fullpath, err)
-		return nil, fuse.ENOENT
+		if err == filer_pb.ErrNotFound {
+			glog.V(4).Infof("lookupEntry not found %s", fullpath)
+			return nil, fuse.ENOENT
+		}
+		glog.Warningf("lookupEntry GetEntry %s: %v", fullpath, err)
+		return nil, fuse.EIO
 	}
 	if entry != nil && entry.Attributes != nil && wfs.option.UidGidMapper != nil {
 		entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid)
@@ -371,31 +377,13 @@ func (wfs *WFS) ClearCacheDir() {
 	os.RemoveAll(wfs.option.getUniqueCacheDirForRead())
 }
 
-func (wfs *WFS) maybeRefreshDirectory(dirPath util.FullPath) {
-	if !wfs.inodeToPath.NeedsRefresh(dirPath) {
+func (wfs *WFS) markDirectoryReadThrough(dirPath util.FullPath) {
+	if !wfs.inodeToPath.MarkDirectoryReadThrough(dirPath, time.Now()) {
 		return
 	}
-	wfs.refreshMu.Lock()
-	if _, exists := wfs.refreshingDirs[dirPath]; exists {
-		wfs.refreshMu.Unlock()
-		return
+	if err := wfs.metaCache.DeleteFolderChildren(context.Background(), dirPath); err != nil {
+		glog.V(2).Infof("clear dir cache %s: %v", dirPath, err)
 	}
-	wfs.refreshingDirs[dirPath] = struct{}{}
-	wfs.refreshMu.Unlock()
-
-	go func() {
-		defer func() {
-			wfs.refreshMu.Lock()
-			delete(wfs.refreshingDirs, dirPath)
-			wfs.refreshMu.Unlock()
-		}()
-		wfs.inodeToPath.InvalidateChildrenCache(dirPath)
-		if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath); err != nil {
-			glog.Warningf("refresh dir cache %s: %v", dirPath, err)
-			return
-		}
-		wfs.inodeToPath.MarkDirectoryRefreshed(dirPath, time.Now())
-	}()
 }
 
 func (wfs *WFS) loopEvictIdleDirCache() {

weed/mount/weedfs_dir_mkrm.go

@@ -2,7 +2,6 @@ package mount
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"strings"
 	"syscall"
@@ -63,19 +62,21 @@ func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out
 		}
 
 		glog.V(1).Infof("mkdir: %v", request)
-		if err := filer_pb.CreateEntry(context.Background(), client, request); err != nil {
+		resp, err := filer_pb.CreateEntryWithResponse(context.Background(), client, request)
+		if err != nil {
 			glog.V(0).Infof("mkdir %s: %v", entryFullPath, err)
 			return err
 		}
 
-		// Only cache the entry if the parent directory is already cached.
-		// This avoids polluting the cache with partial directory data.
-		if wfs.metaCache.IsDirectoryCached(dirFullPath) {
-			wfs.inodeToPath.TouchDirectory(dirFullPath)
-			if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
-				return fmt.Errorf("local mkdir dir %s: %w", entryFullPath, err)
-			}
+		event := resp.GetMetadataEvent()
+		if event == nil {
+			event = metadataCreateEvent(string(dirFullPath), newEntry)
 		}
+		if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil {
+			glog.Warningf("mkdir %s: best-effort metadata apply failed: %v", entryFullPath, applyErr)
+			wfs.inodeToPath.InvalidateChildrenCache(dirFullPath)
+		}
+		wfs.inodeToPath.TouchDirectory(dirFullPath)
 
 		return nil
 	})
@@ -112,7 +113,7 @@ func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string
 
 	glog.V(3).Infof("remove directory: %v", entryFullPath)
 	ignoreRecursiveErr := true // ignore recursion error since the OS should manage it
-	err := filer_pb.Remove(context.Background(), wfs, string(dirFullPath), name, true, false, ignoreRecursiveErr, false, []int32{wfs.signature})
+	resp, err := filer_pb.RemoveWithResponse(context.Background(), wfs, string(dirFullPath), name, true, false, ignoreRecursiveErr, false, []int32{wfs.signature})
 	if err != nil {
 		glog.V(0).Infof("remove %s: %v", entryFullPath, err)
 		if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
@@ -121,7 +122,14 @@ func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string
 		return fuse.ENOENT
 	}
 
-	wfs.metaCache.DeleteEntry(context.Background(), entryFullPath)
+	event := metadataDeleteEvent(string(dirFullPath), name, true)
+	if resp != nil && resp.MetadataEvent != nil {
+		event = resp.MetadataEvent
+	}
+	if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil {
+		glog.Warningf("rmdir %s: best-effort metadata apply failed: %v", entryFullPath, applyErr)
+		wfs.inodeToPath.InvalidateChildrenCache(dirFullPath)
+	}
 	wfs.inodeToPath.RemovePath(entryFullPath)
 	wfs.inodeToPath.TouchDirectory(dirFullPath)
 

weed/mount/weedfs_dir_read.go

@@ -3,11 +3,13 @@ package mount
 import (
 	"context"
 	"sync"
+	"time"
 
 	"github.com/seaweedfs/go-fuse/v2/fuse"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
@@ -26,10 +28,12 @@ type DirectoryHandle struct {
 	isFinished        bool
 	entryStream       []*filer.Entry
 	entryStreamOffset uint64
+	snapshotTsNs      int64 // snapshot timestamp for consistent readdir in direct mode
 }
 
 func (dh *DirectoryHandle) reset() {
 	dh.isFinished = false
+	dh.snapshotTsNs = 0
 	// Nil out pointers to allow garbage collection of old entries,
 	// then reuse the slice's capacity to avoid re-allocations.
 	for i := range dh.entryStream {
@@ -164,7 +168,6 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPl
 		return code
 	}
 	wfs.inodeToPath.TouchDirectory(dirPath)
-	wfs.maybeRefreshDirectory(dirPath)
 
 	var dirEntry fuse.DirEntry
 
@@ -214,6 +217,10 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPl
 
 	var lastEntryName string
 
+	if wfs.inodeToPath.ShouldReadDirectoryDirect(dirPath) {
+		return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
+	}
+
 	// Read from cache first, then load next batch if needed
 	if input.Offset >= dh.entryStreamOffset {
 		// Handle case: new handle with non-zero offset but empty cache
@@ -288,3 +295,90 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPl
 
 	return fuse.OK
 }
+
+func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList, dh *DirectoryHandle, dirPath util.FullPath, processEachEntryFn func(entry *filer.Entry, index int64) bool) fuse.Status {
+	var lastEntryName string
+
+	if input.Offset >= dh.entryStreamOffset {
+		if len(dh.entryStream) == 0 && input.Offset > dh.entryStreamOffset {
+			skipCount := uint32(input.Offset-dh.entryStreamOffset) + batchSize
+			entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs)
+			if err != nil {
+				glog.Errorf("list filer directory: %v", err)
+				return fuse.EIO
+			}
+			dh.entryStream = append(dh.entryStream, entries...)
+			if dh.snapshotTsNs == 0 {
+				dh.snapshotTsNs = snapshotTs
+			}
+		}
+
+		if input.Offset > dh.entryStreamOffset {
+			entryPreviousIndex := (input.Offset - dh.entryStreamOffset) - 1
+			if uint64(len(dh.entryStream)) > entryPreviousIndex {
+				lastEntryName = dh.entryStream[entryPreviousIndex].Name()
+			}
+		}
+
+		entryCurrentIndex := int64(input.Offset - dh.entryStreamOffset)
+		for int64(len(dh.entryStream)) > entryCurrentIndex {
+			entry := dh.entryStream[entryCurrentIndex]
+			if processEachEntryFn(entry, entryCurrentIndex) {
+				lastEntryName = entry.Name()
+				entryCurrentIndex++
+			} else {
+				return fuse.OK
+			}
+		}
+
+		entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs)
+		if err != nil {
+			glog.Errorf("list filer directory: %v", err)
+			return fuse.EIO
+		}
+		if dh.snapshotTsNs == 0 {
+			dh.snapshotTsNs = snapshotTs
+		}
+
+		bufferFull := false
+		for _, entry := range entries {
+			currentIndex := int64(len(dh.entryStream))
+			dh.entryStream = append(dh.entryStream, entry)
+			if !processEachEntryFn(entry, currentIndex) {
+				bufferFull = true
+				break
+			}
+		}
+		if !bufferFull && len(entries) < int(batchSize) {
+			dh.isFinished = true
+			// After a full successful read-through listing, exit direct mode
+			// so subsequent reads can use the cache instead of hitting the filer.
+			wfs.inodeToPath.MarkDirectoryRefreshed(dirPath, time.Now())
+		}
+	}
+
+	return fuse.OK
+}
+
+func loadDirectoryEntriesDirect(ctx context.Context, client filer_pb.FilerClient, uidGidMapper *meta_cache.UidGidMapper, dirPath util.FullPath, startFileName string, includeStart bool, limit uint32, snapshotTsNs int64) ([]*filer.Entry, int64, error) {
+	entries := make([]*filer.Entry, 0, limit)
+	var actualSnapshotTsNs int64
+	err := client.WithFilerClient(false, func(sc filer_pb.SeaweedFilerClient) error {
+		var innerErr error
+		actualSnapshotTsNs, innerErr = filer_pb.DoSeaweedListWithSnapshot(ctx, sc, dirPath, "", func(entry *filer_pb.Entry, isLast bool) error {
+			if meta_cache.IsHiddenSystemEntry(string(dirPath), entry.Name) {
+				return nil
+			}
+			if uidGidMapper != nil && entry.Attributes != nil {
+				entry.Attributes.Uid, entry.Attributes.Gid = uidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid)
+			}
+			entries = append(entries, filer.FromPbEntry(string(dirPath), entry))
+			return nil
+		}, startFileName, includeStart, limit, snapshotTsNs)
+		return innerErr
+	})
+	if err != nil {
+		return nil, actualSnapshotTsNs, err
+	}
+	return entries, actualSnapshotTsNs, nil
+}

weed/mount/weedfs_dir_read_test.go

@@ -0,0 +1,100 @@
+package mount
+
+import (
+	"context"
+	"io"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+type directoryListStream struct {
+	responses []*filer_pb.ListEntriesResponse
+	index     int
+}
+
+func (s *directoryListStream) Recv() (*filer_pb.ListEntriesResponse, error) {
+	if s.index >= len(s.responses) {
+		return nil, io.EOF
+	}
+	resp := s.responses[s.index]
+	s.index++
+	return resp, nil
+}
+
+func (s *directoryListStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
+func (s *directoryListStream) Trailer() metadata.MD         { return metadata.MD{} }
+func (s *directoryListStream) CloseSend() error             { return nil }
+func (s *directoryListStream) Context() context.Context     { return context.Background() }
+func (s *directoryListStream) SendMsg(any) error            { return nil }
+func (s *directoryListStream) RecvMsg(any) error            { return nil }
+
+type directoryListClient struct {
+	filer_pb.SeaweedFilerClient
+	responses []*filer_pb.ListEntriesResponse
+}
+
+func (c *directoryListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
+	return &directoryListStream{responses: c.responses}, nil
+}
+
+type directoryFilerAccessor struct {
+	client filer_pb.SeaweedFilerClient
+}
+
+func (a *directoryFilerAccessor) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return fn(a.client)
+}
+
+func (a *directoryFilerAccessor) AdjustedUrl(*filer_pb.Location) string { return "" }
+func (a *directoryFilerAccessor) GetDataCenter() string                 { return "" }
+
+func TestLoadDirectoryEntriesDirectFiltersHiddenEntriesAndMapsIds(t *testing.T) {
+	mapper, err := meta_cache.NewUidGidMapper("10:1000", "20:2000")
+	if err != nil {
+		t.Fatalf("uid/gid mapper: %v", err)
+	}
+
+	client := &directoryFilerAccessor{
+		client: &directoryListClient{
+			responses: []*filer_pb.ListEntriesResponse{
+				{
+					Entry: &filer_pb.Entry{
+						Name: "topics",
+						Attributes: &filer_pb.FuseAttributes{
+							Uid: 1000,
+							Gid: 2000,
+						},
+					},
+				},
+				{
+					Entry: &filer_pb.Entry{
+						Name: "visible",
+						Attributes: &filer_pb.FuseAttributes{
+							Uid: 1000,
+							Gid: 2000,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, mapper, util.FullPath("/"), "", false, 10, 0)
+	if err != nil {
+		t.Fatalf("loadDirectoryEntriesDirect: %v", err)
+	}
+	if got := len(entries); got != 1 {
+		t.Fatalf("entry count = %d, want 1", got)
+	}
+	if entries[0].Name() != "visible" {
+		t.Fatalf("entry name = %q, want visible", entries[0].Name())
+	}
+	if entries[0].Attr.Uid != 10 || entries[0].Attr.Gid != 20 {
+		t.Fatalf("mapped uid/gid = %d/%d, want 10/20", entries[0].Attr.Uid, entries[0].Attr.Gid)
+	}
+}

weed/mount/weedfs_file_mkrm.go

@@ -2,12 +2,10 @@ package mount
 
 import (
 	"context"
-	"fmt"
 	"syscall"
 	"time"
 
 	"github.com/seaweedfs/go-fuse/v2/fuse"
-	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 )
@@ -83,19 +81,21 @@ func (wfs *WFS) Mknod(cancel <-chan struct{}, in *fuse.MknodIn, name string, out
 		}
 
 		glog.V(1).Infof("mknod: %v", request)
-		if err := filer_pb.CreateEntry(context.Background(), client, request); err != nil {
+		resp, err := filer_pb.CreateEntryWithResponse(context.Background(), client, request)
+		if err != nil {
 			glog.V(0).Infof("mknod %s: %v", entryFullPath, err)
 			return err
 		}
 
-		// Only cache the entry if the parent directory is already cached.
-		// This avoids polluting the cache with partial directory data.
-		if wfs.metaCache.IsDirectoryCached(dirFullPath) {
-			wfs.inodeToPath.TouchDirectory(dirFullPath)
-			if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
-				return fmt.Errorf("local mknod %s: %w", entryFullPath, err)
-			}
+		event := resp.GetMetadataEvent()
+		if event == nil {
+			event = metadataCreateEvent(string(dirFullPath), newEntry)
 		}
+		if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil {
+			glog.Warningf("mknod %s: best-effort metadata apply failed: %v", entryFullPath, applyErr)
+			wfs.inodeToPath.InvalidateChildrenCache(dirFullPath)
+		}
+		wfs.inodeToPath.TouchDirectory(dirFullPath)
 
 		return nil
 	})
@@ -143,16 +143,21 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
 	glog.V(3).Infof("remove file: %v", entryFullPath)
 	// Always let the filer decide whether to delete chunks based on its authoritative data.
 	// The filer has the correct hard link count and will only delete chunks when appropriate.
-	err := filer_pb.Remove(context.Background(), wfs, string(dirFullPath), name, true, false, false, false, []int32{wfs.signature})
+	resp, err := filer_pb.RemoveWithResponse(context.Background(), wfs, string(dirFullPath), name, true, false, false, false, []int32{wfs.signature})
 	if err != nil {
 		glog.V(0).Infof("remove %s: %v", entryFullPath, err)
 		return fuse.OK
 	}
 
-	// then, delete meta cache
-	if err = wfs.metaCache.DeleteEntry(context.Background(), entryFullPath); err != nil {
-		glog.V(3).Infof("local DeleteEntry %s: %v", entryFullPath, err)
-		return fuse.EIO
+	var event *filer_pb.SubscribeMetadataResponse
+	if resp != nil && resp.MetadataEvent != nil {
+		event = resp.MetadataEvent
+	} else {
+		event = metadataDeleteEvent(string(dirFullPath), name, false)
+	}
+	if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil {
+		glog.Warningf("unlink %s: best-effort metadata apply failed: %v", entryFullPath, applyErr)
+		wfs.inodeToPath.InvalidateChildrenCache(dirFullPath)
 	}
 	wfs.inodeToPath.TouchDirectory(dirFullPath)
 

weed/mount/weedfs_file_sync.go

@@ -161,16 +161,19 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
 		wfs.mapPbIdFromLocalToFiler(request.Entry)
 		defer wfs.mapPbIdFromFilerToLocal(request.Entry)
 
-		if err := filer_pb.CreateEntry(context.Background(), client, request); err != nil {
+		resp, err := filer_pb.CreateEntryWithResponse(context.Background(), client, request)
+		if err != nil {
 			glog.Errorf("fh flush create %s: %v", fileFullPath, err)
 			return fmt.Errorf("fh flush create %s: %v", fileFullPath, err)
 		}
 
-		// Only update cache if the parent directory is cached
-		if wfs.metaCache.IsDirectoryCached(util.FullPath(dir)) {
-			if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
-				return fmt.Errorf("update meta cache for %s: %w", fileFullPath, err)
-			}
+		event := resp.GetMetadataEvent()
+		if event == nil {
+			event = metadataUpdateEvent(string(dir), request.Entry)
+		}
+		if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil {
+			glog.Warningf("flush %s: best-effort metadata apply failed: %v", fileFullPath, applyErr)
+			wfs.inodeToPath.InvalidateChildrenCache(util.FullPath(dir))
 		}
 
 		return nil

weed/mount/weedfs_link.go

@@ -2,7 +2,6 @@ package mount
 
 import (
 	"context"
-	"fmt"
 	"syscall"
 	"time"
 
@@ -56,6 +55,8 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
 	}
 
 	// update old file to hardlink mode
+	origHardLinkId := oldEntry.HardLinkId
+	origHardLinkCounter := oldEntry.HardLinkCounter
 	if len(oldEntry.HardLinkId) == 0 {
 		oldEntry.HardLinkId = filer.NewHardLinkId()
 		oldEntry.HardLinkCounter = 1
@@ -90,25 +91,42 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
 		wfs.mapPbIdFromLocalToFiler(request.Entry)
 		defer wfs.mapPbIdFromFilerToLocal(request.Entry)
 
-		if err := filer_pb.UpdateEntry(context.Background(), client, updateOldEntryRequest); err != nil {
+		updateResp, err := filer_pb.UpdateEntryWithResponse(context.Background(), client, updateOldEntryRequest)
+		if err != nil {
 			return err
 		}
-		// Only update cache if the directory is cached
-		if wfs.metaCache.IsDirectoryCached(util.FullPath(updateOldEntryRequest.Directory)) {
-			if err := wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(updateOldEntryRequest.Directory, updateOldEntryRequest.Entry)); err != nil {
-				return fmt.Errorf("update meta cache for %s: %w", oldEntryPath, err)
-			}
+		updateEvent := updateResp.GetMetadataEvent()
+		if updateEvent == nil {
+			updateEvent = metadataUpdateEvent(oldParentPath, updateOldEntryRequest.Entry)
+		}
+		if applyErr := wfs.applyLocalMetadataEvent(context.Background(), updateEvent); applyErr != nil {
+			glog.Warningf("link %s: best-effort metadata apply failed: %v", oldEntryPath, applyErr)
+			wfs.inodeToPath.InvalidateChildrenCache(util.FullPath(oldParentPath))
 		}
 
-		if err := filer_pb.CreateEntry(context.Background(), client, request); err != nil {
+		createResp, err := filer_pb.CreateEntryWithResponse(context.Background(), client, request)
+		if err != nil {
+			// Rollback: restore original HardLinkId/Counter on the source entry
+			oldEntry.HardLinkId = origHardLinkId
+			oldEntry.HardLinkCounter = origHardLinkCounter
+			rollbackReq := &filer_pb.UpdateEntryRequest{
+				Directory:  oldParentPath,
+				Entry:      oldEntry,
+				Signatures: []int32{wfs.signature},
+			}
+			if _, rollbackErr := filer_pb.UpdateEntryWithResponse(context.Background(), client, rollbackReq); rollbackErr != nil {
+				glog.Warningf("link rollback %s: %v", oldEntryPath, rollbackErr)
+			}
 			return err
 		}
 
-		// Only cache the entry if the parent directory is already cached.
-		if wfs.metaCache.IsDirectoryCached(newParentPath) {
-			if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
-				return fmt.Errorf("insert meta cache for %s: %w", newParentPath.Child(name), err)
-			}
+		createEvent := createResp.GetMetadataEvent()
+		if createEvent == nil {
+			createEvent = metadataCreateEvent(string(newParentPath), request.Entry)
+		}
+		if applyErr := wfs.applyLocalMetadataEvent(context.Background(), createEvent); applyErr != nil {
+			glog.Warningf("link %s: best-effort metadata apply failed: %v", newParentPath.Child(name), applyErr)
+			wfs.inodeToPath.InvalidateChildrenCache(newParentPath)
 		}
 
 		return nil

weed/mount/weedfs_metadata_flush.go

@@ -2,7 +2,6 @@ package mount
 
 import (
 	"context"
-	"fmt"
 	"sync"
 	"time"
 
@@ -142,15 +141,18 @@ func (wfs *WFS) flushFileMetadata(fh *FileHandle) error {
 		wfs.mapPbIdFromLocalToFiler(request.Entry)
 		defer wfs.mapPbIdFromFilerToLocal(request.Entry)
 
-		if err := filer_pb.CreateEntry(context.Background(), client, request); err != nil {
+		resp, err := filer_pb.CreateEntryWithResponse(context.Background(), client, request)
+		if err != nil {
 			return err
 		}
 
-		// Only update cache if the parent directory is cached
-		if wfs.metaCache.IsDirectoryCached(util.FullPath(dir)) {
-			if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
-				return fmt.Errorf("update meta cache for %s: %w", fileFullPath, err)
-			}
+		event := resp.GetMetadataEvent()
+		if event == nil {
+			event = metadataUpdateEvent(string(dir), request.Entry)
+		}
+		if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil {
+			glog.Warningf("flushFileMetadata %s: best-effort metadata apply failed: %v", fileFullPath, applyErr)
+			wfs.inodeToPath.InvalidateChildrenCache(util.FullPath(dir))
 		}
 
 		glog.V(3).Infof("flushed metadata for %s with %d chunks", fileFullPath, len(entry.GetChunks()))

weed/mount/weedfs_rename.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/seaweedfs/go-fuse/v2/fs"
 	"github.com/seaweedfs/go-fuse/v2/fuse"
-	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -233,10 +232,12 @@ func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamR
 	glog.V(4).Infof("dir Rename %+v", resp.EventNotification)
 
 	if resp.EventNotification.NewEntry != nil {
-		// with new entry, the old entry name also exists. This is the first step to create new entry
-		newEntry := filer.FromPbEntry(resp.EventNotification.NewParentPath, resp.EventNotification.NewEntry)
-		if err := wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, "", newEntry); err != nil {
-			return err
+		if err := wfs.applyLocalMetadataEvent(ctx, metadataEventFromRenameResponse(resp)); err != nil {
+			glog.Warningf("rename apply metadata event: %v", err)
+			wfs.inodeToPath.InvalidateChildrenCache(util.FullPath(resp.Directory))
+			if resp.EventNotification.NewParentPath != "" {
+				wfs.inodeToPath.InvalidateChildrenCache(util.FullPath(resp.EventNotification.NewParentPath))
+			}
 		}
 
 		oldParent, newParent := util.FullPath(resp.Directory), util.FullPath(resp.EventNotification.NewParentPath)
@@ -245,14 +246,6 @@ func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamR
 		oldPath := oldParent.Child(oldName)
 		newPath := newParent.Child(newName)
 
-		// Keep the renamed destination immediately readable even when the directory
-		// itself is not marked as fully cached.
-		if !wfs.metaCache.IsDirectoryCached(newParent) {
-			if err := wfs.metaCache.InsertEntry(ctx, newEntry); err != nil {
-				return err
-			}
-		}
-
 		sourceInode, targetInode := wfs.inodeToPath.MovePath(oldPath, newPath)
 		if sourceInode != 0 {
 			fh, foundFh := wfs.fhMap.FindFileHandle(sourceInode)
@@ -271,8 +264,9 @@ func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamR
 
 	} else if resp.EventNotification.OldEntry != nil {
 		// without new entry, only old entry name exists. This is the second step to delete old entry
-		if err := wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath(resp.Directory, resp.EventNotification.OldEntry.Name), nil); err != nil {
-			return err
+		if err := wfs.applyLocalMetadataEvent(ctx, metadataEventFromRenameResponse(resp)); err != nil {
+			glog.Warningf("rename apply delete event: %v", err)
+			wfs.inodeToPath.InvalidateChildrenCache(util.FullPath(resp.Directory))
 		}
 	}
 

weed/mount/weedfs_rename_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
-func TestHandleRenameResponseCachesTargetForUncachedDirectory(t *testing.T) {
+func TestHandleRenameResponseLeavesUncachedTargetOutOfCache(t *testing.T) {
 	uidGidMapper, err := meta_cache.NewUidGidMapper("", "")
 	if err != nil {
 		t.Fatalf("create uid/gid mapper: %v", err)
@@ -73,14 +73,11 @@ func TestHandleRenameResponseCachesTargetForUncachedDirectory(t *testing.T) {
 	}
 
 	entry, findErr := mc.FindEntry(context.Background(), targetPath)
-	if findErr != nil {
-		t.Fatalf("find target entry: %v", findErr)
+	if findErr != filer_pb.ErrNotFound {
+		t.Fatalf("find target entry error = %v, want %v", findErr, filer_pb.ErrNotFound)
 	}
-	if entry == nil {
-		t.Fatalf("target entry %s not cached", targetPath)
-	}
-	if entry.FileSize != 53 {
-		t.Fatalf("cached file size = %d, want 53", entry.FileSize)
+	if entry != nil {
+		t.Fatalf("target entry %s should not be cached for an uncached directory", targetPath)
 	}
 
 	updatedInode, found := inodeToPath.GetInode(targetPath)

weed/mount/weedfs_symlink.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/seaweedfs/go-fuse/v2/fuse"
 
-	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 )
@@ -53,15 +52,18 @@ func (wfs *WFS) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target st
 		wfs.mapPbIdFromLocalToFiler(request.Entry)
 		defer wfs.mapPbIdFromFilerToLocal(request.Entry)
 
-		if err := filer_pb.CreateEntry(context.Background(), client, request); err != nil {
+		resp, err := filer_pb.CreateEntryWithResponse(context.Background(), client, request)
+		if err != nil {
 			return fmt.Errorf("symlink %s: %v", entryFullPath, err)
 		}
 
-		// Only cache the entry if the parent directory is already cached.
-		if wfs.metaCache.IsDirectoryCached(dirPath) {
-			if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
-				return fmt.Errorf("insert meta cache for symlink %s: %w", entryFullPath, err)
-			}
+		event := resp.GetMetadataEvent()
+		if event == nil {
+			event = metadataCreateEvent(string(dirPath), request.Entry)
+		}
+		if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil {
+			glog.Warningf("symlink %s: best-effort metadata apply failed: %v", entryFullPath, applyErr)
+			wfs.inodeToPath.InvalidateChildrenCache(dirPath)
 		}
 
 		return nil

weed/mount/wfs_save.go

@@ -6,7 +6,6 @@ import (
 	"syscall"
 
 	"github.com/seaweedfs/go-fuse/v2/fuse"
-	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -28,13 +27,18 @@ func (wfs *WFS) saveEntry(path util.FullPath, entry *filer_pb.Entry) (code fuse.
 		}
 
 		glog.V(1).Infof("save entry: %v", request)
-		_, err := client.UpdateEntry(context.Background(), request)
+		resp, err := filer_pb.UpdateEntryWithResponse(context.Background(), client, request)
 		if err != nil {
 			return fmt.Errorf("UpdateEntry dir %s: %v", path, err)
 		}
 
-		if err := wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
-			return fmt.Errorf("metaCache.UpdateEntry dir %s: %w", path, err)
+		event := resp.GetMetadataEvent()
+		if event == nil {
+			event = metadataUpdateEvent(parentDir, entry)
+		}
+		if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil {
+			glog.Warningf("saveEntry %s: best-effort metadata apply failed: %v", path, applyErr)
+			wfs.inodeToPath.InvalidateChildrenCache(util.FullPath(parentDir))
 		}
 
 		return nil

weed/mq/broker/broker_grpc_lookup.go

@@ -107,7 +107,7 @@ func (b *MessageQueueBroker) ListTopics(ctx context.Context, request *mq_pb.List
 				return err
 			}
 
-			if !resp.Entry.IsDirectory {
+			if resp.Entry == nil || !resp.Entry.IsDirectory {
 				continue
 			}
 
@@ -135,7 +135,7 @@ func (b *MessageQueueBroker) ListTopics(ctx context.Context, request *mq_pb.List
 					break
 				}
 
-				if !topicResp.Entry.IsDirectory {
+				if topicResp.Entry == nil || !topicResp.Entry.IsDirectory {
 					continue
 				}
 

weed/mq/broker/broker_topic_conf_read_write.go

@@ -252,7 +252,7 @@ func (b *MessageQueueBroker) getOffsetRangeFromChunkMetadata(t topic.Topic, part
 			if err != nil {
 				return err
 			}
-			if resp.Entry.IsDirectory && strings.HasPrefix(resp.Entry.Name, "v") {
+			if resp.Entry != nil && resp.Entry.IsDirectory && strings.HasPrefix(resp.Entry.Name, "v") {
 				if latestVersion == "" || resp.Entry.Name > latestVersion {
 					latestVersion = resp.Entry.Name
 				}
@@ -290,7 +290,7 @@ func (b *MessageQueueBroker) getOffsetRangeFromChunkMetadata(t topic.Topic, part
 			if err != nil {
 				return err
 			}
-			if resp.Entry.IsDirectory && resp.Entry.Name == targetPartitionName {
+			if resp.Entry != nil && resp.Entry.IsDirectory && resp.Entry.Name == targetPartitionName {
 				partitionDir = resp.Entry.Name
 				break
 			}
@@ -327,7 +327,7 @@ func (b *MessageQueueBroker) getOffsetRangeFromChunkMetadata(t topic.Topic, part
 			if err != nil {
 				return err
 			}
-			if !resp.Entry.IsDirectory && resp.Entry.Name != "checkpoint.offset" {
+			if resp.Entry != nil && !resp.Entry.IsDirectory && resp.Entry.Name != "checkpoint.offset" {
 				// Check for offset ranges in Extended attributes (both log files and parquet files)
 				if resp.Entry.Extended != nil {
 					fileType := "log"

weed/mq/kafka/consumer_offset/filer_storage.go

@@ -285,7 +285,7 @@ func (f *FilerStorage) listDirectory(path string) ([]string, error) {
 				return err
 			}
 
-			if resp.Entry.IsDirectory {
+			if resp.Entry != nil && resp.Entry.IsDirectory {
 				entries = append(entries, resp.Entry.Name)
 			}
 		}

weed/mq/kafka/gateway/coordinator_registry.go

@@ -678,7 +678,7 @@ func (cr *CoordinatorRegistry) loadCoordinatorAssignmentWithClient(consumerGroup
 	err := clientAccessor.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		// Load from individual file: /topics/kafka/.meta/coordinators/<consumer-group>_assignments.json
 		fileName := fmt.Sprintf("%s_assignments.json", consumerGroup)
-		data, err := filer.ReadInsideFiler(client, CoordinatorAssignmentsDir, fileName)
+		data, err := filer.ReadInsideFiler(context.Background(), client, CoordinatorAssignmentsDir, fileName)
 		if err != nil {
 			return fmt.Errorf("assignment file not found for group %s: %w", consumerGroup, err)
 		}

weed/mq/kafka/integration/broker_client.go

@@ -232,7 +232,7 @@ func (bc *BrokerClient) getOffsetRangeFromChunkMetadata(topic string, partition
 			if err != nil {
 				return err
 			}
-			if resp.Entry.IsDirectory && strings.HasPrefix(resp.Entry.Name, "v") {
+			if resp.Entry != nil && resp.Entry.IsDirectory && strings.HasPrefix(resp.Entry.Name, "v") {
 				if latestVersion == "" || resp.Entry.Name > latestVersion {
 					latestVersion = resp.Entry.Name
 				}
@@ -267,9 +267,15 @@ func (bc *BrokerClient) getOffsetRangeFromChunkMetadata(topic string, partition
 			if err != nil {
 				return err
 			}
-			if resp.Entry.IsDirectory && strings.Contains(resp.Entry.Name, "-") {
-				partitionDir = resp.Entry.Name
-				break // Use the first partition directory we find
+			if resp.Entry != nil && resp.Entry.IsDirectory && strings.Contains(resp.Entry.Name, "-") {
+				// Parse partition range (format: NNNN-NNNN) and match requested partition
+				var pStart, pStop int32
+				if n, scanErr := fmt.Sscanf(resp.Entry.Name, "%04d-%04d", &pStart, &pStop); n == 2 && scanErr == nil {
+					if partition >= pStart && partition < pStop {
+						partitionDir = resp.Entry.Name
+						break
+					}
+				}
 			}
 		}
 		return nil
@@ -303,7 +309,7 @@ func (bc *BrokerClient) getOffsetRangeFromChunkMetadata(topic string, partition
 			if err != nil {
 				return err
 			}
-			if !resp.Entry.IsDirectory && resp.Entry.Name != "checkpoint.offset" {
+			if resp.Entry != nil && !resp.Entry.IsDirectory && resp.Entry.Name != "checkpoint.offset" {
 				// Check for offset ranges in Extended attributes (both log files and parquet files)
 				if resp.Entry.Extended != nil {
 					// Track maximum offset for high water mark

weed/mq/offset/consumer_group_storage.go

@@ -108,7 +108,7 @@ func (f *FilerConsumerGroupOffsetStorage) LoadConsumerGroupPosition(t topic.Topi
 
 	var position *ConsumerGroupPosition
 	err := f.filerClientAccessor.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		data, err := filer.ReadInsideFiler(client, consumersDir, offsetFileName)
+		data, err := filer.ReadInsideFiler(context.Background(), client, consumersDir, offsetFileName)
 		if err != nil {
 			return err
 		}

weed/mq/offset/filer_storage.go

@@ -1,6 +1,7 @@
 package offset
 
 import (
+	"context"
 	"fmt"
 	"time"
 
@@ -48,7 +49,7 @@ func (f *FilerOffsetStorage) LoadCheckpoint(namespace, topicName string, partiti
 
 	var offset int64 = -1
 	err := f.filerClientAccessor.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		data, err := filer.ReadInsideFiler(client, partitionDir, fileName)
+		data, err := filer.ReadInsideFiler(context.Background(), client, partitionDir, fileName)
 		if err != nil {
 			return err
 		}