filer: drop unused constant in oversized-chunk test

filer: fall back to range fetch when a chunk exceeds the reader budget
A ciphered or compressed chunk larger than readerCacheSizeMB can never be read through the whole-chunk path — the budget rejects the buffer — so its partial views must still range-fetch or the GET fails outright.
2026-10-07 07:06:33 +00:00 · 2026-10-03 15:21:22 +08:00 · 2026-10-03 14:59:36 +08:00 · 2026-10-03 12:34:55 +08:00 · 2026-10-03 09:38:40 +08:00 · 2026-10-03 09:36:59 +08:00
102 changed files with 9612 additions and 1677 deletions
@@ -27,7 +27,7 @@ jobs:

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v4.38.1
+        uses: github/codeql-action/init@v4.38.2
        # Override language selection by uncommenting this and choosing your languages
        with:
          languages: go
@@ -35,7 +35,7 @@ jobs:
      # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
      # If this step fails, then you should remove it and run the build manually (see below).
      - name: Autobuild
-        uses: github/codeql-action/autobuild@v4.38.1
+        uses: github/codeql-action/autobuild@v4.38.2

      # ℹ️ Command-line programs to run using the OS shell.
      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -49,4 +49,4 @@ jobs:
      #     make release

      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v4.38.1
+        uses: github/codeql-action/analyze@v4.38.2
@@ -156,7 +156,7 @@ jobs:
    runs-on: ubuntu-latest
    strategy:
      matrix:
-        platform: [amd64, arm64, arm, 386]
+        platform: [amd64, arm64, arm, 386, ppc64le, s390x]
        variant: ${{ fromJSON(needs.setup.outputs.variants) }}

    steps:
@@ -405,7 +405,7 @@ jobs:
          output: trivy-results.sarif
          exit-code: '0'
      - name: Upload Trivy scan results to GitHub Security
-        uses: github/codeql-action/upload-sarif@v4.38.1
+        uses: github/codeql-action/upload-sarif@v4.38.2
        if: always()
        with:
          sarif_file: trivy-results.sarif
@@ -505,7 +505,9 @@ jobs:
            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
-            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
+            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 \
+            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-ppc64le \
+            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-s390x
          # The copy and the signature below use the digest this run pushed, not whatever the tag points at by then.
          DIGEST=$(jq -er '."containerimage.descriptor".digest' /tmp/manifest.json)
          echo "digest=${DIGEST}" >> "$GITHUB_OUTPUT"
@@ -549,13 +551,15 @@ jobs:
            echo "Using skopeo to copy..."
            retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs@${DIGEST} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
          else
-            echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
+            echo "Using docker buildx imagetools (pulling 6 images from Docker Hub)..."
            # Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
            retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
-              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
+              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 \
+              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-ppc64le \
+              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-s390x
          fi

      - name: Sign
@@ -149,12 +149,16 @@ jobs:
          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/arm64,  arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest,    qemu: true }
          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/386,    arch: i386,  runner: ubuntu-latest,    qemu: false }
+          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/ppc64le, arch: ppc64le, runner: ubuntu-latest,  qemu: true }
+          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/s390x,   arch: s390x, runner: ubuntu-latest,    qemu: true }

          # Large disk - multi-arch
          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64,  arch: amd64, runner: ubuntu-latest,    qemu: false }
          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64,  arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest,    qemu: true }
          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386,    arch: i386,  runner: ubuntu-latest,    qemu: false }
+          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/ppc64le, arch: ppc64le, runner: ubuntu-latest,  qemu: true }
+          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/s390x,   arch: s390x, runner: ubuntu-latest,    qemu: true }

          # Full tags - multi-arch
          - { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest,    qemu: false }
@@ -456,7 +460,7 @@ jobs:

      - name: Upload Trivy scan results to GitHub Security
        if: always()
-        uses: github/codeql-action/upload-sarif@v4.38.1
+        uses: github/codeql-action/upload-sarif@v4.38.2
        with:
          sarif_file: trivy-results.sarif
          category: trivy-${{ matrix.variant }}
@@ -289,6 +289,7 @@ jobs:
          s3tests/functional/test_s3.py::test_object_write_check_etag \
          s3tests/functional/test_s3.py::test_object_write_cache_control \
          s3tests/functional/test_s3.py::test_object_write_expires \
+          s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \
          s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
          s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
          s3tests/functional/test_s3.py::test_object_write_file \
@@ -311,6 +312,7 @@ jobs:
          s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
          s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
          s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+          s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
          s3tests/functional/test_s3.py::test_bucket_head \
          s3tests/functional/test_s3.py::test_bucket_head_notexist \
          s3tests/functional/test_s3.py::test_object_raw_authenticated \
@@ -1149,6 +1151,7 @@ jobs:
          s3tests/functional/test_s3.py::test_object_write_check_etag \
          s3tests/functional/test_s3.py::test_object_write_cache_control \
          s3tests/functional/test_s3.py::test_object_write_expires \
+          s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \
          s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
          s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
          s3tests/functional/test_s3.py::test_object_write_file \
@@ -1171,6 +1174,7 @@ jobs:
          s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
          s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
          s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+          s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
          s3tests/functional/test_s3.py::test_bucket_head \
          s3tests/functional/test_s3.py::test_bucket_head_notexist \
          s3tests/functional/test_s3.py::test_object_raw_authenticated \
@@ -14,7 +14,7 @@ require (
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dustin/go-humanize v1.0.1
+	github.com/dustin/go-humanize v1.1.0
 	github.com/eapache/go-resiliency v1.6.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
@@ -94,7 +94,7 @@ require (
 	golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597
 	golang.org/x/image v0.46.0
 	golang.org/x/net v0.58.0
-	golang.org/x/oauth2 v0.36.0
+	golang.org/x/oauth2 v0.37.0
 	golang.org/x/sys v0.48.0
 	golang.org/x/text v0.42.0 // indirect
 	golang.org/x/tools v0.49.0 // indirect
@@ -111,7 +111,7 @@ require (
 )

 require (
-	cloud.google.com/go/kms v1.34.0
+	cloud.google.com/go/kms v1.35.0
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/Jille/raft-grpc-transport v1.6.1
@@ -154,7 +154,7 @@ require (
 	github.com/tarantool/go-tarantool/v3 v3.0.2
 	github.com/testcontainers/testcontainers-go v0.44.0
 	github.com/tikv/client-go/v2 v2.0.7
-	github.com/twmb/avro v1.8.0
+	github.com/twmb/avro v1.9.0
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1
@@ -302,13 +302,13 @@ require (
 	cloud.google.com/go/iam v1.12.0 // indirect
 	cloud.google.com/go/monitoring v1.30.0 // indirect
 	filippo.io/edwards25519 v1.2.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
 	github.com/Azure/go-ntlmssp v0.1.1 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect
 	github.com/Files-com/files-sdk-go/v3 v3.3.194 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect
@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
 cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
 cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
 cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
-cloud.google.com/go/kms v1.34.0 h1:mxWcXEiyjxwFH5gclulLx+B8Y2OEpKJRZ5FOF78c2XE=
-cloud.google.com/go/kms v1.34.0/go.mod h1:FbxZWUiihmyjxlaBha84OK5+fmJHPrS6F5/mBFdJk6A=
+cloud.google.com/go/kms v1.35.0 h1:nJ/ktaqspx1nPM9vIcO0SHbhqCAm8nvAxL1siuVgKm0=
+cloud.google.com/go/kms v1.35.0/go.mod h1:0++71pIHvJL+GmMa8K4jOWFq7gNOX3jm2PRMSJwTKJw=
 cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
 cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
 cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
@@ -553,10 +553,10 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum
 git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4=
@@ -577,8 +577,8 @@ github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDP
 github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d h1:iG9B49Q218F/XxXNRM7k/vWf7MKmLIS8AcJV9cGN4nA=
@@ -953,8 +953,8 @@ github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.4.0/go.mod h1:gDXhl0OElhzYoDs
 github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
 github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
+github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
 github.com/eapache/go-resiliency v1.6.0 h1:CqGDTLtpwuWKn6Nj3uNUdflaq+/kIPsg0gfNzHton30=
 github.com/eapache/go-resiliency v1.6.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
 github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws=
@@ -1965,8 +1965,8 @@ github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365/go.mod h1:zj0GJHGvyf
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
-github.com/twmb/avro v1.8.0 h1:UMWLg+nH4P3yad5Om7yFSohYLy2RG1s7BcFFiOvmK9Q=
-github.com/twmb/avro v1.8.0/go.mod h1:X0fT1dY2xcbV4YuCE4mYro+qljHl4kUF5uA/2z1rgSk=
+github.com/twmb/avro v1.9.0 h1:JSiqewo3AANj7vlVCQXBsIZTA3LiiBble7Xp8T4lbtA=
+github.com/twmb/avro v1.9.0/go.mod h1:X0fT1dY2xcbV4YuCE4mYro+qljHl4kUF5uA/2z1rgSk=
 github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=
 github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4=
@@ -2354,8 +2354,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
 golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
 golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
-golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
-golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
+golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
+golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -25,6 +25,10 @@ spec:
    organizations:
    - "SeaweedFS CA"
  dnsNames:
+    - '{{ include "seaweedfs.fullname" . }}-admin'
+    - '{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}'
+    - '{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}.svc'
+    - '{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}.svc.cluster.local'
    - '*.{{ include "seaweedfs.fullname" . }}-admin'
    - '*.{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}'
    - '*.{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}.svc'
@@ -168,6 +168,7 @@ message VacuumVolumeCheckRequest {
 }
 message VacuumVolumeCheckResponse {
    double garbage_ratio = 1;
+    bool disk_space_low = 4; // the volume is read-only solely because its disk is low on space — a cause compaction itself reclaims
 }

 message VacuumVolumeCompactRequest {
@@ -950,7 +950,7 @@ async fn proxy_request(
        if name.as_str().eq_ignore_ascii_case("server") {
            continue;
        }
-        response_headers.insert(name.clone(), value.clone());
+        response_headers.append(name.clone(), value.clone());
    }

    // Stream the proxy response body instead of buffering it entirely
@@ -4671,6 +4671,64 @@ mod tests {
        server.abort();
    }

+    /// Repeated target response headers (e.g. Set-Cookie) must all reach the
+    /// client, as Go's `w.Header().Add` does; only `Server` is dropped.
+    #[tokio::test]
+    async fn test_proxy_request_keeps_repeated_response_headers() {
+        use axum::{Router, routing::get};
+
+        let app = Router::new().route(
+            "/3,01637037d6",
+            get(|| async {
+                (
+                    axum::response::AppendHeaders([
+                        (header::SET_COOKIE, "a=1"),
+                        (header::SET_COOKIE, "b=2"),
+                        (header::SERVER, "target"),
+                    ]),
+                    "payload",
+                )
+            }),
+        );
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = listener.local_addr().unwrap();
+        let server = tokio::spawn(async move {
+            axum::serve(listener, app).await.unwrap();
+        });
+
+        let info = ProxyRequestInfo {
+            original_headers: HeaderMap::new(),
+            original_query: String::new(),
+            path: "/3,01637037d6".to_string(),
+            vid_str: "3".to_string(),
+            fid_str: "01637037d6".to_string(),
+        };
+        let target = VolumeLocation {
+            url: addr.to_string(),
+            public_url: String::new(),
+            grpc_port: 0,
+            read_only: false,
+            read_only_can_delete: false,
+        };
+
+        let response = proxy_request(&streaming_test_state(), &info, &target).await;
+        assert_eq!(response.status(), StatusCode::OK);
+        let cookies: Vec<_> = response
+            .headers()
+            .get_all(header::SET_COOKIE)
+            .iter()
+            .map(|v| v.to_str().unwrap().to_string())
+            .collect();
+        assert_eq!(cookies, vec!["a=1", "b=2"]);
+        assert!(response.headers().get(header::SERVER).is_none());
+        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+            .await
+            .unwrap();
+        assert_eq!(&body[..], b"payload");
+
+        server.abort();
+    }
+
    fn streaming_test_state() -> Arc<VolumeServerState> {
        use crate::storage::needle_map::NeedleMapKind;
        use crate::storage::store::Store;
@@ -297,7 +297,7 @@ fn apply_master_volume_options(store: &Store, hb_resp: &master_pb::HeartbeatResp
        volume_opts_changed = true;
    }

-    volume_opts_changed && store.maybe_adjust_volume_max()
+    volume_opts_changed
 }

 type EcShardDeltaKey = (u32, String, u32, u32);
@@ -373,6 +373,22 @@ fn diff_ec_shard_delta_messages(
    (new_ec_shards, deleted_ec_shards)
 }

+/// A volume heartbeat carries no shard list, only the expired shards it
+/// deleted, so that is all it may take out of the delta baseline.
+fn forget_reported_ec_deletions(
+    last_ec_shards: &mut HashMap<EcShardDeltaKey, master_pb::VolumeEcShardInformationMessage>,
+    heartbeat: &master_pb::Heartbeat,
+) {
+    last_ec_shards.retain(|(id, collection, disk_id, shard_id), _| {
+        !heartbeat.deleted_ec_shards.iter().any(|deleted| {
+            deleted.id == *id
+                && deleted.collection == *collection
+                && deleted.disk_id == *disk_id
+                && deleted.ec_index_bits & (1u32 << shard_id) != 0
+        })
+    });
+}
+
 /// Perform one heartbeat session with a master server.
 async fn do_heartbeat(
    config: &HeartbeatConfig,
@@ -393,24 +409,22 @@ async fn do_heartbeat(

    let (tx, rx) = tokio::sync::mpsc::channel::<master_pb::Heartbeat>(32);

-    // This master may know nothing about this server, and has not yet said
-    // whether it understands digests, so start from the whole list.
-    state.store.read().unwrap().volume_report.reset();
-
    // Keep track of what we sent, to generate delta updates
-    let (initial_hb, initial_volumes) =
-        off_runtime(config, state, collect_heartbeat_with_snapshot).await?;
+    let (initial_hb, initial_volumes) = off_runtime(config, state, |config, state| {
+        // This master may know nothing about this server, and has not yet said
+        // whether it understands digests, so start from the whole list.
+        state.store.read().unwrap().volume_report.reset();
+        collect_heartbeat_with_snapshot(config, state)
+    })
+    .await?;
    let mut last_volumes: HashMap<u32, VolumeIdentity> = volume_identities(&initial_volumes);
-    let mut last_ec_shards = {
-        let store = state.store.read().unwrap();
-        collect_ec_shard_delta_messages(&store)
-    };

    // Send initial heartbeats BEFORE calling send_heartbeat to avoid deadlock:
    // the server won't send response headers until it receives the first message,
    // but send_heartbeat().await waits for response headers.
    tx.send(initial_hb).await?;
-    tx.send(collect_ec_heartbeat(config, state)).await?;
+    let (initial_ec_hb, mut last_ec_shards) = ec_tick_pass(config, state).await?;
+    tx.send(initial_ec_hb).await?;

    let stream = tokio_stream::wrappers::ReceiverStream::new(rx);
    let mut response_stream = client.send_heartbeat(stream).await?.into_inner();
@@ -445,10 +459,11 @@ async fn do_heartbeat(
                        // Match Go ordering: DuplicatedUuids first, then volume
                        // options, then leader redirect.
                        if !hb_resp.duplicated_uuids.is_empty() {
-                            let duplicate_dirs = {
-                                let store = state.store.read().unwrap();
-                                duplicate_directories(&store, &hb_resp.duplicated_uuids)
-                            };
+                            let uuids = hb_resp.duplicated_uuids.clone();
+                            let duplicate_dirs = off_runtime(config, state, move |_, state| {
+                                duplicate_directories(&state.store.read().unwrap(), &uuids)
+                            })
+                            .await?;
                            error!(
                                "Master reported duplicate volume directories: {:?}",
                                duplicate_dirs
@@ -459,19 +474,14 @@ async fn do_heartbeat(
                            )
                            .into());
                        }
-                        let changed = {
-                            let s = state.store.read().unwrap();
-                            apply_master_volume_options(&s, &hb_resp)
-                        };
-                        if changed {
+                        let options_changed =
+                            volume_options_pass(config, state, &hb_resp).await?;
+                        if options_changed && maybe_adjust_volume_max(config, state).await? {
                            let (adjusted_hb, adjusted_volumes) =
                                off_runtime(config, state, collect_heartbeat_with_snapshot)
                                    .await?;
                            last_volumes = volume_identities(&adjusted_volumes);
-                            last_ec_shards = {
-                                let store = state.store.read().unwrap();
-                                collect_ec_shard_delta_messages(&store)
-                            };
+                            forget_reported_ec_deletions(&mut last_ec_shards, &adjusted_hb);
                            if tx.send(adjusted_hb).await.is_err() {
                                return Ok(None);
                            }
@@ -496,28 +506,19 @@ async fn do_heartbeat(
            }

            _ = volume_tick.tick() => {
-                {
-                    let s = state.store.read().unwrap();
-                    s.maybe_adjust_volume_max();
-                }
+                maybe_adjust_volume_max(config, state).await?;
                let (current_hb, current_volumes) =
                    off_runtime(config, state, collect_heartbeat_with_snapshot).await?;
                last_volumes = volume_identities(&current_volumes);
-                last_ec_shards = {
-                    let store = state.store.read().unwrap();
-                    collect_ec_shard_delta_messages(&store)
-                };
+                forget_reported_ec_deletions(&mut last_ec_shards, &current_hb);
                if tx.send(current_hb).await.is_err() {
                    return Ok(None);
                }
            }

            _ = ec_tick.tick() => {
-                let current_ec_hb = collect_ec_heartbeat(config, state);
-                last_ec_shards = {
-                    let store = state.store.read().unwrap();
-                    collect_ec_shard_delta_messages(&store)
-                };
+                let (current_ec_hb, current_ec_shards) = ec_tick_pass(config, state).await?;
+                last_ec_shards = current_ec_shards;
                if tx.send(current_ec_hb).await.is_err() {
                    return Ok(None);
                }
@@ -530,12 +531,8 @@ async fn do_heartbeat(
                    info!("Heartbeat stopping");
                    return Ok(None);
                }
-                let held_volumes = off_runtime(config, state, collect_volume_snapshot).await?;
+                let (held_volumes, current_ec_shards) = notify_pass(config, state).await?;
                let current_volumes = volume_identities(&held_volumes);
-                let current_ec_shards = {
-                    let store = state.store.read().unwrap();
-                    collect_ec_shard_delta_messages(&store)
-                };

                let mut new_vols = Vec::new();
                let mut del_vols = Vec::new();
@@ -624,30 +621,41 @@ async fn send_deregister_heartbeat(
    state: &Arc<VolumeServerState>,
    tx: &tokio::sync::mpsc::Sender<master_pb::Heartbeat>,
 ) {
-    let empty = {
-        let store = state.store.read().unwrap();
-        // Deregister: no effective max computed, fall back to configured max.
-        let (location_uuids, disk_tags) = collect_location_metadata(&store, &[]);
-        master_pb::Heartbeat {
-            id: store.id.clone(),
-            ip: config.ip.clone(),
-            port: config.port as u32,
-            public_url: config.public_url.clone(),
-            max_file_key: 0,
-            data_center: config.data_center.clone(),
-            rack: config.rack.clone(),
-            has_no_volumes: true,
-            has_no_ec_shards: true,
-            grpc_port: config.grpc_port as u32,
-            location_uuids,
-            disk_tags,
-            ..Default::default()
+    let empty = match off_runtime(config, state, deregister_heartbeat).await {
+        Ok(empty) => empty,
+        Err(e) => {
+            warn!("Deregistration heartbeat not built: {}", e);
+            return;
        }
    };
    let _ = tx.send(empty).await;
    tokio::time::sleep(Duration::from_millis(200)).await;
 }

+fn deregister_heartbeat(
+    config: &HeartbeatConfig,
+    state: &Arc<VolumeServerState>,
+) -> master_pb::Heartbeat {
+    let store = state.store.read().unwrap();
+    // Deregister: no effective max computed, fall back to configured max.
+    let (location_uuids, disk_tags) = collect_location_metadata(&store, &[]);
+    master_pb::Heartbeat {
+        id: store.id.clone(),
+        ip: config.ip.clone(),
+        port: config.port as u32,
+        public_url: config.public_url.clone(),
+        max_file_key: 0,
+        data_center: config.data_center.clone(),
+        rack: config.rack.clone(),
+        has_no_volumes: true,
+        has_no_ec_shards: true,
+        grpc_port: config.grpc_port as u32,
+        location_uuids,
+        disk_tags,
+        ..Default::default()
+    }
+}
+
 fn apply_metrics_push_settings(
    state: &VolumeServerState,
    address: &str,
@@ -786,16 +794,88 @@ fn volume_identities(
        .collect()
 }

-/// Runs a store pass on the blocking pool: it stats every volume's files.
+/// Runs a store pass on the blocking pool, so waiting for the store lock
+/// parks no runtime worker.
 async fn off_runtime<T: Send + 'static>(
    config: &HeartbeatConfig,
    state: &Arc<VolumeServerState>,
-    pass: fn(&HeartbeatConfig, &Arc<VolumeServerState>) -> T,
+    pass: impl FnOnce(&HeartbeatConfig, &Arc<VolumeServerState>) -> T + Send + 'static,
 ) -> Result<T, tokio::task::JoinError> {
    let (config, state) = (config.clone(), state.clone());
    tokio::task::spawn_blocking(move || pass(&config, &state)).await
 }

+/// Applies the master's volume options; whether they changed.
+async fn volume_options_pass(
+    config: &HeartbeatConfig,
+    state: &Arc<VolumeServerState>,
+    hb_resp: &master_pb::HeartbeatResponse,
+) -> Result<bool, tokio::task::JoinError> {
+    let hb_resp = hb_resp.clone();
+    off_runtime(config, state, move |_, state| {
+        apply_master_volume_options(&state.store.read().unwrap(), &hb_resp)
+    })
+    .await
+}
+
+/// The held volumes and EC shards a state notification is diffed from.
+async fn notify_pass(
+    config: &HeartbeatConfig,
+    state: &Arc<VolumeServerState>,
+) -> Result<
+    (
+        Vec<master_pb::VolumeInformationMessage>,
+        HashMap<EcShardDeltaKey, master_pb::VolumeEcShardInformationMessage>,
+    ),
+    tokio::task::JoinError,
+> {
+    off_runtime(config, state, |config, state| {
+        let volumes = collect_volume_snapshot(config, state);
+        #[cfg(test)]
+        {
+            let store_id = state.store.read().unwrap().id.clone();
+            read_phase_hook::park(read_phase_hook::Point::BeforeNotifyEcRead, &store_id);
+        }
+        let shards = collect_ec_shard_delta_messages(&state.store.read().unwrap());
+        (volumes, shards)
+    })
+    .await
+}
+
+/// Go's MaybeAdjustVolumeMax: statvfs on every auto-sized disk, a stat per
+/// writable volume.
+async fn maybe_adjust_volume_max(
+    config: &HeartbeatConfig,
+    state: &Arc<VolumeServerState>,
+) -> Result<bool, tokio::task::JoinError> {
+    off_runtime(config, state, |_, state| {
+        state.store.read().unwrap().maybe_adjust_volume_max()
+    })
+    .await
+}
+
+/// The EC heartbeat and the shard list later deltas are diffed against,
+/// under one guard so no mount lands between them unreported.
+async fn ec_tick_pass(
+    config: &HeartbeatConfig,
+    state: &Arc<VolumeServerState>,
+) -> Result<
+    (
+        master_pb::Heartbeat,
+        HashMap<EcShardDeltaKey, master_pb::VolumeEcShardInformationMessage>,
+    ),
+    tokio::task::JoinError,
+> {
+    off_runtime(config, state, |config, state| {
+        let store = state.store.read().unwrap();
+        (
+            collect_ec_heartbeat(config, &store),
+            collect_ec_shard_delta_messages(&store),
+        )
+    })
+    .await
+}
+
 /// Collect volume information into a Heartbeat message.
 fn collect_heartbeat_with_snapshot(
    config: &HeartbeatConfig,
@@ -906,6 +986,8 @@ mod read_phase_hook {
        BeforeVolumePass,
        /// Inside the volume pass, holding the store read lock.
        VolumePass,
+        /// Between the notify branch's volume snapshot and its EC read.
+        BeforeNotifyEcRead,
    }

    type Park = (Point, String, Sender<()>, Receiver<()>);
@@ -1330,12 +1412,8 @@ fn collect_live_ec_shards(
 }

 /// Collect EC shard information into a Heartbeat message.
-fn collect_ec_heartbeat(
-    config: &HeartbeatConfig,
-    state: &Arc<VolumeServerState>,
-) -> master_pb::Heartbeat {
-    let store = state.store.read().unwrap();
-    let ec_shards = collect_live_ec_shards(&store, true);
+fn collect_ec_heartbeat(config: &HeartbeatConfig, store: &Store) -> master_pb::Heartbeat {
+    let ec_shards = collect_live_ec_shards(store, true);

    let has_no = ec_shards.is_empty();
    master_pb::Heartbeat {
@@ -1984,7 +2062,7 @@ mod tests {
            .unwrap();

        let state = test_state_with_store(store);
-        let heartbeat = collect_ec_heartbeat(&test_config(), &state);
+        let heartbeat = collect_ec_heartbeat(&test_config(), &state.store.read().unwrap());

        assert_eq!(heartbeat.ec_shards.len(), 1);
        assert!(!heartbeat.has_no_ec_shards);
@@ -2177,6 +2255,225 @@ mod tests {
        assert_eq!(volumes.len(), 2);
    }

+    /// Holds the store write lock on another thread until `release` is
+    /// dropped, or for 3s so a parked runtime fails the test instead of
+    /// hanging it. The flag turns true just before the lock is let go.
+    fn hold_store_write_lock(
+        state: &Arc<VolumeServerState>,
+    ) -> (
+        std::sync::mpsc::Sender<()>,
+        Arc<std::sync::atomic::AtomicBool>,
+        std::thread::JoinHandle<()>,
+    ) {
+        let (held_tx, held_rx) = std::sync::mpsc::channel();
+        let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
+        let released = Arc::new(std::sync::atomic::AtomicBool::new(false));
+        let writer = {
+            let (state, released) = (state.clone(), released.clone());
+            std::thread::spawn(move || {
+                let guard = state.store.write().unwrap();
+                held_tx.send(()).unwrap();
+                let _ = release_rx.recv_timeout(Duration::from_secs(3));
+                released.store(true, Ordering::SeqCst);
+                drop(guard);
+            })
+        };
+        held_rx.recv().unwrap();
+        (release_tx, released, writer)
+    }
+
+    // The heartbeat task shares a worker with other tasks; on this
+    // single-threaded runtime, a pass that waits for the store on the worker
+    // stops everything else until the writer lets go.
+    #[tokio::test]
+    async fn test_ec_tick_pass_waits_for_the_store_off_the_runtime() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let dir = temp_dir.path().to_str().unwrap();
+        let mut store = reporting_store(dir, 0);
+        std::fs::write(format!("{}/ec_tick_off_runtime_41.ec00", dir), b"shard").unwrap();
+        std::fs::write(format!("{}/ec_tick_off_runtime_41.ecx", dir), [0u8; 16]).unwrap();
+        store.locations[0]
+            .mount_ec_shards(VolumeId(41), "ec_tick_off_runtime", &[0], "")
+            .unwrap();
+        let state = test_state_with_store(store);
+
+        let (release, released, writer) = hold_store_write_lock(&state);
+        let pass = {
+            let state = state.clone();
+            tokio::spawn(async move { ec_tick_pass(&test_config(), &state).await })
+        };
+        tokio::time::sleep(Duration::from_millis(100)).await;
+        let ran_while_held = !released.load(Ordering::SeqCst);
+        drop(release);
+        writer.join().unwrap();
+        let (heartbeat, shards) = pass.await.unwrap().unwrap();
+
+        assert!(
+            ran_while_held,
+            "the EC pass parked the runtime on the store lock"
+        );
+        assert_eq!(heartbeat.ec_shards.len(), 1);
+        assert_eq!(shards.len(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_volume_max_adjustment_waits_for_the_store_off_the_runtime() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let mut store = reporting_store(temp_dir.path().to_str().unwrap(), 1);
+        // Auto-sized, so the adjustment stats the disk and the volume.
+        store.locations[0].original_max_volume_count = 0;
+        store.locations[0]
+            .max_volume_count
+            .store(0, Ordering::Relaxed);
+        store
+            .volume_size_limit
+            .store(1024 * 1024, Ordering::Relaxed);
+        let state = test_state_with_store(store);
+
+        let (release, released, writer) = hold_store_write_lock(&state);
+        let adjust = {
+            let state = state.clone();
+            tokio::spawn(async move { maybe_adjust_volume_max(&test_config(), &state).await })
+        };
+        tokio::time::sleep(Duration::from_millis(100)).await;
+        let ran_while_held = !released.load(Ordering::SeqCst);
+        drop(release);
+        writer.join().unwrap();
+        let changed = adjust.await.unwrap().unwrap();
+
+        assert!(
+            ran_while_held,
+            "the adjustment parked the runtime on the store lock"
+        );
+        assert!(changed);
+        assert!(
+            state.store.read().unwrap().locations[0]
+                .max_volume_count
+                .load(Ordering::Relaxed)
+                >= 1
+        );
+    }
+
+    #[tokio::test]
+    async fn test_volume_options_wait_for_the_store_off_the_runtime() {
+        let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
+        state
+            .store
+            .read()
+            .unwrap()
+            .volume_size_limit
+            .store(1024, Ordering::Relaxed);
+        let hb_resp = master_pb::HeartbeatResponse {
+            volume_size_limit: 2048,
+            preallocate: true,
+            ..Default::default()
+        };
+
+        let (release, released, writer) = hold_store_write_lock(&state);
+        let apply = {
+            let state = state.clone();
+            tokio::spawn(async move { volume_options_pass(&test_config(), &state, &hb_resp).await })
+        };
+        tokio::time::sleep(Duration::from_millis(100)).await;
+        let ran_while_held = !released.load(Ordering::SeqCst);
+        drop(release);
+        writer.join().unwrap();
+        let changed = apply.await.unwrap().unwrap();
+
+        assert!(
+            ran_while_held,
+            "applying volume options parked the runtime on the store lock"
+        );
+        assert!(changed);
+        let store = state.store.read().unwrap();
+        assert!(store.get_preallocate());
+        assert_eq!(store.volume_size_limit.load(Ordering::Relaxed), 2048);
+    }
+
+    // The writer arrives between the volume snapshot and the EC read.
+    #[tokio::test]
+    async fn test_state_notification_waits_for_the_store_off_the_runtime() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let dir = temp_dir.path().to_str().unwrap();
+        let mut store = reporting_store(dir, 1);
+        store.id = "notify-off-runtime".to_string();
+        let state = test_state_with_store(store);
+        mount_test_ec_shard(&state, dir, "notify_off_runtime", 42);
+
+        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
+        let (resume_tx, resume_rx) = std::sync::mpsc::channel::<()>();
+        read_phase_hook::arm(
+            read_phase_hook::Point::BeforeNotifyEcRead,
+            "notify-off-runtime",
+            entered_tx,
+            resume_rx,
+        );
+        let (held_tx, held_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
+        let released = Arc::new(std::sync::atomic::AtomicBool::new(false));
+        let writer = {
+            let (state, released) = (state.clone(), released.clone());
+            std::thread::spawn(move || {
+                entered_rx.blocking_recv().unwrap();
+                let guard = state.store.write().unwrap();
+                drop(resume_tx);
+                held_tx.send(()).unwrap();
+                let _ = release_rx.recv_timeout(Duration::from_secs(3));
+                released.store(true, Ordering::SeqCst);
+                drop(guard);
+            })
+        };
+        let pass = {
+            let state = state.clone();
+            tokio::spawn(async move { notify_pass(&test_config(), &state).await })
+        };
+        tokio::time::timeout(Duration::from_secs(10), held_rx)
+            .await
+            .expect("the notify pass never reached its EC read")
+            .unwrap();
+        tokio::time::sleep(Duration::from_millis(100)).await;
+        let ran_while_held = !released.load(Ordering::SeqCst);
+        drop(release_tx);
+        writer.join().unwrap();
+        let (volumes, shards) = pass.await.unwrap().unwrap();
+
+        assert!(
+            ran_while_held,
+            "the notify pass parked the runtime on the store lock"
+        );
+        assert_eq!(volumes.len(), 1);
+        assert_eq!(shards.len(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_deregister_heartbeat_waits_for_the_store_off_the_runtime() {
+        let mut store = Store::new(NeedleMapKind::InMemory);
+        store.id = "deregister-off-runtime".to_string();
+        let state = test_state_with_store(store);
+        let (tx, mut rx) = tokio::sync::mpsc::channel(1);
+
+        let (release, released, writer) = hold_store_write_lock(&state);
+        let deregister = {
+            let state = state.clone();
+            tokio::spawn(
+                async move { send_deregister_heartbeat(&test_config(), &state, &tx).await },
+            )
+        };
+        tokio::time::sleep(Duration::from_millis(100)).await;
+        let ran_while_held = !released.load(Ordering::SeqCst);
+        drop(release);
+        writer.join().unwrap();
+        deregister.await.unwrap();
+        let heartbeat = rx.recv().await.unwrap();
+
+        assert!(
+            ran_while_held,
+            "the deregistration heartbeat parked the runtime on the store lock"
+        );
+        assert_eq!(heartbeat.id, "deregister-off-runtime");
+        assert!(heartbeat.has_no_volumes && heartbeat.has_no_ec_shards);
+    }
+
    // What the read pass decided on can go stale before the write lock is
    // taken: each action must re-check its target, not act on whatever now
    // holds the id.
@@ -2321,6 +2618,96 @@ mod tests {
        );
    }

+    fn mount_test_ec_shard(state: &VolumeServerState, dir: &str, collection: &str, id: u32) {
+        std::fs::write(format!("{}/{}_{}.ec00", dir, collection, id), b"shard").unwrap();
+        std::fs::write(format!("{}/{}_{}.ecx", dir, collection, id), [0u8; 16]).unwrap();
+        state.store.write().unwrap().locations[0]
+            .mount_ec_shards(VolumeId(id), collection, &[0], "")
+            .unwrap();
+    }
+
+    /// One pass of the loop's volume tick, then of its notify branch: the
+    /// EC delta the notify branch would send.
+    fn ec_delta_after_volume_tick(
+        state: &Arc<VolumeServerState>,
+        last_ec_shards: &mut HashMap<EcShardDeltaKey, master_pb::VolumeEcShardInformationMessage>,
+    ) -> (
+        master_pb::Heartbeat,
+        Vec<master_pb::VolumeEcShardInformationMessage>,
+        Vec<master_pb::VolumeEcShardInformationMessage>,
+    ) {
+        let (heartbeat, _) = collect_heartbeat_with_snapshot(&test_config(), state);
+        forget_reported_ec_deletions(last_ec_shards, &heartbeat);
+        let current = collect_ec_shard_delta_messages(&state.store.read().unwrap());
+        let (new_ec_shards, deleted_ec_shards) =
+            diff_ec_shard_delta_messages(last_ec_shards, &current);
+        (heartbeat, new_ec_shards, deleted_ec_shards)
+    }
+
+    // A volume heartbeat carries no shard list, so a mount or unmount it
+    // collects past must still go out as the notify branch's delta.
+    #[test]
+    fn test_ec_shard_change_before_a_volume_heartbeat_still_goes_out_as_a_delta() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let dir = temp_dir.path().to_str().unwrap();
+        let state = test_state_with_store(reporting_store(dir, 1));
+        mount_test_ec_shard(&state, dir, "ec_delta_kept", 91);
+        mount_test_ec_shard(&state, dir, "ec_delta_unmounted", 92);
+        let mut last_ec_shards = collect_ec_shard_delta_messages(&state.store.read().unwrap());
+
+        mount_test_ec_shard(&state, dir, "ec_delta_mounted", 93);
+        state
+            .store
+            .write()
+            .unwrap()
+            .unmount_ec_shards(VolumeId(92), &[0]);
+        let (heartbeat, new_ec_shards, deleted_ec_shards) =
+            ec_delta_after_volume_tick(&state, &mut last_ec_shards);
+
+        assert!(heartbeat.ec_shards.is_empty() && !heartbeat.has_no_ec_shards);
+        assert_eq!(
+            new_ec_shards.iter().map(|s| s.id).collect::<Vec<_>>(),
+            vec![93],
+            "a mount collected past by the volume heartbeat was never sent"
+        );
+        assert_eq!(
+            deleted_ec_shards.iter().map(|s| s.id).collect::<Vec<_>>(),
+            vec![92],
+            "an unmount collected past by the volume heartbeat was never sent"
+        );
+    }
+
+    // The volume heartbeat already told the master about the expired EC
+    // volume it destroyed; the next delta must not repeat it.
+    #[test]
+    fn test_ec_volume_expired_by_a_volume_heartbeat_is_not_deleted_again() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let dir = temp_dir.path().to_str().unwrap();
+        let state = test_state_with_store(reporting_store(dir, 0));
+        mount_test_ec_shard(&state, dir, "ec_delta_kept", 94);
+        mount_test_ec_shard(&state, dir, "ec_delta_expired", 95);
+        let mut last_ec_shards = collect_ec_shard_delta_messages(&state.store.read().unwrap());
+        state
+            .store
+            .write()
+            .unwrap()
+            .find_ec_volume_mut(VolumeId(95))
+            .unwrap()
+            .expire_at_sec = 1;
+
+        let (heartbeat, new_ec_shards, deleted_ec_shards) =
+            ec_delta_after_volume_tick(&state, &mut last_ec_shards);
+
+        assert_eq!(heartbeat.deleted_ec_shards.len(), 1);
+        assert_eq!(heartbeat.deleted_ec_shards[0].id, 95);
+        assert!(new_ec_shards.is_empty());
+        assert!(
+            deleted_ec_shards.is_empty(),
+            "an expired EC volume was reported deleted twice"
+        );
+        assert_eq!(last_ec_shards.len(), 1);
+    }
+
    #[test]
    fn test_expired_ec_volume_gone_before_removal_is_not_reported_deleted() {
        let temp_dir = tempfile::tempdir().unwrap();
@@ -2532,7 +2919,7 @@ mod tests {

        assert!(store.get_preallocate());
        assert_eq!(store.volume_size_limit.load(Ordering::Relaxed), 2048);
-        assert!(!changed);
+        assert!(changed);
    }

    #[test]
@@ -27,7 +27,9 @@ impl From<VolumeError> for Status {
            VolumeError::NotFound
            | VolumeError::VolumeNotFound(_)
            | VolumeError::Tier(TierError::NotFound(_)) => Status::not_found(message),
-            VolumeError::ReadOnly | VolumeError::NotEmpty => Status::failed_precondition(message),
+            VolumeError::ReadOnly(_) | VolumeError::NotEmpty => {
+                Status::failed_precondition(message)
+            }
            VolumeError::InsufficientSpace { .. } => Status::resource_exhausted(message),
            VolumeError::AlreadyExists => Status::already_exists(message),
            _ => Status::internal(message),
@@ -69,7 +71,14 @@ mod tests {
            Code::NotFound
        );
        assert_eq!(code(VolumeError::NotFound), Code::NotFound);
-        assert_eq!(code(VolumeError::ReadOnly), Code::FailedPrecondition);
+        assert_eq!(
+            code(VolumeError::ReadOnly(VolumeId(7))),
+            Code::FailedPrecondition
+        );
+        assert_eq!(
+            VolumeError::ReadOnly(VolumeId(7)).to_string(),
+            "volume 7 is read only"
+        );
        assert_eq!(
            code(VolumeError::InsufficientSpace {
                vid: VolumeId(7),
@@ -95,20 +95,41 @@ struct Snapshot {
    encode_ts_ns: i64,
 }

-/// Top-level entry point. Returns `Ok(None)` for "not found" (matches
-/// Go's `ReadEcShardNeedle`); errors propagate as `io::Error`.
+/// Why a distributed EC read has no needle to return.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EcMiss {
+    NotFound,
+    /// Tombstoned in the local `.ecx`/`.ecj`, or reported deleted by a peer.
+    Deleted,
+    VolumeNotFound,
+}
+
+/// Top-level entry point. Returns `Ok(None)` for any miss — absent,
+/// deleted, or volume gone; errors propagate as `io::Error`.
 pub async fn read_ec_shard_needle_distributed(
    state: &Arc<VolumeServerState>,
    vid: VolumeId,
    needle_id: NeedleId,
 ) -> io::Result<Option<Needle>> {
+    Ok(read_ec_shard_needle_or_miss(state, vid, needle_id)
+        .await?
+        .ok())
+}
+
+/// Like `read_ec_shard_needle_distributed`, but says why there is no needle,
+/// as Go's `ReadEcShardNeedle` tells `ErrorDeleted` from not-found.
+pub async fn read_ec_shard_needle_or_miss(
+    state: &Arc<VolumeServerState>,
+    vid: VolumeId,
+    needle_id: NeedleId,
+) -> io::Result<Result<Needle, EcMiss>> {
    // Phase A — under the Store read lock, locate the needle, compute
    // intervals, and read any locally-mounted shard intervals. We must
    // not `.await` while holding this guard (std::sync::RwLockReadGuard
    // is !Send).
    let mut snapshot = match snapshot_under_lock(state, vid, needle_id)? {
-        Some(s) => s,
-        None => return Ok(None),
+        Ok(s) => s,
+        Err(miss) => return Ok(Err(miss)),
    };

    // Phase B — refresh the shard_locations cache from the master if
@@ -210,17 +231,11 @@ pub async fn read_ec_shard_needle_distributed(
        .collect()
        .await;

-    let mut assembled: Vec<Vec<u8>> = Vec::with_capacity(fetched.len());
-    for res in fetched {
-        let (buf, is_deleted) = res?;
-        // A peer reports the needle deleted (a cross-server window where the
-        // local index still shows it live): treat as not-found rather than
-        // serving zeros, mirroring Go's ErrorDeleted.
-        if is_deleted {
-            return Ok(None);
-        }
-        assembled.push(buf);
-    }
+    // A peer reports the needle deleted (a cross-server window where the
+    // local index still shows it live): answer deleted rather than serving zeros.
+    let Some(assembled) = gather_intervals(fetched)? else {
+        return Ok(Err(EcMiss::Deleted));
+    };

    // Phase D — assemble and parse the Needle. Mirrors the tail of
    // `EcVolume::read_ec_shard_needle`.
@@ -252,7 +267,20 @@ pub async fn read_ec_shard_needle_distributed(
        snapshot.version,
    )
    .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{}", e)))?;
-    Ok(Some(n))
+    Ok(Ok(n))
+}
+
+/// `None` when any holder reported the needle deleted. That outranks another
+/// interval's error: deletes are never invented, so the needle is gone either way.
+fn gather_intervals(fetched: Vec<io::Result<(Vec<u8>, bool)>>) -> io::Result<Option<Vec<Vec<u8>>>> {
+    if fetched.iter().any(|r| matches!(r, Ok((_, true)))) {
+        return Ok(None);
+    }
+    fetched
+        .into_iter()
+        .map(|r| r.map(|(buf, _)| buf))
+        .collect::<io::Result<_>>()
+        .map(Some)
 }

 /// What one EC delete RPC carries — `VolumeEcBlobDeleteRequest` minus tonic.
@@ -864,11 +892,10 @@ fn snapshot_under_lock(
    state: &Arc<VolumeServerState>,
    vid: VolumeId,
    needle_id: NeedleId,
-) -> io::Result<Option<Snapshot>> {
+) -> io::Result<Result<Snapshot, EcMiss>> {
    let store = state.store.read().unwrap();
-    let ecv = match store.find_ec_volume(vid) {
-        Some(v) => v,
-        None => return Ok(None),
+    let Some(ecv) = store.find_ec_volume(vid) else {
+        return Ok(Err(EcMiss::VolumeNotFound));
    };

    // Reuse EcVolume::locate_needle for offset/size resolution AND
@@ -876,11 +903,17 @@ fn snapshot_under_lock(
    // local-only read path uses, so we stay byte-identical on the
    // shard-size + interval boundaries. locate_needle applies the runtime
    // delete mask, which is correct for serving reads.
-    let (offset, size, intervals) = match ecv.locate_needle(needle_id)? {
-        Some(v) => v,
-        None => return Ok(None),
+    let Some((offset, size, intervals)) = ecv.locate_needle(needle_id)? else {
+        // locate_needle folds a tombstone into not-found.
+        let deleted =
+            matches!(ecv.find_needle_from_ecx(needle_id)?, Some((_, s)) if s.is_deleted());
+        return Ok(Err(if deleted {
+            EcMiss::Deleted
+        } else {
+            EcMiss::NotFound
+        }));
    };
-    build_snapshot(ecv, offset, size, &intervals).map(Some)
+    build_snapshot(ecv, offset, size, &intervals).map(Ok)
 }

 /// Like `snapshot_under_lock`, but locates intervals from the RAW .ecx
@@ -1790,6 +1823,21 @@ async fn drain_copy_stream(
 mod tests {
    use super::*;

+    #[test]
+    fn gather_intervals_puts_a_reported_deletion_ahead_of_errors() {
+        let failed = || Err(io::Error::other("shard unreachable"));
+        assert!(
+            gather_intervals(vec![failed(), Ok((Vec::new(), true))])
+                .unwrap()
+                .is_none()
+        );
+        assert!(gather_intervals(vec![failed(), Ok((vec![1], false))]).is_err());
+        assert_eq!(
+            gather_intervals(vec![Ok((vec![1], false)), Ok((vec![2], false))]).unwrap(),
+            Some(vec![vec![1], vec![2]])
+        );
+    }
+
    fn locations(count: usize) -> HashMap<ShardId, Vec<String>> {
        (0..count)
            .map(|sid| (sid as ShardId, vec!["127.0.0.1:8080".to_string()]))
@@ -3,8 +3,8 @@
 //! Instead of each upload handler directly calling `write_needle`, writes are
 //! submitted to a queue. A background worker drains the queue in batches (up to
 //! 128 entries), groups them by volume ID, and processes them together under a
-//! single store lock. Requests that asked for `fsync` are flushed by
-//! `write_needle` itself, one flush per durable write.
+//! single store lock. Durable writes to a volume share their .dat and .idx
+//! flushes (see `Volume::write_needles_grouped`).

 use std::sync::Arc;

@@ -159,8 +159,12 @@ fn process_batch(state: Arc<VolumeServerState>, batch: Vec<WriteRequest>) {
    let mut store = state.store.write().unwrap();

    for (vid, entries) in groups {
-        for (mut needle, fsync, response_tx) in entries {
-            let result = store.write_volume_needle(vid, &mut needle, fsync);
+        let (mut writes, senders): (Vec<_>, Vec<_>) = entries
+            .into_iter()
+            .map(|(needle, fsync, response_tx)| ((needle, fsync), response_tx))
+            .unzip();
+        let results = store.write_volume_needles(vid, &mut writes);
+        for (response_tx, result) in senders.into_iter().zip(results) {
            // Send result back; ignore error if receiver dropped.
            let _ = response_tx.send(result);
        }
@@ -312,6 +316,63 @@ mod tests {
        }
    }

+    /// The queue hands a volume's batch to the grouped path, so ten durable
+    /// writes cost one .dat sync and one .idx sync, not ten of each.
+    #[test]
+    fn test_process_batch_group_commits_fsync_writes() {
+        use crate::config::MinFreeSpace;
+        use crate::storage::types::{DiskType, NeedleId};
+        use crate::storage::volume::VolumeSpec;
+
+        let tmp = tempfile::TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let state = make_test_state();
+        {
+            let mut store = state.store.write().unwrap();
+            store
+                .add_location(
+                    dir,
+                    dir,
+                    10,
+                    DiskType::HardDrive,
+                    MinFreeSpace::Percent(1.0),
+                    Vec::new(),
+                )
+                .unwrap();
+            store
+                .add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
+                .unwrap();
+        }
+
+        let mut receivers = Vec::new();
+        let batch = (1..=10u64)
+            .map(|id| {
+                let (response_tx, response_rx) = oneshot::channel();
+                receivers.push(response_rx);
+                WriteRequest {
+                    volume_id: VolumeId(1),
+                    needle: Needle {
+                        id: NeedleId(id),
+                        cookie: 0x1111.into(),
+                        data: vec![id as u8; 8],
+                        data_size: 8,
+                        ..Needle::default()
+                    },
+                    fsync: true,
+                    response_tx,
+                }
+            })
+            .collect();
+        process_batch(state.clone(), batch);
+
+        for mut rx in receivers {
+            assert!(matches!(rx.try_recv().unwrap(), Ok((_, _, false))));
+        }
+        let store = state.store.read().unwrap();
+        let (_, vol) = store.find_volume(VolumeId(1)).unwrap();
+        assert_eq!(vol.sync_counts_for_test(), (1, 1));
+    }
+
    #[tokio::test]
    async fn test_write_queue_dropped_sender() {
        // When the queue is dropped, subsequent submits should fail gracefully.
@@ -3,10 +3,12 @@
 //! Rebuilds the original .dat + .idx files from data shards (.ec00-.ec09)
 //! and the sorted index (.ecx) + deletion journal (.ecj).

+use std::collections::HashSet;
 use std::fs::File;
 use std::io::{self, Read, Write};

 use crate::storage::erasure_coding::ec_shard::*;
+use crate::storage::erasure_coding::ec_volume::read_ecj_ids;
 use crate::storage::idx;
 use crate::storage::needle::needle::get_actual_size;
 use crate::storage::super_block::SUPER_BLOCK_SIZE;
@@ -20,18 +22,21 @@ use crate::storage::volume::{fsync_dir, volume_file_name};
 /// `dir` is used both for reading `.ec00` and `.ecx`. For split-disk
 /// reconciled volumes call [`find_dat_file_size_with_dirs`] instead.
 pub fn find_dat_file_size(dir: &str, collection: &str, volume_id: VolumeId) -> io::Result<i64> {
-    find_dat_file_size_with_dirs(dir, dir, collection, volume_id)
+    let deleted = read_ecj_deletions(&[dir], collection, volume_id)?;
+    find_dat_file_size_with_dirs(dir, dir, collection, volume_id, &deleted)
 }

 /// Like [`find_dat_file_size`] but lets the caller pass separate dirs
 /// for `.ec00` (the data shard) and `.ecx` (the sealed index). This
 /// is the form needed when shards are split across data dirs and the
-/// `.ecx` lives on a sibling disk's idx dir (#9252).
+/// `.ecx` lives on a sibling disk's idx dir (#9252). Needles in `deleted`
+/// count as deleted.
 pub fn find_dat_file_size_with_dirs(
    ec00_dir: &str,
    ecx_dir: &str,
    collection: &str,
    volume_id: VolumeId,
+    deleted: &HashSet<NeedleId>,
 ) -> io::Result<i64> {
    let ec00_base = volume_file_name(ec00_dir, collection, volume_id);
    let ecx_base = volume_file_name(ecx_dir, collection, volume_id);
@@ -53,9 +58,9 @@ pub fn find_dat_file_size_with_dirs(

    for i in 0..entry_count {
        let start = i * NEEDLE_MAP_ENTRY_SIZE;
-        let (_, offset, size) =
+        let (key, offset, size) =
            idx_entry_from_bytes(&ecx_data[start..start + NEEDLE_MAP_ENTRY_SIZE]);
-        if size.is_deleted() {
+        if size.is_deleted() || deleted.contains(&key) {
            continue;
        }
        let entry_stop = offset.to_actual_offset() + get_actual_size(size, version);
@@ -67,6 +72,48 @@ pub fn find_dat_file_size_with_dirs(
    Ok(dat_size)
 }

+/// Whether the `.ecx` in `ecx_dir` indexes a needle deleted neither there nor
+/// in `deleted`.
+pub fn has_live_needles(
+    ecx_dir: &str,
+    collection: &str,
+    volume_id: VolumeId,
+    deleted: &HashSet<NeedleId>,
+) -> io::Result<bool> {
+    let ecx_base = volume_file_name(ecx_dir, collection, volume_id);
+    let ecx_data = std::fs::read(format!("{}.ecx", ecx_base))?;
+    let (entries, _) = ecx_data.as_chunks::<NEEDLE_MAP_ENTRY_SIZE>();
+    Ok(entries.iter().any(|entry| {
+        let (key, _, size) = idx_entry_from_bytes(entry);
+        !size.is_deleted() && !deleted.contains(&key)
+    }))
+}
+
+/// Distinct needle ids journaled in the `.ecj` of any of `dirs`. Go folds the
+/// journal into the `.ecx` (RebuildEcxFile) before a decode; reading it leaves
+/// the sealed index untouched. Only NotFound means "no journal".
+pub fn read_ecj_deletions(
+    dirs: &[&str],
+    collection: &str,
+    volume_id: VolumeId,
+) -> io::Result<HashSet<NeedleId>> {
+    let mut ids = HashSet::new();
+    for (i, dir) in dirs.iter().enumerate() {
+        if dirs[..i].contains(dir) {
+            continue;
+        }
+        let path = format!("{}.ecj", volume_file_name(dir, collection, volume_id));
+        let file = match File::open(&path) {
+            Ok(file) => file,
+            Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
+            Err(e) => return Err(e),
+        };
+        let len = file.metadata()?.len();
+        read_ecj_ids(&file, len, &mut ids)?;
+    }
+    Ok(ids)
+}
+
 /// What it takes to rebuild a volume's .dat from its EC data shards.
 ///
 /// Mirrors Go's `WriteDatFile(baseFileName, datFileSize,
@@ -267,51 +314,61 @@ pub fn write_dat_file_from_shards(spec: &DatRebuild<'_>) -> io::Result<()> {

 /// Write .idx file from .ecx index + .ecj deletion journal.
 ///
-/// Copies sorted .ecx entries to .idx, then appends tombstones for
-/// deleted needles from .ecj.
+/// See [`write_idx_file_from_ec_index_with_dirs`]; everything lives in `dir`.
 pub fn write_idx_file_from_ec_index(
    dir: &str,
    collection: &str,
    volume_id: VolumeId,
 ) -> io::Result<()> {
-    let base = volume_file_name(dir, collection, volume_id);
-    let ecx_path = format!("{}.ecx", base);
-    let ecj_path = format!("{}.ecj", base);
-    let idx_path = format!("{}.idx", base);
+    let deleted = read_ecj_deletions(&[dir], collection, volume_id)?;
+    let dat_file_size = find_dat_file_size_with_dirs(dir, dir, collection, volume_id, &deleted)?;
+    write_idx_file_from_ec_index_with_dirs(dir, dir, collection, volume_id, &deleted, dat_file_size)
+}
+
+/// Write the `.idx` for a `.dat` decoded to `dat_file_size` bytes, from the
+/// `.ecx` in `ecx_dir`, into `idx_dir`.
+///
+/// Copies the `.ecx` rows, then appends one tombstone per row whose needle is
+/// in `deleted`. A deleted needle at or past `dat_file_size` was cut from the
+/// `.dat`, so its row is dropped: a row pointing past the end of the `.dat`
+/// makes the volume load read-only.
+pub fn write_idx_file_from_ec_index_with_dirs(
+    ecx_dir: &str,
+    idx_dir: &str,
+    collection: &str,
+    volume_id: VolumeId,
+    deleted: &HashSet<NeedleId>,
+    dat_file_size: i64,
+) -> io::Result<()> {
+    let ecx_path = format!("{}.ecx", volume_file_name(ecx_dir, collection, volume_id));
+    let idx_path = format!("{}.idx", volume_file_name(idx_dir, collection, volume_id));
    // Write to a temp file and atomically rename into place, so a crash
    // mid-write never leaves a partial .idx at the final name beside the
    // source shards.
    let tmp_path = format!("{}.tmp", idx_path);

    let write_result = (|| -> io::Result<()> {
-        // Copy .ecx to the temp .idx
-        std::fs::copy(&ecx_path, &tmp_path)?;
-
-        // Append deletions from .ecj as tombstones. Read the journal directly
-        // and treat only NotFound as "no journal": Path::exists would also
-        // swallow a permission/IO error and silently skip deletions, which
-        // would resurrect deleted needles as live.
-        let mut idx_file = std::fs::OpenOptions::new().append(true).open(&tmp_path)?;
-        match std::fs::read(&ecj_path) {
-            Ok(ecj_data) => {
-                let count = ecj_data.len() / NEEDLE_ID_SIZE;
-                for i in 0..count {
-                    let start = i * NEEDLE_ID_SIZE;
-                    let needle_id = NeedleId::from_bytes(&ecj_data[start..start + NEEDLE_ID_SIZE]);
-                    idx::write_index_entry(
-                        &mut idx_file,
-                        needle_id,
-                        Offset::default(),
-                        TOMBSTONE_FILE_SIZE,
-                    )?;
-                }
+        let mut ecx_file = File::open(&ecx_path)?;
+        let mut idx_file = io::BufWriter::new(File::create(&tmp_path)?);
+        let mut tombstoned = Vec::new();
+        idx::walk_index_file(&mut ecx_file, 0, |key, offset, size| {
+            let is_deleted = size.is_deleted() || deleted.contains(&key);
+            if is_deleted && offset.to_actual_offset() >= dat_file_size {
+                return Ok(());
            }
-            Err(e) if e.kind() == io::ErrorKind::NotFound => {}
-            Err(e) => return Err(e),
+            idx::write_index_entry(&mut idx_file, key, offset, size)?;
+            if !size.is_deleted() && deleted.contains(&key) {
+                tombstoned.push(key);
+            }
+            Ok(())
+        })?;
+        for key in tombstoned {
+            idx::write_index_entry(&mut idx_file, key, Offset::default(), TOMBSTONE_FILE_SIZE)?;
        }

        // fsync, rename, then fsync the dir so the decoded .idx is durable and
        // atomically published before the caller deletes the source shards.
+        let idx_file = idx_file.into_inner().map_err(|e| e.into_error())?;
        idx_file.sync_all()?;
        drop(idx_file);
        // Windows rename does not replace an existing file on every version;
@@ -694,4 +751,43 @@ mod tests {
            .is_err()
        );
    }
+
+    /// A journal many chunks long that repeats a few ids reads back as those
+    /// ids, from each dir once, whatever its length.
+    #[test]
+    fn test_read_ecj_deletions_collects_distinct_ids_across_dirs() {
+        let tmp = TempDir::new().unwrap();
+        let data = tmp.path().join("data");
+        let idx = tmp.path().join("idx");
+        let missing = tmp.path().join("missing");
+        std::fs::create_dir_all(&data).unwrap();
+        std::fs::create_dir_all(&idx).unwrap();
+        let (data, idx, missing) = (
+            data.to_str().unwrap(),
+            idx.to_str().unwrap(),
+            missing.to_str().unwrap(),
+        );
+
+        let entry = |id: u64| {
+            let mut buf = [0u8; NEEDLE_ID_SIZE];
+            NeedleId(id).to_bytes(&mut buf);
+            buf
+        };
+        // Past two load chunks of three repeating ids, then an id only in the
+        // last chunk and a torn trailing record.
+        let mut ecj = Vec::new();
+        while ecj.len() <= 2 * (1 << 20) {
+            for id in [1, 2, 3] {
+                ecj.extend_from_slice(&entry(id));
+            }
+        }
+        ecj.extend_from_slice(&entry(7));
+        ecj.extend_from_slice(&entry(8)[..3]);
+        std::fs::write(format!("{idx}/1.ecj"), &ecj).unwrap();
+        std::fs::write(format!("{data}/1.ecj"), entry(9)).unwrap();
+
+        let ids = read_ecj_deletions(&[data, idx, idx, missing], "", VolumeId(1)).unwrap();
+        let expected: HashSet<NeedleId> = [1, 2, 3, 7, 9].into_iter().map(NeedleId).collect();
+        assert_eq!(ids, expected);
+    }
 }
@@ -813,7 +813,7 @@ impl EncodeRun<'_> {
        // explicitly.
        for (i, buf) in self.buffers[..self.data_shards].iter_mut().enumerate() {
            let read_offset = offset + (i * block_size) as u64;
-            let n = read_at_most(self.dat_file, buf, read_offset)?;
+            let n = crate::storage::io::read_full_at(self.dat_file, buf, read_offset)?;
            buf[n..].fill(0);
        }

@@ -834,19 +834,6 @@ impl EncodeRun<'_> {
    }
 }

-/// Read into `buf` at `offset` until it is full or EOF; returns bytes read.
-fn read_at_most(dat_file: &File, buf: &mut [u8], offset: u64) -> io::Result<usize> {
-    let mut n = 0;
-    while n < buf.len() {
-        let r = crate::storage::io::read_at(dat_file, &mut buf[n..], offset + n as u64)?;
-        if r == 0 {
-            break;
-        }
-        n += r;
-    }
-    Ok(n)
-}
-
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -87,14 +87,14 @@ impl EcVolumeShard {
        Ok(())
    }

-    /// Read data at a specific offset.
+    /// Read data at a specific offset, filling `buf` unless the shard ends first.
    pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize> {
        let file = self
            .ecd_file
            .as_ref()
            .ok_or_else(|| io::Error::other("shard file not open"))?;

-        crate::storage::io::read_at(file, buf, offset)
+        crate::storage::io::read_full_at(file, buf, offset)
    }

    /// Write data to the shard file (appends).
@@ -48,6 +48,29 @@ pub(crate) struct ShardLocationCache {
 /// A multiple of `NEEDLE_ID_SIZE`; 1 MiB is 131072 entries per syscall.
 const ECJ_LOAD_CHUNK_BYTES: usize = 1 << 20;

+/// Adds every whole needle id in the first `len` bytes of `ecj_file` to `ids`,
+/// reading `ECJ_LOAD_CHUNK_BYTES` at a time; a trailing partial record is
+/// ignored.
+pub(crate) fn read_ecj_ids(
+    ecj_file: &File,
+    len: u64,
+    ids: &mut HashSet<NeedleId>,
+) -> io::Result<()> {
+    let mut buf = vec![0u8; std::cmp::min(ECJ_LOAD_CHUNK_BYTES as u64, len) as usize];
+    let mut off: u64 = 0;
+    while off + NEEDLE_ID_SIZE as u64 <= len {
+        let mut want = std::cmp::min(ECJ_LOAD_CHUNK_BYTES as u64, len - off) as usize;
+        want -= want % NEEDLE_ID_SIZE;
+        // Positional read: the loader's handle is shared with journal appends.
+        read_exact_at(ecj_file, &mut buf[..want], off)?;
+        for entry in buf[..want].as_chunks::<NEEDLE_ID_SIZE>().0 {
+            ids.insert(NeedleId::from_bytes(entry));
+        }
+        off += want as u64;
+    }
+    Ok(())
+}
+
 /// An erasure-coded volume managing its local shards and index.
 pub struct EcVolume {
    pub volume_id: VolumeId,
@@ -810,38 +833,7 @@ impl EcVolume {
        // held the `deleted_needles` write lock for the whole scan, which on a
        // bloated journal is the entire (unbounded) startup.
        let mut loaded: HashSet<NeedleId> = HashSet::new();
-        let mut buf = vec![0u8; ECJ_LOAD_CHUNK_BYTES];
-        let end = self.ecj_file_size as u64;
-        let mut off: u64 = 0;
-        while off + NEEDLE_ID_SIZE as u64 <= end {
-            // Whole entries only; a trailing partial record is ignored, as the
-            // per-entry loop did by construction.
-            let mut want = std::cmp::min(ECJ_LOAD_CHUNK_BYTES as u64, end - off) as usize;
-            want -= want % NEEDLE_ID_SIZE;
-            if want == 0 {
-                break;
-            }
-            #[cfg(unix)]
-            {
-                use std::os::unix::fs::FileExt;
-                ecj_file.read_exact_at(&mut buf[..want], off)?;
-            }
-            #[cfg(windows)]
-            {
-                // Positional read so concurrent readers of the shared .ecj
-                // handle can't interleave seek/read. Mirrors the
-                // read_exact_at helper at the bottom of this file.
-                read_exact_at(ecj_file, &mut buf[..want], off)?;
-            }
-            #[cfg(not(any(unix, windows)))]
-            {
-                compile_error!("Platform not supported: only unix and windows are supported");
-            }
-            for entry in buf[..want].chunks_exact(NEEDLE_ID_SIZE) {
-                loaded.insert(NeedleId::from_bytes(entry));
-            }
-            off += want as u64;
-        }
+        read_ecj_ids(ecj_file, self.ecj_file_size as u64, &mut loaded)?;

        let mut set = self
            .deleted_needles
@@ -4668,7 +4660,7 @@ impl EcLocalShard {
            .file
            .as_ref()
            .map_err(|e| io::Error::new(e.kind(), e.to_string()))?;
-        crate::storage::io::read_at(file, buf, offset)
+        crate::storage::io::read_full_at(file, buf, offset)
    }
 }

@@ -36,23 +36,11 @@ pub(crate) fn read_exact_at(file: &File, buf: &mut [u8], offset: u64) -> io::Res
    }
    #[cfg(windows)]
    {
-        use std::os::windows::fs::FileExt;
-        let mut filled = 0;
-        let mut at = offset;
-        while filled < buf.len() {
-            let n = match file.seek_read(&mut buf[filled..], at) {
-                Ok(n) => n,
-                Err(err) if err.kind() == io::ErrorKind::Interrupted => continue,
-                Err(err) => return Err(err),
-            };
-            if n == 0 {
-                return Err(io::Error::new(
-                    io::ErrorKind::UnexpectedEof,
-                    "unexpected EOF in seek_read",
-                ));
-            }
-            filled += n;
-            at += n as u64;
+        if read_full_at(file, buf, offset)? < buf.len() {
+            return Err(io::Error::new(
+                io::ErrorKind::UnexpectedEof,
+                "unexpected EOF in seek_read",
+            ));
        }
    }
    #[cfg(not(any(unix, windows)))]
@@ -84,9 +72,34 @@ pub(crate) fn read_at(file: &File, buf: &mut [u8], offset: u64) -> io::Result<us
    }
 }

+/// Reads into `buf` at `offset` until it is full or the file ends, retrying
+/// interrupted reads; returns how many bytes were read.
+///
+/// Unlike [`read_at`], a count below `buf.len()` always means end of file.
+pub(crate) fn read_full_at(file: &File, buf: &mut [u8], offset: u64) -> io::Result<usize> {
+    fill_at(|b, at| read_at(file, b, at), buf, offset)
+}
+
+fn fill_at(
+    mut read: impl FnMut(&mut [u8], u64) -> io::Result<usize>,
+    buf: &mut [u8],
+    offset: u64,
+) -> io::Result<usize> {
+    let mut filled = 0;
+    while filled < buf.len() {
+        match read(&mut buf[filled..], offset + filled as u64) {
+            Ok(0) => break,
+            Ok(n) => filled += n,
+            Err(err) if err.kind() == io::ErrorKind::Interrupted => {}
+            Err(err) => return Err(err),
+        }
+    }
+    Ok(filled)
+}
+
 #[cfg(test)]
 mod tests {
-    use super::{read_at, read_exact_at};
+    use super::{fill_at, read_at, read_exact_at, read_full_at};
    use std::io::{ErrorKind, Write};

    fn temp_file(bytes: &[u8]) -> tempfile::NamedTempFile {
@@ -139,4 +152,57 @@ mod tests {
        let n = read_at(f.as_file(), &mut buf, 10).expect("read");
        assert_eq!(n, 0);
    }
+
+    /// A source that returns at most `chunk` bytes per call and fails with
+    /// `Interrupted` on its first call, like a network mount under a signal.
+    fn chunked(src: &[u8], chunk: usize) -> impl FnMut(&mut [u8], u64) -> std::io::Result<usize> {
+        let mut interrupted = false;
+        move |buf, at| {
+            if !interrupted {
+                interrupted = true;
+                return Err(ErrorKind::Interrupted.into());
+            }
+            let at = (at as usize).min(src.len());
+            let n = buf.len().min(chunk).min(src.len() - at);
+            buf[..n].copy_from_slice(&src[at..at + n]);
+            Ok(n)
+        }
+    }
+
+    #[test]
+    fn fill_at_fills_across_short_and_interrupted_reads() {
+        let src: Vec<u8> = (0..=255).collect();
+        let mut buf = [0u8; 100];
+        let n = fill_at(chunked(&src, 7), &mut buf, 50).expect("read");
+        assert_eq!(n, buf.len());
+        assert_eq!(&buf[..], &src[50..150]);
+    }
+
+    #[test]
+    fn fill_at_stops_at_end_of_source() {
+        let src: Vec<u8> = (0..=255).collect();
+        let mut buf = [0u8; 100];
+        let n = fill_at(chunked(&src, 7), &mut buf, 200).expect("read");
+        assert_eq!(n, 56);
+        assert_eq!(&buf[..n], &src[200..]);
+    }
+
+    #[test]
+    fn fill_at_propagates_other_errors() {
+        let mut buf = [0u8; 8];
+        let err = fill_at(|_, _| Err(ErrorKind::PermissionDenied.into()), &mut buf, 0)
+            .expect_err("error");
+        assert_eq!(err.kind(), ErrorKind::PermissionDenied);
+    }
+
+    #[test]
+    fn read_full_at_returns_the_short_count_only_at_eof() {
+        let f = temp_file(b"0123456789");
+        let mut buf = [0u8; 8];
+        assert_eq!(read_full_at(f.as_file(), &mut buf, 0).expect("read"), 8);
+        assert_eq!(&buf, b"01234567");
+        assert_eq!(read_full_at(f.as_file(), &mut buf, 6).expect("read"), 4);
+        assert_eq!(&buf[..4], b"6789");
+        assert_eq!(read_full_at(f.as_file(), &mut buf, 10).expect("read"), 0);
+    }
 }
@@ -3,7 +3,7 @@

 use std::io;
 use std::sync::Mutex;
-use std::sync::atomic::{AtomicBool, AtomicI32, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};

 /// Consecutive storage-media errors allowed before the volume is quarantined.
 pub(crate) const IO_ERROR_TOLERANCE: i32 = 3;
@@ -34,10 +34,30 @@ pub(crate) fn is_storage_io_error(e: &io::Error) -> bool {
 #[derive(Default)]
 pub(crate) struct IoErrorTracker {
    last: Mutex<Option<String>>,
-    count: AtomicI32,
+    /// The consecutive error count in the low 32 bits and, in the high 32,
+    /// how many times it has been cleared. They share one word so that
+    /// `record_success_at` updates both in one step: reads record their
+    /// outcomes here without the volume's write lock.
+    streak: AtomicU64,
    quarantined: AtomicBool,
 }

+const STREAK_COUNT_BITS: u64 = 0xffff_ffff;
+
+fn streak_count(streak: u64) -> i32 {
+    (streak & STREAK_COUNT_BITS) as i32
+}
+
+/// `streak` with its count cleared and one more clear on record.
+fn streak_cleared(streak: u64) -> u64 {
+    (streak >> 32).wrapping_add(1) << 32
+}
+
+/// A point in the error streak, taken where a write landed whose success
+/// is only recorded later. See `IoErrorTracker::record_success_at`.
+#[derive(Clone, Copy)]
+pub(crate) struct StreakMark(u64);
+
 impl IoErrorTracker {
    /// `Some(e)` records a failure, `None` a success. Only storage-media
    /// failures count; every other outcome clears the count and last error.
@@ -45,14 +65,22 @@ impl IoErrorTracker {
        if let Some(e) = err
            && is_storage_io_error(e)
        {
-            self.count.fetch_add(1, Ordering::Relaxed);
+            self.streak.fetch_add(1, Ordering::Relaxed);
            if let Ok(mut guard) = self.last.lock() {
                *guard = Some(e.to_string());
            }
            crate::metrics::STORAGE_IO_ERROR_COUNTER.inc();
            return;
        }
-        self.count.store(0, Ordering::Relaxed);
+        self.clear_count();
+        self.clear_last();
+    }
+
+    fn clear_count(&self) {
+        self.update_streak(|streak| Some(streak_cleared(streak)));
+    }
+
+    fn clear_last(&self) {
        if let Ok(mut guard) = self.last.lock()
            && guard.is_some()
        {
@@ -60,17 +88,58 @@ impl IoErrorTracker {
        }
    }

+    /// Apply `f` to the streak atomically; `None` leaves it as it is.
+    /// Returns the streak `f` produced, if any.
+    fn update_streak(&self, mut f: impl FnMut(u64) -> Option<u64>) -> Option<u64> {
+        let mut updated = None;
+        let _ = self
+            .streak
+            .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |streak| {
+                updated = f(streak);
+                updated
+            });
+        updated
+    }
+
+    pub(crate) fn mark(&self) -> StreakMark {
+        StreakMark(self.streak.load(Ordering::Relaxed))
+    }
+
+    /// Record a success as if it had come at `mark`: the errors counted
+    /// before the mark are cleared and the ones counted since still stand,
+    /// as they would had each outcome been recorded in order. A streak
+    /// cleared since the mark is left as it is.
+    pub(crate) fn record_success_at(&self, mark: StreakMark) {
+        let before = streak_count(mark.0);
+        let updated = self.update_streak(|streak| {
+            if streak >> 32 != mark.0 >> 32 {
+                return None;
+            }
+            if streak_count(streak) <= before {
+                return Some(streak_cleared(streak));
+            }
+            Some(streak - before as u64)
+        });
+        // The last error stays when one counted since the mark is left.
+        if updated.is_some_and(|streak| streak_count(streak) == 0) {
+            self.clear_last();
+        }
+    }
+
    /// The last recorded error, the consecutive count, and the quarantine flag.
    pub(crate) fn get_io_error_state(&self) -> (Option<String>, i32, bool) {
        let err = self.last.lock().ok().and_then(|g| g.clone());
-        let count = self.count.load(Ordering::Relaxed);
+        let count = self.count();
        let quarantined = self.quarantined.load(Ordering::Relaxed);
        (err, count, quarantined)
    }

+    fn count(&self) -> i32 {
+        streak_count(self.streak.load(Ordering::Relaxed))
+    }
+
    pub(crate) fn should_quarantine(&self) -> bool {
-        self.quarantined.load(Ordering::Relaxed)
-            || self.count.load(Ordering::Relaxed) >= IO_ERROR_TOLERANCE
+        self.quarantined.load(Ordering::Relaxed) || self.count() >= IO_ERROR_TOLERANCE
    }

    pub(crate) fn mark_io_quarantined(&self) {
@@ -78,7 +147,7 @@ impl IoErrorTracker {
    }

    pub(crate) fn reset_io_error_state(&self) {
-        self.count.store(0, Ordering::Relaxed);
+        self.clear_count();
        self.quarantined.store(false, Ordering::Relaxed);
        if let Ok(mut guard) = self.last.lock() {
            *guard = None;
@@ -91,9 +160,11 @@ impl IoErrorTracker {
            *guard = err.map(|value| value.to_string());
        }
        if err.is_some() {
-            self.count.store(IO_ERROR_TOLERANCE, Ordering::Relaxed);
+            self.update_streak(|streak| {
+                Some((streak & !STREAK_COUNT_BITS) | IO_ERROR_TOLERANCE as u64)
+            });
        } else {
-            self.count.store(0, Ordering::Relaxed);
+            self.clear_count();
        }
    }
 }
@@ -171,6 +242,87 @@ mod tests {
        assert!(tracker.should_quarantine());
    }

+    #[test]
+    fn success_at_a_mark_keeps_only_the_errors_after_it() {
+        let tracker = IoErrorTracker::default();
+        tracker.check_read_write_error(Some(&media_error()));
+        tracker.check_read_write_error(Some(&media_error()));
+        let mark = tracker.mark();
+        tracker.check_read_write_error(Some(&media_error()));
+        tracker.record_success_at(mark);
+
+        assert_eq!(
+            tracker.get_io_error_state(),
+            (Some(media_error().to_string()), 1, false)
+        );
+    }
+
+    #[test]
+    fn success_at_a_mark_with_nothing_after_it_clears_the_streak() {
+        let tracker = IoErrorTracker::default();
+        tracker.check_read_write_error(Some(&media_error()));
+        let mark = tracker.mark();
+        tracker.record_success_at(mark);
+
+        assert_eq!(tracker.get_io_error_state(), (None, 0, false));
+    }
+
+    #[test]
+    fn success_at_a_mark_leaves_a_streak_cleared_since() {
+        let tracker = IoErrorTracker::default();
+        tracker.check_read_write_error(Some(&media_error()));
+        tracker.check_read_write_error(Some(&media_error()));
+        let mark = tracker.mark();
+        tracker.check_read_write_error(None);
+        tracker.check_read_write_error(Some(&media_error()));
+        tracker.record_success_at(mark);
+
+        assert_eq!(
+            tracker.get_io_error_state(),
+            (Some(media_error().to_string()), 1, false)
+        );
+    }
+
+    /// Reads update the tracker without the volume's write lock, so a
+    /// success replayed at a mark must not lose the errors they record
+    /// while it runs.
+    #[test]
+    fn success_at_a_mark_keeps_concurrent_errors() {
+        use std::sync::{Arc, Barrier};
+        const READERS: i32 = 4;
+        const ERRORS: i32 = 200;
+
+        for _ in 0..500 {
+            let tracker = Arc::new(IoErrorTracker::default());
+            tracker.check_read_write_error(Some(&media_error()));
+            tracker.check_read_write_error(Some(&media_error()));
+            let mark = tracker.mark();
+            let start = Arc::new(Barrier::new(READERS as usize + 1));
+            let readers: Vec<_> = (0..READERS)
+                .map(|_| {
+                    let (tracker, start) = (tracker.clone(), start.clone());
+                    std::thread::spawn(move || {
+                        start.wait();
+                        for _ in 0..ERRORS {
+                            tracker.check_read_write_error(Some(&media_error()));
+                        }
+                    })
+                })
+                .collect();
+            start.wait();
+            while tracker.get_io_error_state().1 < 2 + READERS * ERRORS / 2 {
+                std::hint::spin_loop();
+            }
+            tracker.record_success_at(mark);
+            for reader in readers {
+                reader.join().unwrap();
+            }
+            // The two errors before the mark are cleared; every error the
+            // readers recorded after it stands.
+            assert_eq!(tracker.get_io_error_state().1, READERS * ERRORS);
+        }
+    }
+
    #[test]
    fn reset_io_error_state_lifts_the_quarantine() {
        let tracker = IoErrorTracker::default();
@@ -14,11 +14,11 @@ use crate::pb::master_pb;
 use crate::storage::disk_location::DiskLocation;
 use crate::storage::erasure_coding::ec_shard::{EcVolumeShard, MAX_SHARD_COUNT, ShardId};
 use crate::storage::erasure_coding::ec_volume::{EcVolume, is_usable_ecx_file};
-use crate::storage::needle::needle::Needle;
+use crate::storage::needle::needle::{Needle, get_actual_size};
 use crate::storage::needle_map::NeedleMapKind;
-use crate::storage::super_block::ReplicaPlacement;
+use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE};
 use crate::storage::types::*;
-use crate::storage::volume::{CompactionJob, VifVolumeInfo, VolumeError, VolumeSpec};
+use crate::storage::volume::{CompactionJob, VifVolumeInfo, Volume, VolumeError, VolumeSpec};

 /// Top-level storage manager containing all disk locations and their volumes.
 pub struct Store {
@@ -745,6 +745,30 @@ impl Store {
        n: &mut Needle,
        fsync: bool,
    ) -> Result<(u64, Size, bool), VolumeError> {
+        self.writable_volume_mut(vid)?.write_needle(n, true, fsync)
+    }
+
+    /// Write a batch of needles to one volume, sharing the syncs of its
+    /// durable writes. See `Volume::write_needles_grouped`.
+    pub fn write_volume_needles(
+        &mut self,
+        vid: VolumeId,
+        writes: &mut [(Needle, bool)],
+    ) -> Vec<Result<(u64, Size, bool), VolumeError>> {
+        match self.writable_volume_mut(vid) {
+            Ok(vol) => vol.write_needles_grouped(writes),
+            // The lookup fails only with NotFound or the disk-space ReadOnly.
+            Err(e) => writes
+                .iter()
+                .map(|_| match e {
+                    VolumeError::ReadOnly(vid) => Err(VolumeError::ReadOnly(vid)),
+                    _ => Err(VolumeError::NotFound),
+                })
+                .collect(),
+        }
+    }
+
+    fn writable_volume_mut(&mut self, vid: VolumeId) -> Result<&mut Volume, VolumeError> {
        // Check disk space on the location containing this volume.
        // We do this before the mutable borrow to avoid borrow conflicts.
        let loc_idx = self
@@ -755,11 +779,11 @@ impl Store {
            .is_disk_space_low
            .load(Ordering::Relaxed)
        {
-            return Err(VolumeError::ReadOnly);
+            return Err(VolumeError::ReadOnly(vid));
        }

        let (_, vol) = self.find_volume_mut(vid).ok_or(VolumeError::NotFound)?;
-        vol.write_needle(n, true, fsync)
+        Ok(vol)
    }

    /// Delete a needle from a volume.
@@ -771,7 +795,7 @@ impl Store {
        // Match Go's DeleteVolumeNeedle: check noWriteOrDelete before proceeding.
        let (_, vol) = self.find_volume(vid).ok_or(VolumeError::NotFound)?;
        if vol.is_no_write_or_delete() {
-            return Err(VolumeError::ReadOnly);
+            return Err(VolumeError::ReadOnly(vid));
        }

        let (_, vol) = self.find_volume_mut(vid).ok_or(VolumeError::NotFound)?;
@@ -1216,6 +1240,7 @@ impl Store {
        found_vol.map(|v| (v, dirs))
    }

+    #[cfg(test)]
    pub fn delete_expired_ec_volumes(
        &mut self,
    ) -> (
@@ -1301,6 +1326,7 @@ impl Store {
    }

    /// Remove an EC volume from whichever location has it.
+    #[cfg(test)]
    pub fn remove_ec_volume(&mut self, vid: VolumeId) -> Option<EcVolume> {
        for loc in &mut self.locations {
            if let Some(ecv) = loc.remove_ec_volume(vid) {
@@ -1539,13 +1565,21 @@ impl Store {
        preallocate: u64,
    ) -> Result<Option<CompactionJob>, VolumeError> {
        // Required space matches Go's CompactVolume check: the larger of the
-        // requested preallocation and the estimated volume size.
+        // requested preallocation and the estimated compacted size — the live
+        // needles, not the .dat the garbage already occupies, so a full disk
+        // can still be reclaimed.
        let (loc_idx, space_needed) = {
            let (loc_idx, v) = self
                .find_volume(vid)
                .ok_or(VolumeError::VolumeNotFound(vid))?;
-            let estimated = v.dat_file_size().unwrap_or(0) + v.idx_file_size();
-            (loc_idx, std::cmp::max(preallocate, estimated))
+            let live_count = (v.file_count() - v.deleted_count()).max(0) as u64;
+            let live_bytes = v.content_size().saturating_sub(v.deleted_size());
+            let per_needle = (get_actual_size(Size(0), v.version())
+                + NEEDLE_PADDING_SIZE as i64
+                + NEEDLE_MAP_ENTRY_SIZE as i64) as u64;
+            let estimated = SUPER_BLOCK_SIZE as u64 + live_count * per_needle + live_bytes;
+            let space_needed = std::cmp::max(preallocate, estimated);
+            (loc_idx, space_needed + space_needed / 10)
        };

        let dir = self.locations[loc_idx].directory.clone();
@@ -9,6 +9,7 @@
 //! Matches Go's storage/volume.go, volume_loading.go, volume_read.go,
 //! volume_write.go, volume_super_block.go.

+use std::collections::HashSet;
 use std::fs::{self, File, OpenOptions};
 use std::io::{self, Read, Seek, SeekFrom, Write};
 use std::ops::ControlFlow;
@@ -22,7 +23,7 @@ use tracing::{debug, error, info, warn};

 use crate::storage::idx;
 use crate::storage::io::read_exact_at;
-use crate::storage::io_error::IoErrorTracker;
+use crate::storage::io_error::{IoErrorTracker, StreakMark};
 use crate::storage::needle::needle::{self, Needle, NeedleError, get_actual_size};
 use crate::storage::needle_map::sorted_file::SortedFileNeedleMap;
 use crate::storage::needle_map::{CompactNeedleMap, NeedleMap, NeedleMapKind, RedbNeedleMap};
@@ -60,8 +61,8 @@ pub enum VolumeError {
    #[error("volume already exists")]
    AlreadyExists,

-    #[error("volume is read-only")]
-    ReadOnly,
+    #[error("volume {0} is read only")]
+    ReadOnly(VolumeId),

    #[error("volume is unavailable: {0}")]
    Unavailable(String),
@@ -1146,6 +1147,13 @@ pub struct Volume {
    fail_idx_sync_for_test: bool,
    #[cfg(test)]
    fail_truncate_for_test: bool,
+    /// Needle ids whose .dat append fails with a media error.
+    #[cfg(test)]
+    fail_append_for_test: HashSet<NeedleId>,
+    #[cfg(test)]
+    dat_syncs_for_test: std::sync::atomic::AtomicUsize,
+    #[cfg(test)]
+    idx_syncs_for_test: std::sync::atomic::AtomicUsize,
    needle_map_kind: NeedleMapKind,
    data_file_access_control: Arc<DataFileAccessControl>,

@@ -1244,6 +1252,12 @@ impl Volume {
            fail_idx_sync_for_test: false,
            #[cfg(test)]
            fail_truncate_for_test: false,
+            #[cfg(test)]
+            fail_append_for_test: HashSet::new(),
+            #[cfg(test)]
+            dat_syncs_for_test: Default::default(),
+            #[cfg(test)]
+            idx_syncs_for_test: Default::default(),
            nm: None,
            needle_map_kind,
            data_file_access_control: Arc::new(DataFileAccessControl::default()),
@@ -1288,6 +1302,12 @@ impl Volume {
            fail_idx_sync_for_test: false,
            #[cfg(test)]
            fail_truncate_for_test: false,
+            #[cfg(test)]
+            fail_append_for_test: HashSet::new(),
+            #[cfg(test)]
+            dat_syncs_for_test: Default::default(),
+            #[cfg(test)]
+            idx_syncs_for_test: Default::default(),
            nm: None,
            needle_map_kind: NeedleMapKind::InMemory,
            data_file_access_control: Arc::new(DataFileAccessControl::default()),
@@ -1308,6 +1328,16 @@ impl Volume {
        }
    }

+    /// Identifies this volume instance: a re-created or remounted volume gets
+    /// a new one, even at the same compaction revision.
+    pub(crate) fn instance(&self) -> Arc<DataFileAccessControl> {
+        self.data_file_access_control.clone()
+    }
+
+    pub(crate) fn is_instance(&self, instance: &Arc<DataFileAccessControl>) -> bool {
+        Arc::ptr_eq(instance, &self.data_file_access_control)
+    }
+
    /// Returns true if the volume is currently being compacted.
    pub fn is_compacting(&self) -> bool {
        self.is_compacting.load(Ordering::Acquire)
@@ -2257,20 +2287,152 @@ impl Volume {
        fsync: bool,
    ) -> Result<(u64, Size, bool), VolumeError> {
        let _guard = self.data_file_access_control.write_lock();
+        self.check_writable()?;
+        self.do_write_request(n, check_cookie, fsync)
+    }
+
+    /// Write a batch of needles the way Go's processBatch does: one .dat
+    /// sync and one .idx sync per run of distinct needle ids that holds a
+    /// durable write, instead of two per durable needle. Nothing in such a
+    /// run is published before its sync, so a failed sync takes the whole
+    /// run back off the .dat and fails every entry. A repeated id starts a
+    /// new run, so its dedup and cookie checks see the earlier write.
+    pub fn write_needles_grouped(
+        &mut self,
+        writes: &mut [(Needle, bool)],
+    ) -> Vec<Result<(u64, Size, bool), VolumeError>> {
+        let _guard = self.data_file_access_control.write_lock();
+        let mut results = Vec::with_capacity(writes.len());
+        let mut rest = writes;
+        while !rest.is_empty() {
+            let mut ids = HashSet::new();
+            let len = rest.iter().take_while(|(n, _)| ids.insert(n.id)).count();
+            let (run, tail) = std::mem::take(&mut rest).split_at_mut(len);
+            if run.iter().any(|(_, fsync)| *fsync) {
+                results.extend(self.write_synced_run(run));
+            } else {
+                for (n, _) in run.iter_mut() {
+                    results.push(
+                        self.check_writable()
+                            .and_then(|()| self.do_write_request(n, true, false)),
+                    );
+                }
+            }
+            rest = tail;
+        }
+        results
+    }
+
+    fn write_synced_run(
+        &mut self,
+        run: &mut [(Needle, bool)],
+    ) -> Vec<Result<(u64, Size, bool), VolumeError>> {
+        // Per entry: Some(offset) once appended, None when it dedups.
+        let mut staged = Vec::with_capacity(run.len());
+        // Per entry: the I/O error streak once it is staged, where a write
+        // sent on its own would have recorded its success.
+        let mut marks = Vec::with_capacity(run.len());
+        let mut last_append_at_ns = self.last_append_at_ns;
+        let mut run_start = None;
+        let mut sync = false;
+        for (n, fsync) in run.iter_mut() {
+            let r = self.append_unpublished(n, &mut last_append_at_ns);
+            marks.push(self.io_errors.mark());
+            if let Ok(Some(offset)) = r {
+                run_start.get_or_insert(offset);
+            }
+            // Only a durable entry that got this far needs the sync.
+            sync |= *fsync && r.is_ok();
+            staged.push(r);
+        }
+
+        if sync && let Err(e) = self.flush_dat() {
+            self.check_read_write_error(Some(&e));
+            if let Some(start) = run_start {
+                self.undo_unsynced_append(start);
+            }
+            let e = VolumeError::Io(e);
+            return staged
+                .into_iter()
+                .map(|r| r.and_then(|_| Err(run_error(&e))))
+                .collect();
+        }
+        self.last_append_at_ns = last_append_at_ns;
+
+        for ((n, fsync), r) in run.iter().zip(staged.iter_mut()) {
+            if let Ok(Some(offset)) = *r
+                && let Err(e) = self.publish_write(n, offset, *fsync)
+            {
+                *r = Err(e);
+            }
+        }
+
+        if sync && let Err(e) = self.flush_idx() {
+            return staged
+                .into_iter()
+                .map(|r| r.and_then(|_| Err(run_error(&e))))
+                .collect();
+        }
+
+        let written = run
+            .iter()
+            .zip(&staged)
+            .filter(|(_, r)| matches!(r, Ok(Some(_))))
+            .map(|((n, _), _)| n.last_modified)
+            .max();
+        let last = staged.iter().rposition(|r| matches!(r, Ok(Some(_))));
+        if let (Some(last_modified), Some(last)) = (written, last) {
+            // Sent one at a time, the last write to land would have cleared
+            // the streak of the appends before it, and the ones after it
+            // would have failed on top of that.
+            self.finish_write(last_modified, sync, marks[last]);
+        }
+
+        run.iter()
+            .zip(staged)
+            .map(|((n, _), r)| {
+                let size = Size(n.data_size as i32);
+                r.map(|staged| match staged {
+                    Some(offset) => (offset, size, false),
+                    None => (0, size, true),
+                })
+            })
+            .collect()
+    }
+
+    /// The checks and the append for one entry of a synced run, leaving the
+    /// publish to the caller. Returns the offset, or None when it dedups.
+    fn append_unpublished(
+        &mut self,
+        n: &mut Needle,
+        last_append_at_ns: &mut u64,
+    ) -> Result<Option<u64>, VolumeError> {
+        self.check_writable()?;
+        if self.prepare_write(n, true)? {
+            return Ok(None);
+        }
+        n.append_at_ns = get_append_at_ns(*last_append_at_ns);
+        let (offset, _, _) = self.append_needle(n)?;
+        *last_append_at_ns = n.append_at_ns;
+        Ok(Some(offset))
+    }
+
+    fn check_writable(&self) -> Result<(), VolumeError> {
        if let Some(e) = self.unavailable_error() {
            return Err(e);
        }
        if self.is_read_only() {
-            return Err(VolumeError::ReadOnly);
+            return Err(VolumeError::ReadOnly(self.id));
        }
-
-        self.do_write_request(n, check_cookie, fsync)
+        Ok(())
    }

    /// Flush the .dat, the first half of a durable write. The .idx is flushed
    /// separately by flush_idx once the row is published; the two are split so
    /// nothing is indexed before the bytes it points at are down.
    fn flush_dat(&self) -> io::Result<()> {
+        #[cfg(test)]
+        self.dat_syncs_for_test.fetch_add(1, Ordering::Relaxed);
        #[cfg(test)]
        if self.fail_fsync_for_test {
            return Err(io::Error::other("injected fsync failure"));
@@ -2293,6 +2455,8 @@ impl Volume {
    /// record whose index may not survive, and the master routes writes
    /// elsewhere once the volume heartbeats read only.
    fn flush_idx(&mut self) -> Result<(), VolumeError> {
+        #[cfg(test)]
+        self.idx_syncs_for_test.fetch_add(1, Ordering::Relaxed);
        #[cfg(test)]
        if self.fail_idx_sync_for_test {
            let e = io::Error::other("injected idx sync failure");
@@ -2335,6 +2499,54 @@ impl Volume {
        check_cookie: bool,
        fsync: bool,
    ) -> Result<(u64, Size, bool), VolumeError> {
+        if self.prepare_write(n, check_cookie)? {
+            // Nothing to append, but the write this matched may have been
+            // non-durable, and the caller is asking for the content to be on
+            // disk. Its .idx row can be sitting in the page cache too, so both
+            // files get flushed exactly as they would for a fresh append.
+            if fsync {
+                self.flush_dat().map_err(|e| {
+                    self.check_read_write_error(Some(&e));
+                    VolumeError::Io(e)
+                })?;
+                self.flush_idx()?;
+            }
+            return Ok((0, Size(n.data_size as i32), true));
+        }
+
+        // Update append timestamp
+        n.append_at_ns = get_append_at_ns(self.last_append_at_ns);
+
+        // Append to .dat file
+        let (offset, _body_size, _actual_size) = self.append_needle(n)?;
+
+        // Nothing is published until the bytes are down: an index entry for an
+        // unflushed append would resolve past the end of the file after a crash,
+        // and undoing it afterwards would double-count the volume's metrics.
+        if fsync && let Err(e) = self.flush_dat() {
+            self.check_read_write_error(Some(&e));
+            self.undo_unsynced_append(offset);
+            return Err(VolumeError::Io(e));
+        }
+
+        self.last_append_at_ns = n.append_at_ns;
+
+        self.publish_write(n, offset, fsync)?;
+
+        if fsync {
+            self.flush_idx()?;
+        }
+
+        self.finish_write(n.last_modified, fsync, self.io_errors.mark());
+
+        // Return Size(n.DataSize) as the logical size, matching Go's doWriteRequest
+        Ok((offset, Size(n.data_size as i32), false))
+    }
+
+    /// The checks a write passes before anything is appended: TTL
+    /// inheritance, checksum, dedup and cookie. Returns true when the needle
+    /// matches the stored copy and there is nothing to append.
+    fn prepare_write(&self, n: &mut Needle, check_cookie: bool) -> Result<bool, VolumeError> {
        // TTL inheritance from volume (matching Go's writeNeedle2)
        {
            use crate::storage::needle::ttl::TTL;
@@ -2353,18 +2565,7 @@ impl Volume {
        // Dedup check (matches Go: n.DataSize = oldNeedle.DataSize on dedup)
        if let Some(old_data_size) = self.is_file_unchanged(n) {
            n.data_size = old_data_size;
-            // Nothing to append, but the write this matched may have been
-            // non-durable, and the caller is asking for the content to be on
-            // disk. Its .idx row can be sitting in the page cache too, so both
-            // files get flushed exactly as they would for a fresh append.
-            if fsync {
-                self.flush_dat().map_err(|e| {
-                    self.check_read_write_error(Some(&e));
-                    VolumeError::Io(e)
-                })?;
-                self.flush_idx()?;
-            }
-            return Ok((0, Size(n.data_size as i32), true));
+            return Ok(true);
        }

        // Cookie validation for existing needle (matches Go: check whenever nm.Get returns ok)
@@ -2382,32 +2583,24 @@ impl Volume {
                return Err(VolumeError::CookieMismatch(n.cookie.0));
            }
        }
+        Ok(false)
+    }

-        // Update append timestamp
-        n.append_at_ns = get_append_at_ns(self.last_append_at_ns);
-
-        // Append to .dat file
-        let (offset, _body_size, _actual_size) = self.append_needle(n)?;
-
-        // Nothing is published until the bytes are down: an index entry for an
-        // unflushed append would resolve past the end of the file after a crash,
-        // and undoing it afterwards would double-count the volume's metrics.
-        if fsync && let Err(e) = self.flush_dat() {
-            self.check_read_write_error(Some(&e));
-            if let Err(te) = self.truncate_dat(offset) {
-                // The rejected record is still on the end. A later append
-                // would bury it mid-file, where the .dat tail check cannot
-                // see it, so the volume fails closed instead.
-                self.mark_io_unavailable(format!(
-                    "failed to truncate back to {} after a failed fsync: {}",
-                    offset, te
-                ));
-            }
-            return Err(VolumeError::Io(e));
+    /// Take an append whose sync failed back off the .dat.
+    fn undo_unsynced_append(&mut self, offset: u64) {
+        if let Err(te) = self.truncate_dat(offset) {
+            // The rejected record is still on the end. A later append
+            // would bury it mid-file, where the .dat tail check cannot
+            // see it, so the volume fails closed instead.
+            self.mark_io_unavailable(format!(
+                "failed to truncate back to {} after a failed fsync: {}",
+                offset, te
+            ));
        }
+    }

-        self.last_append_at_ns = n.append_at_ns;
-
+    /// Index an appended needle. `fsync` means its record is already down.
+    fn publish_write(&mut self, n: &Needle, offset: u64, fsync: bool) -> Result<(), VolumeError> {
        // Update needle map (uses n.size = full body size, matching Go's nm.Put)
        let prior = match self.nm.as_ref() {
            Some(nm) => nm.get(n.id),
@@ -2455,26 +2648,25 @@ impl Volume {
                return Err(VolumeError::Io(e));
            }
        }
+        Ok(())
+    }

-        if fsync {
-            self.flush_idx()?;
+    /// The bookkeeping after a write is fully down. `landed_at` is the point
+    /// in the I/O error streak where the write landed; errors counted after
+    /// it, by later appends of the same run, are not cleared.
+    fn finish_write(&mut self, last_modified: u64, idx_synced: bool, landed_at: StreakMark) {
+        if self.last_modified_ts_seconds < last_modified {
+            self.last_modified_ts_seconds = last_modified;
        }

-        if self.last_modified_ts_seconds < n.last_modified {
-            self.last_modified_ts_seconds = n.last_modified;
-        }
-
-        let checkpoint_ok = self.maybe_checkpoint_index(fsync);
+        let checkpoint_ok = self.maybe_checkpoint_index(idx_synced);

        // Clear the EIO streak only after the full write (data + flush +
        // index + checkpoint) succeeds, so a failed fsync or checkpoint
        // does not get its EIO erased by the success reset.
        if checkpoint_ok {
-            self.check_read_write_error(None);
+            self.io_errors.record_success_at(landed_at);
        }
-
-        // Return Size(n.DataSize) as the logical size, matching Go's doWriteRequest
-        Ok((offset, Size(n.data_size as i32), false))
    }

    /// Take the index checkpoint the needle map asked for, data first: the
@@ -2586,9 +2778,15 @@ impl Volume {
            });
        }

-        if let Err(e) = dat_file.write_all(&bytes) {
-            // Truncate back to pre-write position on error (matching Go)
-            let _ = dat_file.set_len(offset);
+        let written = dat_file.write_all(&bytes);
+        #[cfg(test)]
+        let written = if self.fail_append_for_test.contains(&n.id) {
+            Err(media_error_for_test())
+        } else {
+            written
+        };
+        if let Err(e) = written {
+            self.undo_unsynced_append(offset);
            self.check_read_write_error(Some(&e));
            return Err(VolumeError::Io(e));
        }
@@ -2605,7 +2803,7 @@ impl Volume {
            return Err(e);
        }
        if self.no_write_or_delete {
-            return Err(VolumeError::ReadOnly);
+            return Err(VolumeError::ReadOnly(self.id));
        }
        self.do_delete_request(n)
    }
@@ -2690,6 +2888,19 @@ impl Volume {
            || self.location_disk_space_low.load(Ordering::Relaxed)
    }

+    /// Mirrors Go's ReadOnlyReasons: `no_write_or_delete` already covers the
+    /// io_unavailable quarantine.
+    pub fn read_only_reasons(&self) -> (bool, bool, bool, bool) {
+        let no_write_or_delete = self.no_write_or_delete || self.io_unavailable.is_some();
+        let disk_space_low = self.location_disk_space_low.load(Ordering::Relaxed);
+        (
+            no_write_or_delete || self.no_write_can_delete || disk_space_low,
+            no_write_or_delete,
+            self.no_write_can_delete,
+            disk_space_low,
+        )
+    }
+
    /// The reason the volume refuses all I/O, when a failed recovery left the
    /// .dat/index pair unverified. Mirrors Go's unavailableError.
    pub fn unavailable_error(&self) -> Option<VolumeError> {
@@ -3944,7 +4155,7 @@ impl Volume {
        needle_blob: &[u8],
    ) -> Result<(), VolumeError> {
        if self.is_read_only() {
-            return Err(VolumeError::ReadOnly);
+            return Err(VolumeError::ReadOnly(self.id));
        }
        let dat_file = self
            .dat_file
@@ -3965,7 +4176,7 @@ impl Volume {
    ) -> Result<(), VolumeError> {
        // nm.put on a read-only volume fails only after the blob is appended to .dat.
        if self.is_read_only() {
-            return Err(VolumeError::ReadOnly);
+            return Err(VolumeError::ReadOnly(self.id));
        }
        // Storage guard: negativity-only (Go parity). See parse_needle_at.
        if size.0 < 0 {
@@ -4235,6 +4446,15 @@ impl Volume {
        let Some(_claim) = CompactionClaim::try_claim(&self.is_compacting) else {
            return Ok(()); // already compacting, silently skip (matches Go)
        };
+        // The reload would read the remote object through the compacted .idx.
+        if self.has_remote_file() {
+            let _ = fs::remove_file(self.file_name(".cpd"));
+            let _ = fs::remove_file(self.file_name(".cpx"));
+            return Err(VolumeError::Io(io::Error::other(format!(
+                "volume {} is tiered to remote storage, cannot commit compaction",
+                self.id
+            ))));
+        }
        self.do_commit_compact()
    }

@@ -4798,6 +5018,20 @@ impl Volume {
        self.fail_truncate_for_test = fail;
    }

+    #[cfg(test)]
+    pub(crate) fn fail_append_for_test(&mut self, ids: &[NeedleId]) {
+        self.fail_append_for_test = ids.iter().copied().collect();
+    }
+
+    /// (.dat syncs, .idx syncs) attempted since the volume was opened.
+    #[cfg(test)]
+    pub(crate) fn sync_counts_for_test(&self) -> (usize, usize) {
+        (
+            self.dat_syncs_for_test.load(Ordering::Relaxed),
+            self.idx_syncs_for_test.load(Ordering::Relaxed),
+        )
+    }
+
    #[cfg(test)]
    pub(crate) fn set_last_modified_ts_for_test(&mut self, ts_seconds: u64) {
        self.last_modified_ts_seconds = ts_seconds;
@@ -4815,6 +5049,14 @@ impl Volume {
 // Helpers
 // ============================================================================

+/// A copy of a run-wide failure for each entry it fails.
+fn run_error(e: &VolumeError) -> VolumeError {
+    match e {
+        VolumeError::Io(e) => VolumeError::Io(io::Error::new(e.kind(), e.to_string())),
+        e => VolumeError::Io(io::Error::other(e.to_string())),
+    }
+}
+
 /// Generate volume file base name: dir/collection_id or dir/id
 /// Byte offset just past the needle's on-disk record. Mirrors Go's
 /// needleDiskEnd.
@@ -5064,6 +5306,25 @@ fn preallocate_file(file: &File, size: u64) {
 // Tests
 // ============================================================================

+/// An OS error the platform reports for failing storage media, which is
+/// what counts toward the I/O error streak.
+#[cfg(test)]
+fn media_error_for_test() -> io::Error {
+    #[cfg(unix)]
+    {
+        io::Error::from_raw_os_error(libc::EIO)
+    }
+    #[cfg(windows)]
+    {
+        const ERROR_IO_DEVICE: i32 = 1117;
+        io::Error::from_raw_os_error(ERROR_IO_DEVICE)
+    }
+    #[cfg(not(any(unix, windows)))]
+    {
+        io::Error::other("injected media error")
+    }
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -5926,7 +6187,7 @@ mod tests {
        assert!(
            matches!(
                v.write_needle(&mut later, true, false),
-                Err(VolumeError::ReadOnly)
+                Err(VolumeError::ReadOnly(_))
            ),
            "later writes must not append past the record whose index is in doubt"
        );
@@ -6042,6 +6303,309 @@ mod tests {
        assert!(v.read_needle(&mut read_n).is_err());
    }

+    fn batch_needle(id: u64, cookie: u32, data: &[u8]) -> Needle {
+        Needle {
+            id: NeedleId(id),
+            cookie: Cookie(cookie),
+            data: data.to_vec(),
+            data_size: data.len() as u32,
+            ..Needle::default()
+        }
+    }
+
+    fn dat_len(v: &Volume) -> u64 {
+        std::fs::metadata(v.file_name(".dat")).unwrap().len()
+    }
+
+    /// A batch of durable writes shares one .dat sync and one .idx sync,
+    /// where writing them one at a time syncs both files per needle.
+    #[test]
+    fn test_grouped_fsync_writes_share_one_sync() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let mut v = make_test_volume(dir);
+
+        let mut writes: Vec<_> = (1..=10u64)
+            .map(|id| {
+                (
+                    batch_needle(id, 0xaa, format!("body-{id}").as_bytes()),
+                    true,
+                )
+            })
+            .collect();
+        let results = v.write_needles_grouped(&mut writes);
+        assert!(results.iter().all(|r| matches!(r, Ok((_, _, false)))));
+        assert_eq!(v.sync_counts_for_test(), (1, 1));
+        for id in 1..=10u64 {
+            let mut n = Needle {
+                id: NeedleId(id),
+                ..Needle::default()
+            };
+            v.read_needle(&mut n).unwrap();
+            assert_eq!(n.data, format!("body-{id}").as_bytes());
+        }
+
+        // A batch with no durable write syncs nothing, as before.
+        let mut writes: Vec<_> = (11..=20u64)
+            .map(|id| (batch_needle(id, 0xaa, b"lazy"), false))
+            .collect();
+        let results = v.write_needles_grouped(&mut writes);
+        assert!(results.iter().all(|r| r.is_ok()));
+        assert_eq!(v.sync_counts_for_test(), (1, 1));
+
+        // The per-needle path pays both syncs for every durable write.
+        let tmp2 = TempDir::new().unwrap();
+        let mut one_by_one = make_test_volume(tmp2.path().to_str().unwrap());
+        for id in 1..=10u64 {
+            let mut n = batch_needle(id, 0xaa, format!("body-{id}").as_bytes());
+            one_by_one.write_needle(&mut n, true, true).unwrap();
+        }
+        assert_eq!(one_by_one.sync_counts_for_test(), (10, 10));
+    }
+
+    /// A failed shared sync fails every entry of the run, durable or not,
+    /// and leaves the volume as it was before the run: the .dat back at the
+    /// run start, the clocks where they were, nothing published.
+    #[test]
+    fn test_grouped_failed_sync_rolls_back_the_run() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let mut v = make_test_volume(dir);
+
+        let mut kept = batch_needle(1, 0xaa, b"first-copy");
+        v.write_needle(&mut kept, true, true).unwrap();
+        let prior = v.nm.as_ref().unwrap().get(NeedleId(1)).unwrap().unwrap();
+        let dat_len_before = dat_len(&v);
+        let last_append_before = v.last_append_at_ns;
+        let last_modified_before = v.last_modified_ts_seconds;
+        let file_count_before = v.file_count();
+
+        let mut writes = vec![
+            (batch_needle(1, 0xaa, b"second-copy"), false),
+            (batch_needle(2, 0xbb, b"durable"), true),
+            (batch_needle(3, 0xcc, b"lazy"), false),
+        ];
+        for (n, _) in writes.iter_mut() {
+            n.last_modified = last_modified_before + 1000;
+        }
+        v.fail_next_fsync_for_test(true);
+        let results = v.write_needles_grouped(&mut writes);
+        v.fail_next_fsync_for_test(false);
+
+        assert!(
+            results.iter().all(|r| matches!(r, Err(VolumeError::Io(_)))),
+            "every entry of the run shares its failed sync: {results:?}"
+        );
+        assert_eq!(dat_len(&v), dat_len_before, "the run is off the .dat");
+        assert_eq!(v.last_append_at_ns, last_append_before);
+        assert_eq!(v.last_modified_ts_seconds, last_modified_before);
+        let now = v.nm.as_ref().unwrap().get(NeedleId(1)).unwrap().unwrap();
+        assert_eq!((now.offset, now.size), (prior.offset, prior.size));
+        assert!(v.nm.as_ref().unwrap().get(NeedleId(2)).unwrap().is_none());
+        assert!(v.nm.as_ref().unwrap().get(NeedleId(3)).unwrap().is_none());
+        assert_eq!(v.file_count(), file_count_before);
+        assert!(
+            !v.is_read_only(),
+            "a rolled-back run keeps the volume writable"
+        );
+
+        let mut read_n = Needle {
+            id: NeedleId(1),
+            ..Needle::default()
+        };
+        v.read_needle(&mut read_n).unwrap();
+        assert_eq!(read_n.data, b"first-copy");
+
+        // The same run goes through once the disk recovers.
+        let results = v.write_needles_grouped(&mut writes);
+        assert!(results.iter().all(|r| r.is_ok()));
+        assert!(v.last_append_at_ns > last_append_before);
+    }
+
+    /// A repeated id starts a new run, so the second write sees the first
+    /// one published: a different cookie is refused and the same content
+    /// dedups, exactly as two sequential writes would.
+    #[test]
+    fn test_grouped_repeated_id_behaves_as_sequential_writes() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let mut v = make_test_volume(dir);
+
+        let mut writes = vec![
+            (batch_needle(1, 0xaa, b"one"), true),
+            (batch_needle(1, 0xbb, b"imposter"), true),
+            (batch_needle(2, 0xcc, b"two"), true),
+            (batch_needle(2, 0xcc, b"two"), true),
+        ];
+        let results = v.write_needles_grouped(&mut writes);
+
+        assert!(matches!(results[0], Ok((_, _, false))));
+        assert!(matches!(results[1], Err(VolumeError::CookieMismatch(0xbb))));
+        assert!(matches!(results[2], Ok((_, _, false))));
+        assert!(
+            matches!(results[3], Ok((0, _, true))),
+            "the same content dedups against the write just before it"
+        );
+        // Runs [1], [1, 2], [2]: one pair of syncs each.
+        assert_eq!(v.sync_counts_for_test(), (3, 3));
+
+        let mut read_n = Needle {
+            id: NeedleId(1),
+            ..Needle::default()
+        };
+        v.read_needle(&mut read_n).unwrap();
+        assert_eq!(read_n.data, b"one");
+    }
+
+    /// A failed shared .idx sync fails every published entry and stops the
+    /// volume taking writes, as it does for a single durable write.
+    #[test]
+    fn test_grouped_failed_idx_sync_quarantines_volume() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let mut v = make_test_volume(dir);
+
+        let mut writes = vec![
+            (batch_needle(1, 0xaa, b"durable"), true),
+            (batch_needle(2, 0xbb, b"lazy"), false),
+        ];
+        v.fail_next_idx_sync_for_test(true);
+        let results = v.write_needles_grouped(&mut writes);
+        v.fail_next_idx_sync_for_test(false);
+
+        assert!(results.iter().all(|r| matches!(r, Err(VolumeError::Io(_)))));
+        assert_eq!(v.sync_counts_for_test(), (1, 1));
+        assert!(v.is_read_only());
+    }
+
+    /// A failed shared sync whose truncate also fails leaves an unverified
+    /// tail, so the volume fails closed exactly as a single write does.
+    #[test]
+    fn test_grouped_failed_rollback_marks_volume_unavailable() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let mut v = make_test_volume(dir);
+
+        let mut writes = vec![
+            (batch_needle(1, 0xaa, b"never"), true),
+            (batch_needle(2, 0xbb, b"landed"), false),
+        ];
+        v.fail_next_fsync_for_test(true);
+        v.fail_next_truncate_for_test(true);
+        let results = v.write_needles_grouped(&mut writes);
+        v.fail_next_fsync_for_test(false);
+        v.fail_next_truncate_for_test(false);
+
+        assert!(results.iter().all(|r| r.is_err()));
+        assert!(v.unavailable_error().is_some());
+        assert!(v.is_read_only());
+        assert!(v.should_quarantine());
+        assert!(v.nm.as_ref().unwrap().get(NeedleId(1)).unwrap().is_none());
+
+        let mut later = vec![(batch_needle(3, 0xcc, b"refused"), true)];
+        let results = v.write_needles_grouped(&mut later);
+        assert!(matches!(results[0], Err(VolumeError::Unavailable(_))));
+    }
+
+    /// The I/O error streak after writing four durable needles, the ones in
+    /// `failing` with a media error on their append, first one at a time and
+    /// then as one grouped run. Also returns which grouped writes landed.
+    #[cfg(any(unix, windows))]
+    fn io_error_streaks(failing: &[u64]) -> (i32, i32, Vec<bool>) {
+        let writes = || -> Vec<_> {
+            (1..=4u64)
+                .map(|id| {
+                    (
+                        batch_needle(id, 0xaa, format!("body-{id}").as_bytes()),
+                        true,
+                    )
+                })
+                .collect()
+        };
+        let failing: Vec<_> = failing.iter().map(|&id| NeedleId(id)).collect();
+
+        let tmp = TempDir::new().unwrap();
+        let mut one_by_one = make_test_volume(tmp.path().to_str().unwrap());
+        one_by_one.fail_append_for_test(&failing);
+        for (mut n, fsync) in writes() {
+            let _ = one_by_one.write_needle(&mut n, true, fsync);
+        }
+
+        let tmp2 = TempDir::new().unwrap();
+        let mut v = make_test_volume(tmp2.path().to_str().unwrap());
+        v.fail_append_for_test(&failing);
+        let mut grouped = writes();
+        let results = v.write_needles_grouped(&mut grouped);
+        assert!(!v.is_read_only(), "the failed appends were truncated back");
+
+        let landed = results
+            .iter()
+            .map(|r| matches!(r, Ok((_, _, false))))
+            .collect();
+        (
+            one_by_one.get_io_error_state().1,
+            v.get_io_error_state().1,
+            landed,
+        )
+    }
+
+    /// A run counts I/O errors as the same writes sent one at a time would:
+    /// a success early in the run must not wipe out the streak that the
+    /// failed appends after it built up, or the volume escapes quarantine.
+    #[cfg(any(unix, windows))]
+    #[test]
+    fn test_grouped_run_keeps_the_io_error_streak_of_later_appends() {
+        use crate::storage::io_error::IO_ERROR_TOLERANCE;
+
+        let (one_by_one, grouped, landed) = io_error_streaks(&[2, 3, 4]);
+
+        assert_eq!(landed, [true, false, false, false]);
+        assert_eq!(one_by_one, IO_ERROR_TOLERANCE);
+        assert_eq!(grouped, IO_ERROR_TOLERANCE);
+    }
+
+    /// The other half: a write that lands clears the errors of the appends
+    /// queued before it, even when another append after it fails, or the
+    /// run reaches a quarantine that the same writes one at a time do not.
+    #[cfg(any(unix, windows))]
+    #[test]
+    fn test_grouped_run_clears_the_io_error_streak_of_earlier_appends() {
+        let (one_by_one, grouped, landed) = io_error_streaks(&[1, 2, 4]);
+
+        assert_eq!(landed, [false, false, true, false]);
+        assert_eq!(one_by_one, 1);
+        assert_eq!(grouped, 1);
+    }
+
+    /// An append whose partial bytes cannot be truncated back leaves the .dat
+    /// tail unverified, so the volume fails closed rather than let a later
+    /// append bury them mid-file.
+    #[test]
+    fn test_failed_append_rollback_marks_volume_unavailable() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let mut v = make_test_volume(dir);
+
+        let mut first = batch_needle(1, 0xaa, b"landed");
+        v.write_needle(&mut first, true, true).unwrap();
+
+        // A read-only .dat handle fails the append and the truncate-back alike.
+        v.dat_file = Some(File::open(v.file_name(".dat")).unwrap());
+        let mut n = batch_needle(2, 0xbb, b"never-lands");
+        v.write_needle(&mut n, true, false).unwrap_err();
+
+        assert!(v.unavailable_error().is_some());
+        assert!(v.is_read_only());
+        assert!(v.should_quarantine());
+        assert!(v.nm.as_ref().unwrap().get(NeedleId(2)).unwrap().is_none());
+
+        let mut later = batch_needle(3, 0xcc, b"refused");
+        assert!(matches!(
+            v.write_needle(&mut later, true, true),
+            Err(VolumeError::Unavailable(_))
+        ));
+    }
+
    #[test]
    fn test_volume_write_dedup() {
        let tmp = TempDir::new().unwrap();
@@ -7830,7 +8394,7 @@ mod tests {
                false,
            )
            .unwrap_err();
-        assert!(matches!(err, VolumeError::ReadOnly));
+        assert!(matches!(err, VolumeError::ReadOnly(_)));

        let deleted_size = v
            .delete_needle(&mut Needle {
@@ -8754,7 +9318,7 @@ mod tests {
                    false,
                )
                .unwrap_err();
-            assert!(matches!(err, VolumeError::ReadOnly));
+            assert!(matches!(err, VolumeError::ReadOnly(_)));

            let deleted = v
                .delete_needle(&mut Needle {
@@ -8795,7 +9359,7 @@ mod tests {
                false,
            )
            .unwrap_err();
-        assert!(matches!(err, VolumeError::ReadOnly));
+        assert!(matches!(err, VolumeError::ReadOnly(_)));

        let deleted = v
            .delete_needle(&mut Needle {
@@ -8862,7 +9426,7 @@ mod tests {
            })
            .unwrap_err();
        assert!(
-            matches!(err, VolumeError::ReadOnly),
+            matches!(err, VolumeError::ReadOnly(_)),
            "plain readonly must reject deletes"
        );

@@ -477,6 +477,74 @@ async fn delete_then_get_returns_404() {
    );
 }

+// Go answers both with 500 and an error containing "volume N is read only"
+// (the write behind "failed to write to local disk: ").
+#[tokio::test]
+async fn write_and_delete_on_read_only_volume_say_is_read_only() {
+    let (state, _tmp) = test_state();
+    let uri = "/1,01637037d6";
+
+    let request = |method: &str, body: &[u8]| {
+        Request::builder()
+            .method(method)
+            .uri(uri)
+            .body(Body::from(body.to_vec()))
+            .unwrap()
+    };
+    let error_of = |body: Vec<u8>| -> String {
+        let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
+        json["error"].as_str().unwrap_or_default().to_string()
+    };
+
+    let response = build_admin_router(state.clone())
+        .oneshot(request("POST", b"written before read-only"))
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::CREATED);
+
+    state
+        .store
+        .write()
+        .unwrap()
+        .find_volume_mut(VolumeId(1))
+        .unwrap()
+        .1
+        .set_no_write_or_delete(true);
+
+    let response = build_admin_router(state.clone())
+        .oneshot(request("POST", b"refused"))
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
+    let error = error_of(body_bytes(response).await);
+    assert!(error.contains("volume 1 is read only"), "{error}");
+
+    let response = build_admin_router(state.clone())
+        .oneshot(request("DELETE", b""))
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
+    assert_eq!(
+        error_of(body_bytes(response).await),
+        "Deletion Failed: volume 1 is read only"
+    );
+
+    // The needle is still there to delete once the volume is writable again.
+    state
+        .store
+        .write()
+        .unwrap()
+        .find_volume_mut(VolumeId(1))
+        .unwrap()
+        .1
+        .set_no_write_or_delete(false);
+    let response = build_admin_router(state.clone())
+        .oneshot(request("DELETE", b""))
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::ACCEPTED);
+}
+
 // ============================================================================
 // 6. HEAD returns headers without body
 // ============================================================================
@@ -21,7 +21,7 @@
        <scala.binary.version>2.12</scala.binary.version>
        <junit.version>4.13.2</junit.version>
        <seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
-        <jackson.version>2.22.1</jackson.version> <!-- Upgraded from 2.15.3 -->
+        <jackson.version>2.22.3</jackson.version> <!-- Upgraded from 2.15.3 -->
        <!-- jackson-annotations dropped the patch digit in 2.20; matches jackson.version's major.minor -->
        <jackson.annotations.version>2.22</jackson.annotations.version>
        <netty.version>4.2.16.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
@@ -0,0 +1,258 @@
+package iam
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/xml"
+	"io"
+	"math/big"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/golang-jwt/jwt/v5"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	webIdentityAudience = "web-identity-api-test"
+	webIdentitySubject  = "spiffe://example.org/ns/app/sa/app"
+)
+
+// webIdentityIssuer is an OIDC issuer served by the test: discovery document
+// and JWKS for one RSA key, so the server under test fetches real keys.
+type webIdentityIssuer struct {
+	server *httptest.Server
+	key    *rsa.PrivateKey
+}
+
+func newWebIdentityIssuer(t *testing.T) *webIdentityIssuer {
+	t.Helper()
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	require.NoError(t, err)
+	issuer := &webIdentityIssuer{key: key}
+	mux := http.NewServeMux()
+	mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"issuer":                                issuer.server.URL,
+			"jwks_uri":                              issuer.server.URL + "/keys",
+			"id_token_signing_alg_values_supported": []string{"RS256"},
+		})
+	})
+	mux.HandleFunc("/keys", func(w http.ResponseWriter, _ *http.Request) {
+		b64 := func(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) }
+		_ = json.NewEncoder(w).Encode(map[string]any{"keys": []any{map[string]any{
+			"kty": "RSA", "kid": "k1", "use": "sig", "alg": "RS256",
+			"n": b64(key.PublicKey.N.Bytes()), "e": b64(big.NewInt(int64(key.PublicKey.E)).Bytes()),
+		}}})
+	})
+	issuer.server = httptest.NewServer(mux)
+	t.Cleanup(issuer.server.Close)
+	return issuer
+}
+
+func (i *webIdentityIssuer) claims(sub string) jwt.MapClaims {
+	now := time.Now()
+	return jwt.MapClaims{"iss": i.server.URL, "sub": sub, "aud": webIdentityAudience,
+		"iat": now.Unix(), "exp": now.Add(10 * time.Minute).Unix()}
+}
+
+func (i *webIdentityIssuer) token(t *testing.T, sub string, key *rsa.PrivateKey) string {
+	t.Helper()
+	tok := jwt.NewWithClaims(jwt.SigningMethodRS256, i.claims(sub))
+	tok.Header["kid"] = "k1"
+	signed, err := tok.SignedString(key)
+	require.NoError(t, err)
+	return signed
+}
+
+func (i *webIdentityIssuer) tokenForAudience(t *testing.T, sub, aud string) string {
+	t.Helper()
+	claims := i.claims(sub)
+	claims["aud"] = aud
+	tok := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)
+	tok.Header["kid"] = "k1"
+	signed, err := tok.SignedString(i.key)
+	require.NoError(t, err)
+	return signed
+}
+
+func (i *webIdentityIssuer) unsignedToken(t *testing.T, sub string) string {
+	t.Helper()
+	signed, err := jwt.NewWithClaims(jwt.SigningMethodNone, i.claims(sub)).SignedString(jwt.UnsafeAllowNoneSignatureType)
+	require.NoError(t, err)
+	return signed
+}
+
+func trustPolicyFor(issuerURL, sub string) string {
+	doc, _ := json.Marshal(map[string]any{"Version": "2012-10-17", "Statement": []any{map[string]any{
+		"Effect": "Allow", "Principal": map[string]any{"Federated": issuerURL},
+		"Action":    []string{"sts:AssumeRoleWithWebIdentity"},
+		"Condition": map[string]any{"StringEquals": map[string]any{"oidc:sub": sub}},
+	}}})
+	return string(doc)
+}
+
+// assumeWithWebIdentity returns session credentials, or nil and the error body.
+func assumeWithWebIdentity(t *testing.T, roleArn, token string) (*credentials.Credentials, string) {
+	t.Helper()
+	resp, err := callSTSAPI(t, url.Values{
+		"Action": {"AssumeRoleWithWebIdentity"}, "Version": {"2011-06-15"},
+		"RoleArn": {roleArn}, "RoleSessionName": {"web-identity-api"}, "WebIdentityToken": {token},
+	})
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	if resp.StatusCode != http.StatusOK {
+		return nil, string(body)
+	}
+	var out AssumeRoleWithWebIdentityTestResponse
+	require.NoError(t, xml.Unmarshal(body, &out), "body: %s", body)
+	c := out.Result.Credentials
+	return credentials.NewStaticCredentials(c.AccessKeyId, c.SecretAccessKey, c.SessionToken), ""
+}
+
+func s3ClientWith(t *testing.T, creds *credentials.Credentials) *s3.S3 {
+	t.Helper()
+	sess, err := session.NewSession(&aws.Config{
+		Region: aws.String(TestRegion), Endpoint: aws.String(TestS3Endpoint),
+		Credentials: creds, S3ForcePathStyle: aws.Bool(true), DisableSSL: aws.Bool(true),
+	})
+	require.NoError(t, err)
+	return s3.New(sess)
+}
+
+// TestWebIdentityWithProviderAndRoleManagedThroughIAMAPI configures STS
+// federation entirely at runtime — an OIDC provider, a managed policy and a role
+// created through the IAM API, with no static configuration — and checks that
+// the role admits exactly the subject its trust policy names, with exactly the
+// permissions of its attached policy.
+func TestWebIdentityWithProviderAndRoleManagedThroughIAMAPI(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	if !isSTSEndpointRunning(t) {
+		t.Fatal("SeaweedFS STS endpoint is not running at", TestSTSEndpoint, "- please run 'make setup-all-tests' first")
+	}
+
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+	admin, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+	adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	issuer := newWebIdentityIssuer(t)
+	bucket := framework.GenerateUniqueBucketName("web-identity")
+	require.NoError(t, framework.CreateBucketWithCleanup(adminS3, bucket))
+
+	provider, err := admin.CreateOpenIDConnectProvider(&iam.CreateOpenIDConnectProviderInput{
+		Url:          aws.String(issuer.server.URL),
+		ClientIDList: []*string{aws.String(webIdentityAudience)},
+		// Required by this SDK version; pinning only applies to a TLS issuer,
+		// and the test issuer is plain HTTP.
+		ThumbprintList: []*string{aws.String("0000000000000000000000000000000000000000")},
+	})
+	require.NoError(t, err)
+	defer admin.DeleteOpenIDConnectProvider(&iam.DeleteOpenIDConnectProviderInput{OpenIDConnectProviderArn: provider.OpenIDConnectProviderArn})
+
+	policyDoc, _ := json.Marshal(map[string]any{"Version": "2012-10-17", "Statement": []any{map[string]any{
+		"Effect": "Allow", "Action": []string{"s3:*"},
+		"Resource": []string{"arn:aws:s3:::" + bucket, "arn:aws:s3:::" + bucket + "/*"},
+	}}})
+	policy, err := admin.CreatePolicy(&iam.CreatePolicyInput{
+		PolicyName: aws.String(bucket + "-rw"), PolicyDocument: aws.String(string(policyDoc)),
+	})
+	require.NoError(t, err)
+	defer admin.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policy.Policy.Arn})
+
+	// Role names are at most 64 characters; the bucket name is longer.
+	roleName := "web-identity-" + strconv.FormatInt(time.Now().UnixNano(), 36)
+	role, err := admin.CreateRole(&iam.CreateRoleInput{
+		RoleName:                 aws.String(roleName),
+		AssumeRolePolicyDocument: aws.String(trustPolicyFor(issuer.server.URL, webIdentitySubject)),
+	})
+	require.NoError(t, err)
+	defer admin.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
+	_, err = admin.AttachRolePolicy(&iam.AttachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
+	require.NoError(t, err)
+	defer admin.DetachRolePolicy(&iam.DetachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
+	roleArn := aws.StringValue(role.Role.Arn)
+
+	t.Run("the trusted subject gets credentials scoped to the attached policy", func(t *testing.T) {
+		creds, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
+		require.NotNil(t, creds, "AssumeRoleWithWebIdentity refused the trusted subject: %s", failure)
+		client := s3ClientWith(t, creds)
+		_, err := client.PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("federated.txt")})
+		assert.NoError(t, err, "the session cannot write the bucket its policy grants")
+		_, err = client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket + "-other")})
+		assert.Error(t, err, "the session created a bucket its policy does not grant")
+	})
+
+	refusals := map[string]string{
+		"another subject": issuer.token(t, "spiffe://example.org/ns/other/sa/other", issuer.key),
+		"a token signed by another key": func() string {
+			other, err := rsa.GenerateKey(rand.Reader, 2048)
+			require.NoError(t, err)
+			return issuer.token(t, webIdentitySubject, other)
+		}(),
+		"an unsigned token": issuer.unsignedToken(t, webIdentitySubject),
+		// The provider accepts only its registered client IDs as audiences.
+		"a token issued for another audience": issuer.tokenForAudience(t, webIdentitySubject, "some-other-service"),
+	}
+	for name, token := range refusals {
+		t.Run("refuses "+name, func(t *testing.T) {
+			creds, _ := assumeWithWebIdentity(t, roleArn, token)
+			assert.Nil(t, creds, "AssumeRoleWithWebIdentity issued credentials for %s", name)
+		})
+	}
+
+	t.Run("an updated trust policy takes effect", func(t *testing.T) {
+		moved := "spiffe://example.org/ns/app/sa/moved"
+		_, err := admin.UpdateAssumeRolePolicy(&iam.UpdateAssumeRolePolicyInput{
+			RoleName: aws.String(roleName), PolicyDocument: aws.String(trustPolicyFor(issuer.server.URL, moved)),
+		})
+		require.NoError(t, err)
+		creds, _ := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
+		assert.Nil(t, creds, "the subject removed from the trust policy still assumes the role")
+		creds, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, moved, issuer.key))
+		assert.NotNil(t, creds, "the subject added to the trust policy was refused: %s", failure)
+	})
+
+	t.Run("a session does not survive its role being deleted and created again", func(t *testing.T) {
+		trust := trustPolicyFor(issuer.server.URL, webIdentitySubject)
+		_, err := admin.UpdateAssumeRolePolicy(&iam.UpdateAssumeRolePolicyInput{RoleName: aws.String(roleName), PolicyDocument: aws.String(trust)})
+		require.NoError(t, err)
+		before, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
+		require.NotNil(t, before, "precondition: %s", failure)
+
+		_, err = admin.DetachRolePolicy(&iam.DetachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
+		require.NoError(t, err)
+		_, err = admin.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
+		require.NoError(t, err)
+		recreated, err := admin.CreateRole(&iam.CreateRoleInput{RoleName: aws.String(roleName), AssumeRolePolicyDocument: aws.String(trust)})
+		require.NoError(t, err)
+		_, err = admin.AttachRolePolicy(&iam.AttachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
+		require.NoError(t, err)
+		assert.NotEqual(t, aws.StringValue(role.Role.RoleId), aws.StringValue(recreated.Role.RoleId), "the recreated role reuses the deleted role's ID")
+
+		_, err = s3ClientWith(t, before).PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("revived.txt")})
+		assert.Error(t, err, "a session of the deleted role works again under the new role of the same name")
+		after, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
+		require.NotNil(t, after, "the new role refuses its trusted subject: %s", failure)
+		_, err = s3ClientWith(t, after).PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("fresh.txt")})
+		assert.NoError(t, err, "a session of the new role cannot use its policy")
+	})
+}
@@ -0,0 +1,231 @@
+package remote_cache
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	evictRemoteS3      = "http://localhost:28334"
+	evictRemoteMaster  = "30334"
+	evictRemoteFiler   = "28889"
+	evictRemoteVolume  = "30341"
+	evictRemoteWebdav  = "27334"
+	evictRemoteMetrics = "30326"
+
+	evictPrimaryS3      = "http://localhost:28333"
+	evictPrimaryMaster  = "30333"
+	evictPrimaryFiler   = "28887"
+	evictPrimaryVolume  = "30340"
+	evictPrimaryWebdav  = "27333"
+	evictPrimaryMetrics = "30327"
+
+	evictBucket = "evictsrc"
+	evictMount  = "evictmnt"
+)
+
+var miniLogPaths []string
+
+func startMini(t *testing.T, dir string, args ...string) {
+	t.Helper()
+	require.NoError(t, os.MkdirAll(dir, 0755))
+	logPath := filepath.Join(dir, "weed.log")
+	logFile, err := os.Create(logPath)
+	require.NoError(t, err)
+	miniLogPaths = append(miniLogPaths, logPath)
+	cmd := exec.Command(weedBinary, append([]string{"mini",
+		"-dir=" + dir,
+		"-s3.config=s3_config.json",
+		"-s3.allowDeleteBucketNotEmpty=true",
+		"-ip=127.0.0.1", "-ip.bind=127.0.0.1",
+	}, args...)...)
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	require.NoError(t, cmd.Start())
+	t.Cleanup(func() {
+		cmd.Process.Kill()
+		cmd.Wait()
+		logFile.Close()
+	})
+}
+
+func waitForHTTP(t *testing.T, url string) {
+	t.Helper()
+	deadline := time.Now().Add(90 * time.Second)
+	for time.Now().Before(deadline) {
+		if resp, err := http.Get(url); err == nil {
+			resp.Body.Close()
+			return
+		}
+		time.Sleep(time.Second)
+	}
+	for _, p := range miniLogPaths {
+		if data, err := os.ReadFile(p); err == nil {
+			lines := strings.Split(string(data), "\n")
+			if len(lines) > 30 {
+				lines = lines[len(lines)-30:]
+			}
+			t.Logf("last lines of %s:\n%s", p, strings.Join(lines, "\n"))
+		}
+	}
+	t.Fatalf("timed out waiting for %s", url)
+}
+
+func shellOn(t *testing.T, masterPort, command string) string {
+	t.Helper()
+	cmd := exec.Command(weedBinary, "shell", "-master=localhost:"+masterPort)
+	cmd.Stdin = strings.NewReader(command + "\nexit\n")
+	out, err := cmd.CombinedOutput()
+	require.NoErrorf(t, err, "shell %q failed: %s", command, out)
+	return stripLogs(string(out))
+}
+
+func chunkCountOn(t *testing.T, masterPort, path string) string {
+	meta := shellOn(t, masterPort, "fs.meta.cat "+path)
+	idx := strings.LastIndex(meta, "chunks ")
+	require.GreaterOrEqualf(t, idx, 0, "no chunk count in %s", meta)
+	return strings.Fields(meta[idx+len("chunks "):])[0]
+}
+
+func volumeStats(t *testing.T, volumePort string) (size, garbage uint64) {
+	t.Helper()
+	resp, err := http.Get("http://localhost:" + volumePort + "/status")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	var status struct {
+		Volumes []struct {
+			Size             uint64 `json:"Size"`
+			DeletedByteCount uint64 `json:"DeletedByteCount"`
+		} `json:"Volumes"`
+	}
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&status))
+	for _, v := range status.Volumes {
+		size += v.Size
+		garbage += v.DeletedByteCount
+	}
+	return size, garbage
+}
+
+func readViaFiler(t *testing.T, filerPort, path string) []byte {
+	t.Helper()
+	resp, err := http.Get("http://localhost:" + filerPort + path)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	require.Equal(t, http.StatusOK, resp.StatusCode, "read %s", path)
+	data, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	return data
+}
+
+// TestRemoteCacheEvictUnderPressure fills a cache constrained to two small
+// volumes with remote-mounted objects until writes fail, then verifies the
+// filer evicts the oldest synced entry, vacuums the garbage, and a later read
+// caches again.
+func TestRemoteCacheEvictUnderPressure(t *testing.T) {
+	if testing.Short() {
+		t.Skip("spawns two weed mini clusters")
+	}
+	if _, err := os.Stat(weedBinary); err != nil {
+		t.Skipf("weed binary not found at %s; run make build-weed", weedBinary)
+	}
+	if isServerRunning(evictRemoteS3) || isServerRunning(evictPrimaryS3) {
+		t.Skip("eviction test ports are already in use")
+	}
+
+	tmp := t.TempDir()
+	startMini(t, filepath.Join(tmp, "remote"),
+		"-s3.port=28334", "-master.port="+evictRemoteMaster,
+		"-filer.port="+evictRemoteFiler, "-volume.port="+evictRemoteVolume,
+		"-webdav.port="+evictRemoteWebdav, "-metricsPort="+evictRemoteMetrics)
+	waitForHTTP(t, evictRemoteS3)
+	startMini(t, filepath.Join(tmp, "primary"),
+		"-s3.port=28333", "-master.port="+evictPrimaryMaster,
+		"-filer.port="+evictPrimaryFiler, "-volume.port="+evictPrimaryVolume,
+		"-webdav.port="+evictPrimaryWebdav, "-metricsPort="+evictPrimaryMetrics,
+		"-volume.allowUntrustedRemoteEndpoints", "-filer.allowUntrustedRemoteEndpoints",
+		"-s3.allowUntrustedRemoteEndpoints",
+		"-master.volumeSizeLimitMB=32", "-volume.max=2",
+		"-filer.remoteCacheEvictThreshold=0.99")
+	waitForHTTP(t, evictPrimaryS3)
+
+	remote := createS3Client(evictRemoteS3)
+	_, err := remote.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(evictBucket)})
+	require.NoError(t, err)
+	for i := 0; i < 5; i++ {
+		data := make([]byte, 16*1024*1024)
+		for j := range data {
+			data[j] = byte(i + j%251)
+		}
+		_, err = remote.PutObject(&s3.PutObjectInput{
+			Bucket: aws.String(evictBucket),
+			Key:    aws.String(fmt.Sprintf("obj%d.bin", i)),
+			Body:   bytes.NewReader(data),
+		})
+		require.NoError(t, err)
+	}
+
+	shellOn(t, evictPrimaryMaster, fmt.Sprintf(
+		"remote.configure -name=evictremote -type=s3 -s3.access_key=%s -s3.secret_key=%s -s3.endpoint=%s -s3.region=us-east-1",
+		accessKey, secretKey, evictRemoteS3))
+	shellOn(t, evictPrimaryMaster, fmt.Sprintf(
+		"remote.mount -dir=/buckets/%s -remote=evictremote/%s -nonempty", evictMount, evictBucket))
+	shellOn(t, evictPrimaryMaster, fmt.Sprintf("remote.meta.sync -dir=/buckets/%s", evictMount))
+	time.Sleep(2 * time.Second)
+
+	mount := "/buckets/" + evictMount
+
+	// obj0 caches alone so it is the oldest evictable entry.
+	first := readViaFiler(t, evictPrimaryFiler, mount+"/obj0.bin")
+	require.Len(t, first, 16*1024*1024)
+	require.NotEqual(t, "0", chunkCountOn(t, evictPrimaryMaster, mount+"/obj0.bin"), "obj0 should be cached")
+
+	// 4 x 16MB against ~64MB of capacity: the fills overrun, hit the
+	// capacity error path, and trigger eviction + vacuum.
+	var wg sync.WaitGroup
+	for i := 1; i <= 4; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			data := readViaFiler(t, evictPrimaryFiler, fmt.Sprintf("%s/obj%d.bin", mount, i))
+			assert.Len(t, data, 16*1024*1024)
+		}(i)
+	}
+	wg.Wait()
+
+	// Eviction must have dropped obj0's local chunks and vacuumed the
+	// tombstoned bytes, leaving only live data within the two volumes.
+	require.True(t, waitForCondition(t, func() bool {
+		size, garbage := volumeStats(t, evictPrimaryVolume)
+		return garbage == 0 && size <= 72*1024*1024
+	}, 2*time.Minute, "volumes to be reclaimed by eviction+vacuum"),
+		"evicted chunks were not reclaimed")
+
+	assert.Equal(t, "0", chunkCountOn(t, evictPrimaryMaster, mount+"/obj0.bin"),
+		"oldest cached entry should be evicted back to remote-only")
+
+	// The cache self-heals: reading the evicted object re-caches it. Fills
+	// still running from the concurrent wave may evict it again, so retry
+	// the read until a commit sticks.
+	var again []byte
+	require.True(t, waitForCondition(t, func() bool {
+		again = readViaFiler(t, evictPrimaryFiler, mount+"/obj0.bin")
+		return chunkCountOn(t, evictPrimaryMaster, mount+"/obj0.bin") != "0"
+	}, 2*time.Minute, "obj0 to re-cache"),
+		"evicted object did not re-cache on read")
+	assert.Equal(t, first, again)
+}
@@ -89,6 +89,7 @@ type FilerOptions struct {
 	s3ConfigFile              *string // optional path to static S3 identity config

 	allowUntrustedRemoteEndpoints *bool
+	remoteCacheEvictThreshold     *float64
 	// shutdownCtx, when non-nil, tells startFiler to gracefully shut down its
 	// HTTP/gRPC servers once the ctx is cancelled. Used by integration tests
 	// and by weed mini; nil for standalone weed filer.
@@ -134,6 +135,7 @@ func init() {
 	f.tusMaxSizeMB = cmdFiler.Flag.Int("tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
 	f.tusSessionExpiry = cmdFiler.Flag.Duration("tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"")
 	f.allowUntrustedRemoteEndpoints = cmdFiler.Flag.Bool("allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
+	f.remoteCacheEvictThreshold = cmdFiler.Flag.Float64("remoteCacheEvictThreshold", 0.9, "evict remote-cached objects (oldest first) when any volume disk exceeds this usage fraction; 0 disables")

 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -407,6 +409,7 @@ func (fo *FilerOptions) startFiler() {
 		CredentialManager:         credentialManager,

 		AllowUntrustedRemoteEndpoints: *fo.allowUntrustedRemoteEndpoints,
+		RemoteCacheEvictThreshold:     *fo.remoteCacheEvictThreshold,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)
@@ -467,6 +467,7 @@ func initMiniFilerFlags() {
 	miniFilerOptions.tusMaxSizeMB = cmdMini.Flag.Int("filer.tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
 	miniFilerOptions.tusSessionExpiry = cmdMini.Flag.Duration("filer.tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration")
 	miniFilerOptions.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("filer.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
+	miniFilerOptions.remoteCacheEvictThreshold = cmdMini.Flag.Float64("filer.remoteCacheEvictThreshold", 0.9, "evict remote-cached objects (oldest first) when any volume disk exceeds this usage fraction; 0 disables")
 }

 // initMiniVolumeFlags initializes Volume server flag options
@@ -136,6 +136,7 @@ func init() {
 	filerOptions.tusMaxSizeMB = cmdServer.Flag.Int("filer.tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
 	filerOptions.tusSessionExpiry = cmdServer.Flag.Duration("filer.tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"")
 	filerOptions.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("filer.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
+	filerOptions.remoteCacheEvictThreshold = cmdServer.Flag.Float64("filer.remoteCacheEvictThreshold", 0.9, "evict remote-cached objects (oldest first) when any volume disk exceeds this usage fraction; 0 disables")

 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
@@ -380,15 +380,6 @@ func fetchWholeChunk(ctx context.Context, bytesBuffer *bytes.Buffer, lookupFileI
 	})
 }

-func fetchChunkRange(ctx context.Context, buffer []byte, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool, offset int64, refreshUrls util_http.RefreshUrlsFunc) (int, error) {
-	urlStrings, err := lookupFileIdFn(ctx, fileId)
-	if err != nil {
-		glog.ErrorfCtx(ctx, "operation LookupFileId %s failed, err: %v", fileId, err)
-		return 0, err
-	}
-	return util_http.RetriedFetchChunkData(ctx, buffer, urlStrings, cipherKey, isGzipped, false, offset, fileId, refreshUrls)
-}
-
 // retriedStreamFetchChunkData streams a chunk from the first location that
 // answers. refreshUrls may be nil; when a location failed and a later one
 // answered, it is called so the reads that follow start from a fresh list.
@@ -185,6 +185,15 @@ func (cv *ChunkView) IsFullChunk() bool {
 	return cv.OffsetInChunk == 0 && cv.ViewSize == cv.ChunkSize
 }

+// CanRangeFetch reports whether fetching just the view's byte range avoids
+// reading more than the view needs. Ciphered and compressed chunks are
+// stored and served whole — a range on either still costs a full read plus
+// decrypt or decompress on the volume server — so partial views of them
+// take the shared whole-chunk path instead.
+func (cv *ChunkView) CanRangeFetch() bool {
+	return cv.CipherKey == nil && !cv.IsGzipped
+}
+
 func ViewFromChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (chunkViews *IntervalList[*ChunkView]) {

 	visibles, _ := NonOverlappingVisibleIntervals(ctx, lookupFileIdFn, chunks, offset, offset+size)
@@ -23,6 +23,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"

 	"google.golang.org/grpc"
+	"google.golang.org/protobuf/proto"

 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -327,7 +328,7 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
 			return fmt.Errorf("%s: %w", entry.FullPath, filer_pb.ErrEntryAlreadyExists)
 		}
 		glog.V(4).InfofCtx(ctx, "UpdateEntry %s: old entry: %v", entry.FullPath, oldEntry.Name())
-		if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
+		if err := f.UpdateEntry(ctx, oldEntry, entry, isFromOtherCluster); err != nil {
 			if errors.Is(err, filer_pb.ErrExistingIsDirectory) || errors.Is(err, filer_pb.ErrExistingIsFile) {
 				glog.V(2).InfofCtx(ctx, "update entry %s: %v", entry.FullPath, err)
 			} else {
@@ -482,7 +483,7 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
 		narrowed := existing.ShallowClone()
 		narrowed.Mode = existing.Mode&^restorableModeBits | kept
 		glog.V(1).InfofCtx(ctx, "restore directory %s: narrowing %v to %v", dirPath, existing.Mode, narrowed.Mode)
-		if err := f.UpdateEntry(ctx, existing, narrowed); err != nil {
+		if err := f.UpdateEntry(ctx, existing, narrowed, false); err != nil {
 			return err
 		}
 		f.NotifyUpdateEvent(ctx, existing, narrowed, false, false, nil)
@@ -515,7 +516,7 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
 	return nil
 }

-func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
+func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromOtherCluster bool) (err error) {
 	if oldEntry != nil {
 		entry.Attr.Crtime = oldEntry.Attr.Crtime
 		if oldEntry.Attr.Inode != 0 {
@@ -535,6 +536,17 @@ func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err er
 			glog.V(2).InfofCtx(ctx, "existing %s is a file", oldEntry.FullPath)
 			return fmt.Errorf("%s: %w", oldEntry.FullPath, filer_pb.ErrExistingIsFile)
 		}
+		// A local write to a remote-backed entry leaves the copy on remote
+		// stale until a sync re-uploads it. Content changes that arrive
+		// without a fresh sync stamp are unsynced; clear the stamp so nothing
+		// mistakes the still-local chunks for a re-fetchable cache copy.
+		// Replicated updates carry the writer's authoritative stamp.
+		if !isFromOtherCluster && oldEntry.Remote != nil && entry.Remote != nil &&
+			oldEntry.Remote.LastLocalSyncTsNs == entry.Remote.LastLocalSyncTsNs &&
+			!chunksEqual(oldEntry.Chunks, entry.Chunks) {
+			entry.Remote = proto.Clone(entry.Remote).(*filer_pb.RemoteEntry)
+			entry.Remote.LastLocalSyncTsNs = 0
+		}
 	}
 	if entry.Attr.Atime.IsZero() {
 		entry.Attr.Atime = entryInitialAtime(entry.Attr)
@@ -306,24 +306,29 @@ func (f *Filer) loopProcessingDeletion() {
 			glog.V(0).Infof("deletion processor shutting down")
 			return
 		case <-ticker.C:
-			f.FileIdDeletionQueue.Consume(func(fileIds []string) {
-				for i := 0; i < len(fileIds); i += DeletionBatchSize {
-					end := i + DeletionBatchSize
-					if end > len(fileIds) {
-						end = len(fileIds)
-					}
-					toDeleteFileIds := fileIds[i:end]
-					f.processDeletionBatch(toDeleteFileIds, lookupFunc)
-				}
-			})
+			f.FlushFileIdDeletionQueue(context.Background(), lookupFunc)
 		}
 	}
 }

+func (f *Filer) FlushFileIdDeletionQueue(ctx context.Context, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) (consumed []string) {
+	f.FileIdDeletionQueue.Consume(func(fileIds []string) {
+		consumed = fileIds
+		for i := 0; i < len(fileIds); i += DeletionBatchSize {
+			end := i + DeletionBatchSize
+			if end > len(fileIds) {
+				end = len(fileIds)
+			}
+			f.processDeletionBatch(ctx, fileIds[i:end], lookupFunc)
+		}
+	})
+	return consumed
+}
+
 // processDeletionBatch handles deletion of a batch of file IDs and processes results.
 // It classifies errors into retryable and permanent categories, adds retryable failures
 // to the retry queue, and logs appropriate messages.
-func (f *Filer) processDeletionBatch(toDeleteFileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) {
+func (f *Filer) processDeletionBatch(ctx context.Context, toDeleteFileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) {
 	// Deduplicate file IDs to prevent incorrect retry count increments for the same file ID within a single batch.
 	uniqueFileIdsSlice := make([]string, 0, len(toDeleteFileIds))
 	processed := make(map[string]struct{}, len(toDeleteFileIds))
@@ -339,7 +344,7 @@ func (f *Filer) processDeletionBatch(toDeleteFileIds []string, lookupFunc func([
 	}

 	// Delete files and classify outcomes
-	outcomes := deleteFilesAndClassify(f.GrpcDialOption, uniqueFileIdsSlice, lookupFunc)
+	outcomes := deleteFilesAndClassify(ctx, f.GrpcDialOption, uniqueFileIdsSlice, lookupFunc)

 	// Process outcomes
 	var successCount, notFoundCount, retryableErrorCount, permanentErrorCount int
@@ -405,9 +410,9 @@ type deletionOutcome struct {
 }

 // deleteFilesAndClassify performs deletion and classifies outcomes for a list of file IDs
-func deleteFilesAndClassify(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) map[string]deletionOutcome {
+func deleteFilesAndClassify(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) map[string]deletionOutcome {
 	// Perform deletion
-	results := operation.DeleteFileIdsWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
+	results := operation.DeleteFileIdsWithLookupVolumeId(ctx, grpcDialOption, fileIds, lookupFunc)

 	// Group results by file ID to handle multiple results for replicated volumes
 	resultsByFileId := make(map[string][]*volume_server_pb.DeleteResult)
@@ -545,7 +550,7 @@ func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc fu
 	}

 	// Delete files and classify outcomes
-	outcomes := deleteFilesAndClassify(f.GrpcDialOption, fileIds, lookupFunc)
+	outcomes := deleteFilesAndClassify(context.Background(), f.GrpcDialOption, fileIds, lookupFunc)

 	// Process outcomes - iterate over readyItems to ensure all items are accounted for
 	var successCount, notFoundCount, retryCount, permanentErrorCount int
@@ -173,7 +173,7 @@ func TestUpdateEntryPreservesExistingInode(t *testing.T) {
 		},
 	}

-	err := f.UpdateEntry(context.Background(), original, updated)
+	err := f.UpdateEntry(context.Background(), original, updated, false)
 	require.Error(t, err)

 	updated = &Entry{
@@ -182,7 +182,7 @@ func TestUpdateEntryPreservesExistingInode(t *testing.T) {
 			Mode: 0o600,
 		},
 	}
-	err = f.UpdateEntry(context.Background(), original, updated)
+	err = f.UpdateEntry(context.Background(), original, updated, false)
 	require.NoError(t, err)

 	stored, findErr := store.FindEntry(context.Background(), original.FullPath)
@@ -208,7 +208,7 @@ func TestUpdateEntryBackfillsMissingLegacyInode(t *testing.T) {
 			Mode: 0o640,
 		},
 	}
-	err := f.UpdateEntry(context.Background(), original, updated)
+	err := f.UpdateEntry(context.Background(), original, updated, false)
 	require.NoError(t, err)

 	stored, findErr := store.FindEntry(context.Background(), original.FullPath)
@@ -0,0 +1,89 @@
+package filer
+
+import (
+	"context"
+	"sort"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func chunksEqual(a, b []*filer_pb.FileChunk) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i].GetFileIdString() != b[i].GetFileIdString() ||
+			a[i].Size != b[i].Size ||
+			a[i].Offset != b[i].Offset ||
+			a[i].ModifiedTsNs != b[i].ModifiedTsNs {
+			return false
+		}
+	}
+	return true
+}
+
+// IsEvictableRemoteEntry reports whether an entry's local chunks are backed by
+// a synchronized remote copy, mirroring the checks remote.uncache applies:
+// remote-backed, chunks present, and not ahead of the remote version.
+func IsEvictableRemoteEntry(entry *Entry) bool {
+	if entry.IsDirectory() || entry.Remote == nil {
+		return false
+	}
+	if entry.Remote.LastLocalSyncTsNs <= 0 || len(entry.GetChunks()) == 0 {
+		return false
+	}
+	if entry.Remote.LastLocalSyncTsNs < entry.Mtime.UnixNano() {
+		return false
+	}
+	return true
+}
+
+// ListEvictableRemoteEntries walks every mounted directory and returns
+// synchronized remote entries holding local chunks, oldest cached first.
+func (f *Filer) ListEvictableRemoteEntries(ctx context.Context, mounts []util.FullPath, minCacheAge time.Duration) (out []*Entry) {
+	cutoffNs := time.Now().UnixNano() - minCacheAge.Nanoseconds()
+	for _, dir := range mounts {
+		if err := f.collectEvictableRemoteEntries(ctx, dir, cutoffNs, &out); err != nil {
+			glog.WarningfCtx(ctx, "list evictable remote entries under %s: %v", dir, err)
+		}
+	}
+	sort.Slice(out, func(i, j int) bool {
+		return out[i].Remote.LastLocalSyncTsNs < out[j].Remote.LastLocalSyncTsNs
+	})
+	return out
+}
+
+func (f *Filer) collectEvictableRemoteEntries(ctx context.Context, dir util.FullPath, cutoffNs int64, out *[]*Entry) error {
+	startFileName := ""
+	for {
+		var subDirs []util.FullPath
+		lastFileName, err := f.Store.ListDirectoryEntries(ctx, dir, startFileName, false, 1024, func(entry *Entry) (bool, error) {
+			if err := ctx.Err(); err != nil {
+				return false, err
+			}
+			if entry.IsDirectory() {
+				subDirs = append(subDirs, entry.FullPath)
+				return true, nil
+			}
+			if entry.Remote != nil && entry.Remote.LastLocalSyncTsNs > 0 && entry.Remote.LastLocalSyncTsNs <= cutoffNs && IsEvictableRemoteEntry(entry) {
+				*out = append(*out, entry)
+			}
+			return true, nil
+		})
+		if err != nil {
+			return err
+		}
+		for _, subDir := range subDirs {
+			if err := f.collectEvictableRemoteEntries(ctx, subDir, cutoffNs, out); err != nil {
+				return err
+			}
+		}
+		if lastFileName == "" {
+			return nil
+		}
+		startFileName = lastFileName
+	}
+}
@@ -0,0 +1,164 @@
+package filer
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func remoteCachedEntry(path string, syncTsNs int64, mtime time.Time, chunks int) *Entry {
+	entry := &Entry{
+		FullPath: util.FullPath(path),
+		Attr: Attr{
+			Mtime:    mtime,
+			Crtime:   mtime,
+			Mode:     0644,
+			Uid:      1,
+			Gid:      1,
+			Mime:     "application/octet-stream",
+			Md5:      nil,
+			FileSize: 0,
+		},
+		Remote: &filer_pb.RemoteEntry{
+			RemoteMtime:       mtime.Unix(),
+			LastLocalSyncTsNs: syncTsNs,
+			RemoteETag:        "etag",
+			RemoteSize:        100,
+		},
+	}
+	for i := 0; i < chunks; i++ {
+		entry.Chunks = append(entry.Chunks, &filer_pb.FileChunk{FileId: "1,01637037d6", Size: 100})
+	}
+	return entry
+}
+
+func TestIsEvictableRemoteEntry(t *testing.T) {
+	now := time.Now()
+	synced := now.Add(-time.Hour).UnixNano()
+	mtime := now.Add(-time.Hour)
+
+	tests := []struct {
+		name     string
+		entry    *Entry
+		expected bool
+	}{
+		{"directory", &Entry{FullPath: "/d", Attr: Attr{Mode: os.ModeDir | 0755}}, false},
+		{"local only, no remote", &Entry{FullPath: "/f", Attr: Attr{Mtime: mtime, Mode: 0644}}, false},
+		{"remote only, never cached", remoteCachedEntry("/f", 0, mtime, 0), false},
+		{"remote, chunks cleared", remoteCachedEntry("/f", synced, mtime, 0), false},
+		{"dirty, newer than remote", remoteCachedEntry("/f", synced, now, 1), false},
+		{"same-second write after sync", remoteCachedEntry("/f",
+			now.Truncate(time.Second).Add(100*time.Millisecond).UnixNano(),
+			now.Truncate(time.Second).Add(900*time.Millisecond), 1), false},
+		{"same-second sync after write", remoteCachedEntry("/f",
+			now.Truncate(time.Second).Add(900*time.Millisecond).UnixNano(),
+			now.Truncate(time.Second).Add(100*time.Millisecond), 1), true},
+		{"synced and cached", remoteCachedEntry("/f", synced, mtime, 1), true},
+	}
+	for _, tt := range tests {
+		assert.Equal(t, tt.expected, IsEvictableRemoteEntry(tt.entry), tt.name)
+	}
+}
+
+func TestListEvictableRemoteEntries(t *testing.T) {
+	now := time.Now()
+	mtime := now.Add(-2 * time.Hour)
+	oldSync := now.Add(-time.Hour).UnixNano()
+	newSync := now.Add(-time.Second).UnixNano()
+
+	store := newStubFilerStore()
+	f := newTestFiler(t, store, NewFilerRemoteStorage())
+	mounts := []util.FullPath{"/buckets/mybucket"}
+
+	seed := func(e *Entry) {
+		require.NoError(t, f.CreateEntry(context.Background(), e, nil, false, false, nil, false, 255))
+	}
+	seed(remoteCachedEntry("/buckets/mybucket/old.bin", oldSync, mtime, 1))
+	seed(remoteCachedEntry("/buckets/mybucket/fresh.bin", newSync, mtime, 1))
+	seed(remoteCachedEntry("/buckets/mybucket/remoteonly.bin", 0, mtime, 0))
+	seed(remoteCachedEntry("/buckets/mybucket/dirty.bin", oldSync, now, 1))
+	seed(&Entry{FullPath: "/buckets/mybucket/plain.bin", Attr: Attr{Mtime: mtime, Mode: 0644}})
+
+	store.entries["/buckets/mybucket/sub"] = &Entry{
+		FullPath: "/buckets/mybucket/sub",
+		Attr:     Attr{Mode: os.ModeDir | 0755, Mtime: mtime},
+	}
+	seed(remoteCachedEntry("/buckets/mybucket/sub/nested.bin", oldSync-1000, mtime, 1))
+
+	got := f.ListEvictableRemoteEntries(context.Background(), mounts, 0)
+	var paths []string
+	for _, e := range got {
+		paths = append(paths, string(e.FullPath))
+	}
+	assert.Equal(t, []string{
+		"/buckets/mybucket/sub/nested.bin",
+		"/buckets/mybucket/old.bin",
+		"/buckets/mybucket/fresh.bin",
+	}, paths, "oldest LastLocalSyncTsNs first; remote-only, dirty, local entries skipped")
+
+	got = f.ListEvictableRemoteEntries(context.Background(), mounts, 30*time.Second)
+	paths = paths[:0]
+	for _, e := range got {
+		paths = append(paths, string(e.FullPath))
+	}
+	assert.Equal(t, []string{
+		"/buckets/mybucket/sub/nested.bin",
+		"/buckets/mybucket/old.bin",
+	}, paths, "minCacheAge excludes freshly cached entries")
+}
+
+func TestUpdateEntryInvalidatesStaleSyncStamp(t *testing.T) {
+	now := time.Now()
+	synced := now.Add(-time.Hour).UnixNano()
+	mtime := now.Add(-time.Hour)
+	path := "/buckets/mybucket/a.bin"
+
+	newFiler := func() *Filer {
+		f := newTestFiler(t, newStubFilerStore(), NewFilerRemoteStorage())
+		require.NoError(t, f.CreateEntry(context.Background(), remoteCachedEntry(path, synced, mtime, 1), nil, false, false, nil, false, 255))
+		return f
+	}
+	syncStamp := func(f *Filer) int64 {
+		entry, err := f.FindEntry(context.Background(), util.FullPath(path))
+		require.NoError(t, err)
+		return entry.Remote.LastLocalSyncTsNs
+	}
+
+	t.Run("local chunk change clears stamp", func(t *testing.T) {
+		f := newFiler()
+		update := remoteCachedEntry(path, synced, mtime, 1)
+		update.Chunks[0].FileId = "2,01637037d6"
+		require.NoError(t, f.CreateEntry(context.Background(), update, nil, false, false, nil, false, 255))
+		assert.Zero(t, syncStamp(f))
+	})
+
+	t.Run("metadata-only update keeps stamp", func(t *testing.T) {
+		f := newFiler()
+		update := remoteCachedEntry(path, synced, mtime, 1)
+		update.Attr.Mime = "text/plain"
+		require.NoError(t, f.CreateEntry(context.Background(), update, nil, false, false, nil, false, 255))
+		assert.Equal(t, synced, syncStamp(f))
+	})
+
+	t.Run("fresh sync stamp survives chunk change", func(t *testing.T) {
+		f := newFiler()
+		update := remoteCachedEntry(path, now.UnixNano(), mtime, 1)
+		update.Chunks[0].FileId = "2,01637037d6"
+		require.NoError(t, f.CreateEntry(context.Background(), update, nil, false, false, nil, false, 255))
+		assert.Equal(t, now.UnixNano(), syncStamp(f))
+	})
+
+	t.Run("replicated update keeps authoritative stamp", func(t *testing.T) {
+		f := newFiler()
+		update := remoteCachedEntry(path, synced, mtime, 1)
+		update.Chunks[0].FileId = "2,01637037d6"
+		require.NoError(t, f.CreateEntry(context.Background(), update, nil, false, true, nil, false, 255))
+		assert.Equal(t, synced, syncStamp(f))
+	})
+}
@@ -349,14 +349,23 @@ func (c *ChunkReadAt) doReadAt(ctx context.Context, p []byte, offset int64) (n i

 func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunkView *ChunkView, nextChunkViews *Interval[*ChunkView], offset uint64) (n int, err error) {

-	if c.readerPattern.IsRandomMode() {
+	// A view clipped to part of its chunk (e.g. the edge of a ranged GET,
+	// whose views ViewFromVisibleIntervals clips to the request) only ever
+	// needs that part: fetch it as a range no matter the detected pattern.
+	// Fetching the chunk whole would multiply volume-server reads. Ciphered
+	// and compressed chunks are the exception: the volume server reads the
+	// whole blob to serve a range, so they take the shared whole-chunk path
+	// where one download serves every buffer — unless the whole chunk cannot
+	// even fit the reader budget, in which case a range fetch is the only
+	// way to serve the request.
+	rangeFetch := chunkView.CanRangeFetch() || !c.readerCache.budget.canFit(int(chunkView.ChunkSize))
+	if rangeFetch && (!chunkView.IsFullChunk() || c.readerPattern.IsRandomMode()) {
 		c.readerCache.releaseStream(&c.stream)
 		n, err := c.readerCache.chunkCache.ReadChunkAt(buffer, chunkView.FileId, offset)
 		if n > 0 {
 			return n, err
 		}
-		return fetchChunkRange(ctx, buffer, c.readerCache.lookupFileIdFn, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset),
-			refreshUrls(ctx, c.readerCache.cacheInvalidator, c.readerCache.lookupFileIdFn, chunkView.FileId))
+		return c.readerCache.fetchChunkRange(ctx, buffer, chunkView, int64(offset))
 	}

 	shouldCache := (uint64(chunkView.ViewOffset) + chunkView.ChunkSize) <= c.readerCache.chunkCache.GetMaxFilePartSizeInCache()
@@ -380,6 +389,13 @@ func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunk
 // readChunkSliceAtForParallel is a simplified version for parallel chunk fetching
 // It doesn't update lastChunkFid or trigger prefetch (handled by the caller)
 func (c *ChunkReadAt) readChunkSliceAtForParallel(ctx context.Context, buffer []byte, chunkView *ChunkView, offset uint64) (n int, err error) {
+	if (chunkView.CanRangeFetch() || !c.readerCache.budget.canFit(int(chunkView.ChunkSize))) && !chunkView.IsFullChunk() {
+		n, err = c.readerCache.chunkCache.ReadChunkAt(buffer, chunkView.FileId, offset)
+		if n > 0 {
+			return n, err
+		}
+		return c.readerCache.fetchChunkRange(ctx, buffer, chunkView, int64(offset))
+	}
 	shouldCache := (uint64(chunkView.ViewOffset) + chunkView.ChunkSize) <= c.readerCache.chunkCache.GetMaxFilePartSizeInCache()
 	return c.readerCache.ReadChunkAt(ctx, buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), shouldCache)
 }
@@ -208,6 +208,200 @@ func TestChunkStreamConcurrentReadsOnOneReader(t *testing.T) {
 	}
 }

+type recordedFetch struct {
+	fileId      string
+	isFullChunk bool
+	offset      int64
+	size        int
+}
+
+// fetchRecorder stubs the volume fetch and records how each chunk was
+// requested: isFullChunk=false is a range fetch of just the view's slice,
+// isFullChunk=true is a whole-chunk download into the shared cache.
+func fetchRecorder(rc *ReaderCache) (fetches *[]recordedFetch) {
+	var mu sync.Mutex
+	recorded := &[]recordedFetch{}
+	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, isFullChunk bool, offset int64, fileId string, _ util_http.RefreshUrlsFunc) (int, error) {
+		mu.Lock()
+		*recorded = append(*recorded, recordedFetch{fileId, isFullChunk, offset, len(buffer)})
+		mu.Unlock()
+		for i := range buffer {
+			buffer[i] = fileId[len(fileId)-1]
+		}
+		return len(buffer), nil
+	}
+	return recorded
+}
+
+// A reader whose views are clipped to a request window — how the S3 gateway
+// builds a ranged GET — must fetch only the covered part of each chunk:
+// clipped edge views take range fetches, a fully covered chunk keeps the
+// shared whole-chunk path. This is what keeps a ranged GET larger than a
+// buffer from multiplying volume-server reads (issue #11564), without giving
+// up whole-chunk caching where the whole chunk is actually wanted.
+func TestChunkReadAtClippedViewsFetchOnlyCoveredParts(t *testing.T) {
+	const chunkSize = 64 << 10
+
+	rc := NewReaderCache(64, (*chunk_cache.TieredChunkCache)(nil), func(context.Context, string) ([]string, error) {
+		return []string{"unused"}, nil
+	}, nil)
+	defer rc.destroy()
+	fetches := fetchRecorder(rc)
+
+	// Window [56KiB, 152KiB): tail of chunk0, all of chunk1, head of
+	// chunk2, head of ciphered chunk3, head of compressed chunk4 (file
+	// chunks need not be aligned).
+	views := NewIntervalList[*ChunkView]()
+	views.AppendInterval(&Interval[*ChunkView]{
+		StartOffset: chunkSize - 8<<10,
+		StopOffset:  chunkSize,
+		Value:       &ChunkView{FileId: "chunk0", OffsetInChunk: chunkSize - 8<<10, ViewSize: 8 << 10, ViewOffset: chunkSize - 8<<10, ChunkSize: chunkSize},
+	})
+	views.AppendInterval(&Interval[*ChunkView]{
+		StartOffset: chunkSize,
+		StopOffset:  2 * chunkSize,
+		Value:       &ChunkView{FileId: "chunk1", ViewSize: chunkSize, ViewOffset: chunkSize, ChunkSize: chunkSize},
+	})
+	views.AppendInterval(&Interval[*ChunkView]{
+		StartOffset: 2 * chunkSize,
+		StopOffset:  2*chunkSize + 8<<10,
+		Value:       &ChunkView{FileId: "chunk2", ViewSize: 8 << 10, ViewOffset: 2 * chunkSize, ChunkSize: chunkSize},
+	})
+	views.AppendInterval(&Interval[*ChunkView]{
+		StartOffset: 2*chunkSize + 8<<10,
+		StopOffset:  2*chunkSize + 16<<10,
+		Value:       &ChunkView{FileId: "chunk3", ViewSize: 8 << 10, ViewOffset: 2*chunkSize + 8<<10, ChunkSize: chunkSize, CipherKey: []byte("key")},
+	})
+	views.AppendInterval(&Interval[*ChunkView]{
+		StartOffset: 2*chunkSize + 16<<10,
+		StopOffset:  2*chunkSize + 24<<10,
+		Value:       &ChunkView{FileId: "chunk4", ViewSize: 8 << 10, ViewOffset: 2*chunkSize + 16<<10, ChunkSize: chunkSize, IsGzipped: true},
+	})
+
+	reader := NewChunkReaderAtFromClient(context.Background(), rc, views, 4*chunkSize, 0)
+	buf := make([]byte, chunkSize+32<<10)
+	if n, err := reader.ReadAt(buf, chunkSize-8<<10); err != nil || n != len(buf) {
+		t.Fatalf("window read: n=%d err=%v", n, err)
+	}
+	// buf holds [56KiB, 152KiB): chunk0's tail, chunk1, and the heads of
+	// chunk2, chunk3 and chunk4.
+	for i, b := range buf {
+		want := byte('1')
+		if i < 8<<10 {
+			want = '0'
+		} else if i >= 24<<10+chunkSize {
+			want = '4'
+		} else if i >= 16<<10+chunkSize {
+			want = '3'
+		} else if i >= 8<<10+chunkSize {
+			want = '2'
+		}
+		if b != want {
+			t.Fatalf("buf[%d]=%q, want %q", i, b, want)
+		}
+	}
+
+	want := []recordedFetch{
+		{fileId: "chunk0", isFullChunk: false, offset: chunkSize - 8<<10, size: 8 << 10},
+		{fileId: "chunk1", isFullChunk: true, offset: 0, size: chunkSize},
+		{fileId: "chunk2", isFullChunk: false, offset: 0, size: 8 << 10},
+		// partial views, but ciphered and compressed chunks download whole
+		// either way and the shared path decrypts/decompresses once for
+		// every buffer
+		{fileId: "chunk3", isFullChunk: true, offset: 0, size: chunkSize},
+		{fileId: "chunk4", isFullChunk: true, offset: 0, size: chunkSize},
+	}
+	got := map[string]recordedFetch{}
+	for _, f := range *fetches {
+		if _, dup := got[f.fileId]; dup {
+			t.Fatalf("chunk %s fetched more than once: %+v", f.fileId, *fetches)
+		}
+		got[f.fileId] = f
+	}
+	for _, w := range want {
+		if g, ok := got[w.fileId]; !ok {
+			t.Fatalf("chunk %s never fetched: %+v", w.fileId, *fetches)
+		} else if g != w {
+			t.Fatalf("chunk %s fetched as %+v, want %+v", w.fileId, g, w)
+		}
+	}
+}
+
+// The regression from issue #11564: a ranged GET sitting inside one big
+// chunk. Every buffer of the request must stay a range fetch — none may
+// escalate into a whole-chunk download once the reads look sequential.
+func TestChunkReadAtRangeInsideOneChunkStaysRangeFetch(t *testing.T) {
+	const chunkSize = 1 << 20
+	const sliceSize = 16 << 10
+
+	rc := NewReaderCache(64, (*chunk_cache.TieredChunkCache)(nil), func(context.Context, string) ([]string, error) {
+		return []string{"unused"}, nil
+	}, nil)
+	defer rc.destroy()
+	fetches := fetchRecorder(rc)
+
+	// Range [32KiB, 96KiB) inside one 1MiB chunk: a single clipped view.
+	views := NewIntervalList[*ChunkView]()
+	views.AppendInterval(&Interval[*ChunkView]{
+		StartOffset: 32 << 10,
+		StopOffset:  96 << 10,
+		Value:       &ChunkView{FileId: "chunk0", OffsetInChunk: 32 << 10, ViewSize: 64 << 10, ViewOffset: 32 << 10, ChunkSize: chunkSize},
+	})
+
+	reader := NewChunkReaderAtFromClient(context.Background(), rc, views, chunkSize, 0)
+	for offset := int64(32 << 10); offset < 96<<10; offset += sliceSize {
+		buf := make([]byte, sliceSize)
+		if n, err := reader.ReadAt(buf, offset); err != nil || n != sliceSize {
+			t.Fatalf("read at %d: n=%d err=%v", offset, n, err)
+		}
+	}
+
+	if len(*fetches) != 4 {
+		t.Fatalf("got %d fetches, want 4 range fetches: %+v", len(*fetches), *fetches)
+	}
+	for i, f := range *fetches {
+		wantOffset := int64(32<<10) + int64(i)*sliceSize
+		if f.isFullChunk || f.offset != wantOffset || f.size != sliceSize {
+			t.Fatalf("fetch %d = %+v, want range fetch offset=%d size=%d", i, f, wantOffset, sliceSize)
+		}
+	}
+}
+
+// A compressed chunk larger than the reader cache budget can never be
+// downloaded whole — the budget rejects the buffer — so its partial view
+// must fall back to a range fetch even though each range costs a full
+// decompress server-side. The alternative is a failed GET.
+func TestChunkReadAtOversizedCompressedChunkFallsBackToRange(t *testing.T) {
+	const chunkSize = 1 << 20
+
+	budget := NewReaderCacheBudget(64 << 10) // smaller than the chunk
+	rc := NewReaderCache(64, (*chunk_cache.TieredChunkCache)(nil), func(context.Context, string) ([]string, error) {
+		return []string{"unused"}, nil
+	}, nil, budget)
+	defer rc.destroy()
+	fetches := fetchRecorder(rc)
+
+	views := NewIntervalList[*ChunkView]()
+	views.AppendInterval(&Interval[*ChunkView]{
+		StartOffset: 32 << 10,
+		StopOffset:  64 << 10,
+		Value:       &ChunkView{FileId: "chunk0", OffsetInChunk: 32 << 10, ViewSize: 32 << 10, ViewOffset: 32 << 10, ChunkSize: chunkSize, IsGzipped: true},
+	})
+
+	reader := NewChunkReaderAtFromClient(context.Background(), rc, views, chunkSize, 0)
+	buf := make([]byte, 32<<10)
+	if n, err := reader.ReadAt(buf, 32<<10); err != nil || n != len(buf) {
+		t.Fatalf("read: n=%d err=%v", n, err)
+	}
+
+	if len(*fetches) != 1 {
+		t.Fatalf("got %d fetches, want 1 range fetch: %+v", len(*fetches), *fetches)
+	}
+	if f := (*fetches)[0]; f.isFullChunk || f.offset != 32<<10 || f.size != 32<<10 {
+		t.Fatalf("fetch = %+v, want range fetch offset=%d size=%d", f, 32<<10, 32<<10)
+	}
+}
+
 // A chunk a stream is positioned in must outlast downloader-limit eviction:
 // otherwise a busy cache drops the buffer mid-stream and forces a refetch.
 func TestChunkReadAtPinnedChunkSurvivesEviction(t *testing.T) {
@@ -100,6 +100,14 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView], count int) {
 			// abort when slots are filled
 			return
 		}
+		if (chunkView.CanRangeFetch() || !rc.budget.canFit(int(chunkView.ChunkSize))) && !chunkView.IsFullChunk() {
+			// the view is clipped to part of the chunk and will be
+			// range-fetched, so prefetching it whole would download bytes
+			// nobody needs; a ciphered or compressed partial view needs
+			// the whole blob anyway and is worth prefetching, but not when
+			// it cannot fit the budget at all
+			continue
+		}

 		// glog.V(4).Infof("prefetch %s offset %d", chunkView.FileId, chunkView.ViewOffset)
 		// cache this chunk if not yet
@@ -114,6 +122,20 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView], count int) {
 	return
 }

+// fetchChunkRange downloads only [offset, offset+len(buffer)) of a chunk,
+// for views clipped to part of their chunk and for random-mode reads. It
+// goes through fetchChunkDataFn so tests observe range fetches the same way
+// they observe whole-chunk downloads.
+func (rc *ReaderCache) fetchChunkRange(ctx context.Context, buffer []byte, chunkView *ChunkView, offset int64) (int, error) {
+	urlStrings, err := rc.lookupFileIdFn(ctx, chunkView.FileId)
+	if err != nil {
+		glog.ErrorfCtx(ctx, "operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
+		return 0, err
+	}
+	return rc.fetchChunkDataFn(ctx, buffer, urlStrings, chunkView.CipherKey, chunkView.IsGzipped, false, offset, chunkView.FileId,
+		refreshUrls(ctx, rc.cacheInvalidator, rc.lookupFileIdFn, chunkView.FileId))
+}
+
 // chunkStream is one sequential reader's position in a shared ReaderCache.
 // The chunk it is reading stays pinned until the stream reads it to the end or
 // moves elsewhere, so another stream finishing or leaving the same chunk does
@@ -93,6 +93,13 @@ func (b *ReaderCacheBudget) reserve(s *SingleChunkCacher) error {
 	}
 }

+// canFit reports whether a whole-chunk buffer of this size can ever be
+// reserved. A chunk bigger than the budget cannot be read through the
+// whole-chunk path at all, so callers must fall back to range fetches.
+func (b *ReaderCacheBudget) canFit(size int) bool {
+	return b == nil || int64(mem.AllocationSize(size)) <= b.limit
+}
+
 func (b *ReaderCacheBudget) complete(s *SingleChunkCacher) {
 	if b == nil {
 		return
@@ -72,7 +72,7 @@ func TestReaderCacheBudgetInFlight(t *testing.T) {
 					return len(buffer), nil
 				}
 				if prefetch {
-					rc.MaybeCache(&Interval[*ChunkView]{Value: &ChunkView{FileId: "chunk", ChunkSize: 3 << 10}}, 1)
+					rc.MaybeCache(&Interval[*ChunkView]{Value: &ChunkView{FileId: "chunk", ViewSize: 3 << 10, ChunkSize: 3 << 10}}, 1)
 				} else {
 					readers.Add(1)
 					go func() {
@@ -180,7 +180,7 @@ func TestReaderCacheFailedPrefetchReleasesBudget(t *testing.T) {
 			rc.fetchChunkDataFn = func(_ context.Context, _ []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
 				return 0, fmt.Errorf("fetch failed")
 			}
-			rc.MaybeCache(&Interval[*ChunkView]{Value: &ChunkView{FileId: "failed", ChunkSize: 1024}}, 1)
+			rc.MaybeCache(&Interval[*ChunkView]{Value: &ChunkView{FileId: "failed", ViewSize: 1024, ChunkSize: 1024}}, 1)
 			deadline := time.Now().Add(5 * time.Second)
 			for {
 				rc.Lock()
@@ -278,7 +278,7 @@ func TestReaderCachePrefetchBufferDroppedAfterRead(t *testing.T) {
 		buffer[0] = 42
 		return len(buffer), nil
 	}
-	rc.MaybeCache(&Interval[*ChunkView]{Value: &ChunkView{FileId: "chunk", ChunkSize: 4 << 10}}, 1)
+	rc.MaybeCache(&Interval[*ChunkView]{Value: &ChunkView{FileId: "chunk", ViewSize: 4 << 10, ChunkSize: 4 << 10}}, 1)

 	buf := make([]byte, 4<<10)
 	n, err := rc.ReadChunkAt(context.Background(), buf, "chunk", nil, false, 0, 4<<10, false)
@@ -54,10 +54,13 @@ func (rp *ReaderPattern) MonitorReadAt(offset int64, size int) {
 		if counter < ModeChangeLimit {
 			atomic.AddInt64(&rp.isSequentialCounter, 1)
 		}
+	} else if counter <= 0 {
+		// Entering random mode is a strong verdict: drop to the bottom of
+		// the window so the contiguous tail of one ranged request cannot
+		// flip it back on the next buffer read and pay a whole-chunk fetch.
+		atomic.StoreInt64(&rp.isSequentialCounter, -ModeChangeLimit)
 	} else {
-		if counter > -ModeChangeLimit {
-			atomic.AddInt64(&rp.isSequentialCounter, -1)
-		}
+		atomic.AddInt64(&rp.isSequentialCounter, -1)
 	}
 }

@@ -102,3 +102,23 @@ func TestReaderPatternRecoversFromRandom(t *testing.T) {
 		t.Fatal("sustained near reads must recover sequential mode")
 	}
 }
+
+// A ranged request's first read lands far from the frontier, but its
+// remaining buffer reads are contiguous. The random verdict must stick for
+// them — otherwise the tail of every range >256KiB pays a whole-chunk fetch.
+func TestReaderPatternRangedReadStaysRandom(t *testing.T) {
+	rp := NewReaderPattern()
+	rp.MonitorReadAt(500*mb, 256*1024) // far first read -> -ModeChangeLimit
+	for i := 1; i <= 2; i++ {
+		rp.MonitorReadAt(500*mb+int64(i)*256*1024, 256*1024)
+		if !rp.IsRandomMode() {
+			t.Fatalf("contiguous read %d of a ranged request flipped back to sequential", i+1)
+		}
+	}
+	for i := 3; i < 10; i++ {
+		rp.MonitorReadAt(500*mb+int64(i)*256*1024, 256*1024)
+	}
+	if rp.IsRandomMode() {
+		t.Fatal("sustained sequential reads should restore sequential mode")
+	}
+}
@@ -116,6 +116,14 @@ func (c *GenericCachedRoleStore) DeleteRole(ctx context.Context, filerAddress st
 	return c.Delete(ctx, filerAddress, roleName)
 }

+// UpdateRole implements RoleStore interface. The update reads the filer, not
+// the cache, and the cache is dropped afterwards whatever the outcome: a
+// refused write may mean the cached role is stale.
+func (c *GenericCachedRoleStore) UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error {
+	defer c.ClearCache()
+	return c.adapter.store.UpdateRole(ctx, filerAddress, roleName, update)
+}
+
 // genericCopyRoleDefinition creates a deep copy of a RoleDefinition for the generic cache
 func genericCopyRoleDefinition(role *RoleDefinition) *RoleDefinition {
 	if role == nil {
@@ -127,6 +135,9 @@ func genericCopyRoleDefinition(role *RoleDefinition) *RoleDefinition {
 		RoleArn:            role.RoleArn,
 		Description:        role.Description,
 		MaxSessionDuration: role.MaxSessionDuration,
+		Source:             role.Source,
+		CreatedAt:          role.CreatedAt,
+		RoleId:             role.RoleId,
 	}

 	// Deep copy trust policy if it exists
@@ -2,7 +2,10 @@ package integration

 import (
 	"context"
+	"crypto/rand"
+	"crypto/sha256"
 	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -27,11 +30,30 @@ const maxPoliciesForEvaluation = 1024

 // IAMManager orchestrates all IAM components
 type IAMManager struct {
-	stsService           *sts.STSService
-	policyEngine         *policy.PolicyEngine
-	roleStore            RoleStore
-	userStore            UserStore
-	oidcProviderStore    OIDCProviderStore
+	stsService        *sts.STSService
+	policyEngine      *policy.PolicyEngine
+	roleStore         RoleStore
+	userStore         UserStore
+	oidcProviderStore OIDCProviderStore
+	// staticOIDCProviders are the OIDC providers of this server's IAM config
+	// file, by ARN. With an in-memory store they are also written to the
+	// store; a persistent store never holds them (see installOIDCProviderStore).
+	staticOIDCProviders map[string]*OIDCProviderRecord
+	// staticRoles are the roles of this server's IAM config file, by name, once
+	// LoadStaticRoles has run (see installRoleStore).
+	staticRoles map[string]*RoleDefinition
+	// oidcRetryMu guards the background refresh retry and which store is
+	// current: cancelOIDCLoad stops the retry, oidcRetryGen names the one
+	// running (0 when none) so at most one runs, and oidcRetryAgain records a
+	// refresh that failed while it ran, so the retry runs once more.
+	oidcRetryMu    sync.Mutex
+	cancelOIDCLoad context.CancelFunc
+	oidcRetryGen   uint64
+	oidcRetrySeq   uint64
+	oidcRetryAgain bool
+	// oidcRefreshMu serializes refreshes from reading the store to handing
+	// STS the result, so an older snapshot cannot replace a newer one.
+	oidcRefreshMu        sync.Mutex
 	oidcAuditSink        OIDCProviderAuditSink
 	revocationStore      SessionRevocationStore
 	filerAddressProvider func() string // Function to get current filer address
@@ -108,12 +130,13 @@ func (m *IAMManager) PurgeRevokedSessions(ctx context.Context) (int, error) {
 }

 // SetOIDCProviderStore configures the IAM-managed OIDC provider store. When
-// nil, OIDC provider IAM actions return ServiceNotReady. The store is the
-// source of truth for AssumeRoleWithWebIdentity provider resolution once
-// Phase 2b lands; in Phase 2a it is read-only and populated from static
-// configuration at boot.
+// nil, OIDC provider IAM actions return ServiceNotReady.
 func (m *IAMManager) SetOIDCProviderStore(store OIDCProviderStore) {
-	m.oidcProviderStore = store
+	var stsConfig *sts.STSConfig
+	if m.stsService != nil {
+		stsConfig = m.stsService.Config
+	}
+	m.installOIDCProviderStore(store, stsConfig)
 }

 // GetOIDCProviderStore returns the configured store (may be nil).
@@ -121,13 +144,33 @@ func (m *IAMManager) GetOIDCProviderStore() OIDCProviderStore {
 	return m.oidcProviderStore
 }

+// GetRoleStore returns the configured role store.
+func (m *IAMManager) GetRoleStore() RoleStore {
+	return m.roleStore
+}
+
+// SetRoleStore replaces the role store. An S3 server builds the list of
+// directories it watches for peers' changes once, at startup, from the store
+// installed then (RoleStoreDirectory): a filer-backed store installed later
+// with a different basePath is not watched, so peers' changes to it reach this
+// server's cached roles only when the cache expires.
+func (m *IAMManager) SetRoleStore(store RoleStore) {
+	m.installRoleStore(context.Background(), store)
+}
+
 // GetOIDCProvider returns the record for the given ARN, or an error if the
 // store is not configured or the record is missing.
 func (m *IAMManager) GetOIDCProvider(ctx context.Context, arn string) (*OIDCProviderRecord, error) {
 	if m.oidcProviderStore == nil {
 		return nil, fmt.Errorf("OIDC provider store not configured")
 	}
-	return m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	if errors.Is(err, ErrOIDCProviderNotFound) {
+		if static, ok := m.staticOIDCProviders[arn]; ok {
+			return copyOIDCProviderRecord(static), nil
+		}
+	}
+	return rec, err
 }

 // ListOIDCProviders enumerates all configured OIDC providers.
@@ -135,7 +178,33 @@ func (m *IAMManager) ListOIDCProviders(ctx context.Context) ([]*OIDCProviderReco
 	if m.oidcProviderStore == nil {
 		return nil, fmt.Errorf("OIDC provider store not configured")
 	}
-	return m.oidcProviderStore.ListProviders(ctx, m.getFilerAddress())
+	records, err := m.oidcProviderStore.ListProviders(ctx, m.getFilerAddress())
+	if err != nil {
+		return nil, err
+	}
+	// A persistent store does not hold the config file's providers.
+	seen := make(map[string]bool, len(records))
+	for _, rec := range records {
+		seen[rec.ARN] = true
+	}
+	for arn, static := range m.staticOIDCProviders {
+		if !seen[arn] {
+			records = append(records, copyOIDCProviderRecord(static))
+		}
+	}
+	return records, nil
+}
+
+// mutableOIDCProvider loads a stored provider for a change. A provider only
+// the IAM config file defines is refused with ErrOIDCProviderStatic.
+func (m *IAMManager) mutableOIDCProvider(ctx context.Context, arn string) (*OIDCProviderRecord, error) {
+	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	if errors.Is(err, ErrOIDCProviderNotFound) {
+		if _, ok := m.staticOIDCProviders[arn]; ok {
+			return nil, fmt.Errorf("%w: %s", ErrOIDCProviderStatic, arn)
+		}
+	}
+	return rec, err
 }

 // CreateOIDCProvider persists a new IAM-managed OIDC provider record. Refuses
@@ -150,6 +219,9 @@ func (m *IAMManager) CreateOIDCProvider(ctx context.Context, rec *OIDCProviderRe
 	if err := validateOIDCProviderRecord(rec); err != nil {
 		return err
 	}
+	if _, static := m.staticOIDCProviders[rec.ARN]; static {
+		return fmt.Errorf("%w: %s", ErrOIDCProviderAlreadyExists, rec.ARN)
+	}
 	existing, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), rec.ARN)
 	if err == nil && existing != nil {
 		return fmt.Errorf("%w: %s", ErrOIDCProviderAlreadyExists, rec.ARN)
@@ -173,6 +245,9 @@ func (m *IAMManager) DeleteOIDCProvider(ctx context.Context, arn string) error {
 	if m.oidcProviderStore == nil {
 		return fmt.Errorf("OIDC provider store not configured")
 	}
+	if _, err := m.mutableOIDCProvider(ctx, arn); errors.Is(err, ErrOIDCProviderStatic) {
+		return err
+	}
 	if err := m.oidcProviderStore.DeleteProvider(ctx, m.getFilerAddress(), arn); err != nil {
 		return err
 	}
@@ -190,7 +265,7 @@ func (m *IAMManager) AddClientIDToOIDCProvider(ctx context.Context, arn, clientI
 	if clientID == "" {
 		return fmt.Errorf("ClientID cannot be empty")
 	}
-	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	rec, err := m.mutableOIDCProvider(ctx, arn)
 	if err != nil {
 		return err
 	}
@@ -218,7 +293,7 @@ func (m *IAMManager) RemoveClientIDFromOIDCProvider(ctx context.Context, arn, cl
 	if m.oidcProviderStore == nil {
 		return fmt.Errorf("OIDC provider store not configured")
 	}
-	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	rec, err := m.mutableOIDCProvider(ctx, arn)
 	if err != nil {
 		return err
 	}
@@ -255,7 +330,7 @@ func (m *IAMManager) UpdateOIDCProviderThumbprints(ctx context.Context, arn stri
 			return fmt.Errorf("invalid thumbprint %q: must be 40-character SHA-1 hex", tp)
 		}
 	}
-	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	rec, err := m.mutableOIDCProvider(ctx, arn)
 	if err != nil {
 		return err
 	}
@@ -274,7 +349,7 @@ func (m *IAMManager) TagOIDCProvider(ctx context.Context, arn string, tags map[s
 	if m.oidcProviderStore == nil {
 		return fmt.Errorf("OIDC provider store not configured")
 	}
-	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	rec, err := m.mutableOIDCProvider(ctx, arn)
 	if err != nil {
 		return err
 	}
@@ -297,7 +372,7 @@ func (m *IAMManager) UntagOIDCProvider(ctx context.Context, arn string, keys []s
 	if m.oidcProviderStore == nil {
 		return fmt.Errorf("OIDC provider store not configured")
 	}
-	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	rec, err := m.mutableOIDCProvider(ctx, arn)
 	if err != nil {
 		return err
 	}
@@ -415,6 +490,88 @@ type RoleDefinition struct {
 	// set it must satisfy AWS bounds: 3600 ≤ MaxSessionDuration ≤ 43200.
 	// Honoured by AssumeRole, AssumeRoleWithWebIdentity, AssumeRoleWithCredentials.
 	MaxSessionDuration int64 `json:"maxSessionDuration,omitempty"`
+
+	// Source records where the role came from. RoleSourceStaticConfig marks a
+	// role loaded from the IAM config file; empty means it was created at
+	// runtime. Only static-config roles are pruned when they leave the file.
+	Source string `json:"source,omitempty"`
+
+	// CreatedAt is when the role was created through the IAM API. Zero for
+	// roles loaded from the config file.
+	CreatedAt time.Time `json:"createdAt,omitempty"`
+
+	// RoleId uniquely identifies this role, as AWS's RoleId does. A role
+	// deleted and created again under the same name gets a new ID, and a
+	// session is honoured only while the role it was issued for still has
+	// the ID the session carries — so a session outlives neither the role's
+	// deletion nor a later role that reuses its name.
+	RoleId string `json:"roleId,omitempty"`
+}
+
+// NewRoleID returns a fresh, random role ID in AWS's AROA form.
+func NewRoleID() string {
+	const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+	b := make([]byte, 17)
+	if _, err := rand.Read(b); err != nil {
+		panic(fmt.Sprintf("read random role id: %v", err))
+	}
+	for i := range b {
+		b[i] = alphabet[int(b[i])%len(alphabet)]
+	}
+	return "AROA" + string(b)
+}
+
+// StaticRoleID is the ID of a role defined in the IAM config file. Such a
+// role is created again at every start, so its ID is derived rather than
+// random, to keep sessions valid across restarts. It is derived from the name
+// and the trust policy together: a role removed from the file and replaced by
+// a different one under the same name trusts different principals, and must
+// not inherit the old role's sessions. Restoring the same role restores its ID.
+func StaticRoleID(role *RoleDefinition) string {
+	h := sha256.New()
+	h.Write([]byte("static-role:" + role.RoleName + "\x00"))
+	if role.TrustPolicy != nil {
+		trust, err := json.Marshal(role.TrustPolicy)
+		if err != nil {
+			// A trust policy that cannot be encoded cannot be matched on
+			// either; give the role an ID no session can carry.
+			return NewRoleID()
+		}
+		h.Write(trust)
+	}
+	return "AROA" + strings.ToUpper(hex.EncodeToString(h.Sum(nil)))[:17]
+}
+
+// RoleSourceStaticConfig is the Source of a role loaded from the IAM config
+// file.
+const RoleSourceStaticConfig = "static-config"
+
+// checkSessionRoleBinding refuses a session issued for a role that has since
+// been deleted, or replaced by a role reusing its name: the role's current ID
+// must be the one the session carries. It runs for every session carrying a
+// role ID, whatever policies the session embeds — the policies a session
+// embeds are the ones its role had, and outlive the role otherwise. A session
+// issued before role IDs were recorded carries none and is not bound. It
+// returns the role it checked, nil for an unbound session.
+func (m *IAMManager) checkSessionRoleBinding(ctx context.Context, sessionInfo *sts.SessionInfo) (*RoleDefinition, error) {
+	if sessionInfo == nil || sessionInfo.RoleId == "" {
+		return nil, nil
+	}
+	roleName := utils.ExtractRoleNameFromArn(sessionInfo.RoleArn)
+	if roleName == "" {
+		return nil, nil
+	}
+	role, err := m.roleStore.GetRole(ctx, m.getFilerAddress(), roleName)
+	if errors.Is(err, ErrRoleNotFound) {
+		return nil, fmt.Errorf("session was issued for role %s, which no longer exists", roleName)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("resolve role %s for session: %w", roleName, err)
+	}
+	if role.RoleId != sessionInfo.RoleId {
+		return nil, fmt.Errorf("session was issued for an earlier role named %s", roleName)
+	}
+	return role, nil
 }

 // ActionRequest represents a request to perform an action
@@ -557,55 +714,174 @@ func (m *IAMManager) initOIDCProviderStore(config *IAMConfig) error {
 	if err != nil {
 		return err
 	}
-	m.oidcProviderStore = store
+	m.installOIDCProviderStore(store, config.STS)
+	return nil
+}

-	if config.STS == nil {
-		return nil
+// installOIDCProviderStore makes store the manager's OIDC provider store.
+//
+// The IAM config file's providers are reported by the IAM API alongside the
+// stored ones. An in-memory store holds them as records, as it always has. A
+// persistent store never does: it outlives this process and may be shared by
+// S3 servers with different config files, so a record written from one file
+// would outlive its removal from that file and be trusted by every server.
+// Those providers stay in memory (staticOIDCProviders) and STS keeps serving
+// them from its static configuration; the store holds only providers created
+// through the IAM API, and those are loaded into STS here.
+//
+// A provider stored under the same ARN as a config-file provider takes
+// precedence, in the IAM API as in STS, which already prefers IAM-managed
+// providers so that an API call can shadow a bootstrap entry. Deleting the
+// stored provider brings the config-file one back.
+func (m *IAMManager) installOIDCProviderStore(store OIDCProviderStore, stsConfig *sts.STSConfig) {
+	// Cancel the old store's retry and switch stores in one step, so a failed
+	// refresh of the old store cannot start a retry after the cancel.
+	m.oidcRetryMu.Lock()
+	m.stopOIDCRetryLocked()
+	m.oidcProviderStore = store
+	m.oidcRetryMu.Unlock()
+	m.staticOIDCProviders = staticOIDCProviderRecords(stsConfig)
+	if _, inMemory := store.(*MemoryOIDCProviderStore); inMemory {
+		ctx := context.Background()
+		now := time.Now().UTC()
+		for _, rec := range m.staticOIDCProviders {
+			mirrored := copyOIDCProviderRecord(rec)
+			mirrored.CreatedAt, mirrored.UpdatedAt = now, now
+			if err := store.StoreProvider(ctx, m.getFilerAddress(), mirrored); err != nil {
+				glog.Warningf("mirror static OIDC provider %s into store: %v", rec.ARN, err)
+			}
+		}
+		// The store now holds them and the API may change them, as before;
+		// the overlay is only for stores that must not hold them.
+		m.staticOIDCProviders = nil
+		return
 	}
-	for _, pc := range config.STS.Providers {
+	// The metadata subscription only reports changes made from now on, so
+	// providers already in the store would stay unknown until one changes; a
+	// failed load is retried (RefreshOIDCProvidersFromStore).
+	if err := m.RefreshOIDCProvidersFromStore(context.Background()); err != nil {
+		glog.Warningf("load OIDC providers from the store at startup: %v; retrying in the background", err)
+	}
+}
+
+// startOIDCRetry retries loading store in the background until it succeeds.
+// A store that is no longer current gets no retry: nothing would cancel it,
+// and its eventual success would replace the current store's providers. When
+// a retry is already running, it is asked to run once more instead, because
+// it may already have listed a snapshot older than this failure.
+func (m *IAMManager) startOIDCRetry(store OIDCProviderStore) {
+	m.oidcRetryMu.Lock()
+	defer m.oidcRetryMu.Unlock()
+	if store != m.oidcProviderStore {
+		return
+	}
+	if m.oidcRetryGen != 0 {
+		m.oidcRetryAgain = true
+		return
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	m.oidcRetrySeq++
+	gen := m.oidcRetrySeq
+	m.cancelOIDCLoad, m.oidcRetryGen = cancel, gen
+	bounds := oidcHydrateRetry // read here, not in the goroutine: it outlives its caller
+	go func() {
+		defer cancel()
+		for {
+			m.retryOIDCProviderLoad(ctx, store, bounds)
+			m.oidcRetryMu.Lock()
+			if m.oidcRetryGen != gen {
+				m.oidcRetryMu.Unlock()
+				return // cancelled: another store was installed
+			}
+			if m.oidcRetryAgain && ctx.Err() == nil {
+				m.oidcRetryAgain = false
+				m.oidcRetryMu.Unlock()
+				continue
+			}
+			m.cancelOIDCLoad, m.oidcRetryGen, m.oidcRetryAgain = nil, 0, false
+			m.oidcRetryMu.Unlock()
+			return
+		}
+	}()
+}
+
+// stopOIDCRetryLocked cancels a running retry, as installing another store
+// must. The caller holds oidcRetryMu.
+func (m *IAMManager) stopOIDCRetryLocked() {
+	if m.cancelOIDCLoad != nil {
+		m.cancelOIDCLoad()
+	}
+	m.cancelOIDCLoad, m.oidcRetryGen, m.oidcRetryAgain = nil, 0, false
+}
+
+// currentOIDCProviderStore is the installed store, read under oidcRetryMu.
+func (m *IAMManager) currentOIDCProviderStore() OIDCProviderStore {
+	m.oidcRetryMu.Lock()
+	defer m.oidcRetryMu.Unlock()
+	return m.oidcProviderStore
+}
+
+// staticOIDCProviderRecords describes the enabled OIDC providers of the IAM
+// config file as provider records.
+func staticOIDCProviderRecords(stsConfig *sts.STSConfig) map[string]*OIDCProviderRecord {
+	out := map[string]*OIDCProviderRecord{}
+	if stsConfig == nil {
+		return out
+	}
+	for _, pc := range stsConfig.Providers {
 		if pc == nil || !pc.Enabled || pc.Type != sts.ProviderTypeOIDC {
 			continue
 		}
 		issuer, _ := pc.Config["issuer"].(string)
 		if issuer == "" {
-			glog.Warningf("OIDC provider %s in static config has empty issuer; skipping mirror to store", pc.Name)
+			glog.Warningf("OIDC provider %s in static config has empty issuer; skipping", pc.Name)
 			continue
 		}
-		accountID := ""
-		if config.STS != nil {
-			accountID = config.STS.AccountId
-		}
-		arn, err := DeriveOIDCProviderARN(accountID, issuer)
+		arn, err := DeriveOIDCProviderARN(stsConfig.AccountId, issuer)
 		if err != nil {
 			glog.Warningf("derive ARN for static OIDC provider %s: %v", pc.Name, err)
 			continue
 		}
-		clientIDs := extractClientIDs(pc.Config)
-		ctx := context.Background()
-		// Preserve CreatedAt across reboots when a persistent store already
-		// has this provider — IAM's GetOpenIDConnectProvider response
-		// shouldn't shift its CreateDate every time the server restarts.
-		now := time.Now().UTC()
-		createdAt := now
-		if existing, err := store.GetProviderByARN(ctx, m.getFilerAddress(), arn); err == nil && existing != nil && !existing.CreatedAt.IsZero() {
-			createdAt = existing.CreatedAt
-		}
-		rec := &OIDCProviderRecord{
-			AccountID:               accountID,
+		out[arn] = &OIDCProviderRecord{
+			AccountID:               stsConfig.AccountId,
 			ARN:                     arn,
 			URL:                     issuer,
-			ClientIDs:               clientIDs,
+			ClientIDs:               extractClientIDs(pc.Config),
 			Thumbprints:             extractStringList(pc.Config, "thumbprints"),
 			AllowedPrincipalTagKeys: extractStringList(pc.Config, "allowedPrincipalTagKeys"),
 			PolicyClaim:             extractString(pc.Config, "policyClaim"),
-			CreatedAt:               createdAt,
-			UpdatedAt:               now,
+			// No CreatedAt: a config-file provider has no creation time the
+			// server could report consistently across restarts.
 		}
-		if err := store.StoreProvider(ctx, m.getFilerAddress(), rec); err != nil {
-			glog.Warningf("mirror static OIDC provider %s into store: %v", pc.Name, err)
+	}
+	return out
+}
+
+// oidcHydrateRetry bounds the backoff between startup load attempts.
+var oidcHydrateRetry = struct{ initial, max time.Duration }{initial: time.Second, max: 30 * time.Second}
+
+// retryOIDCProviderLoad retries loading store, the store it was started for,
+// until it succeeds or ctx is cancelled because another store was installed.
+// It calls refreshOIDCProvidersFrom, not RefreshOIDCProvidersFromStore, so it
+// never schedules a retry of its own.
+func (m *IAMManager) retryOIDCProviderLoad(ctx context.Context, store OIDCProviderStore, bounds struct{ initial, max time.Duration }) {
+	delay := bounds.initial
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(delay):
+		}
+		err := m.refreshOIDCProvidersFrom(ctx, store)
+		if err == nil {
+			glog.V(0).Infof("loaded OIDC providers from the store after retrying")
+			return
+		}
+		glog.V(1).Infof("load OIDC providers from the store: %v; retrying in %v", err, delay)
+		if delay *= 2; delay > bounds.max {
+			delay = bounds.max
 		}
 	}
-	return nil
 }

 // refreshOIDCProvidersBestEffort calls RefreshOIDCProvidersFromStore and
@@ -626,14 +902,46 @@ func (m *IAMManager) refreshOIDCProvidersBestEffort(ctx context.Context, op, arn
 // the store is empty (clears the IAM-managed map). Records with empty URLs
 // or invalid configuration are logged and skipped so a single bad entry
 // does not stop the rest from refreshing.
+//
+// A refresh that fails keeps retrying in the background until the store
+// answers. Every caller needs that: a metadata-subscription event reports each
+// change ONCE, so a refresh that found the filer unreachable on it would leave
+// a peer's new provider untrusted, or a deleted one trusted, until an unrelated
+// later change; the refresh after a local IAM API mutation and the startup load
+// have the same shape. At most one retry runs.
 func (m *IAMManager) RefreshOIDCProvidersFromStore(ctx context.Context) error {
-	if m.oidcProviderStore == nil || m.stsService == nil {
+	store := m.currentOIDCProviderStore()
+	err := m.refreshOIDCProvidersFrom(ctx, store)
+	if err != nil && store != nil {
+		m.startOIDCRetry(store)
+	}
+	return err
+}
+
+// refreshOIDCProvidersFrom is RefreshOIDCProvidersFromStore for a given store.
+func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCProviderStore) error {
+	if store == nil || m.stsService == nil {
 		return nil
 	}
-	records, err := m.oidcProviderStore.ListProviders(ctx, m.getFilerAddress())
+	// Refreshes run concurrently: after an IAM API change, on a peer's change
+	// and in the startup retry. Unserialized, a refresh that read the store
+	// before a DeleteOIDCProvider could finish after that call's own refresh
+	// and keep the deleted provider trusted.
+	m.oidcRefreshMu.Lock()
+	defer m.oidcRefreshMu.Unlock()
+	records, err := store.ListProviders(ctx, m.getFilerAddress())
 	if err != nil {
 		return fmt.Errorf("list OIDC providers: %w", err)
 	}
+	// A snapshot of a store that has since been replaced must not replace the
+	// current store's providers: a retry is cancelled when another store is
+	// installed, and a refresh may have listed the old store just before.
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if store != m.currentOIDCProviderStore() {
+		return fmt.Errorf("list OIDC providers: the store was replaced during the refresh")
+	}
 	byIssuer := make(map[string][]sts.ScopedOIDCProvider, len(records))
 	for _, rec := range records {
 		if rec == nil || rec.URL == "" {
@@ -664,9 +972,11 @@ func buildOIDCProviderFromRecord(rec *OIDCProviderRecord) (*oidc.OIDCProvider, e
 		return nil, fmt.Errorf("record cannot be nil")
 	}
 	cfg := &oidc.OIDCConfig{
-		Issuer:      rec.URL,
-		ClientIDs:   append([]string(nil), rec.ClientIDs...),
-		Thumbprints: append([]string(nil), rec.Thumbprints...),
+		Issuer:                  rec.URL,
+		ClientIDs:               append([]string(nil), rec.ClientIDs...),
+		Thumbprints:             append([]string(nil), rec.Thumbprints...),
+		AllowedPrincipalTagKeys: append([]string(nil), rec.AllowedPrincipalTagKeys...),
+		PolicyClaim:             rec.PolicyClaim,
 	}
 	provider := oidc.NewOIDCProvider(rec.ARN)
 	if err := provider.Initialize(cfg); err != nil {
@@ -800,14 +1110,51 @@ func (m *IAMManager) CreateRole(ctx context.Context, filerAddress string, roleNa
 	if !m.initialized {
 		return fmt.Errorf("IAM manager not initialized")
 	}
+	if err := PrepareRoleDefinition(roleName, roleDef); err != nil {
+		return err
+	}
+	if roleDef.RoleId == "" {
+		roleDef.RoleId = NewRoleID()
+	}

+	// Store role definition
+	return m.roleStore.StoreRole(ctx, "", roleName, roleDef)
+}
+
+// UpdateRole changes a role atomically in the role store (see
+// RoleStore.UpdateRole). update receives the role's current definition, nil
+// when it does not exist, and its result is validated like CreateRole's; a
+// nil result deletes the role.
+// The IAM API's role actions use it, so a change made on one S3 server is
+// neither lost to a concurrent change on another nor written over a delete.
+func (m *IAMManager) UpdateRole(ctx context.Context, roleName string, update RoleUpdate) error {
+	if !m.initialized {
+		return fmt.Errorf("IAM manager not initialized")
+	}
+	return m.roleStore.UpdateRole(ctx, "", roleName, func(current *RoleDefinition) (*RoleDefinition, error) {
+		next, err := update(current)
+		if err != nil || next == nil {
+			return next, err
+		}
+		if err := PrepareRoleDefinition(roleName, next); err != nil {
+			return nil, err
+		}
+		return next, nil
+	})
+}
+
+// PrepareRoleDefinition applies CreateRole's defaults and validation to a role
+// about to be stored or loaded.
+func PrepareRoleDefinition(roleName string, roleDef *RoleDefinition) error {
 	if roleName == "" {
 		return fmt.Errorf("role name cannot be empty")
 	}
-
 	if roleDef == nil {
 		return fmt.Errorf("role definition cannot be nil")
 	}
+	if roleDef.RoleName == "" {
+		roleDef.RoleName = roleName
+	}

 	// Set role ARN if not provided
 	if roleDef.RoleArn == "" {
@@ -827,12 +1174,122 @@ func (m *IAMManager) CreateRole(ctx context.Context, filerAddress string, roleNa
 			return fmt.Errorf("MaxSessionDuration must be between 3600 and 43200 seconds, got %d", roleDef.MaxSessionDuration)
 		}
 	}
+	return nil
+}

-	// Store role definition
-	return m.roleStore.StoreRole(ctx, "", roleName, roleDef)
+// LoadStaticRoles installs the roles of the IAM config file.
+//
+// An in-memory store holds them as records, as it always has. A persistent
+// store never does: it outlives this process and may be shared by S3 servers
+// with different config files, so a record written from one file would
+// outlive its removal from that file and be honoured by every server. Those
+// roles are served from memory instead, ahead of the store, and cannot be
+// changed or deleted through the store (ErrRoleStatic). A role stored under
+// the same name takes precedence. They report no creation time.
+func (m *IAMManager) LoadStaticRoles(ctx context.Context, roles []*RoleDefinition) {
+	defs := make(map[string]*RoleDefinition, len(roles))
+	for _, role := range roles {
+		if role == nil {
+			continue
+		}
+		role.Source = RoleSourceStaticConfig
+		if role.RoleId == "" {
+			role.RoleId = StaticRoleID(role)
+		}
+		if err := PrepareRoleDefinition(role.RoleName, role); err != nil {
+			glog.Warningf("Failed to load role %s: %v", role.RoleName, err)
+			continue
+		}
+		defs[role.RoleName] = role
+	}
+	m.staticRoles = defs
+	m.installRoleStore(ctx, m.roleStore)
+}
+
+// installRoleStore makes store the role store, with the config-file roles
+// installed in it as LoadStaticRoles describes, so a store set after startup
+// behaves like the one set at startup.
+func (m *IAMManager) installRoleStore(ctx context.Context, store RoleStore) {
+	if overlay, ok := store.(*staticRoleOverlay); ok {
+		store = overlay.inner
+	}
+	if store == nil || m.staticRoles == nil {
+		m.roleStore = store
+		return
+	}
+	if _, inMemory := store.(*MemoryRoleStore); inMemory {
+		for name, role := range m.staticRoles {
+			if err := store.StoreRole(ctx, "", name, role); err != nil {
+				glog.Warningf("Failed to create role %s: %v", name, err)
+			}
+		}
+		m.roleStore = store
+		return
+	}
+	m.roleStore = &staticRoleOverlay{static: m.staticRoles, inner: store}
 }

 // GetRole retrieves a role definition by name.
+// ListRoles returns every stored role definition.
+func (m *IAMManager) ListRoles(ctx context.Context) ([]*RoleDefinition, error) {
+	if !m.initialized {
+		return nil, fmt.Errorf("IAM manager not initialized")
+	}
+	names, err := m.roleStore.ListRoles(ctx, m.getFilerAddress())
+	if err != nil {
+		return nil, fmt.Errorf("list roles: %w", err)
+	}
+	roles := make([]*RoleDefinition, 0, len(names))
+	for _, name := range names {
+		role, err := m.roleStore.GetRole(ctx, m.getFilerAddress(), name)
+		if errors.Is(err, ErrRoleNotFound) {
+			continue // deleted between list and read
+		}
+		if err != nil {
+			return nil, fmt.Errorf("get role %s: %w", name, err)
+		}
+		roles = append(roles, role)
+	}
+	return roles, nil
+}
+
+// DeleteRole removes a role definition.
+func (m *IAMManager) DeleteRole(ctx context.Context, roleName string) error {
+	if !m.initialized {
+		return fmt.Errorf("IAM manager not initialized")
+	}
+	if roleName == "" {
+		return fmt.Errorf("role name cannot be empty")
+	}
+	return m.roleStore.DeleteRole(ctx, m.getFilerAddress(), roleName)
+}
+
+// InvalidateRoleCache drops any cached role definitions, so a change written
+// to the store by a peer is seen on the next lookup rather than after the
+// cache TTL.
+func (m *IAMManager) InvalidateRoleCache() {
+	if cached, ok := m.roleStore.(interface{ ClearCache() }); ok {
+		cached.ClearCache()
+	}
+}
+
+// RoleStoreDirectory is the filer directory the role store keeps roles in,
+// its configured basePath; empty when the store is not filer-backed. S3
+// servers watch it to drop cached roles when a peer changes one.
+func (m *IAMManager) RoleStoreDirectory() string {
+	store := m.roleStore
+	if overlay, ok := store.(*staticRoleOverlay); ok {
+		store = overlay.inner
+	}
+	if cached, ok := store.(*GenericCachedRoleStore); ok {
+		store = cached.adapter.store
+	}
+	if filerStore, ok := store.(*FilerRoleStore); ok {
+		return filerStore.basePath
+	}
+	return ""
+}
+
 func (m *IAMManager) GetRole(ctx context.Context, roleName string) (*RoleDefinition, error) {
 	if !m.initialized {
 		return nil, fmt.Errorf("IAM manager not initialized")
@@ -943,6 +1400,7 @@ func (m *IAMManager) AssumeRoleWithWebIdentity(ctx context.Context, request *sts
 	// the global MaxSessionLength and the source-token-expiry cap on top of
 	// this; per-role takes precedence whenever it is the tightest bound.
 	request.DurationSeconds = capDurationByRole(request.DurationSeconds, roleDef.MaxSessionDuration, m.defaultTokenDurationSeconds(), m.maxSessionLengthSeconds())
+	request.RoleId = roleDef.RoleId

 	// Use STS service to assume the role
 	return m.stsService.AssumeRoleWithWebIdentity(ctx, request)
@@ -1079,6 +1537,7 @@ func (m *IAMManager) AssumeRoleWithCredentials(ctx context.Context, request *sts
 	request.DurationSeconds = capDurationByRole(request.DurationSeconds, roleDef.MaxSessionDuration, m.defaultTokenDurationSeconds(), m.maxSessionLengthSeconds())

 	// Use STS service to assume the role
+	request.RoleId = roleDef.RoleId
 	return m.stsService.AssumeRoleWithCredentials(ctx, request)
 }

@@ -1092,6 +1551,10 @@ func (m *IAMManager) IsActionAllowed(ctx context.Context, request *ActionRequest
 	// We always try to validate with the internal STS service first if it's a SeaweedFS token.
 	// This ensures that session policies embedded in the token are correctly extracted and enforced.
 	var sessionInfo *sts.SessionInfo
+	// boundRole is the role a session carrying a role ID was checked against;
+	// its policies are the ones evaluated, so the check and the evaluation
+	// see one definition even if the role is replaced in between.
+	var boundRole *RoleDefinition
 	if request.SessionToken != "" {
 		// Parse unverified to check issuer
 		parsed, _, err := new(jwt.Parser).ParseUnverified(request.SessionToken, jwt.MapClaims{})
@@ -1124,6 +1587,9 @@ func (m *IAMManager) IsActionAllowed(ctx context.Context, request *ActionRequest
 					return false, fmt.Errorf("session has been revoked")
 				}
 			}
+			if boundRole, err = m.checkSessionRoleBinding(ctx, sessionInfo); err != nil {
+				return false, err
+			}
 		}
 	}

@@ -1220,9 +1686,12 @@ func (m *IAMManager) IsActionAllowed(ctx context.Context, request *ActionRequest
 				policies = user.GetPolicyNames()
 			} else {
 				// Get role definition
-				roleDef, err := m.roleStore.GetRole(ctx, m.getFilerAddress(), roleName)
-				if err != nil {
-					return false, fmt.Errorf("role not found: %s", roleName)
+				roleDef := boundRole
+				if roleDef == nil || roleDef.RoleName != roleName {
+					roleDef, err = m.roleStore.GetRole(ctx, m.getFilerAddress(), roleName)
+					if err != nil {
+						return false, fmt.Errorf("role not found: %s", roleName)
+					}
 				}

 				hasManagedSubject = true
@@ -2,16 +2,30 @@ package integration

 import (
 	"context"
+	"errors"
 	"fmt"

 	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
 	"github.com/seaweedfs/seaweedfs/weed/iam/utils"
 )

+// ErrTrustPolicyDenied is wrapped when a role's trust policy does not admit
+// the principal.
+var ErrTrustPolicyDenied = errors.New("trust policy denies access to principal")
+
 // ValidateTrustPolicyForPrincipal validates if a principal is allowed to assume a role
 func (m *IAMManager) ValidateTrustPolicyForPrincipal(ctx context.Context, roleArn, principalArn string) error {
+	_, err := m.ResolveRoleForPrincipal(ctx, roleArn, principalArn)
+	return err
+}
+
+// ResolveRoleForPrincipal returns the role roleArn names if its trust policy
+// admits principalArn. Issuing a session from the definition returned binds
+// the session to the role whose trust was evaluated, not to a role that
+// replaced it under the same name in between.
+func (m *IAMManager) ResolveRoleForPrincipal(ctx context.Context, roleArn, principalArn string) (*RoleDefinition, error) {
 	if !m.initialized {
-		return fmt.Errorf("IAM manager not initialized")
+		return nil, fmt.Errorf("IAM manager not initialized")
 	}

 	// Extract role name from ARN
@@ -20,11 +34,11 @@ func (m *IAMManager) ValidateTrustPolicyForPrincipal(ctx context.Context, roleAr
 	// Get role definition
 	roleDef, err := m.roleStore.GetRole(ctx, m.getFilerAddress(), roleName)
 	if err != nil {
-		return fmt.Errorf("failed to get role %s: %w", roleName, err)
+		return nil, fmt.Errorf("failed to get role %s: %w", roleName, err)
 	}

 	if roleDef.TrustPolicy == nil {
-		return fmt.Errorf("role has no trust policy")
+		return nil, fmt.Errorf("%w: role has no trust policy", ErrTrustPolicyDenied)
 	}

 	// Create evaluation context with RequestContext populated so that
@@ -43,8 +57,8 @@ func (m *IAMManager) ValidateTrustPolicyForPrincipal(ctx context.Context, roleAr

 	// Evaluate the trust policy
 	if !m.evaluateTrustPolicy(roleDef.TrustPolicy, evalCtx) {
-		return fmt.Errorf("trust policy denies access to principal: %s", principalArn)
+		return nil, fmt.Errorf("%w: %s", ErrTrustPolicyDenied, principalArn)
 	}

-	return nil
+	return roleDef, nil
 }
@@ -0,0 +1,561 @@
+package integration
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/golang-jwt/jwt/v5"
+	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
+	"github.com/seaweedfs/seaweedfs/weed/iam/sts"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// persistentTestStore stands in for a store that outlives the process and may
+// be shared (the filer store): anything but *MemoryOIDCProviderStore is
+// treated as persistent.
+type persistentTestStore struct{ *MemoryOIDCProviderStore }
+
+const (
+	persistTestStaticIssuer = "https://static.example"
+	persistTestAPIIssuer    = "https://api.example"
+)
+
+// persistTestConfig is an IAM config whose file defines the given OIDC
+// issuers.
+func persistTestConfig(issuers ...string) *IAMConfig {
+	var providers []*sts.ProviderConfig
+	for i, issuer := range issuers {
+		providers = append(providers, &sts.ProviderConfig{
+			Name:    "static-" + string(rune('a'+i)),
+			Type:    sts.ProviderTypeOIDC,
+			Enabled: true,
+			Config:  map[string]interface{}{"issuer": issuer, "clientId": "aud"},
+		})
+	}
+	return &IAMConfig{
+		STS: &sts.STSConfig{
+			TokenDuration:    sts.FlexibleDuration{Duration: time.Hour},
+			MaxSessionLength: sts.FlexibleDuration{Duration: 12 * time.Hour},
+			Issuer:           "test-sts",
+			SigningKey:       []byte("test-signing-key-32-characters-long"),
+			Providers:        providers,
+		},
+		Policy: &policy.PolicyEngineConfig{DefaultEffect: "Deny", StoreType: "memory"},
+		Roles:  &RoleStoreConfig{StoreType: "memory"},
+	}
+}
+
+// startServer initializes a manager as an S3 server would at boot, with the
+// given config file providers and OIDC provider store.
+func startServer(t *testing.T, store OIDCProviderStore, issuers ...string) *IAMManager {
+	t.Helper()
+	cfg := persistTestConfig(issuers...)
+	mgr := NewIAMManager()
+	require.NoError(t, mgr.Initialize(cfg, func() string { return "localhost:8888" }))
+	mgr.installOIDCProviderStore(store, cfg.STS)
+	return mgr
+}
+
+func arnOf(t *testing.T, issuer string) string {
+	t.Helper()
+	arn, err := DeriveOIDCProviderARN("", issuer)
+	require.NoError(t, err)
+	return arn
+}
+
+func listedARNs(t *testing.T, mgr *IAMManager) map[string]bool {
+	t.Helper()
+	recs, err := mgr.ListOIDCProviders(context.Background())
+	require.NoError(t, err)
+	out := map[string]bool{}
+	for _, r := range recs {
+		out[r.ARN] = true
+	}
+	return out
+}
+
+func storedARNs(t *testing.T, store OIDCProviderStore) map[string]bool {
+	t.Helper()
+	recs, err := store.ListProviders(context.Background(), "")
+	require.NoError(t, err)
+	out := map[string]bool{}
+	for _, r := range recs {
+		out[r.ARN] = true
+	}
+	return out
+}
+
+func createAPIProvider(t *testing.T, mgr *IAMManager, issuer string) {
+	t.Helper()
+	require.NoError(t, mgr.CreateOIDCProvider(context.Background(), &OIDCProviderRecord{
+		ARN: arnOf(t, issuer), URL: issuer, ClientIDs: []string{"aud"},
+	}))
+}
+
+// stsKnowsIssuer reports whether STS resolves a provider for the issuer. The
+// token is not validly signed; the only question is which error comes back.
+func stsKnowsIssuer(t *testing.T, mgr *IAMManager, issuer string) bool {
+	t.Helper()
+	tok, err := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
+		"iss": issuer, "sub": "probe", "aud": "aud", "exp": time.Now().Add(time.Hour).Unix(),
+	}).SignedString([]byte("not-the-providers-key"))
+	require.NoError(t, err)
+	_, _, err = mgr.GetSTSService().ValidateWebIdentityToken(context.Background(), tok)
+	require.Error(t, err, "an unsigned-by-provider token for %s was accepted", issuer)
+	return !strings.Contains(err.Error(), "no identity provider registered")
+}
+
+// A persistent store never receives the config file's providers: it outlives
+// the file and may be shared with servers whose files differ.
+func TestPersistentStoreNeverHoldsConfigFileProviders(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, store, persistTestStaticIssuer)
+
+	assert.Empty(t, storedARNs(t, store), "a config-file provider was written to the persistent store")
+	assert.True(t, listedARNs(t, mgr)[arnOf(t, persistTestStaticIssuer)], "the IAM API no longer lists the config-file provider")
+	rec, err := mgr.GetOIDCProvider(context.Background(), arnOf(t, persistTestStaticIssuer))
+	require.NoError(t, err)
+	assert.Equal(t, persistTestStaticIssuer, rec.URL)
+	assert.True(t, stsKnowsIssuer(t, mgr, persistTestStaticIssuer), "STS stopped trusting the config-file provider")
+}
+
+// Servers sharing a store, with different config files, must not remove or
+// replace each other's providers — including a zero-config server.
+func TestServersSharingAStoreKeepEachOthersProviders(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	configured := startServer(t, store, persistTestStaticIssuer)
+	createAPIProvider(t, configured, persistTestAPIIssuer)
+
+	zeroConfig := startServer(t, store)
+
+	assert.True(t, storedARNs(t, store)[arnOf(t, persistTestAPIIssuer)], "a peer's start removed an API-created provider")
+	assert.True(t, listedARNs(t, configured)[arnOf(t, persistTestStaticIssuer)], "the configured server lost its config-file provider")
+	assert.False(t, listedARNs(t, zeroConfig)[arnOf(t, persistTestStaticIssuer)], "a server lists a provider only its peer's config file defines")
+	assert.True(t, stsKnowsIssuer(t, zeroConfig, persistTestAPIIssuer), "the peer does not trust the shared API-created provider")
+	assert.False(t, stsKnowsIssuer(t, zeroConfig, persistTestStaticIssuer), "the peer trusts a provider only another server's config file defines")
+}
+
+// Removing a provider from the config file revokes it at the next start.
+func TestRemovingAProviderFromTheConfigFileRevokesIt(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	startServer(t, store, persistTestStaticIssuer)
+
+	restarted := startServer(t, store) // the file no longer lists it
+
+	assert.False(t, listedARNs(t, restarted)[arnOf(t, persistTestStaticIssuer)])
+	assert.False(t, stsKnowsIssuer(t, restarted, persistTestStaticIssuer), "a provider removed from the config file is still trusted")
+}
+
+// The API cannot change a provider the config file defines, nor create one
+// with its ARN; the file is where it changes.
+func TestConfigFileProvidersCannotBeChangedThroughTheAPI(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, store, persistTestStaticIssuer)
+	ctx := context.Background()
+	arn := arnOf(t, persistTestStaticIssuer)
+
+	for name, err := range map[string]error{
+		"add client ID":     mgr.AddClientIDToOIDCProvider(ctx, arn, "other"),
+		"remove client ID":  mgr.RemoveClientIDFromOIDCProvider(ctx, arn, "aud"),
+		"update thumbprint": mgr.UpdateOIDCProviderThumbprints(ctx, arn, []string{"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"}),
+		"tag":               mgr.TagOIDCProvider(ctx, arn, map[string]string{"k": "v"}),
+		"untag":             mgr.UntagOIDCProvider(ctx, arn, []string{"k"}),
+		"delete":            mgr.DeleteOIDCProvider(ctx, arn),
+	} {
+		assert.ErrorIs(t, err, ErrOIDCProviderStatic, name)
+	}
+	err := mgr.CreateOIDCProvider(ctx, &OIDCProviderRecord{ARN: arn, URL: persistTestStaticIssuer, ClientIDs: []string{"aud"}})
+	assert.ErrorIs(t, err, ErrOIDCProviderAlreadyExists)
+	assert.Empty(t, storedARNs(t, store), "a refused change still wrote to the store")
+}
+
+// Providers created through the API on an earlier boot, or on a peer, are
+// trusted at startup rather than after the next change.
+func TestStartupLoadsStoredProvidersIntoSTS(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	createAPIProvider(t, startServer(t, store), persistTestAPIIssuer)
+
+	restarted := startServer(t, store)
+	assert.True(t, stsKnowsIssuer(t, restarted, persistTestAPIIssuer))
+}
+
+// An in-memory store keeps its behaviour: the config file's providers are
+// records in it, as before.
+func TestInMemoryStoreStillHoldsConfigFileProviders(t *testing.T) {
+	store := NewMemoryOIDCProviderStore()
+	startServer(t, store, persistTestStaticIssuer)
+	assert.True(t, storedARNs(t, store)[arnOf(t, persistTestStaticIssuer)])
+}
+
+// unreachableThenReadyStore fails its first reads, as a filer that is not up
+// yet when the S3 server starts.
+type unreachableThenReadyStore struct {
+	*MemoryOIDCProviderStore
+	mu        sync.Mutex
+	failsLeft int
+}
+
+func (s *unreachableThenReadyStore) ListProviders(ctx context.Context, addr string) ([]*OIDCProviderRecord, error) {
+	s.mu.Lock()
+	if s.failsLeft > 0 {
+		s.failsLeft--
+		s.mu.Unlock()
+		return nil, errors.New("filer unavailable")
+	}
+	s.mu.Unlock()
+	return s.MemoryOIDCProviderStore.ListProviders(ctx, addr)
+}
+
+// A store that cannot be read at startup is retried: the metadata
+// subscription only reports later changes, so without a retry the providers
+// already stored would stay unknown to STS.
+func TestStartupLoadRetriesUntilTheStoreIsReadable(t *testing.T) {
+	saved := oidcHydrateRetry
+	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, 5*time.Millisecond
+	t.Cleanup(func() { oidcHydrateRetry = saved })
+
+	seeded := NewMemoryOIDCProviderStore()
+	require.NoError(t, seeded.StoreProvider(context.Background(), "", &OIDCProviderRecord{
+		ARN: arnOf(t, persistTestAPIIssuer), URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
+	}))
+	mgr := startServer(t, &unreachableThenReadyStore{MemoryOIDCProviderStore: seeded, failsLeft: 3})
+
+	deadline := time.Now().Add(2 * time.Second)
+	for !stsKnowsIssuer(t, mgr, persistTestAPIIssuer) {
+		if time.Now().After(deadline) {
+			t.Fatal("STS never learned the stored provider after the store became readable")
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+}
+
+// A provider stored under a config-file provider's ARN takes precedence, as it
+// does in STS; the API then changes the stored one, and deleting it brings the
+// config-file provider back.
+func TestStoredProviderTakesPrecedenceOverTheConfigFileOne(t *testing.T) {
+	ctx := context.Background()
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	configured := startServer(t, store, persistTestStaticIssuer)
+	peer := startServer(t, store)
+	arn := arnOf(t, persistTestStaticIssuer)
+	require.NoError(t, peer.CreateOIDCProvider(ctx, &OIDCProviderRecord{ARN: arn, URL: persistTestStaticIssuer, ClientIDs: []string{"stored"}}))
+
+	rec, err := configured.GetOIDCProvider(ctx, arn)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"stored"}, rec.ClientIDs, "the config-file provider hides the stored one")
+	assert.NoError(t, configured.AddClientIDToOIDCProvider(ctx, arn, "more"), "the stored provider cannot be changed")
+
+	require.NoError(t, configured.DeleteOIDCProvider(ctx, arn))
+	rec, err = configured.GetOIDCProvider(ctx, arn)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"aud"}, rec.ClientIDs, "deleting the stored provider did not bring the config-file one back")
+}
+
+// A store installed through SetOIDCProviderStore behaves like one installed
+// at startup: stored providers load into STS and config-file providers stay
+// visible to the IAM API.
+func TestSetOIDCProviderStoreInstallsLikeStartup(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	require.NoError(t, store.StoreProvider(context.Background(), "", &OIDCProviderRecord{
+		ARN: arnOf(t, persistTestAPIIssuer), URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
+	}))
+
+	cfg := persistTestConfig(persistTestStaticIssuer)
+	mgr := NewIAMManager()
+	require.NoError(t, mgr.Initialize(cfg, func() string { return "localhost:8888" }))
+	mgr.SetOIDCProviderStore(store)
+
+	assert.True(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer), "a stored provider was not trusted after install")
+	assert.True(t, listedARNs(t, mgr)[arnOf(t, persistTestStaticIssuer)], "the IAM API no longer lists the config-file provider")
+	assert.ErrorIs(t, mgr.DeleteOIDCProvider(context.Background(), arnOf(t, persistTestStaticIssuer)), ErrOIDCProviderStatic)
+}
+
+// A config-file provider reports no creation time: a time taken at startup
+// would change with every restart.
+func TestConfigFileProvidersReportNoCreationTime(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, store, persistTestStaticIssuer)
+	rec, err := mgr.GetOIDCProvider(context.Background(), arnOf(t, persistTestStaticIssuer))
+	require.NoError(t, err)
+	assert.True(t, rec.CreatedAt.IsZero())
+}
+
+// countingUnreadableStore never becomes readable and counts the attempts.
+type countingUnreadableStore struct {
+	*MemoryOIDCProviderStore
+	mu    sync.Mutex
+	reads int
+}
+
+func (s *countingUnreadableStore) ListProviders(context.Context, string) ([]*OIDCProviderRecord, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.reads++
+	return nil, errors.New("filer unavailable")
+}
+
+func (s *countingUnreadableStore) readCount() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.reads
+}
+
+// Installing another store stops the previous one's startup retry.
+func TestInstallingAnotherStoreStopsThePreviousRetry(t *testing.T) {
+	saved := oidcHydrateRetry
+	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, time.Millisecond
+	t.Cleanup(func() { oidcHydrateRetry = saved })
+
+	first := &countingUnreadableStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, first)
+	deadline := time.Now().Add(2 * time.Second)
+	for first.readCount() <= 1 {
+		if time.Now().After(deadline) {
+			t.Fatal("precondition: the first store is never retried")
+		}
+		time.Sleep(time.Millisecond)
+	}
+
+	mgr.installOIDCProviderStore(&persistentTestStore{NewMemoryOIDCProviderStore()}, persistTestConfig().STS)
+	time.Sleep(10 * time.Millisecond) // let an in-flight attempt finish
+	settled := first.readCount()
+	time.Sleep(30 * time.Millisecond)
+	assert.Equal(t, settled, first.readCount(), "the superseded store is still being retried")
+}
+
+// blockingListStore blocks the first ListProviders call after arm, having
+// already read its snapshot, until release is closed.
+type blockingListStore struct {
+	*MemoryOIDCProviderStore
+	mu      sync.Mutex
+	armed   bool
+	entered chan struct{}
+	release chan struct{}
+}
+
+func (s *blockingListStore) arm() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.armed, s.entered, s.release = true, make(chan struct{}), make(chan struct{})
+}
+
+func (s *blockingListStore) ListProviders(ctx context.Context, addr string) ([]*OIDCProviderRecord, error) {
+	records, err := s.MemoryOIDCProviderStore.ListProviders(ctx, addr)
+	s.mu.Lock()
+	block := s.armed
+	s.armed = false
+	s.mu.Unlock()
+	if block {
+		close(s.entered)
+		<-s.release
+	}
+	return records, err
+}
+
+// A refresh that read the store before a provider was deleted cannot leave the
+// deleted provider trusted by finishing after the deletion's own refresh.
+func TestAnOlderRefreshCannotRestoreADeletedProvider(t *testing.T) {
+	store := &blockingListStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, store)
+	createAPIProvider(t, mgr, persistTestAPIIssuer)
+	require.True(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer), "precondition: the provider is trusted")
+
+	store.arm()
+	stale := make(chan struct{})
+	go func() {
+		defer close(stale)
+		_ = mgr.RefreshOIDCProvidersFromStore(context.Background())
+	}()
+	<-store.entered // the stale refresh holds a snapshot with the provider
+
+	deleted := make(chan error, 1)
+	go func() { deleted <- mgr.DeleteOIDCProvider(context.Background(), arnOf(t, persistTestAPIIssuer)) }()
+	// Serialized, the deletion's refresh waits for the stale one; otherwise
+	// let it finish first, which is the ordering that went wrong.
+	var deleteErr error
+	select {
+	case deleteErr = <-deleted:
+		deleted = nil
+	case <-time.After(200 * time.Millisecond):
+	}
+	close(store.release)
+	<-stale
+	if deleted != nil {
+		deleteErr = <-deleted
+	}
+	require.NoError(t, deleteErr)
+	assert.False(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer), "a refresh older than the deletion left the deleted provider trusted")
+}
+
+// A refresh that fails is retried until the store answers. A change event reports each mutation once, so
+// a subscription refresh that found the filer unreachable — mid-restart, say — left the provider set stale
+// until some unrelated later change; a peer's new provider stayed untrusted and a deleted one trusted.
+func TestAFailedRefreshIsRetriedUntilTheStoreAnswers(t *testing.T) {
+	saved := oidcHydrateRetry
+	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, 5*time.Millisecond
+	t.Cleanup(func() { oidcHydrateRetry = saved })
+
+	store := &unreachableThenReadyStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, store)
+	require.NoError(t, store.StoreProvider(context.Background(), "", &OIDCProviderRecord{
+		ARN: arnOf(t, persistTestAPIIssuer), URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
+	}))
+	store.mu.Lock()
+	store.failsLeft = 3
+	store.mu.Unlock()
+	require.Error(t, mgr.RefreshOIDCProvidersFromStore(context.Background()), "precondition: the refresh fails")
+
+	deadline := time.Now().Add(2 * time.Second)
+	for !stsKnowsIssuer(t, mgr, persistTestAPIIssuer) {
+		if time.Now().After(deadline) {
+			t.Fatal("a failed refresh was never retried: the stored provider stays untrusted until an unrelated change")
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+}
+
+// Failures during an outage start ONE retry, not one per event: a filer that is down for a while produces a
+// change event per mutation, and each would otherwise add a goroutine polling the same store. Once the store
+// answers and the retry ends, a later failure starts a new one.
+func TestFailedRefreshesShareOneRetry(t *testing.T) {
+	saved := oidcHydrateRetry
+	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, time.Millisecond
+	t.Cleanup(func() { oidcHydrateRetry = saved })
+
+	store := &unreachableThenReadyStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, store)
+	store.mu.Lock()
+	store.failsLeft = 1 << 30
+	store.mu.Unlock()
+	for range 20 {
+		require.Error(t, mgr.RefreshOIDCProvidersFromStore(context.Background()))
+	}
+	mgr.oidcRetryMu.Lock()
+	started := mgr.oidcRetrySeq
+	mgr.oidcRetryMu.Unlock()
+	assert.Equal(t, uint64(1), started, "twenty failed refreshes started more than one retry")
+
+	store.mu.Lock()
+	store.failsLeft = 0
+	store.mu.Unlock()
+	deadline := time.Now().Add(2 * time.Second)
+	for {
+		mgr.oidcRetryMu.Lock()
+		running := mgr.oidcRetryGen != 0
+		mgr.oidcRetryMu.Unlock()
+		if !running {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("the retry never ended after the store answered")
+		}
+		time.Sleep(time.Millisecond)
+	}
+	store.mu.Lock()
+	store.failsLeft = 1
+	store.mu.Unlock()
+	require.Error(t, mgr.RefreshOIDCProvidersFromStore(context.Background()))
+	mgr.oidcRetryMu.Lock()
+	assert.Equal(t, uint64(2), mgr.oidcRetrySeq, "a failure after the retry ended starts a new one")
+	mgr.oidcRetryMu.Unlock()
+}
+
+// flakyCountingStore fails while fail is set and counts the lists that succeed.
+type flakyCountingStore struct {
+	*MemoryOIDCProviderStore
+	mu   sync.Mutex
+	fail bool
+	ok   int
+}
+
+func (s *flakyCountingStore) ListProviders(ctx context.Context, addr string) ([]*OIDCProviderRecord, error) {
+	s.mu.Lock()
+	if s.fail {
+		s.mu.Unlock()
+		return nil, errors.New("filer unavailable")
+	}
+	s.ok++
+	s.mu.Unlock()
+	return s.MemoryOIDCProviderStore.ListProviders(ctx, addr)
+}
+
+func (s *flakyCountingStore) set(fail bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.fail = fail
+}
+
+func (s *flakyCountingStore) successes() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.ok
+}
+
+// A refresh of store A that fails as store B is installed must not start a retry for A: nothing would cancel
+// it, and when A answered it would replace B's providers (a removed one trusted again, B's own untrusted).
+func TestAFailedRefreshOfASupersededStoreStartsNoRetry(t *testing.T) {
+	a := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, a)
+	mgr.SetOIDCProviderStore(&persistentTestStore{NewMemoryOIDCProviderStore()})
+	mgr.oidcRetryMu.Lock()
+	before := mgr.oidcRetrySeq
+	mgr.oidcRetryMu.Unlock()
+
+	mgr.startOIDCRetry(a) // the late call of A's failed refresh, after B's install
+
+	mgr.oidcRetryMu.Lock()
+	defer mgr.oidcRetryMu.Unlock()
+	assert.Equal(t, before, mgr.oidcRetrySeq, "a retry was started for the store that was replaced")
+	assert.Zero(t, mgr.oidcRetryGen)
+}
+
+// A refresh that listed store A before B was installed must not hand STS A's providers afterwards.
+func TestASupersededSnapshotIsNotApplied(t *testing.T) {
+	a := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	require.NoError(t, a.StoreProvider(context.Background(), "", &OIDCProviderRecord{
+		ARN: arnOf(t, persistTestAPIIssuer), URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
+	}))
+	b := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	require.NoError(t, b.StoreProvider(context.Background(), "", &OIDCProviderRecord{
+		ARN: arnOf(t, persistTestStaticIssuer), URL: persistTestStaticIssuer, ClientIDs: []string{"aud"},
+	}))
+	mgr := startServer(t, a)
+	mgr.SetOIDCProviderStore(b)
+
+	assert.Error(t, mgr.refreshOIDCProvidersFrom(context.Background(), a), "a snapshot of the replaced store was applied")
+	assert.False(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer), "the replaced store's provider is trusted")
+	assert.True(t, stsKnowsIssuer(t, mgr, persistTestStaticIssuer), "the current store's provider is not")
+}
+
+// A refresh that fails while a retry runs is not dropped: the retry may already have listed an older snapshot,
+// so after its success it runs once more and picks up whatever the failed refresh would have seen.
+func TestAFailureDuringARetryIsNotDropped(t *testing.T) {
+	saved := oidcHydrateRetry
+	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, 2*time.Millisecond
+	t.Cleanup(func() { oidcHydrateRetry = saved })
+
+	store := &flakyCountingStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore(), fail: true}
+	mgr := startServer(t, store) // the startup load fails: a retry is running
+	require.Error(t, mgr.RefreshOIDCProvidersFromStore(context.Background()), "a later refresh fails while it runs")
+	store.set(false)
+
+	deadline := time.Now().Add(2 * time.Second)
+	for {
+		mgr.oidcRetryMu.Lock()
+		running := mgr.oidcRetryGen != 0
+		mgr.oidcRetryMu.Unlock()
+		if !running {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("the retry never ended")
+		}
+		time.Sleep(time.Millisecond)
+	}
+	assert.GreaterOrEqual(t, store.successes(), 2, "the retry ended on its first success, dropping the refresh that failed while it ran")
+}
@@ -27,6 +27,9 @@ import (
 var (
 	ErrOIDCProviderNotFound      = errors.New("OIDC provider not found")
 	ErrOIDCProviderAlreadyExists = errors.New("OIDC provider already exists")
+	// ErrOIDCProviderStatic refuses a change to a provider defined in the
+	// server's IAM config file: change it there instead.
+	ErrOIDCProviderStatic = errors.New("OIDC provider is defined in the IAM config file")
 )

 // OIDCProviderRecord is the persisted, IAM-managed view of an OIDC identity
@@ -267,15 +270,20 @@ func (f *FilerOIDCProviderStore) GetProviderByARN(ctx context.Context, filerAddr

 	var data []byte
 	err := f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
-		resp, err := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
+		resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
 			Directory: f.basePath,
 			Name:      f.fileName(arn),
 		})
+		// Only a confirmed absence is ErrOIDCProviderNotFound: callers create
+		// on it, so an unreachable filer must not read as "no such provider".
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			return fmt.Errorf("%w: %s", ErrOIDCProviderNotFound, arn)
+		}
 		if err != nil {
-			return fmt.Errorf("%w: %v", ErrOIDCProviderNotFound, err)
+			return fmt.Errorf("lookup OIDC provider %s: %w", arn, err)
 		}
 		if resp.Entry == nil {
-			return fmt.Errorf("OIDC provider not found: %s", arn)
+			return fmt.Errorf("%w: %s", ErrOIDCProviderNotFound, arn)
 		}
 		data = resp.Entry.Content
 		return nil
@@ -0,0 +1,165 @@
+package integration
+
+import (
+	"context"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
+	"github.com/seaweedfs/seaweedfs/weed/iam/sts"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A session is bound to the role it was issued for, not to the role's name:
+// deleting the role ends it, and a role created later under the same name —
+// by whoever may create roles — must not inherit it.
+func TestSessionIsBoundToTheRoleNotItsName(t *testing.T) {
+	ctx := context.Background()
+	m := setupIntegratedIAMSystem(t)
+
+	assume := func() (token, principal string) {
+		resp, err := m.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
+			RoleArn:          "arn:aws:iam::role/S3ReadOnlyRole",
+			WebIdentityToken: createTestJWT(t, "https://test-issuer.com", "test-user-123", "test-signing-key"),
+			RoleSessionName:  "binding-test",
+		})
+		require.NoError(t, err)
+		return resp.Credentials.SessionToken, resp.AssumedRoleUser.Arn
+	}
+	allowed := func(token, principal string) bool {
+		ok, _ := m.IsActionAllowed(ctx, &ActionRequest{
+			Principal:    principal,
+			Action:       "s3:GetObject",
+			Resource:     "arn:aws:s3:::test-bucket/file.txt",
+			SessionToken: token,
+		})
+		return ok
+	}
+
+	original, err := m.GetRole(ctx, "S3ReadOnlyRole")
+	require.NoError(t, err)
+	require.NotEmpty(t, original.RoleId, "a created role has an ID")
+
+	token, principal := assume()
+	require.True(t, allowed(token, principal), "precondition: the session works")
+
+	require.NoError(t, m.DeleteRole(ctx, "S3ReadOnlyRole"))
+	assert.False(t, allowed(token, principal), "the session outlived its role's deletion")
+
+	recreated := *original
+	recreated.RoleId = ""
+	require.NoError(t, m.CreateRole(ctx, "", "S3ReadOnlyRole", &recreated))
+	assert.NotEqual(t, original.RoleId, recreated.RoleId, "a role created again under the same name got the same ID")
+	assert.False(t, allowed(token, principal), "a session of the deleted role works again under a new role of the same name")
+
+	fresh, freshPrincipal := assume()
+	assert.True(t, allowed(fresh, freshPrincipal), "a session issued for the new role is refused")
+}
+
+// A session carries its role's attached policies, and S3 evaluates those
+// rather than looking the role up; the binding must hold on that path too.
+func TestASessionCarryingItsPoliciesIsStillBoundToItsRole(t *testing.T) {
+	ctx := context.Background()
+	m := setupIntegratedIAMSystem(t)
+
+	original, err := m.GetRole(ctx, "S3ReadOnlyRole")
+	require.NoError(t, err)
+	require.NotEmpty(t, original.AttachedPolicies, "precondition: the role attaches a policy")
+	resp, err := m.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
+		RoleArn:          "arn:aws:iam::role/S3ReadOnlyRole",
+		WebIdentityToken: createTestJWT(t, "https://test-issuer.com", "test-user-123", "test-signing-key"),
+		RoleSessionName:  "binding-test",
+	})
+	require.NoError(t, err)
+	allowed := func() bool {
+		ok, _ := m.IsActionAllowed(ctx, &ActionRequest{
+			Principal:    resp.AssumedRoleUser.Arn,
+			Action:       "s3:GetObject",
+			Resource:     "arn:aws:s3:::test-bucket/file.txt",
+			SessionToken: resp.Credentials.SessionToken,
+			PolicyNames:  original.AttachedPolicies,
+		})
+		return ok
+	}
+	require.True(t, allowed(), "precondition: the session works")
+
+	require.NoError(t, m.DeleteRole(ctx, "S3ReadOnlyRole"))
+	assert.False(t, allowed(), "the session's embedded policies outlived its role's deletion")
+
+	recreated := *original
+	recreated.RoleId = ""
+	require.NoError(t, m.CreateRole(ctx, "", "S3ReadOnlyRole", &recreated))
+	assert.False(t, allowed(), "the session's embedded policies work again under a new role of the same name")
+}
+
+func TestStaticRoleIDIsStableAndRuntimeIDsAreUnique(t *testing.T) {
+	trusting := func(principal string) *policy.PolicyDocument {
+		return &policy.PolicyDocument{Version: "2012-10-17", Statement: []policy.Statement{{
+			Effect: "Allow", Action: []string{"sts:AssumeRoleWithWebIdentity"},
+			Principal: map[string]interface{}{"Federated": principal},
+		}}}
+	}
+	app := &RoleDefinition{RoleName: "app", TrustPolicy: trusting("https://a.example")}
+	restarted := &RoleDefinition{RoleName: "app", TrustPolicy: trusting("https://a.example")}
+	replaced := &RoleDefinition{RoleName: "app", TrustPolicy: trusting("https://b.example")}
+
+	assert.Equal(t, StaticRoleID(app), StaticRoleID(restarted), "a config-file role must keep its ID across restarts")
+	assert.NotEqual(t, StaticRoleID(app), StaticRoleID(replaced), "a different role under the same name inherits the old one's ID")
+	assert.NotEqual(t, StaticRoleID(app), StaticRoleID(&RoleDefinition{RoleName: "other", TrustPolicy: trusting("https://a.example")}))
+	assert.Regexp(t, `^AROA[A-Z0-9]{17}$`, StaticRoleID(app))
+	assert.NotEqual(t, NewRoleID(), NewRoleID())
+	assert.Regexp(t, `^AROA[A-Z2-7]{17}$`, NewRoleID())
+}
+
+// replacedAfterFirstReadStore serves the stored role on its first read and a
+// replacement of the same name afterwards: a role replaced while a request
+// is being authorized.
+type replacedAfterFirstReadStore struct {
+	RoleStore
+	reads       int
+	replacement *RoleDefinition
+}
+
+func (s *replacedAfterFirstReadStore) GetRole(ctx context.Context, addr, name string) (*RoleDefinition, error) {
+	s.reads++
+	if s.reads > 1 && name == s.replacement.RoleName {
+		return copyRoleDefinition(s.replacement), nil
+	}
+	return s.RoleStore.GetRole(ctx, addr, name)
+}
+
+// The session's binding is checked against one definition of its role, and
+// that definition's policies are the ones evaluated: a replacement read in
+// between must not lend the session its permissions.
+func TestAuthorizationEvaluatesTheRoleTheBindingCheckSaw(t *testing.T) {
+	ctx := context.Background()
+	m := setupIntegratedIAMSystem(t)
+	require.NoError(t, m.CreatePolicy(ctx, "", "S3WritePolicy", &policy.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy.Statement{{
+			Effect: "Allow", Action: []string{"s3:PutObject"},
+			Resource: []string{"arn:aws:s3:::test-bucket/*"},
+		}},
+	}))
+	resp, err := m.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
+		RoleArn:          "arn:aws:iam::role/S3ReadOnlyRole",
+		WebIdentityToken: createTestJWT(t, "https://test-issuer.com", "test-user-123", "test-signing-key"),
+		RoleSessionName:  "snapshot-test",
+	})
+	require.NoError(t, err)
+
+	original, err := m.GetRole(ctx, "S3ReadOnlyRole")
+	require.NoError(t, err)
+	replacement := *original
+	replacement.RoleId = NewRoleID()
+	replacement.AttachedPolicies = []string{"S3WritePolicy"}
+	m.roleStore = &replacedAfterFirstReadStore{RoleStore: m.roleStore, replacement: &replacement}
+
+	allowed, _ := m.IsActionAllowed(ctx, &ActionRequest{
+		Principal:    resp.AssumedRoleUser.Arn,
+		Action:       "s3:PutObject",
+		Resource:     "arn:aws:s3:::test-bucket/file.txt",
+		SessionToken: resp.Credentials.SessionToken,
+	})
+	assert.False(t, allowed, "the session was authorized by the replacement role's policies")
+}
@@ -3,7 +3,10 @@ package integration
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -13,9 +16,38 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )

+// ErrRoleNotFound is wrapped by every RoleStore's GetRole when the role does
+// not exist, so callers can tell a missing role from a store that could not
+// be read.
+var ErrRoleNotFound = errors.New("role not found")
+
+// ErrRoleStatic refuses a change to a role defined in the server's IAM config
+// file: change it there instead.
+var ErrRoleStatic = errors.New("role is defined in the IAM config file")
+
+// ErrRoleExists is returned by an UpdateRole whose update refuses a role that
+// already exists (CreateRole).
+var ErrRoleExists = errors.New("role already exists")
+
+// RoleUpdate computes a role's new definition from its current one, nil when
+// the role does not exist. It returns nil to delete the role, and an error to
+// leave it unchanged. It may run more than once: it is called again with the
+// fresh definition when another writer changed the role in between.
+type RoleUpdate func(current *RoleDefinition) (*RoleDefinition, error)
+
+// maxRoleUpdateAttempts bounds UpdateRole's retries under contention.
+const maxRoleUpdateAttempts = 10
+
+// errRoleUpdateContended is returned when the role kept changing under
+// UpdateRole for maxRoleUpdateAttempts reads.
+var errRoleUpdateContended = errors.New("role changed concurrently; retry")
+
 // RoleStore defines the interface for storing IAM role definitions
 type RoleStore interface {
 	// StoreRole stores a role definition (filerAddress ignored for memory stores)
@@ -29,6 +61,14 @@ type RoleStore interface {

 	// DeleteRole deletes a role definition (filerAddress ignored for memory stores)
 	DeleteRole(ctx context.Context, filerAddress string, roleName string) error
+
+	// UpdateRole replaces a role with update's result, atomically: the write
+	// lands only if the role is still as update saw it, absent included, and
+	// update is retried against the current role otherwise. Every change the
+	// IAM API and the filer IAM service make to a role goes through it, so
+	// writers on different servers neither lose each other's changes nor bring
+	// back a role deleted in between.
+	UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error
 }

 // MemoryRoleStore implements RoleStore using in-memory storage
@@ -72,7 +112,7 @@ func (m *MemoryRoleStore) GetRole(ctx context.Context, filerAddress string, role

 	role, exists := m.roles[roleName]
 	if !exists {
-		return nil, fmt.Errorf("role not found: %s", roleName)
+		return nil, fmt.Errorf("%w: %s", ErrRoleNotFound, roleName)
 	}

 	// Return a copy to prevent external modifications
@@ -105,6 +145,26 @@ func (m *MemoryRoleStore) DeleteRole(ctx context.Context, filerAddress string, r
 	return nil
 }

+// UpdateRole applies update under the store's lock (filerAddress ignored for
+// memory store).
+func (m *MemoryRoleStore) UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error {
+	if roleName == "" {
+		return fmt.Errorf("role name cannot be empty")
+	}
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	next, err := update(copyRoleDefinition(m.roles[roleName]))
+	if err != nil {
+		return err
+	}
+	if next == nil {
+		delete(m.roles, roleName)
+		return nil
+	}
+	m.roles[roleName] = copyRoleDefinition(next)
+	return nil
+}
+
 // copyRoleDefinition creates a deep copy of a role definition
 func copyRoleDefinition(original *RoleDefinition) *RoleDefinition {
 	if original == nil {
@@ -116,6 +176,9 @@ func copyRoleDefinition(original *RoleDefinition) *RoleDefinition {
 		RoleArn:            original.RoleArn,
 		Description:        original.Description,
 		MaxSessionDuration: original.MaxSessionDuration,
+		Source:             original.Source,
+		CreatedAt:          original.CreatedAt,
+		RoleId:             original.RoleId,
 	}

 	// Deep copy trust policy if it exists
@@ -136,6 +199,10 @@ func copyRoleDefinition(original *RoleDefinition) *RoleDefinition {
 	return copied
 }

+// roleListPageSize is the number of entries FilerRoleStore.ListRoles asks the
+// filer for per page.
+const roleListPageSize = 1000
+
 // FilerRoleStore implements RoleStore using SeaweedFS filer
 type FilerRoleStore struct {
 	grpcDialOption       grpc.DialOption
@@ -214,6 +281,139 @@ func (f *FilerRoleStore) StoreRole(ctx context.Context, filerAddress string, rol
 	})
 }

+// UpdateRole reads the role's entry, applies update, and writes the result on
+// the condition that the entry is unchanged since the read — absent, when the
+// role did not exist — so the filer refuses a write racing another writer's
+// change or delete, and update is applied again to what that writer left. A
+// delete is made on the same condition, so it removes the role update saw and
+// not one written after it.
+func (f *FilerRoleStore) UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error {
+	if filerAddress == "" && f.filerAddressProvider != nil {
+		filerAddress = f.filerAddressProvider()
+	}
+	if filerAddress == "" {
+		return fmt.Errorf("filer address is required for FilerRoleStore")
+	}
+	if roleName == "" {
+		return fmt.Errorf("role name cannot be empty")
+	}
+	return f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
+		for attempt := 0; attempt < maxRoleUpdateAttempts; attempt++ {
+			var entry *filer_pb.Entry
+			var current *RoleDefinition
+			resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
+				Directory: f.basePath,
+				Name:      f.getRoleFileName(roleName),
+			})
+			switch {
+			case errors.Is(err, filer_pb.ErrNotFound):
+			case err != nil:
+				return fmt.Errorf("lookup role %s: %w", roleName, err)
+			case resp.Entry != nil:
+				entry = resp.Entry
+				current = &RoleDefinition{}
+				if err := json.Unmarshal(entry.Content, current); err != nil {
+					return fmt.Errorf("failed to deserialize role %s: %v", roleName, err)
+				}
+			}
+
+			next, err := update(current)
+			if err != nil {
+				return err
+			}
+			if next == nil {
+				if entry == nil {
+					return nil
+				}
+				deleted, err := f.deleteRoleEntryIfUnchanged(ctx, client, entry)
+				if err != nil {
+					return fmt.Errorf("failed to delete role %s: %w", roleName, err)
+				}
+				if !deleted {
+					glog.V(3).Infof("Role %s changed before its delete; retrying", roleName)
+					continue
+				}
+				return nil
+			}
+			roleData, err := json.MarshalIndent(next, "", "  ")
+			if err != nil {
+				return fmt.Errorf("failed to serialize role: %v", err)
+			}
+
+			clause := &filer_pb.WriteCondition_Clause{Kind: filer_pb.WriteCondition_IF_NOT_EXISTS}
+			if entry != nil {
+				clause = &filer_pb.WriteCondition_Clause{Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, ExpectedEntry: entry}
+			}
+			now := time.Now().Unix()
+			created, err := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
+				Directory: f.basePath,
+				Entry: &filer_pb.Entry{
+					Name: f.getRoleFileName(roleName),
+					Attributes: &filer_pb.FuseAttributes{
+						Mtime:    now,
+						Crtime:   now,
+						FileMode: uint32(0600),
+					},
+					Content: roleData,
+				},
+				Condition: &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{clause}},
+			})
+			if isRoleWriteConflict(created, err) {
+				glog.V(3).Infof("Role %s changed during update; retrying", roleName)
+				continue
+			}
+			if err != nil {
+				return fmt.Errorf("failed to store role %s: %v", roleName, err)
+			}
+			if created.Error != "" {
+				return fmt.Errorf("failed to store role %s: %s", roleName, created.Error)
+			}
+			return nil
+		}
+		return fmt.Errorf("update role %s: %w", roleName, errRoleUpdateContended)
+	})
+}
+
+// deleteRoleEntryIfUnchanged deletes the role's entry if it still equals
+// entry, reporting false when it changed. The delete is routed and locked as
+// the conditional CreateEntry of the same path is, so the two serialize.
+func (f *FilerRoleStore) deleteRoleEntryIfUnchanged(ctx context.Context, client filer_pb.SeaweedFilerClient, entry *filer_pb.Entry) (bool, error) {
+	fullPath := f.basePath + "/" + entry.Name
+	resp, err := client.ObjectTransaction(ctx, &filer_pb.ObjectTransactionRequest{
+		LockKey:  fullPath,
+		RouteKey: s3_constants.ObjectWriteRouteKeyPrefix + fullPath,
+		Condition: &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{
+			Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, ExpectedEntry: entry,
+		}}},
+		Mutations: []*filer_pb.ObjectMutation{{
+			Type: filer_pb.ObjectMutation_DELETE, Directory: f.basePath, Name: entry.Name, IsDeleteData: true,
+		}},
+	})
+	if err != nil {
+		if status.Code(err) == codes.FailedPrecondition {
+			return false, nil
+		}
+		return false, err
+	}
+	if resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED {
+		return false, nil
+	}
+	if resp.Error != "" {
+		return false, errors.New(resp.Error)
+	}
+	return true, nil
+}
+
+// isRoleWriteConflict reports a write the filer refused because its condition
+// no longer held: in the response, or as FailedPrecondition when the write
+// was forwarded to the entry's owner filer.
+func isRoleWriteConflict(resp *filer_pb.CreateEntryResponse, err error) bool {
+	if err != nil {
+		return status.Code(err) == codes.FailedPrecondition
+	}
+	return resp != nil && resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED
+}
+
 // GetRole retrieves a role definition from filer
 func (f *FilerRoleStore) GetRole(ctx context.Context, filerAddress string, roleName string) (*RoleDefinition, error) {
 	// Use provider function if filerAddress is not provided
@@ -235,13 +435,16 @@ func (f *FilerRoleStore) GetRole(ctx context.Context, filerAddress string, roleN
 		}

 		glog.V(3).Infof("Looking up role %s", roleName)
-		response, err := client.LookupDirectoryEntry(ctx, request)
+		response, err := filer_pb.LookupEntry(ctx, client, request)
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			return fmt.Errorf("%w: %s", ErrRoleNotFound, roleName)
+		}
 		if err != nil {
-			return fmt.Errorf("role not found: %v", err)
+			return fmt.Errorf("lookup role %s: %w", roleName, err)
 		}

 		if response.Entry == nil {
-			return fmt.Errorf("role not found")
+			return fmt.Errorf("%w: %s", ErrRoleNotFound, roleName)
 		}

 		roleData = response.Entry.Content
@@ -271,46 +474,50 @@ func (f *FilerRoleStore) ListRoles(ctx context.Context, filerAddress string) ([]
 		return nil, fmt.Errorf("filer address is required for FilerRoleStore")
 	}

+	// Page through the directory, and fail on a broken stream rather than
+	// return what arrived: a truncated list would let DeletePolicy miss a role
+	// that still attaches the policy (RolesAttachingPolicy).
 	var roleNames []string
-
 	err := f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
-		request := &filer_pb.ListEntriesRequest{
-			Directory:          f.basePath,
-			Prefix:             "",
-			StartFromFileName:  "",
-			InclusiveStartFrom: false,
-			Limit:              1000, // Process in batches of 1000
-		}
-
 		glog.V(3).Infof("Listing roles in %s", f.basePath)
-		stream, err := client.ListEntries(ctx, request)
-		if err != nil {
-			return fmt.Errorf("failed to list roles: %v", err)
-		}
-
+		startFrom := ""
 		for {
-			resp, err := stream.Recv()
+			stream, err := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
+				Directory:         f.basePath,
+				StartFromFileName: startFrom,
+				Limit:             roleListPageSize,
+			})
 			if err != nil {
-				break // End of stream or error
+				return err
 			}
-
-			if resp.Entry == nil || resp.Entry.IsDirectory {
-				continue
+			received := 0
+			for {
+				resp, err := stream.Recv()
+				if errors.Is(err, io.EOF) {
+					break
+				}
+				if err != nil {
+					return err
+				}
+				if resp.Entry == nil {
+					continue
+				}
+				received++
+				startFrom = resp.Entry.Name
+				if resp.Entry.IsDirectory {
+					continue
+				}
+				if name, ok := strings.CutSuffix(resp.Entry.Name, ".json"); ok {
+					roleNames = append(roleNames, name)
+				}
 			}
-
-			// Extract role name from filename
-			filename := resp.Entry.Name
-			if strings.HasSuffix(filename, ".json") {
-				roleName := strings.TrimSuffix(filename, ".json")
-				roleNames = append(roleNames, roleName)
+			if received < roleListPageSize {
+				return nil
 			}
 		}
-
-		return nil
 	})
-
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to list roles: %w", err)
 	}

 	return roleNames, nil
@@ -389,3 +596,146 @@ type CachedFilerRoleStoreConfig struct {
 	ListTTL      string `json:"listTtl,omitempty"`      // e.g., "1m", "30s"
 	MaxCacheSize int    `json:"maxCacheSize,omitempty"` // Maximum number of cached roles
 }
+
+// RolesAttachingPolicy returns the names of the roles that attach the policy.
+// A policy is attached to a role by name, so deleting it while attached would
+// let a policy created later under the same name take effect on the role;
+// callers refuse the delete instead, as AWS does (DeleteConflict).
+//
+// It sees the stored roles, which every server sharing the store sees, and
+// this server's config-file roles. A role defined only in another server's
+// config file is invisible here: servers sharing a role store may have
+// different config files, but a config-file role that attaches a managed
+// (API-created) policy is protected only on the servers whose file defines it.
+// Keep such roles in every server's file, or attach only config-file policies
+// to config-file roles.
+func RolesAttachingPolicy(ctx context.Context, store RoleStore, policyName string) ([]string, error) {
+	names, err := store.ListRoles(ctx, "")
+	if err != nil {
+		return nil, fmt.Errorf("list roles: %w", err)
+	}
+	var attaching []string
+	for _, name := range names {
+		role, err := store.GetRole(ctx, "", name)
+		if errors.Is(err, ErrRoleNotFound) {
+			continue // deleted between list and read
+		}
+		if err != nil {
+			return nil, fmt.Errorf("get role %s: %w", name, err)
+		}
+		for _, attached := range role.AttachedPolicies {
+			if attached == policyName {
+				attaching = append(attaching, name)
+				break
+			}
+		}
+	}
+	return attaching, nil
+}
+
+// MaxManagedPoliciesPerRole caps the managed policies attached to one role,
+// as AWS's default quota does (and as MaxManagedPoliciesPerUser does for users).
+const MaxManagedPoliciesPerRole = 10
+
+var roleNamePattern = regexp.MustCompile(`^[\w+=,.@-]{1,64}$`)
+
+// ValidateRoleName checks a role name against AWS's rules. A role is stored as
+// <name>.json in the filer, so the rules also keep a name from leaving the
+// role store's directory.
+func ValidateRoleName(name string) error {
+	if !roleNamePattern.MatchString(name) {
+		return fmt.Errorf("invalid role name %q: must be 1-64 characters of letters, digits and +=,.@_-", name)
+	}
+	return nil
+}
+
+// staticRoleOverlay serves the IAM config file's roles from memory beside a
+// persistent store, which never holds them (see IAMManager.LoadStaticRoles).
+// A role stored under the same name takes precedence, as a stored OIDC
+// provider does over a config-file one; deleting it brings the config-file
+// role back. A config-file role itself cannot be stored over or deleted.
+type staticRoleOverlay struct {
+	static map[string]*RoleDefinition
+	inner  RoleStore
+}
+
+// storedRoleExists reports whether the store itself holds the role.
+func (o *staticRoleOverlay) storedRoleExists(ctx context.Context, filerAddress, roleName string) (bool, error) {
+	_, err := o.inner.GetRole(ctx, filerAddress, roleName)
+	if errors.Is(err, ErrRoleNotFound) {
+		return false, nil
+	}
+	return err == nil, err
+}
+
+func (o *staticRoleOverlay) StoreRole(ctx context.Context, filerAddress string, roleName string, role *RoleDefinition) error {
+	if _, ok := o.static[roleName]; ok {
+		stored, err := o.storedRoleExists(ctx, filerAddress, roleName)
+		if err != nil {
+			return err
+		}
+		if !stored {
+			return fmt.Errorf("%w: %s", ErrRoleStatic, roleName)
+		}
+	}
+	return o.inner.StoreRole(ctx, filerAddress, roleName, role)
+}
+
+// UpdateRole refuses to create a stored role over a config-file role, as
+// StoreRole does; a stored role of that name may be changed.
+func (o *staticRoleOverlay) UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error {
+	_, static := o.static[roleName]
+	return o.inner.UpdateRole(ctx, filerAddress, roleName, func(current *RoleDefinition) (*RoleDefinition, error) {
+		if current == nil && static {
+			return nil, fmt.Errorf("%w: %s", ErrRoleStatic, roleName)
+		}
+		return update(current)
+	})
+}
+
+func (o *staticRoleOverlay) GetRole(ctx context.Context, filerAddress string, roleName string) (*RoleDefinition, error) {
+	role, err := o.inner.GetRole(ctx, filerAddress, roleName)
+	if errors.Is(err, ErrRoleNotFound) {
+		if static, ok := o.static[roleName]; ok {
+			return copyRoleDefinition(static), nil
+		}
+	}
+	return role, err
+}
+
+func (o *staticRoleOverlay) ListRoles(ctx context.Context, filerAddress string) ([]string, error) {
+	names, err := o.inner.ListRoles(ctx, filerAddress)
+	if err != nil {
+		return nil, err
+	}
+	seen := make(map[string]bool, len(names))
+	for _, name := range names {
+		seen[name] = true
+	}
+	for name := range o.static {
+		if !seen[name] {
+			names = append(names, name)
+		}
+	}
+	return names, nil
+}
+
+func (o *staticRoleOverlay) DeleteRole(ctx context.Context, filerAddress string, roleName string) error {
+	if _, ok := o.static[roleName]; ok {
+		stored, err := o.storedRoleExists(ctx, filerAddress, roleName)
+		if err != nil {
+			return err
+		}
+		if !stored {
+			return fmt.Errorf("%w: %s", ErrRoleStatic, roleName)
+		}
+	}
+	return o.inner.DeleteRole(ctx, filerAddress, roleName)
+}
+
+// ClearCache forwards cache invalidation to the store underneath.
+func (o *staticRoleOverlay) ClearCache() {
+	if cached, ok := o.inner.(interface{ ClearCache() }); ok {
+		cached.ClearCache()
+	}
+}
@@ -0,0 +1,286 @@
+package integration
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"slices"
+	"sort"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+)
+
+// roleStoreTestFiler is a filer holding one directory. It evaluates the write
+// conditions FilerRoleStore sends as the filer does, and can run another
+// writer between a role's read and its write (afterLookup) or break a listing
+// stream partway (failListAfter).
+type roleStoreTestFiler struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	mu            sync.Mutex
+	entries       map[string]*filer_pb.Entry
+	afterLookup   func()
+	failListAfter int
+}
+
+func (s *roleStoreTestFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	s.mu.Lock()
+	entry, found := s.entries[req.Name]
+	hook := s.afterLookup
+	s.afterLookup = nil
+	s.mu.Unlock()
+	if hook != nil {
+		defer hook()
+	}
+	if !found {
+		return nil, status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())
+	}
+	return &filer_pb.LookupDirectoryEntryResponse{Entry: proto.Clone(entry).(*filer_pb.Entry)}, nil
+}
+
+func (s *roleStoreTestFiler) CreateEntry(_ context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	current, exists := s.entries[req.Entry.Name]
+	for _, c := range req.GetCondition().GetClauses() {
+		var ok bool
+		switch c.Kind {
+		case filer_pb.WriteCondition_IF_NOT_EXISTS:
+			ok = !exists
+		case filer_pb.WriteCondition_IF_ENTRY_EQUAL:
+			ok = exists && proto.Equal(current, c.ExpectedEntry)
+		default:
+			return nil, fmt.Errorf("unexpected condition %v", c.Kind)
+		}
+		if !ok {
+			return &filer_pb.CreateEntryResponse{Error: "precondition failed", ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
+		}
+	}
+	s.entries[req.Entry.Name] = proto.Clone(req.Entry).(*filer_pb.Entry)
+	return &filer_pb.CreateEntryResponse{}, nil
+}
+
+// ObjectTransaction applies the conditional delete FilerRoleStore sends.
+func (s *roleStoreTestFiler) ObjectTransaction(_ context.Context, req *filer_pb.ObjectTransactionRequest) (*filer_pb.ObjectTransactionResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if len(req.Mutations) != 1 || req.Mutations[0].Type != filer_pb.ObjectMutation_DELETE {
+		return nil, fmt.Errorf("unexpected mutations %v", req.Mutations)
+	}
+	name := req.Mutations[0].Name
+	current, exists := s.entries[name]
+	for _, c := range req.GetCondition().GetClauses() {
+		if c.Kind != filer_pb.WriteCondition_IF_ENTRY_EQUAL {
+			return nil, fmt.Errorf("unexpected condition %v", c.Kind)
+		}
+		if !exists || !proto.Equal(current, c.ExpectedEntry) {
+			return &filer_pb.ObjectTransactionResponse{Error: "precondition failed", ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
+		}
+	}
+	delete(s.entries, name)
+	return &filer_pb.ObjectTransactionResponse{}, nil
+}
+
+func (s *roleStoreTestFiler) ListEntries(req *filer_pb.ListEntriesRequest, stream grpc.ServerStreamingServer[filer_pb.ListEntriesResponse]) error {
+	s.mu.Lock()
+	var names []string
+	for name := range s.entries {
+		if name > req.StartFromFileName {
+			names = append(names, name)
+		}
+	}
+	sort.Strings(names)
+	if req.Limit > 0 && len(names) > int(req.Limit) {
+		names = names[:req.Limit]
+	}
+	page := make([]*filer_pb.Entry, 0, len(names))
+	for _, name := range names {
+		page = append(page, proto.Clone(s.entries[name]).(*filer_pb.Entry))
+	}
+	failAfter := s.failListAfter
+	s.mu.Unlock()
+	for i, entry := range page {
+		if failAfter > 0 && i == failAfter {
+			return status.Error(codes.Unavailable, "filer went away")
+		}
+		if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: entry}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *roleStoreTestFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	delete(s.entries, req.Name)
+	return &filer_pb.DeleteEntryResponse{}, nil
+}
+
+func newTestFilerRoleStore(t *testing.T) (*FilerRoleStore, *roleStoreTestFiler) {
+	t.Helper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	filer := &roleStoreTestFiler{entries: map[string]*filer_pb.Entry{}}
+	server := pb.NewGrpcServer()
+	filer_pb.RegisterSeaweedFilerServer(server, filer)
+	go func() { _ = server.Serve(lis) }()
+	t.Cleanup(func() {
+		server.Stop()
+		_ = lis.Close()
+	})
+	host, port, err := net.SplitHostPort(lis.Addr().String())
+	require.NoError(t, err)
+	grpcPort, err := strconv.Atoi(port)
+	require.NoError(t, err)
+	store, err := NewFilerRoleStore(nil, func() string { return string(pb.NewServerAddress(host, 1, grpcPort)) })
+	require.NoError(t, err)
+	store.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
+	return store, filer
+}
+
+func attachPolicy(policyName string) RoleUpdate {
+	return func(current *RoleDefinition) (*RoleDefinition, error) {
+		if current == nil {
+			return nil, ErrRoleNotFound
+		}
+		current.AttachedPolicies = append(current.AttachedPolicies, policyName)
+		return current, nil
+	}
+}
+
+func createRole(roleID string) RoleUpdate {
+	return func(current *RoleDefinition) (*RoleDefinition, error) {
+		if current != nil {
+			return nil, ErrRoleExists
+		}
+		return &RoleDefinition{RoleName: "app", RoleId: roleID}, nil
+	}
+}
+
+// Two servers changing one role: the change written second is applied to
+// what the first left, not to the role as it read it.
+func TestFilerRoleUpdateIsNotLostToAConcurrentChange(t *testing.T) {
+	ctx := context.Background()
+	store, filer := newTestFilerRoleStore(t)
+	require.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA1")))
+
+	filer.afterLookup = func() { assert.NoError(t, store.UpdateRole(ctx, "", "app", attachPolicy("peer"))) }
+	require.NoError(t, store.UpdateRole(ctx, "", "app", attachPolicy("mine")))
+
+	role, err := store.GetRole(ctx, "", "app")
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"peer", "mine"}, role.AttachedPolicies, "one server's change was lost")
+}
+
+// A change racing a delete must not write the role back.
+func TestFilerRoleUpdateDoesNotReviveADeletedRole(t *testing.T) {
+	ctx := context.Background()
+	store, filer := newTestFilerRoleStore(t)
+	require.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA1")))
+
+	filer.afterLookup = func() { assert.NoError(t, store.DeleteRole(ctx, "", "app")) }
+	err := store.UpdateRole(ctx, "", "app", attachPolicy("mine"))
+	assert.ErrorIs(t, err, ErrRoleNotFound)
+
+	_, err = store.GetRole(ctx, "", "app")
+	assert.ErrorIs(t, err, ErrRoleNotFound, "the deleted role was written back")
+}
+
+// Of two creates of one name, the second sees the first's role.
+func TestFilerRoleCreateRefusesARoleCreatedConcurrently(t *testing.T) {
+	ctx := context.Background()
+	store, filer := newTestFilerRoleStore(t)
+
+	filer.afterLookup = func() { assert.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA-FIRST"))) }
+	err := store.UpdateRole(ctx, "", "app", createRole("AROA-SECOND"))
+	assert.ErrorIs(t, err, ErrRoleExists)
+
+	role, err := store.GetRole(ctx, "", "app")
+	require.NoError(t, err)
+	assert.Equal(t, "AROA-FIRST", role.RoleId, "the second create replaced the first role")
+}
+
+var errRoleHasPolicies = errors.New("role has attached policies")
+
+// deleteUnattached deletes the role only if it attaches no policy, as
+// DeleteRole requires.
+func deleteUnattached(current *RoleDefinition) (*RoleDefinition, error) {
+	if current == nil {
+		return nil, ErrRoleNotFound
+	}
+	if len(current.AttachedPolicies) > 0 {
+		return nil, errRoleHasPolicies
+	}
+	return nil, nil
+}
+
+func TestFilerRoleUpdateCanDeleteTheRole(t *testing.T) {
+	ctx := context.Background()
+	store, _ := newTestFilerRoleStore(t)
+	require.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA1")))
+
+	require.NoError(t, store.UpdateRole(ctx, "", "app", deleteUnattached))
+	_, err := store.GetRole(ctx, "", "app")
+	assert.ErrorIs(t, err, ErrRoleNotFound)
+}
+
+// A delete is decided against the role it removes: a policy attached between
+// the check and the delete makes it refuse, not delete the role anyway.
+func TestFilerRoleDeleteIsDecidedAgainstTheRoleItRemoves(t *testing.T) {
+	ctx := context.Background()
+	store, filer := newTestFilerRoleStore(t)
+	require.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA1")))
+
+	filer.afterLookup = func() { assert.NoError(t, store.UpdateRole(ctx, "", "app", attachPolicy("peer"))) }
+	err := store.UpdateRole(ctx, "", "app", deleteUnattached)
+	assert.ErrorIs(t, err, errRoleHasPolicies)
+
+	role, err := store.GetRole(ctx, "", "app")
+	require.NoError(t, err, "the role was deleted although a policy was attached to it")
+	assert.Equal(t, []string{"peer"}, role.AttachedPolicies)
+}
+
+func TestFilerRoleListingPagesPastTheFirstThousand(t *testing.T) {
+	store, filer := newTestFilerRoleStore(t)
+	for i := range roleListPageSize + 1 {
+		name := fmt.Sprintf("role-%04d.json", i)
+		filer.entries[name] = &filer_pb.Entry{Name: name}
+	}
+	filer.entries["role-0500"] = &filer_pb.Entry{Name: "role-0500", IsDirectory: true}
+
+	names, err := store.ListRoles(context.Background(), "")
+	require.NoError(t, err)
+	assert.Len(t, names, roleListPageSize+1)
+	assert.True(t, slices.Contains(names, fmt.Sprintf("role-%04d", roleListPageSize)), "the role past the first page is missing")
+}
+
+// A listing cut short must fail: DeletePolicy decides from it whether any
+// role still attaches the policy.
+func TestFilerRoleListingFailsOnABrokenStream(t *testing.T) {
+	store, filer := newTestFilerRoleStore(t)
+	for i := range 5 {
+		name := fmt.Sprintf("role-%d.json", i)
+		filer.entries[name] = &filer_pb.Entry{Name: name}
+	}
+	filer.failListAfter = 3
+
+	_, err := store.ListRoles(context.Background(), "")
+	require.Error(t, err, "a partial listing was returned as complete")
+
+	attaching, err := RolesAttachingPolicy(context.Background(), store, "read")
+	assert.Error(t, err)
+	assert.Nil(t, attaching)
+	assert.False(t, errors.Is(err, ErrRoleNotFound))
+}
@@ -0,0 +1,14 @@
+package integration
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+func TestMemoryRoleStoreWrapsErrRoleNotFound(t *testing.T) {
+	_, err := NewMemoryRoleStore().GetRole(context.Background(), "", "missing")
+	if !errors.Is(err, ErrRoleNotFound) {
+		t.Fatalf("missing role error does not wrap ErrRoleNotFound: %v", err)
+	}
+}
@@ -0,0 +1,196 @@
+package integration
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// persistentTestRoleStore stands in for a role store that outlives the process
+// and may be shared (the filer store): anything but *MemoryRoleStore is
+// treated as persistent.
+type persistentTestRoleStore struct {
+	*MemoryRoleStore
+	cleared int
+}
+
+func (s *persistentTestRoleStore) ClearCache() { s.cleared++ }
+
+// startRoleServer initializes a manager as an S3 server would at boot, with the
+// given role store and config file roles.
+func startRoleServer(t *testing.T, store RoleStore, staticRoles ...string) *IAMManager {
+	t.Helper()
+	mgr := NewIAMManager()
+	require.NoError(t, mgr.Initialize(persistTestConfig(), func() string { return "localhost:8888" }))
+	mgr.roleStore = store
+	var defs []*RoleDefinition
+	for _, name := range staticRoles {
+		defs = append(defs, &RoleDefinition{RoleName: name})
+	}
+	mgr.LoadStaticRoles(context.Background(), defs)
+	return mgr
+}
+
+func roleNames(t *testing.T, store RoleStore) map[string]bool {
+	t.Helper()
+	names, err := store.ListRoles(context.Background(), "")
+	require.NoError(t, err)
+	out := map[string]bool{}
+	for _, n := range names {
+		out[n] = true
+	}
+	return out
+}
+
+// A persistent store never receives the config file's roles: it outlives the
+// file and may be shared with servers whose files differ.
+func TestPersistentRoleStoreNeverHoldsConfigFileRoles(t *testing.T) {
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	mgr := startRoleServer(t, store, "from-file")
+
+	assert.Empty(t, roleNames(t, store), "a config-file role was written to the persistent store")
+	assert.True(t, roleNames(t, mgr.GetRoleStore())["from-file"], "the config-file role is not listed")
+	role, err := mgr.GetRole(context.Background(), "from-file")
+	require.NoError(t, err)
+	assert.Equal(t, RoleSourceStaticConfig, role.Source)
+	assert.Equal(t, StaticRoleID(&RoleDefinition{RoleName: "from-file"}), role.RoleId)
+}
+
+// Servers sharing a store, with different config files, must not remove or
+// replace each other's roles — including a zero-config server.
+func TestServersSharingARoleStoreKeepEachOthersRoles(t *testing.T) {
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	configured := startRoleServer(t, store, "from-file")
+	require.NoError(t, configured.CreateRole(context.Background(), "", "created-at-runtime", &RoleDefinition{}))
+
+	zeroConfig := startRoleServer(t, store)
+
+	assert.True(t, roleNames(t, store)["created-at-runtime"], "a peer's start removed a role created at runtime")
+	assert.True(t, roleNames(t, configured.GetRoleStore())["from-file"], "the configured server lost its config-file role")
+	_, err := zeroConfig.GetRole(context.Background(), "from-file")
+	assert.ErrorIs(t, err, ErrRoleNotFound, "a server sees a role only its peer's config file defines")
+	_, err = zeroConfig.GetRole(context.Background(), "created-at-runtime")
+	assert.NoError(t, err, "the peer does not see the shared role created at runtime")
+}
+
+// Removing a role from the config file removes it at the next start.
+func TestRemovingARoleFromTheConfigFileRemovesIt(t *testing.T) {
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	startRoleServer(t, store, "from-file")
+
+	restarted := startRoleServer(t, store) // the file no longer lists it
+
+	_, err := restarted.GetRole(context.Background(), "from-file")
+	assert.ErrorIs(t, err, ErrRoleNotFound, "a role removed from the config file is still served")
+}
+
+// The store refuses to replace or delete a role the config file defines; the
+// file is where it changes.
+func TestConfigFileRolesCannotBeReplacedOrDeletedThroughTheStore(t *testing.T) {
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	mgr := startRoleServer(t, store, "from-file")
+
+	err := mgr.CreateRole(context.Background(), "", "from-file", &RoleDefinition{})
+	assert.ErrorIs(t, err, ErrRoleStatic)
+	assert.ErrorIs(t, mgr.DeleteRole(context.Background(), "from-file"), ErrRoleStatic)
+	assert.Empty(t, roleNames(t, store), "a refused change still wrote to the store")
+}
+
+// An in-memory store keeps its behaviour: the config file's roles are records
+// in it.
+func TestInMemoryRoleStoreStillHoldsConfigFileRoles(t *testing.T) {
+	store := NewMemoryRoleStore()
+	startRoleServer(t, store, "from-file")
+	assert.True(t, roleNames(t, store)["from-file"])
+}
+
+// Invalidating the role cache reaches the store beneath the config-file roles.
+func TestRoleCacheInvalidationReachesTheStoreBeneathTheOverlay(t *testing.T) {
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	mgr := startRoleServer(t, store, "from-file")
+	mgr.InvalidateRoleCache()
+	assert.Equal(t, 1, store.cleared)
+}
+
+func TestRoleCopiesKeepSource(t *testing.T) {
+	role := &RoleDefinition{RoleName: "r", Source: RoleSourceStaticConfig}
+	assert.Equal(t, RoleSourceStaticConfig, copyRoleDefinition(role).Source)
+	assert.Equal(t, RoleSourceStaticConfig, genericCopyRoleDefinition(role).Source)
+}
+
+// Config-file roles report no creation time: a time taken at load would
+// change with every restart.
+func TestConfigFileRolesReportNoCreationTime(t *testing.T) {
+	for name, store := range map[string]RoleStore{
+		"persistent": &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()},
+		"in-memory":  NewMemoryRoleStore(),
+	} {
+		t.Run(name, func(t *testing.T) {
+			mgr := startRoleServer(t, store, "from-file")
+			role, err := mgr.GetRole(context.Background(), "from-file")
+			require.NoError(t, err)
+			assert.True(t, role.CreatedAt.IsZero())
+		})
+	}
+}
+
+// A role stored under a config-file role's name takes precedence, as a stored
+// OIDC provider does; it can be changed, and deleting it brings the
+// config-file role back.
+func TestStoredRoleTakesPrecedenceOverTheConfigFileOne(t *testing.T) {
+	ctx := context.Background()
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	configured := startRoleServer(t, store, "shared-name")
+	peer := startRoleServer(t, store)
+	require.NoError(t, peer.CreateRole(ctx, "", "shared-name", &RoleDefinition{Description: "stored"}))
+
+	role, err := configured.GetRole(ctx, "shared-name")
+	require.NoError(t, err)
+	assert.Equal(t, "stored", role.Description, "the config-file role hides the stored one")
+
+	role.Description = "changed"
+	assert.NoError(t, configured.CreateRole(ctx, "", "shared-name", role), "the stored role cannot be changed")
+
+	require.NoError(t, configured.DeleteRole(ctx, "shared-name"))
+	role, err = configured.GetRole(ctx, "shared-name")
+	require.NoError(t, err)
+	assert.Equal(t, RoleSourceStaticConfig, role.Source, "deleting the stored role did not bring the config-file one back")
+}
+
+// A store installed through SetRoleStore behaves like one installed at
+// startup: the config-file roles stay visible and protected, and it serves
+// the roles it holds.
+func TestSetRoleStoreInstallsLikeStartup(t *testing.T) {
+	mgr := startRoleServer(t, &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}, "from-file")
+
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	require.NoError(t, store.StoreRole(context.Background(), "", "stored", &RoleDefinition{RoleName: "stored"}))
+	mgr.SetRoleStore(store)
+
+	names := roleNames(t, mgr.GetRoleStore())
+	assert.True(t, names["from-file"], "the config-file role is no longer listed")
+	assert.True(t, names["stored"], "the new store's role is not listed")
+	assert.ErrorIs(t, mgr.GetRoleStore().DeleteRole(context.Background(), "", "from-file"), ErrRoleStatic)
+	assert.False(t, roleNames(t, store)["from-file"], "the config-file role was written into the new store")
+}
+
+// S3 servers watch the directory a filer-backed role store keeps roles in, so
+// a store configured with its own basePath must report it, through the
+// cache and the config-file overlay alike.
+func TestRoleStoreDirectoryIsTheConfiguredBasePath(t *testing.T) {
+	provider := func() string { return "localhost:8888" }
+	cached, err := NewGenericCachedRoleStore(map[string]interface{}{"basePath": "/custom/roles"}, provider)
+	require.NoError(t, err)
+	mgr := startRoleServer(t, cached, "from-file")
+	assert.Equal(t, "/custom/roles", mgr.RoleStoreDirectory())
+
+	uncached, err := NewFilerRoleStore(nil, provider)
+	require.NoError(t, err)
+	mgr.SetRoleStore(uncached)
+	assert.Equal(t, "/etc/iam/roles", mgr.RoleStoreDirectory())
+
+	mgr.SetRoleStore(NewMemoryRoleStore())
+	assert.Empty(t, mgr.RoleStoreDirectory(), "a memory store has no directory")
+}
@@ -473,8 +473,11 @@ func (e *PolicyEngine) EvaluateTrustPolicy(ctx context.Context, trustPolicy *Pol
 		return nil, fmt.Errorf("trust policy cannot be nil")
 	}

+	// A trust policy is deny-by-default, whatever the engine's DefaultEffect:
+	// a role may be assumed only by a principal a trust statement explicitly
+	// allows.
 	result := &EvaluationResult{
-		Effect: Effect(e.config.DefaultEffect),
+		Effect: EffectDeny,
 		EvaluationDetails: &EvaluationDetails{
 			Principal:         evalCtx.Principal,
 			Action:            evalCtx.Action,
@@ -511,7 +514,7 @@ func (e *PolicyEngine) EvaluateTrustPolicy(ctx context.Context, trustPolicy *Pol
 	// AWS IAM evaluation logic:
 	// 1. If there's an explicit Deny, the result is Deny
 	// 2. If there's an Allow and no Deny, the result is Allow
-	// 3. Otherwise, use the default effect
+	// 3. Otherwise, the implicit deny stands
 	if explicitDeny {
 		result.Effect = EffectDeny
 	} else if hasAllow {
@@ -0,0 +1,80 @@
+package policy
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+)
+
+const subjectBoundTrustPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+	"Principal":{"Federated":"https://oidc.example"},"Action":["sts:AssumeRoleWithWebIdentity"],
+	"Condition":{"StringEquals":{"oidc:sub":"spiffe://example.org/ns/app/sa/app"}}}]}`
+
+func webIdentityContext(sub string) *EvaluationContext {
+	return &EvaluationContext{
+		Principal: "web-identity-user",
+		Action:    "sts:AssumeRoleWithWebIdentity",
+		Resource:  "arn:aws:iam::role/app",
+		RequestContext: map[string]interface{}{
+			"aws:FederatedProvider": "https://oidc.example",
+			"oidc:iss":              "https://oidc.example",
+			"oidc:sub":              sub,
+		},
+	}
+}
+
+// A trust policy is deny-by-default. The engine's DefaultEffect decides
+// requests no identity policy speaks to; it must not let a principal the trust
+// policy does not allow assume the role.
+func TestEvaluateTrustPolicyIsImplicitDenyWhateverTheDefaultEffect(t *testing.T) {
+	var trust PolicyDocument
+	if err := json.Unmarshal([]byte(subjectBoundTrustPolicy), &trust); err != nil {
+		t.Fatalf("parse test trust policy: %v", err)
+	}
+	for _, defaultEffect := range []string{"Allow", "Deny"} {
+		t.Run("defaultEffect="+defaultEffect, func(t *testing.T) {
+			engine := NewPolicyEngine()
+			if err := engine.Initialize(&PolicyEngineConfig{DefaultEffect: defaultEffect, StoreType: "memory"}); err != nil {
+				t.Fatalf("initialize policy engine: %v", err)
+			}
+
+			res, err := engine.EvaluateTrustPolicy(context.Background(), &trust, webIdentityContext("spiffe://example.org/ns/other/sa/x"))
+			if err != nil {
+				t.Fatalf("evaluate trust policy: %v", err)
+			}
+			if res.Effect != EffectDeny {
+				t.Errorf("a subject the trust policy does not allow got %s", res.Effect)
+			}
+
+			res, err = engine.EvaluateTrustPolicy(context.Background(), &trust, webIdentityContext("spiffe://example.org/ns/app/sa/app"))
+			if err != nil {
+				t.Fatalf("evaluate trust policy: %v", err)
+			}
+			if res.Effect != EffectAllow {
+				t.Errorf("the subject the trust policy allows got %s", res.Effect)
+			}
+		})
+	}
+}
+
+func TestEvaluateTrustPolicyExplicitDenyWins(t *testing.T) {
+	var trust PolicyDocument
+	doc := `{"Version":"2012-10-17","Statement":[
+		{"Effect":"Allow","Principal":{"Federated":"https://oidc.example"},"Action":["sts:AssumeRoleWithWebIdentity"]},
+		{"Effect":"Deny","Principal":{"Federated":"https://oidc.example"},"Action":["sts:AssumeRoleWithWebIdentity"],
+		 "Condition":{"StringEquals":{"oidc:sub":"spiffe://example.org/ns/app/sa/app"}}}]}`
+	if err := json.Unmarshal([]byte(doc), &trust); err != nil {
+		t.Fatalf("parse test trust policy: %v", err)
+	}
+	engine := NewPolicyEngine()
+	if err := engine.Initialize(&PolicyEngineConfig{DefaultEffect: "Allow", StoreType: "memory"}); err != nil {
+		t.Fatalf("initialize policy engine: %v", err)
+	}
+	res, err := engine.EvaluateTrustPolicy(context.Background(), &trust, webIdentityContext("spiffe://example.org/ns/app/sa/app"))
+	if err != nil {
+		t.Fatalf("evaluate trust policy: %v", err)
+	}
+	if res.Effect != EffectDeny {
+		t.Errorf("explicit Deny did not win: %s", res.Effect)
+	}
+}
@@ -532,3 +532,83 @@ type ListUserTagsResponse struct {
 	} `xml:"ListUserTagsResult"`
 	CommonResponse
 }
+
+// IAMRole is the Role element returned by the role actions.
+type IAMRole struct {
+	Path                     string `xml:"Path"`
+	RoleName                 string `xml:"RoleName"`
+	RoleId                   string `xml:"RoleId"`
+	Arn                      string `xml:"Arn"`
+	CreateDate               string `xml:"CreateDate,omitempty"`
+	AssumeRolePolicyDocument string `xml:"AssumeRolePolicyDocument,omitempty"`
+	Description              string `xml:"Description,omitempty"`
+	MaxSessionDuration       int64  `xml:"MaxSessionDuration,omitempty"`
+}
+
+// IAMAttachedPolicy is one element of ListAttachedRolePolicies.
+type IAMAttachedPolicy struct {
+	PolicyName string `xml:"PolicyName"`
+	PolicyArn  string `xml:"PolicyArn"`
+}
+
+// CreateRoleResponse is the response for CreateRole.
+type CreateRoleResponse struct {
+	XMLName          xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateRoleResponse"`
+	CreateRoleResult struct {
+		Role IAMRole `xml:"Role"`
+	} `xml:"CreateRoleResult"`
+	CommonResponse
+}
+
+// GetRoleResponse is the response for GetRole.
+type GetRoleResponse struct {
+	XMLName       xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetRoleResponse"`
+	GetRoleResult struct {
+		Role IAMRole `xml:"Role"`
+	} `xml:"GetRoleResult"`
+	CommonResponse
+}
+
+// ListRolesResponse is the response for ListRoles.
+type ListRolesResponse struct {
+	XMLName         xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListRolesResponse"`
+	ListRolesResult struct {
+		Roles       []*IAMRole `xml:"Roles>member"`
+		IsTruncated bool       `xml:"IsTruncated"`
+	} `xml:"ListRolesResult"`
+	CommonResponse
+}
+
+// DeleteRoleResponse is the response for DeleteRole.
+type DeleteRoleResponse struct {
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteRoleResponse"`
+	CommonResponse
+}
+
+// UpdateAssumeRolePolicyResponse is the response for UpdateAssumeRolePolicy.
+type UpdateAssumeRolePolicyResponse struct {
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ UpdateAssumeRolePolicyResponse"`
+	CommonResponse
+}
+
+// AttachRolePolicyResponse is the response for AttachRolePolicy.
+type AttachRolePolicyResponse struct {
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ AttachRolePolicyResponse"`
+	CommonResponse
+}
+
+// DetachRolePolicyResponse is the response for DetachRolePolicy.
+type DetachRolePolicyResponse struct {
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DetachRolePolicyResponse"`
+	CommonResponse
+}
+
+// ListAttachedRolePoliciesResponse is the response for ListAttachedRolePolicies.
+type ListAttachedRolePoliciesResponse struct {
+	XMLName                        xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListAttachedRolePoliciesResponse"`
+	ListAttachedRolePoliciesResult struct {
+		AttachedPolicies []*IAMAttachedPolicy `xml:"AttachedPolicies>member"`
+		IsTruncated      bool                 `xml:"IsTruncated"`
+	} `xml:"ListAttachedRolePoliciesResult"`
+	CommonResponse
+}
@@ -65,9 +65,10 @@ type STSSessionClaims struct {
 	TokenType   string `json:"typ"`  // token_type

 	// Role information
-	RoleArn     string `json:"role"`      // role_arn
-	AssumedRole string `json:"assumed"`   // assumed_role_user
-	Principal   string `json:"principal"` // principal_arn
+	RoleArn     string `json:"role"`          // role_arn
+	RoleId      string `json:"rid,omitempty"` // unique ID of the assumed role
+	AssumedRole string `json:"assumed"`       // assumed_role_user
+	Principal   string `json:"principal"`     // principal_arn

 	// Authorization data
 	Policies []string `json:"pol,omitempty"` // policies (abbreviated)
@@ -136,6 +137,7 @@ func (c *STSSessionClaims) ToSessionInfo(credGen *CredentialGenerator) *SessionI
 		SessionId:        c.SessionId,
 		SessionName:      c.SessionName,
 		RoleArn:          c.RoleArn,
+		RoleId:           c.RoleId,
 		AssumedRoleUser:  c.AssumedRole,
 		Principal:        c.Principal,
 		Policies:         c.Policies,
@@ -196,6 +198,13 @@ func (c *STSSessionClaims) WithRoleInfo(roleArn, assumedRole, principal string)
 }

 // WithPolicies sets the policies associated with this session
+// WithRoleId binds the session to the assumed role's unique ID. An empty ID
+// leaves the session unbound (roles that predate role IDs).
+func (c *STSSessionClaims) WithRoleId(roleId string) *STSSessionClaims {
+	c.RoleId = roleId
+	return c
+}
+
 func (c *STSSessionClaims) WithPolicies(policies []string) *STSSessionClaims {
 	c.Policies = policies
 	return c
@@ -187,6 +187,11 @@ type AssumeRoleWithWebIdentityRequest struct {

 	// Policy is an optional session policy (optional)
 	Policy *string `json:"Policy,omitempty"`
+
+	// RoleId is the assumed role's unique ID, set by the IAM manager after it
+	// resolves the role. It is embedded in the session so the session is bound
+	// to this role and not to whatever role later holds the same name.
+	RoleId string `json:"-"`
 }

 // AssumeRoleWithCredentialsRequest represents a request to assume role with username/password
@@ -211,6 +216,9 @@ type AssumeRoleWithCredentialsRequest struct {

 	// Policy is an optional session policy (optional)
 	Policy *string `json:"Policy,omitempty"`
+
+	// RoleId is set by the IAM manager; see AssumeRoleWithWebIdentityRequest.
+	RoleId string `json:"-"`
 }

 // AssumeRoleResponse represents the response from assume role operations
@@ -304,6 +312,10 @@ type SessionInfo struct {

 	// ParentUser is the stable hashed identity (sub+iss) derived at federation time.
 	ParentUser string `json:"parentUser,omitempty"`
+
+	// RoleId is the unique ID of the role the session was issued for; empty
+	// for sessions issued before role IDs were recorded.
+	RoleId string `json:"roleId,omitempty"`
 }

 // NewSTSService creates a new STS service
@@ -688,6 +700,7 @@ func (s *STSService) AssumeRoleWithWebIdentity(ctx context.Context, request *Ass
 	sessionClaims := NewSTSSessionClaims(sessionId, s.Config.Issuer, expiresAt).
 		WithSessionName(request.RoleSessionName).
 		WithRoleInfo(effectiveRoleArn, assumedRoleUser.Arn, assumedRoleUser.Arn).
+		WithRoleId(request.RoleId).
 		WithIdentityProvider(provider.Name(), externalIdentity.UserID, externalIdentity.Issuer).
 		WithMaxDuration(sessionDuration).
 		WithRequestContext(requestContext)
@@ -760,13 +773,13 @@ func (s *STSService) AssumeRoleWithCredentials(ctx context.Context, request *Ass
 	}

 	// 4-7. Mint the session
-	return s.issueSession(request.RoleArn, request.RoleSessionName, sessionPolicy,
+	return s.issueSession(request.RoleArn, request.RoleId, request.RoleSessionName, sessionPolicy,
 		request.DurationSeconds, provider.Name(), externalIdentity.UserID)
 }

 // issueSession mints temporary credentials and the self-contained JWT that
 // carries the whole session, shared by every assume-role entry point.
-func (s *STSService) issueSession(roleArn, roleSessionName, sessionPolicy string,
+func (s *STSService) issueSession(roleArn, roleId, roleSessionName, sessionPolicy string,
 	durationSeconds *int64, providerName, subject string) (*AssumeRoleResponse, error) {

 	sessionDuration := s.CalculateSessionDuration(durationSeconds)
@@ -791,6 +804,7 @@ func (s *STSService) issueSession(roleArn, roleSessionName, sessionPolicy string
 	sessionClaims := NewSTSSessionClaims(sessionId, s.Config.Issuer, expiresAt).
 		WithSessionName(roleSessionName).
 		WithRoleInfo(roleArn, assumedRoleUser.Arn, assumedRoleUser.Arn).
+		WithRoleId(roleId).
 		WithIdentityProvider(providerName, subject, "").
 		WithMaxDuration(sessionDuration)
 	if sessionPolicy != "" {
@@ -891,7 +905,9 @@ func (s *STSService) AssumeRoleForPrincipal(ctx context.Context, request *Assume
 		return nil, fmt.Errorf("role assumption denied: %w", err)
 	}

-	return s.issueSession(request.RoleArn, request.RoleSessionName, sessionPolicy,
+	// Iceberg credential vending resolves no role definition, so its sessions
+	// carry no role ID and stay bound by name.
+	return s.issueSession(request.RoleArn, "", request.RoleSessionName, sessionPolicy,
 		request.DurationSeconds, request.ProviderName, request.Principal)
 }

@@ -45,11 +45,11 @@ func DeleteFileIds(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc.
 		return
 	}

-	return DeleteFileIdsWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
+	return DeleteFileIdsWithLookupVolumeId(context.Background(), grpcDialOption, fileIds, lookupFunc)

 }

-func DeleteFileIdsWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]*LookupResult, error)) []*volume_server_pb.DeleteResult {
+func DeleteFileIdsWithLookupVolumeId(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]*LookupResult, error)) []*volume_server_pb.DeleteResult {

 	var ret []*volume_server_pb.DeleteResult

@@ -117,7 +117,7 @@ func DeleteFileIdsWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []s
 		go func(server pb.ServerAddress, fidList []string) {
 			defer wg.Done()

-			resultChan <- DeleteFileIdsAtOneVolumeServer(server, grpcDialOption, fidList, false)
+			resultChan <- DeleteFileIdsAtOneVolumeServer(ctx, server, grpcDialOption, fidList, false)

 		}(server, fidList)
 	}
@@ -133,7 +133,7 @@ func DeleteFileIdsWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []s

 // DeleteFileIdsAtOneVolumeServer deletes a list of files that is on one volume server via gRpc
 // Returns individual results for each file ID. Check result.Error for per-file failures.
-func DeleteFileIdsAtOneVolumeServer(volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) []*volume_server_pb.DeleteResult {
+func DeleteFileIdsAtOneVolumeServer(ctx context.Context, volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) []*volume_server_pb.DeleteResult {

 	var ret []*volume_server_pb.DeleteResult

@@ -144,7 +144,7 @@ func DeleteFileIdsAtOneVolumeServer(volumeServer pb.ServerAddress, grpcDialOptio
 			SkipCookieCheck: !includeCookie,
 		}

-		resp, err := volumeServerClient.BatchDelete(context.Background(), req)
+		resp, err := volumeServerClient.BatchDelete(ctx, req)

 		// fmt.Printf("deleted %v %v: %v\n", fileIds, err, resp)

@@ -168,6 +168,7 @@ message VacuumVolumeCheckRequest {
 }
 message VacuumVolumeCheckResponse {
    double garbage_ratio = 1;
+    bool disk_space_low = 4; // the volume is read-only solely because its disk is low on space — a cause compaction itself reclaims
 }

 message VacuumVolumeCompactRequest {
@@ -436,6 +436,7 @@ func (x *VacuumVolumeCheckRequest) GetVolumeId() uint32 {
 type VacuumVolumeCheckResponse struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	GarbageRatio  float64                `protobuf:"fixed64,1,opt,name=garbage_ratio,json=garbageRatio,proto3" json:"garbage_ratio,omitempty"`
+	DiskSpaceLow  bool                   `protobuf:"varint,4,opt,name=disk_space_low,json=diskSpaceLow,proto3" json:"disk_space_low,omitempty"` // the volume is read-only solely because its disk is low on space — a cause compaction itself reclaims
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -477,6 +478,13 @@ func (x *VacuumVolumeCheckResponse) GetGarbageRatio() float64 {
 	return 0
 }

+func (x *VacuumVolumeCheckResponse) GetDiskSpaceLow() bool {
+	if x != nil {
+		return x.DiskSpaceLow
+	}
+	return false
+}
+
 type VacuumVolumeCompactRequest struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	VolumeId      uint32                 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"`
@@ -7276,9 +7284,10 @@ const file_volume_server_proto_rawDesc = "" +
 	"\aversion\x18\x05 \x01(\rR\aversion\"\a\n" +
 	"\x05Empty\"7\n" +
 	"\x18VacuumVolumeCheckRequest\x12\x1b\n" +
-	"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\"@\n" +
+	"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\"f\n" +
 	"\x19VacuumVolumeCheckResponse\x12#\n" +
-	"\rgarbage_ratio\x18\x01 \x01(\x01R\fgarbageRatio\"[\n" +
+	"\rgarbage_ratio\x18\x01 \x01(\x01R\fgarbageRatio\x12$\n" +
+	"\x0edisk_space_low\x18\x04 \x01(\bR\fdiskSpaceLow\"[\n" +
 	"\x1aVacuumVolumeCompactRequest\x12\x1b\n" +
 	"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12 \n" +
 	"\vpreallocate\x18\x02 \x01(\x03R\vpreallocate\"f\n" +
@@ -14,6 +14,9 @@ import (

 const oidcProvidersDir = filer.IamConfigDirectory + "/oidc-providers"

+// rolesDir is the filer role store's default base path (sts.DefaultRoleBasePath).
+const rolesDir = filer.IamConfigDirectory + "/roles"
+
 func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, lastTsNs int64, prefix string, directoriesToWatch []string) {

 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
@@ -33,6 +36,7 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, lastTsNs int64, p
 		_ = s3a.onBucketMetadataChange(dir, message.OldEntry, message.NewEntry)
 		_ = s3a.onIamConfigChange(dir, message.OldEntry, message.NewEntry)
 		_ = s3a.onOIDCProviderChange(dir, message.OldEntry, message.NewEntry)
+		s3a.onRoleChange(dir)
 		_ = s3a.onCircuitBreakerConfigChange(dir, message.OldEntry, message.NewEntry)

 		// For moves across directories, replay a delete event for the source directory
@@ -40,6 +44,7 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, lastTsNs int64, p
 			_ = s3a.onBucketMetadataChange(resp.Directory, message.OldEntry, nil)
 			_ = s3a.onIamConfigChange(resp.Directory, message.OldEntry, nil)
 			_ = s3a.onOIDCProviderChange(resp.Directory, message.OldEntry, nil)
+			s3a.onRoleChange(resp.Directory)
 			_ = s3a.onCircuitBreakerConfigChange(resp.Directory, message.OldEntry, nil)
 		}

@@ -137,6 +142,40 @@ func (s3a *S3ApiServer) onOIDCProviderChange(dir string, oldEntry *filer_pb.Entr
 	return nil
 }

+// roleStoreDir is the directory the persisted role store keeps roles in: its
+// configured basePath, else the default. The metadata subscription watches it.
+func (s3a *S3ApiServer) roleStoreDir() string {
+	if s3a.iam != nil {
+		if provider, ok := s3a.iam.iamIntegration.(IAMManagerProvider); ok {
+			if mgr := provider.GetIAMManager(); mgr != nil {
+				if dir := mgr.RoleStoreDirectory(); dir != "" {
+					return dir
+				}
+			}
+		}
+	}
+	return rolesDir
+}
+
+// onRoleChange drops the cached role definitions when the persisted role
+// store's directory changes, so a role created, changed or deleted on a peer
+// takes effect here on the next lookup instead of after the cache TTL.
+func (s3a *S3ApiServer) onRoleChange(dir string) {
+	base := s3a.roleStoreDir()
+	if dir != base && !strings.HasPrefix(dir, base+"/") {
+		return
+	}
+	if s3a.iam == nil || s3a.iam.iamIntegration == nil {
+		return
+	}
+	s3iam, ok := s3a.iam.iamIntegration.(*S3IAMIntegration)
+	if !ok || s3iam.iamManager == nil {
+		return
+	}
+	s3iam.iamManager.InvalidateRoleCache()
+	glog.V(2).Infof("Invalidated cached roles after %s change", dir)
+}
+
 // onCircuitBreakerConfigChange handles circuit breaker config file changes (create, update, delete)
 func (s3a *S3ApiServer) onCircuitBreakerConfigChange(dir string, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error {
 	if dir != s3_constants.CircuitBreakerConfigDir {
@@ -277,3 +277,66 @@ func TestLoadIAMManagerFromConfig_ExplicitFileEnforcesUserScopedPolicy(t *testin
 	assert.NoError(t, err)
 	assert.True(t, allowed, "user-scoped bucket creation should be allowed")
 }
+
+func TestLoadIAMManagerFromConfig_HonorsOIDCProviderStore(t *testing.T) {
+	// The documented oidcProviderStore key must reach the IAM manager; without
+	// it, providers created through the IAM API live in one gateway's memory.
+	cases := []struct {
+		name  string
+		store string
+		filer bool
+	}{
+		{"absent keeps memory", ``, false},
+		{"filer persists", `,"oidcProviderStore":{"storeType":"filer"}`, true},
+		{"no config file persists", "no-file", true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			configPath := ""
+			if tc.store != "no-file" {
+				configPath = filepath.Join(t.TempDir(), "iam_config.json")
+				configContent := `{"sts":{"providers":[]},"policy":{"storeType":"memory","defaultEffect":"Deny"}` + tc.store + `}`
+				assert.NoError(t, os.WriteFile(configPath, []byte(configContent), 0644))
+			}
+
+			manager, err := loadIAMManagerFromConfig(configPath, func() string { return "localhost:8888" }, func() string { return "oidc-store-signing-key" })
+			assert.NoError(t, err)
+			_, isFiler := manager.GetOIDCProviderStore().(*integration.FilerOIDCProviderStore)
+			assert.Equal(t, tc.filer, isFiler)
+		})
+	}
+}
+
+func TestLoadIAMManagerFromConfig_HonorsRoleStoreAndMarksStaticRoles(t *testing.T) {
+	cases := []struct {
+		name  string
+		store string
+		filer bool
+	}{
+		{"absent keeps memory", ``, false},
+		{"filer persists", `,"roleStore":{"storeType":"filer"}`, true},
+		{"no config file persists", "no-file", true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			configPath := ""
+			if tc.store != "no-file" {
+				configPath = filepath.Join(t.TempDir(), "iam_config.json")
+				configContent := `{"sts":{"providers":[]},"policy":{"storeType":"memory","defaultEffect":"Deny"},` +
+					`"roles":[{"roleName":"from-file","trustPolicy":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"https://issuer.example"},"Action":["sts:AssumeRoleWithWebIdentity"]}]}}]` +
+					tc.store + `}`
+				assert.NoError(t, os.WriteFile(configPath, []byte(configContent), 0644))
+			}
+
+			manager, err := loadIAMManagerFromConfig(configPath, func() string { return "localhost:8888" }, func() string { return "role-store-signing-key" })
+			assert.NoError(t, err)
+			_, inMemory := manager.GetRoleStore().(*integration.MemoryRoleStore)
+			assert.Equal(t, tc.filer, !inMemory)
+			if !tc.filer {
+				role, err := manager.GetRole(context.Background(), "from-file")
+				assert.NoError(t, err)
+				assert.Equal(t, integration.RoleSourceStaticConfig, role.Source)
+			}
+		})
+	}
+}
@@ -2,6 +2,7 @@ package s3api

 import (
 	"bytes"
+	"net/http"
 	"net/http/httptest"
 	"testing"

@@ -202,3 +203,57 @@ func TestContentEncodingWithOtherHeaders(t *testing.T) {
 	assert.Equal(t, "max-age=3600", getResp.Header().Get("Cache-Control"))
 	assert.Equal(t, "attachment; filename=test.txt", getResp.Header().Get("Content-Disposition"))
 }
+
+// TestContentEncodingDropsAwsChunked verifies that aws-chunked, the SigV4
+// streaming framing of the request body, is not stored with the object, also
+// when the encodings come in separate Content-Encoding fields
+func TestContentEncodingDropsAwsChunked(t *testing.T) {
+	testCases := []struct {
+		name   string
+		fields []string
+		stored string
+	}{
+		{"last", []string{"gzip, aws-chunked"}, "gzip"},
+		{"first", []string{"aws-chunked, gzip"}, "gzip"},
+		{"no spaces", []string{"aws-chunked,gzip,br"}, "gzip, br"},
+		{"alone", []string{"aws-chunked"}, ""},
+		{"capitals", []string{"AWS-Chunked"}, ""},
+		{"twice", []string{"aws-chunked, aws-chunked"}, ""},
+		{"separate fields", []string{"aws-chunked", "gzip"}, "gzip"},
+		{"separate fields, alone", []string{"aws-chunked", "aws-chunked"}, ""},
+		{"without aws-chunked", []string{"deflate, gzip"}, "deflate, gzip"},
+		{"without aws-chunked, separate fields", []string{"deflate", "gzip"}, "deflate, gzip"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.stored, storedContentEncoding(tc.fields))
+
+			// CreateMultipartUpload
+			putReq := httptest.NewRequest("PUT", "/test-bucket/test-object.txt", bytes.NewBufferString("body"))
+			for _, field := range tc.fields {
+				putReq.Header.Add("Content-Encoding", field)
+			}
+			metadata, errCode := ParseS3Metadata(putReq, nil, false)
+			require.Equal(t, 0, int(errCode))
+			if tc.stored == "" {
+				assert.NotContains(t, metadata, "Content-Encoding")
+			} else {
+				assert.Equal(t, []byte(tc.stored), metadata["Content-Encoding"])
+			}
+
+			// CopyObject with the REPLACE metadata directive
+			copyReq := http.Header{}
+			for _, field := range tc.fields {
+				copyReq.Add("Content-Encoding", field)
+			}
+			metadata, err := processMetadataBytes(copyReq, map[string][]byte{"Content-Encoding": []byte("br")}, true, false)
+			require.NoError(t, err)
+			if tc.stored == "" {
+				assert.NotContains(t, metadata, "Content-Encoding")
+			} else {
+				assert.Equal(t, []byte(tc.stored), metadata["Content-Encoding"])
+			}
+		})
+	}
+}
@@ -30,7 +30,7 @@ func ParseS3Metadata(r *http.Request, existing map[string][]byte, isReplace bool
 	}

 	// Content-Encoding (standard HTTP header used by S3)
-	if ce := r.Header.Get("Content-Encoding"); ce != "" {
+	if ce := storedContentEncoding(r.Header.Values("Content-Encoding")); ce != "" {
 		metadata["Content-Encoding"] = []byte(ce)
 	}

@@ -108,3 +108,28 @@ func ParseS3Metadata(r *http.Request, existing map[string][]byte, isReplace bool

 	return metadata, s3err.ErrNone
 }
+
+// storedContentEncoding returns the Content-Encoding to keep with an object,
+// from the values of the request's Content-Encoding fields, which it combines
+// as one list. aws-chunked names the SigV4 streaming framing of the request
+// body, which is decoded on upload, so S3 does not store it: "gzip,
+// aws-chunked" is kept as "gzip", and "aws-chunked" alone as no
+// Content-Encoding at all.
+func storedContentEncoding(values []string) string {
+	value := strings.Join(values, ", ")
+	var kept []string
+	chunked := false
+	for _, encoding := range strings.Split(value, ",") {
+		encoding = strings.TrimSpace(encoding)
+		switch {
+		case strings.EqualFold(encoding, "aws-chunked"):
+			chunked = true
+		case encoding != "":
+			kept = append(kept, encoding)
+		}
+	}
+	if !chunked {
+		return value
+	}
+	return strings.Join(kept, ", ")
+}
@@ -22,6 +22,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/credential"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	iamlib "github.com/seaweedfs/seaweedfs/weed/iam"
+	"github.com/seaweedfs/seaweedfs/weed/iam/integration"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
@@ -225,6 +226,8 @@ func (e *EmbeddedIamApi) writeIamErrorResponse(w http.ResponseWriter, r *http.Re
 		s3err.WriteXMLResponse(w, r, http.StatusNotImplemented, errorResp)
 	case iam.ErrCodeDeleteConflictException:
 		s3err.WriteXMLResponse(w, r, http.StatusConflict, errorResp)
+	case iam.ErrCodeUnmodifiableEntityException:
+		s3err.WriteXMLResponse(w, r, http.StatusBadRequest, errorResp)
 	default:
 		s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, internalErrorResponse)
 	}
@@ -644,6 +647,20 @@ func (e *EmbeddedIamApi) DeletePolicy(ctx context.Context, values url.Values) (*
 			}
 		}
 	}
+	// Roles attach policies by name too; see integration.RolesAttachingPolicy,
+	// including why a role only in a peer's IAM config file is not seen here.
+	if mgr := e.oidcIAMManager(); mgr != nil && mgr.GetRoleStore() != nil {
+		roles, err := integration.RolesAttachingPolicy(ctx, mgr.GetRoleStore(), policyName)
+		if err != nil {
+			return resp, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+		}
+		if len(roles) > 0 {
+			return resp, &iamError{
+				Code:  iam.ErrCodeDeleteConflictException,
+				Error: fmt.Errorf("policy %s is attached to role %s", policyName, roles[0]),
+			}
+		}
+	}
 	if err := e.credentialManager.DeletePolicy(ctx, policyName); err != nil {
 		return resp, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
 	}
@@ -2582,7 +2599,10 @@ func (e *EmbeddedIamApi) AuthIam(f http.HandlerFunc, _ Action) http.HandlerFunc
 }

 // ExecuteAction executes an IAM action with the given values.
-// If skipPersist is true, the changed configuration is not saved to the persistent store.
+// If skipPersist is true, the changed S3ApiConfiguration is not saved to the
+// persistent store. OIDC provider and role actions do not change that
+// configuration: they write to the IAM manager's own stores, whichever the
+// server was configured with, and skipPersist does not apply to them.
 // reqID is set on the response; if empty, a new request ID is generated.
 func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, skipPersist bool, reqID string) (iamlib.RequestIDSetter, *iamError) {
 	if reqID == "" {
@@ -2598,7 +2618,8 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
 		case "ListUsers", "ListAccessKeys", "GetUser", "GetUserPolicy", "ListUserPolicies", "ListAttachedUserPolicies", "ListPolicies", "GetPolicy", "ListPolicyVersions", "GetPolicyVersion", "ListServiceAccounts", "GetServiceAccount",
 			"GetGroup", "ListGroups", "ListAttachedGroupPolicies", "GetGroupPolicy", "ListGroupPolicies", "ListGroupsForUser",
 			"ListUserTags",
-			actionListOpenIDConnectProviders, actionGetOpenIDConnectProvider:
+			actionListOpenIDConnectProviders, actionGetOpenIDConnectProvider,
+			actionGetRole, actionListRoles, actionListAttachedRolePolicies:
 			// Allowed read-only actions
 		default:
 			return nil, &iamError{Code: s3err.GetAPIError(s3err.ErrAccessDenied).Code, Error: fmt.Errorf("IAM write operations are disabled on this server")}
@@ -2615,6 +2636,15 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
 		return response, nil
 	}

+	// Role actions operate on the IAM manager's role store, likewise.
+	if response, iamErr, ok := e.dispatchRoleAction(ctx, values); ok {
+		if iamErr != nil {
+			return nil, iamErr
+		}
+		response.SetRequestId(reqID)
+		return response, nil
+	}
+
 	s3cfg := &iam_pb.S3ApiConfiguration{}
 	if err := e.GetS3ApiConfiguration(s3cfg); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
 		return nil, &iamError{Code: s3err.GetAPIError(s3err.ErrInternalError).Code, Error: fmt.Errorf("failed to get s3 api configuration: %v", err)}
@@ -161,7 +161,7 @@ func (e *EmbeddedIamApi) deleteOpenIDConnectProvider(ctx context.Context, mgr *i
 		return nil, iamErr
 	}
 	if err := mgr.DeleteOIDCProvider(ctx, arn); err != nil {
-		return nil, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+		return nil, oidcMutationError(err)
 	}
 	return &iamlib.DeleteOpenIDConnectProviderResponse{}, nil
 }
@@ -176,10 +176,7 @@ func (e *EmbeddedIamApi) addClientIDToOpenIDConnectProvider(ctx context.Context,
 		return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: errors.New("ClientID is required")}
 	}
 	if err := mgr.AddClientIDToOIDCProvider(ctx, arn, clientID); err != nil {
-		if errors.Is(err, integration.ErrOIDCProviderNotFound) {
-			return nil, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: err}
-		}
-		return nil, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+		return nil, oidcMutationError(err)
 	}
 	return &iamlib.AddClientIDToOpenIDConnectProviderResponse{}, nil
 }
@@ -194,10 +191,7 @@ func (e *EmbeddedIamApi) removeClientIDFromOpenIDConnectProvider(ctx context.Con
 		return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: errors.New("ClientID is required")}
 	}
 	if err := mgr.RemoveClientIDFromOIDCProvider(ctx, arn, clientID); err != nil {
-		if errors.Is(err, integration.ErrOIDCProviderNotFound) {
-			return nil, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: err}
-		}
-		return nil, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+		return nil, oidcMutationError(err)
 	}
 	return &iamlib.RemoveClientIDFromOpenIDConnectProviderResponse{}, nil
 }
@@ -212,6 +206,9 @@ func (e *EmbeddedIamApi) updateOpenIDConnectProviderThumbprint(ctx context.Conte
 		return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: errors.New("ThumbprintList must contain at least one entry")}
 	}
 	if err := mgr.UpdateOIDCProviderThumbprints(ctx, arn, thumbprints); err != nil {
+		if errors.Is(err, integration.ErrOIDCProviderStatic) {
+			return nil, oidcMutationError(err)
+		}
 		if errors.Is(err, integration.ErrOIDCProviderNotFound) {
 			return nil, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: err}
 		}
@@ -230,10 +227,7 @@ func (e *EmbeddedIamApi) tagOpenIDConnectProvider(ctx context.Context, mgr *inte
 		return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: errors.New("Tags must contain at least one Key/Value pair")}
 	}
 	if err := mgr.TagOIDCProvider(ctx, arn, tags); err != nil {
-		if errors.Is(err, integration.ErrOIDCProviderNotFound) {
-			return nil, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: err}
-		}
-		return nil, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+		return nil, oidcMutationError(err)
 	}
 	return &iamlib.TagOpenIDConnectProviderResponse{}, nil
 }
@@ -248,10 +242,7 @@ func (e *EmbeddedIamApi) untagOpenIDConnectProvider(ctx context.Context, mgr *in
 		return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: errors.New("TagKeys must contain at least one entry")}
 	}
 	if err := mgr.UntagOIDCProvider(ctx, arn, keys); err != nil {
-		if errors.Is(err, integration.ErrOIDCProviderNotFound) {
-			return nil, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: err}
-		}
-		return nil, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+		return nil, oidcMutationError(err)
 	}
 	return &iamlib.UntagOpenIDConnectProviderResponse{}, nil
 }
@@ -339,3 +330,17 @@ func (e *EmbeddedIamApi) getOpenIDConnectProvider(ctx context.Context, mgr *inte
 	}
 	return resp, nil
 }
+
+// oidcMutationError maps an IAMManager error from a provider change to its
+// IAM error code. A provider defined in the IAM config file is changed there,
+// not through the API.
+func oidcMutationError(err error) *iamError {
+	switch {
+	case errors.Is(err, integration.ErrOIDCProviderStatic):
+		return &iamError{Code: iam.ErrCodeUnmodifiableEntityException, Error: err}
+	case errors.Is(err, integration.ErrOIDCProviderNotFound):
+		return &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: err}
+	default:
+		return &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+}
@@ -0,0 +1,419 @@
+package s3api
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/iam"
+	iamlib "github.com/seaweedfs/seaweedfs/weed/iam"
+	"github.com/seaweedfs/seaweedfs/weed/iam/integration"
+	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
+)
+
+// Role IAM actions handled by this file.
+const (
+	actionCreateRole               = "CreateRole"
+	actionGetRole                  = "GetRole"
+	actionListRoles                = "ListRoles"
+	actionDeleteRole               = "DeleteRole"
+	actionUpdateAssumeRolePolicy   = "UpdateAssumeRolePolicy"
+	actionAttachRolePolicy         = "AttachRolePolicy"
+	actionDetachRolePolicy         = "DetachRolePolicy"
+	actionListAttachedRolePolicies = "ListAttachedRolePolicies"
+)
+
+// isRoleAction reports whether an action belongs to the role family.
+func isRoleAction(action string) bool {
+	switch action {
+	case actionCreateRole, actionGetRole, actionListRoles, actionDeleteRole,
+		actionUpdateAssumeRolePolicy, actionAttachRolePolicy, actionDetachRolePolicy,
+		actionListAttachedRolePolicies:
+		return true
+	default:
+		return false
+	}
+}
+
+// dispatchRoleAction handles the role IAM actions. Roles live in the IAM
+// manager's role store, not in S3ApiConfiguration, so like the OIDC provider
+// actions they are dispatched before the configuration load. The boolean
+// reports whether the action was recognised.
+func (e *EmbeddedIamApi) dispatchRoleAction(ctx context.Context, values url.Values) (iamlib.RequestIDSetter, *iamError, bool) {
+	if !isRoleAction(values.Get("Action")) {
+		return nil, nil, false
+	}
+	mgr := e.oidcIAMManager()
+	if mgr == nil {
+		return nil, &iamError{Code: iam.ErrCodeServiceFailureException, Error: errors.New("role store not configured: start the S3 server with an IAM config")}, true
+	}
+
+	switch values.Get("Action") {
+	case actionCreateRole:
+		resp, err := e.createRole(ctx, mgr, values)
+		return resp, err, true
+	case actionGetRole:
+		resp, err := e.getRole(ctx, mgr, values)
+		return resp, err, true
+	case actionListRoles:
+		resp, err := e.listRoles(ctx, mgr)
+		return resp, err, true
+	case actionDeleteRole:
+		resp, err := e.deleteRole(ctx, mgr, values)
+		return resp, err, true
+	case actionUpdateAssumeRolePolicy:
+		resp, err := e.updateAssumeRolePolicy(ctx, mgr, values)
+		return resp, err, true
+	case actionAttachRolePolicy:
+		resp, err := e.attachRolePolicy(ctx, mgr, values)
+		return resp, err, true
+	case actionDetachRolePolicy:
+		resp, err := e.detachRolePolicy(ctx, mgr, values)
+		return resp, err, true
+	case actionListAttachedRolePolicies:
+		resp, err := e.listAttachedRolePolicies(ctx, mgr, values)
+		return resp, err, true
+	}
+	return nil, nil, false
+}
+
+// parseTrustPolicy decodes and validates an AssumeRolePolicyDocument.
+func parseTrustPolicy(document string) (*policy.PolicyDocument, *iamError) {
+	if strings.TrimSpace(document) == "" {
+		return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: errors.New("AssumeRolePolicyDocument is required")}
+	}
+	var doc policy.PolicyDocument
+	if err := json.Unmarshal([]byte(document), &doc); err != nil {
+		return nil, &iamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: fmt.Errorf("parse trust policy: %w", err)}
+	}
+	if err := policy.ValidateTrustPolicyDocument(&doc); err != nil {
+		return nil, &iamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err}
+	}
+	return &doc, nil
+}
+
+// requireRole loads the named role, mapping a missing role to NoSuchEntity.
+func requireRole(ctx context.Context, mgr *integration.IAMManager, values url.Values) (*integration.RoleDefinition, *iamError) {
+	name := strings.TrimSpace(values.Get("RoleName"))
+	if name == "" {
+		return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: errors.New("RoleName is required")}
+	}
+	role, err := mgr.GetRole(ctx, name)
+	if errors.Is(err, integration.ErrRoleNotFound) || (err == nil && role == nil) {
+		return nil, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("role %s not found", name)}
+	}
+	if err != nil {
+		return nil, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+	return role, nil
+}
+
+// requireMutableRole is requireRole for actions that change or delete the
+// role. A role loaded from the IAM config file is reloaded from it at every
+// start, so a change made through the API would be silently reverted; it is
+// refused instead.
+func requireMutableRole(ctx context.Context, mgr *integration.IAMManager, values url.Values) (*integration.RoleDefinition, *iamError) {
+	role, iamErr := requireRole(ctx, mgr, values)
+	if iamErr != nil {
+		return nil, iamErr
+	}
+	if role.Source == integration.RoleSourceStaticConfig {
+		return nil, &iamError{Code: iam.ErrCodeUnmodifiableEntityException, Error: fmt.Errorf("role %s is defined in the IAM config file; change it there", role.RoleName)}
+	}
+	return role, nil
+}
+
+// errRoleUpdateRefused aborts a RoleStore update whose refusal the handler
+// has already recorded as an iamError.
+var errRoleUpdateRefused = errors.New("role update refused")
+
+// updateRole applies change to the named role through the role store's atomic
+// update, so a concurrent change on another server is neither lost nor
+// written over a delete. change sees the current role and returns an iamError
+// to refuse; a missing role is NoSuchEntity and a config-file role is
+// UnmodifiableEntity, as requireMutableRole reports them.
+func updateRole(ctx context.Context, mgr *integration.IAMManager, name string, change func(role *integration.RoleDefinition) *iamError) *iamError {
+	var refused *iamError
+	err := mgr.UpdateRole(ctx, name, func(current *integration.RoleDefinition) (*integration.RoleDefinition, error) {
+		if current == nil {
+			refused = &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("role %s not found", name)}
+			return nil, errRoleUpdateRefused
+		}
+		if refused = change(current); refused != nil {
+			return nil, errRoleUpdateRefused
+		}
+		return current, nil
+	})
+	return roleWriteError(err, refused)
+}
+
+// roleWriteError maps a role store write's outcome to the IAM error to
+// report: the handler's own refusal, else the store's error by kind. A write
+// that failed for another reason is a service failure, which clients retry.
+func roleWriteError(err error, refused *iamError) *iamError {
+	switch {
+	case err == nil:
+		return nil
+	case errors.Is(err, errRoleUpdateRefused) && refused != nil:
+		return refused
+	case errors.Is(err, integration.ErrRoleStatic):
+		return &iamError{Code: iam.ErrCodeUnmodifiableEntityException, Error: fmt.Errorf("%w; change it there", err)}
+	case errors.Is(err, integration.ErrRoleExists):
+		return &iamError{Code: iam.ErrCodeEntityAlreadyExistsException, Error: err}
+	default:
+		return &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+}
+
+// roleID is the role's stored ID. A role stored before IDs were recorded has
+// none; it is reported with the ID StaticRoleID derives for it.
+func roleID(role *integration.RoleDefinition) string {
+	if role.RoleId != "" {
+		return role.RoleId
+	}
+	return integration.StaticRoleID(role)
+}
+
+func toIAMRole(role *integration.RoleDefinition) iamlib.IAMRole {
+	out := iamlib.IAMRole{
+		Path:               "/",
+		RoleName:           role.RoleName,
+		RoleId:             roleID(role),
+		Arn:                role.RoleArn,
+		Description:        role.Description,
+		MaxSessionDuration: role.MaxSessionDuration,
+	}
+	if !role.CreatedAt.IsZero() {
+		out.CreateDate = role.CreatedAt.UTC().Format(time.RFC3339)
+	}
+	if role.TrustPolicy != nil {
+		if doc, err := json.Marshal(role.TrustPolicy); err == nil {
+			// AWS returns the document URL-encoded.
+			out.AssumeRolePolicyDocument = url.PathEscape(string(doc))
+		}
+	}
+	return out
+}
+
+func (e *EmbeddedIamApi) createRole(ctx context.Context, mgr *integration.IAMManager, values url.Values) (*iamlib.CreateRoleResponse, *iamError) {
+	name := strings.TrimSpace(values.Get("RoleName"))
+	if name == "" {
+		return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: errors.New("RoleName is required")}
+	}
+	if err := integration.ValidateRoleName(name); err != nil {
+		return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: err}
+	}
+	if path := values.Get("Path"); path != "" && path != "/" {
+		return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("role paths are not supported: %s", path)}
+	}
+	for key := range values {
+		if strings.HasPrefix(key, "Tags.") {
+			return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: errors.New("role tags are not supported")}
+		}
+	}
+	// A config-file role is served beside the store, not from it, so the
+	// store's create-if-absent cannot see it.
+	existing, err := mgr.GetRole(ctx, name)
+	if err == nil && existing != nil {
+		return nil, &iamError{Code: iam.ErrCodeEntityAlreadyExistsException, Error: fmt.Errorf("role %s already exists", name)}
+	}
+	if err != nil && !errors.Is(err, integration.ErrRoleNotFound) {
+		return nil, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+	trust, iamErr := parseTrustPolicy(values.Get("AssumeRolePolicyDocument"))
+	if iamErr != nil {
+		return nil, iamErr
+	}
+	var maxSession int64
+	if raw := values.Get("MaxSessionDuration"); raw != "" {
+		n, err := strconv.ParseInt(raw, 10, 64)
+		if err != nil {
+			return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("MaxSessionDuration: %w", err)}
+		}
+		maxSession = n
+	}
+
+	role := &integration.RoleDefinition{
+		RoleName:           name,
+		TrustPolicy:        trust,
+		Description:        values.Get("Description"),
+		MaxSessionDuration: maxSession,
+		CreatedAt:          time.Now().UTC(),
+		RoleId:             integration.NewRoleID(),
+	}
+	// Validation first: only a bad request is InvalidInput. The write below
+	// is created only if no role of this name exists by then, so of two
+	// concurrent creates, one fails with EntityAlreadyExists.
+	if err := integration.PrepareRoleDefinition(name, role); err != nil {
+		return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: err}
+	}
+	err = mgr.UpdateRole(ctx, name, func(current *integration.RoleDefinition) (*integration.RoleDefinition, error) {
+		if current != nil {
+			return nil, fmt.Errorf("%w: %s", integration.ErrRoleExists, name)
+		}
+		return role, nil
+	})
+	if iamErr := roleWriteError(err, nil); iamErr != nil {
+		return nil, iamErr
+	}
+	resp := &iamlib.CreateRoleResponse{}
+	resp.CreateRoleResult.Role = toIAMRole(role)
+	return resp, nil
+}
+
+func (e *EmbeddedIamApi) getRole(ctx context.Context, mgr *integration.IAMManager, values url.Values) (*iamlib.GetRoleResponse, *iamError) {
+	role, iamErr := requireRole(ctx, mgr, values)
+	if iamErr != nil {
+		return nil, iamErr
+	}
+	resp := &iamlib.GetRoleResponse{}
+	resp.GetRoleResult.Role = toIAMRole(role)
+	return resp, nil
+}
+
+func (e *EmbeddedIamApi) listRoles(ctx context.Context, mgr *integration.IAMManager) (*iamlib.ListRolesResponse, *iamError) {
+	roles, err := mgr.ListRoles(ctx)
+	if err != nil {
+		return nil, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+	resp := &iamlib.ListRolesResponse{}
+	resp.ListRolesResult.Roles = make([]*iamlib.IAMRole, 0, len(roles))
+	for _, role := range roles {
+		view := toIAMRole(role)
+		resp.ListRolesResult.Roles = append(resp.ListRolesResult.Roles, &view)
+	}
+	return resp, nil
+}
+
+func (e *EmbeddedIamApi) deleteRole(ctx context.Context, mgr *integration.IAMManager, values url.Values) (*iamlib.DeleteRoleResponse, *iamError) {
+	role, iamErr := requireMutableRole(ctx, mgr, values)
+	if iamErr != nil {
+		return nil, iamErr
+	}
+	// The delete is decided against the role as it is when deleted, in the
+	// store's atomic update: a policy attached meanwhile on another server is
+	// a DeleteConflict, as AWS reports a role with managed policies attached.
+	var refused *iamError
+	err := mgr.UpdateRole(ctx, role.RoleName, func(current *integration.RoleDefinition) (*integration.RoleDefinition, error) {
+		if current == nil {
+			refused = &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("role %s not found", role.RoleName)}
+			return nil, errRoleUpdateRefused
+		}
+		if len(current.AttachedPolicies) > 0 {
+			refused = &iamError{Code: iam.ErrCodeDeleteConflictException, Error: fmt.Errorf("role %s has attached policies; detach them first", role.RoleName)}
+			return nil, errRoleUpdateRefused
+		}
+		return nil, nil
+	})
+	if iamErr := roleWriteError(err, refused); iamErr != nil {
+		return nil, iamErr
+	}
+	return &iamlib.DeleteRoleResponse{}, nil
+}
+
+func (e *EmbeddedIamApi) updateAssumeRolePolicy(ctx context.Context, mgr *integration.IAMManager, values url.Values) (*iamlib.UpdateAssumeRolePolicyResponse, *iamError) {
+	role, iamErr := requireMutableRole(ctx, mgr, values)
+	if iamErr != nil {
+		return nil, iamErr
+	}
+	trust, iamErr := parseTrustPolicy(values.Get("PolicyDocument"))
+	if iamErr != nil {
+		return nil, iamErr
+	}
+	if iamErr := updateRole(ctx, mgr, role.RoleName, func(current *integration.RoleDefinition) *iamError {
+		current.TrustPolicy = trust
+		return nil
+	}); iamErr != nil {
+		return nil, iamErr
+	}
+	return &iamlib.UpdateAssumeRolePolicyResponse{}, nil
+}
+
+// rolePolicyName resolves PolicyArn to the name of an existing managed policy.
+func (e *EmbeddedIamApi) rolePolicyName(ctx context.Context, values url.Values) (string, *iamError) {
+	name, err := iamPolicyNameFromArn(values.Get("PolicyArn"))
+	if err != nil {
+		return "", &iamError{Code: iam.ErrCodeInvalidInputException, Error: err}
+	}
+	if e.credentialManager == nil {
+		return "", &iamError{Code: iam.ErrCodeServiceFailureException, Error: errors.New("credential manager not configured")}
+	}
+	existing, err := e.credentialManager.GetPolicy(ctx, name)
+	if err != nil {
+		return "", &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+	if existing == nil {
+		return "", &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("policy %s not found", name)}
+	}
+	return name, nil
+}
+
+func (e *EmbeddedIamApi) attachRolePolicy(ctx context.Context, mgr *integration.IAMManager, values url.Values) (*iamlib.AttachRolePolicyResponse, *iamError) {
+	role, iamErr := requireMutableRole(ctx, mgr, values)
+	if iamErr != nil {
+		return nil, iamErr
+	}
+	name, iamErr := e.rolePolicyName(ctx, values)
+	if iamErr != nil {
+		return nil, iamErr
+	}
+	if slices.Contains(role.AttachedPolicies, name) {
+		return &iamlib.AttachRolePolicyResponse{}, nil
+	}
+	if iamErr := updateRole(ctx, mgr, role.RoleName, func(current *integration.RoleDefinition) *iamError {
+		if slices.Contains(current.AttachedPolicies, name) {
+			return nil
+		}
+		if len(current.AttachedPolicies) >= integration.MaxManagedPoliciesPerRole {
+			return &iamError{Code: iam.ErrCodeLimitExceededException,
+				Error: fmt.Errorf("cannot attach more than %d managed policies to role %s", integration.MaxManagedPoliciesPerRole, current.RoleName)}
+		}
+		current.AttachedPolicies = append(current.AttachedPolicies, name)
+		return nil
+	}); iamErr != nil {
+		return nil, iamErr
+	}
+	return &iamlib.AttachRolePolicyResponse{}, nil
+}
+
+func (e *EmbeddedIamApi) detachRolePolicy(ctx context.Context, mgr *integration.IAMManager, values url.Values) (*iamlib.DetachRolePolicyResponse, *iamError) {
+	role, iamErr := requireMutableRole(ctx, mgr, values)
+	if iamErr != nil {
+		return nil, iamErr
+	}
+	name, err := iamPolicyNameFromArn(values.Get("PolicyArn"))
+	if err != nil {
+		return nil, &iamError{Code: iam.ErrCodeInvalidInputException, Error: err}
+	}
+	if iamErr := updateRole(ctx, mgr, role.RoleName, func(current *integration.RoleDefinition) *iamError {
+		idx := slices.Index(current.AttachedPolicies, name)
+		if idx < 0 {
+			return &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("policy %s is not attached to role %s", name, current.RoleName)}
+		}
+		current.AttachedPolicies = slices.Delete(current.AttachedPolicies, idx, idx+1)
+		return nil
+	}); iamErr != nil {
+		return nil, iamErr
+	}
+	return &iamlib.DetachRolePolicyResponse{}, nil
+}
+
+func (e *EmbeddedIamApi) listAttachedRolePolicies(ctx context.Context, mgr *integration.IAMManager, values url.Values) (*iamlib.ListAttachedRolePoliciesResponse, *iamError) {
+	role, iamErr := requireRole(ctx, mgr, values)
+	if iamErr != nil {
+		return nil, iamErr
+	}
+	resp := &iamlib.ListAttachedRolePoliciesResponse{}
+	resp.ListAttachedRolePoliciesResult.AttachedPolicies = make([]*iamlib.IAMAttachedPolicy, 0, len(role.AttachedPolicies))
+	for _, name := range role.AttachedPolicies {
+		resp.ListAttachedRolePoliciesResult.AttachedPolicies = append(resp.ListAttachedRolePoliciesResult.AttachedPolicies,
+			&iamlib.IAMAttachedPolicy{PolicyName: name, PolicyArn: iamPolicyArn(name)})
+	}
+	return resp, nil
+}
@@ -2,6 +2,10 @@ package s3api

 import (
 	"context"
+	"fmt"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"testing"
 	"time"
@@ -394,3 +398,18 @@ func TestUpdateThumbprintAndTags(t *testing.T) {
 		t.Fatalf("Tags should be empty after untag, got: %v", gr.Tags)
 	}
 }
+
+// A refusal to change a config-file provider reaches the client as AWS sends
+// it (400 UnmodifiableEntity), not as an internal error clients retry.
+func TestUnmodifiableEntityIsAClientError(t *testing.T) {
+	api := NewEmbeddedIamApiForTest()
+	rec := httptest.NewRecorder()
+	api.writeIamErrorResponse(rec, httptest.NewRequest(http.MethodPost, "/", nil), "req-1",
+		oidcMutationError(fmt.Errorf("%w: arn:aws:iam:::oidc-provider/static.example", integration.ErrOIDCProviderStatic)))
+	if rec.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want 400", rec.Code)
+	}
+	if code, _ := extractEmbeddedIamErrorCodeAndMessage(rec); code != iam.ErrCodeUnmodifiableEntityException {
+		t.Fatalf("code = %q, want UnmodifiableEntity", code)
+	}
+}
@@ -0,0 +1,340 @@
+package s3api
+
+import (
+	"context"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/iam"
+	iamlib "github.com/seaweedfs/seaweedfs/weed/iam"
+	"github.com/seaweedfs/seaweedfs/weed/iam/integration"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const spiffeTrustPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow",` +
+	`"Principal":{"Federated":"https://oidc.example"},"Action":["sts:AssumeRoleWithWebIdentity"],` +
+	`"Condition":{"StringEquals":{"oidc:sub":"spiffe://example.org/ns/app/sa/app"}}}]}`
+
+func roleAction(t *testing.T, api *EmbeddedIamApiForTest, params map[string]string) (iamlib.RequestIDSetter, *iamError) {
+	t.Helper()
+	values := url.Values{}
+	for k, v := range params {
+		values.Set(k, v)
+	}
+	return api.ExecuteAction(context.Background(), values, true, "role-test")
+}
+
+func requireIamCode(t *testing.T, iamErr *iamError, code string) {
+	t.Helper()
+	require.NotNil(t, iamErr, "expected %s", code)
+	assert.Equal(t, code, iamErr.Code, "error: %v", iamErr.Error)
+}
+
+func newRoleTestAPI(t *testing.T) (*EmbeddedIamApiForTest, *integration.IAMManager) {
+	t.Helper()
+	api, mgr := newOIDCTestAPI(t)
+	doc := policy_engine.PolicyDocument{Version: "2012-10-17", Statement: []policy_engine.PolicyStatement{{
+		Effect:   policy_engine.PolicyEffectAllow,
+		Action:   policy_engine.NewStringOrStringSlice("s3:GetObject"),
+		Resource: policy_engine.NewStringOrStringSlicePtr("arn:aws:s3:::bucket/*"),
+	}}}
+	require.NoError(t, api.credentialManager.CreatePolicy(context.Background(), "read-bucket", doc))
+	return api, mgr
+}
+
+func TestRoleLifecycle(t *testing.T) {
+	api, mgr := newRoleTestAPI(t)
+
+	resp, iamErr := roleAction(t, api, map[string]string{
+		"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy,
+		"Description": "app runtime", "MaxSessionDuration": "3600",
+	})
+	require.Nil(t, iamErr)
+	created := resp.(*iamlib.CreateRoleResponse).CreateRoleResult.Role
+	assert.Equal(t, "arn:aws:iam::role/app", created.Arn)
+	assert.NotEmpty(t, created.RoleId)
+	assert.NotEmpty(t, created.CreateDate)
+	decoded, err := url.PathUnescape(created.AssumeRolePolicyDocument)
+	require.NoError(t, err)
+	assert.Contains(t, decoded, "spiffe://example.org/ns/app/sa/app")
+
+	// The role is the one STS evaluates.
+	stored, err := mgr.GetRole(context.Background(), "app")
+	require.NoError(t, err)
+	assert.Equal(t, "https://oidc.example", stored.TrustPolicy.Statement[0].Principal.(map[string]interface{})["Federated"])
+
+	_, iamErr = roleAction(t, api, map[string]string{"Action": actionAttachRolePolicy, "RoleName": "app", "PolicyArn": "arn:aws:iam:::policy/read-bucket"})
+	require.Nil(t, iamErr)
+	resp, iamErr = roleAction(t, api, map[string]string{"Action": actionListAttachedRolePolicies, "RoleName": "app"})
+	require.Nil(t, iamErr)
+	attached := resp.(*iamlib.ListAttachedRolePoliciesResponse).ListAttachedRolePoliciesResult.AttachedPolicies
+	require.Len(t, attached, 1)
+	assert.Equal(t, "read-bucket", attached[0].PolicyName)
+
+	_, iamErr = roleAction(t, api, map[string]string{"Action": actionDeleteRole, "RoleName": "app"})
+	requireIamCode(t, iamErr, iam.ErrCodeDeleteConflictException)
+
+	_, iamErr = roleAction(t, api, map[string]string{"Action": actionDetachRolePolicy, "RoleName": "app", "PolicyArn": "arn:aws:iam:::policy/read-bucket"})
+	require.Nil(t, iamErr)
+	_, iamErr = roleAction(t, api, map[string]string{"Action": actionDeleteRole, "RoleName": "app"})
+	require.Nil(t, iamErr)
+	_, iamErr = roleAction(t, api, map[string]string{"Action": actionGetRole, "RoleName": "app"})
+	requireIamCode(t, iamErr, iam.ErrCodeNoSuchEntityException)
+}
+
+func TestUpdateAssumeRolePolicyReplacesTheTrustPolicy(t *testing.T) {
+	api, mgr := newRoleTestAPI(t)
+	_, iamErr := roleAction(t, api, map[string]string{"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy})
+	require.Nil(t, iamErr)
+
+	updated := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"https://other.example"},"Action":["sts:AssumeRoleWithWebIdentity"]}]}`
+	_, iamErr = roleAction(t, api, map[string]string{"Action": actionUpdateAssumeRolePolicy, "RoleName": "app", "PolicyDocument": updated})
+	require.Nil(t, iamErr)
+	stored, err := mgr.GetRole(context.Background(), "app")
+	require.NoError(t, err)
+	assert.Equal(t, "https://other.example", stored.TrustPolicy.Statement[0].Principal.(map[string]interface{})["Federated"])
+}
+
+func TestRoleActionsRefuseWhatTheyCannotHonour(t *testing.T) {
+	api, _ := newRoleTestAPI(t)
+	_, iamErr := roleAction(t, api, map[string]string{"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy})
+	require.Nil(t, iamErr)
+
+	cases := []struct {
+		name   string
+		params map[string]string
+		code   string
+	}{
+		{"duplicate", map[string]string{"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy}, iam.ErrCodeEntityAlreadyExistsException},
+		{"no trust policy", map[string]string{"Action": actionCreateRole, "RoleName": "b"}, iam.ErrCodeInvalidInputException},
+		{"malformed trust policy", map[string]string{"Action": actionCreateRole, "RoleName": "b", "AssumeRolePolicyDocument": "{"}, iam.ErrCodeMalformedPolicyDocumentException},
+		{"path", map[string]string{"Action": actionCreateRole, "RoleName": "b", "Path": "/team/", "AssumeRolePolicyDocument": spiffeTrustPolicy}, iam.ErrCodeInvalidInputException},
+		{"tags", map[string]string{"Action": actionCreateRole, "RoleName": "b", "Tags.member.1.Key": "k", "Tags.member.1.Value": "v", "AssumeRolePolicyDocument": spiffeTrustPolicy}, iam.ErrCodeInvalidInputException},
+		{"session out of bounds", map[string]string{"Action": actionCreateRole, "RoleName": "b", "MaxSessionDuration": "60", "AssumeRolePolicyDocument": spiffeTrustPolicy}, iam.ErrCodeInvalidInputException},
+		{"attach missing policy", map[string]string{"Action": actionAttachRolePolicy, "RoleName": "app", "PolicyArn": "arn:aws:iam:::policy/nope"}, iam.ErrCodeNoSuchEntityException},
+		{"detach unattached", map[string]string{"Action": actionDetachRolePolicy, "RoleName": "app", "PolicyArn": "arn:aws:iam:::policy/read-bucket"}, iam.ErrCodeNoSuchEntityException},
+		{"missing role", map[string]string{"Action": actionDeleteRole, "RoleName": "nope"}, iam.ErrCodeNoSuchEntityException},
+		{"name leaving the role store", map[string]string{"Action": actionCreateRole, "RoleName": "../identities/admin", "AssumeRolePolicyDocument": spiffeTrustPolicy}, iam.ErrCodeInvalidInputException},
+		{"name too long", map[string]string{"Action": actionCreateRole, "RoleName": strings.Repeat("a", 65), "AssumeRolePolicyDocument": spiffeTrustPolicy}, iam.ErrCodeInvalidInputException},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, iamErr := roleAction(t, api, tc.params)
+			requireIamCode(t, iamErr, tc.code)
+		})
+	}
+}
+
+// A role from the IAM config file is reloaded from it at every start, so an
+// API change to it would be silently reverted; it is refused instead.
+func TestConfigFileRolesAreUnmodifiable(t *testing.T) {
+	api, mgr := newRoleTestAPI(t)
+	trust, iamErr := parseTrustPolicy(spiffeTrustPolicy)
+	require.Nil(t, iamErr)
+	require.NoError(t, mgr.CreateRole(context.Background(), "", "from-file", &integration.RoleDefinition{
+		RoleName: "from-file", TrustPolicy: trust, Source: integration.RoleSourceStaticConfig,
+	}))
+
+	for _, params := range []map[string]string{
+		{"Action": actionDeleteRole, "RoleName": "from-file"},
+		{"Action": actionUpdateAssumeRolePolicy, "RoleName": "from-file", "PolicyDocument": spiffeTrustPolicy},
+		{"Action": actionAttachRolePolicy, "RoleName": "from-file", "PolicyArn": "arn:aws:iam:::policy/read-bucket"},
+		{"Action": actionDetachRolePolicy, "RoleName": "from-file", "PolicyArn": "arn:aws:iam:::policy/read-bucket"},
+	} {
+		t.Run(params["Action"], func(t *testing.T) {
+			_, iamErr := roleAction(t, api, params)
+			requireIamCode(t, iamErr, iam.ErrCodeUnmodifiableEntityException)
+		})
+	}
+	_, iamErr = roleAction(t, api, map[string]string{"Action": actionGetRole, "RoleName": "from-file"})
+	assert.Nil(t, iamErr, "reading a config-file role is allowed")
+}
+
+func TestReadOnlyAllowsRoleReadsAndDeniesMutations(t *testing.T) {
+	api, _ := newRoleTestAPI(t)
+	_, iamErr := roleAction(t, api, map[string]string{"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy})
+	require.Nil(t, iamErr)
+	api.readOnly = true
+
+	for _, action := range []string{actionGetRole, actionListRoles, actionListAttachedRolePolicies} {
+		_, iamErr := roleAction(t, api, map[string]string{"Action": action, "RoleName": "app"})
+		assert.Nil(t, iamErr, "%s must be allowed in read-only mode", action)
+	}
+	for _, action := range []string{actionCreateRole, actionDeleteRole, actionUpdateAssumeRolePolicy, actionAttachRolePolicy, actionDetachRolePolicy} {
+		_, iamErr := roleAction(t, api, map[string]string{"Action": action, "RoleName": "app"})
+		assert.NotNil(t, iamErr, "%s must be denied in read-only mode", action)
+	}
+}
+
+// unreadableRoleStore fails every read the way an unreachable filer does.
+type unreadableRoleStore struct{ *integration.MemoryRoleStore }
+
+func (unreadableRoleStore) GetRole(context.Context, string, string) (*integration.RoleDefinition, error) {
+	return nil, errors.New("lookup role: filer unavailable")
+}
+
+// An unreadable store is not an absent role: CreateRole must not write over a
+// role it could not see, and the reads must not report NoSuchEntity.
+func TestRoleActionsTreatAnUnreadableStoreAsAFailureNotAnAbsence(t *testing.T) {
+	api, mgr := newRoleTestAPI(t)
+	store := unreadableRoleStore{integration.NewMemoryRoleStore()}
+	mgr.SetRoleStore(store)
+
+	_, iamErr := roleAction(t, api, map[string]string{"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy})
+	requireIamCode(t, iamErr, iam.ErrCodeServiceFailureException)
+	names, err := store.ListRoles(context.Background(), "")
+	require.NoError(t, err)
+	assert.Empty(t, names, "CreateRole wrote through a store it could not read")
+
+	_, iamErr = roleAction(t, api, map[string]string{"Action": actionGetRole, "RoleName": "app"})
+	requireIamCode(t, iamErr, iam.ErrCodeServiceFailureException)
+}
+
+// A policy is attached to a role by name, so deleting it while attached would
+// let a policy created later under that name take effect on the role. It is
+// refused, as for users and groups.
+func TestDeletePolicyAttachedToARoleIsAConflict(t *testing.T) {
+	api, _ := newRoleTestAPI(t)
+	// The test API replaces the credential store with mockConfig on the first
+	// action that loads the configuration, so the policy has to be declared
+	// there too.
+	api.mockConfig.Policies = append(api.mockConfig.Policies, &iam_pb.Policy{
+		Name:    "read-bucket",
+		Content: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::bucket/*"]}]}`,
+	})
+	_, iamErr := roleAction(t, api, map[string]string{"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy})
+	require.Nil(t, iamErr)
+	_, iamErr = roleAction(t, api, map[string]string{"Action": actionAttachRolePolicy, "RoleName": "app", "PolicyArn": "arn:aws:iam:::policy/read-bucket"})
+	require.Nil(t, iamErr)
+
+	_, iamErr = roleAction(t, api, map[string]string{"Action": "DeletePolicy", "PolicyArn": "arn:aws:iam:::policy/read-bucket"})
+	requireIamCode(t, iamErr, iam.ErrCodeDeleteConflictException)
+
+	_, iamErr = roleAction(t, api, map[string]string{"Action": actionDetachRolePolicy, "RoleName": "app", "PolicyArn": "arn:aws:iam:::policy/read-bucket"})
+	require.Nil(t, iamErr)
+	_, iamErr = roleAction(t, api, map[string]string{"Action": "DeletePolicy", "PolicyArn": "arn:aws:iam:::policy/read-bucket"})
+	assert.Nil(t, iamErr, "a detached policy could not be deleted")
+}
+
+func TestAttachRolePolicyStopsAtTheRoleQuota(t *testing.T) {
+	api, _ := newRoleTestAPI(t)
+	_, iamErr := roleAction(t, api, map[string]string{"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy})
+	require.Nil(t, iamErr)
+	doc := policy_engine.PolicyDocument{Version: "2012-10-17", Statement: []policy_engine.PolicyStatement{{
+		Effect: policy_engine.PolicyEffectAllow, Action: policy_engine.NewStringOrStringSlice("s3:GetObject"),
+		Resource: policy_engine.NewStringOrStringSlicePtr("arn:aws:s3:::bucket/*"),
+	}}}
+	for i := 0; i <= integration.MaxManagedPoliciesPerRole; i++ {
+		name := fmt.Sprintf("p%d", i)
+		require.NoError(t, api.credentialManager.CreatePolicy(context.Background(), name, doc))
+		_, iamErr = roleAction(t, api, map[string]string{"Action": actionAttachRolePolicy, "RoleName": "app", "PolicyArn": "arn:aws:iam:::policy/" + name})
+		if i < integration.MaxManagedPoliciesPerRole {
+			require.Nil(t, iamErr, "attach %d", i)
+		}
+	}
+	requireIamCode(t, iamErr, iam.ErrCodeLimitExceededException)
+}
+
+// A role without a creation time omits CreateDate rather than sending it
+// empty, which clients cannot parse.
+func TestRoleWithoutACreationTimeOmitsCreateDate(t *testing.T) {
+	out, err := xml.Marshal(toIAMRole(&integration.RoleDefinition{RoleName: "r", RoleArn: "arn:aws:iam::role/r"}))
+	require.NoError(t, err)
+	assert.NotContains(t, string(out), "CreateDate")
+}
+
+// Roles take no tags, so any tag parameter is refused rather than dropped.
+func TestCreateRoleRefusesEveryTagParameter(t *testing.T) {
+	api, _ := newRoleTestAPI(t)
+	for _, key := range []string{"Tags.member.1.Key", "Tags.member.2.Key", "Tags.member.1.Value"} {
+		_, iamErr := roleAction(t, api, map[string]string{
+			"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy, key: "x",
+		})
+		requireIamCode(t, iamErr, iam.ErrCodeInvalidInputException)
+	}
+}
+
+// unwritableRoleStore fails every write the way an unreachable filer does.
+type unwritableRoleStore struct{ *integration.MemoryRoleStore }
+
+func (unwritableRoleStore) UpdateRole(context.Context, string, string, integration.RoleUpdate) error {
+	return errors.New("store role: filer unavailable")
+}
+
+// A store that cannot be written is a service failure, which clients retry;
+// only a bad request is InvalidInput.
+func TestCreateRoleReportsAFailedWriteAsAServiceFailure(t *testing.T) {
+	api, mgr := newRoleTestAPI(t)
+	mgr.SetRoleStore(unwritableRoleStore{integration.NewMemoryRoleStore()})
+
+	_, iamErr := roleAction(t, api, map[string]string{"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy})
+	requireIamCode(t, iamErr, iam.ErrCodeServiceFailureException)
+
+	_, iamErr = roleAction(t, api, map[string]string{
+		"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy, "MaxSessionDuration": "60",
+	})
+	requireIamCode(t, iamErr, iam.ErrCodeInvalidInputException)
+}
+
+// racingRoleStore reports every role absent to plain reads while the store
+// holds them: the view of a server whose create raced another server's.
+type racingRoleStore struct{ *integration.MemoryRoleStore }
+
+func (racingRoleStore) GetRole(_ context.Context, _ string, name string) (*integration.RoleDefinition, error) {
+	return nil, fmt.Errorf("%w: %s", integration.ErrRoleNotFound, name)
+}
+
+// Of two creates of one name, the second is told the role exists; it does
+// not replace the first role.
+func TestCreateRoleRacingAnotherCreateIsEntityAlreadyExists(t *testing.T) {
+	api, mgr := newRoleTestAPI(t)
+	store := racingRoleStore{integration.NewMemoryRoleStore()}
+	require.NoError(t, store.StoreRole(context.Background(), "", "app", &integration.RoleDefinition{RoleName: "app", RoleId: "AROA-FIRST"}))
+	mgr.SetRoleStore(store)
+
+	_, iamErr := roleAction(t, api, map[string]string{"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy})
+	requireIamCode(t, iamErr, iam.ErrCodeEntityAlreadyExistsException)
+	role, err := store.MemoryRoleStore.GetRole(context.Background(), "", "app")
+	require.NoError(t, err)
+	assert.Equal(t, "AROA-FIRST", role.RoleId, "the second create replaced the first role")
+}
+
+// vanishingRoleStore lists a role that is gone by the time it is read.
+type vanishingRoleStore struct{ *integration.MemoryRoleStore }
+
+func (s vanishingRoleStore) ListRoles(ctx context.Context, filerAddress string) ([]string, error) {
+	names, err := s.MemoryRoleStore.ListRoles(ctx, filerAddress)
+	return append(names, "deleted-meanwhile"), err
+}
+
+func TestListRolesSkipsARoleDeletedWhileListing(t *testing.T) {
+	api, mgr := newRoleTestAPI(t)
+	mgr.SetRoleStore(vanishingRoleStore{integration.NewMemoryRoleStore()})
+	_, iamErr := roleAction(t, api, map[string]string{"Action": actionCreateRole, "RoleName": "app", "AssumeRolePolicyDocument": spiffeTrustPolicy})
+	require.Nil(t, iamErr)
+
+	resp, iamErr := roleAction(t, api, map[string]string{"Action": actionListRoles})
+	require.Nil(t, iamErr)
+	roles := resp.(*iamlib.ListRolesResponse).ListRolesResult.Roles
+	require.Len(t, roles, 1)
+	assert.Equal(t, "app", roles[0].RoleName)
+}
+
+// Peer role changes are watched in the role store's own directory.
+func TestRoleChangesAreWatchedInTheRoleStoresDirectory(t *testing.T) {
+	api, mgr := newRoleTestAPI(t)
+	s3a := &S3ApiServer{iam: api.iam}
+	assert.Equal(t, rolesDir, s3a.roleStoreDir(), "a memory store falls back to the default directory")
+
+	store, err := integration.NewGenericCachedRoleStore(map[string]interface{}{"basePath": "/custom/roles"}, func() string { return "localhost:8888" })
+	require.NoError(t, err)
+	mgr.SetRoleStore(store)
+	assert.Equal(t, "/custom/roles", s3a.roleStoreDir())
+}
@@ -29,10 +29,8 @@ type GetObjectAttributesResponse struct {

 // ObjectAttributesChecksum holds checksum info for GetObjectAttributes.
 type ObjectAttributesChecksum struct {
-	ChecksumCRC32  string `xml:"ChecksumCRC32,omitempty"`
-	ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
-	ChecksumSHA1   string `xml:"ChecksumSHA1,omitempty"`
-	ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
+	ChecksumResult
+	ChecksumType string `xml:"ChecksumType,omitempty"`
 }

 // ObjectAttributesParts holds parts info for GetObjectAttributes.
@@ -77,6 +75,24 @@ func validateObjectAttributes(attrs map[string]struct{}) bool {
 	return true
 }

+// objectAttributesChecksum returns the additional checksum that PutObject or
+// CompleteMultipartUpload stored with the object, or nil if it has none.
+func objectAttributesChecksum(entry *filer_pb.Entry) *ObjectAttributesChecksum {
+	if entry == nil || entry.Extended == nil {
+		return nil
+	}
+	value := string(entry.Extended[s3_constants.ExtChecksumValue])
+	if value == "" {
+		return nil
+	}
+	checksum := &ObjectAttributesChecksum{ChecksumType: string(entry.Extended[s3_constants.ExtChecksumType])}
+	checksum.SetChecksum(string(entry.Extended[s3_constants.ExtChecksumAlgorithm]), value)
+	if checksum.ChecksumResult == (ChecksumResult{}) {
+		return nil
+	}
+	return checksum
+}
+
 func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) {
 	bucket, object := s3_constants.GetBucketAndObject(r)
 	glog.V(3).Infof("GetObjectAttributesHandler %s %s", bucket, object)
@@ -245,10 +261,9 @@ func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *htt
 		resp.StorageClass = storageClass
 	}

-	// Checksum: accepted in validation so clients don't get a 400, but SeaweedFS
-	// does not yet store S3 checksums (CRC32, CRC32C, SHA1, SHA256), so
-	// resp.Checksum is intentionally left nil. When checksum storage is added,
-	// populate resp.Checksum here.
+	if _, ok := requestedAttrs["Checksum"]; ok {
+		resp.Checksum = objectAttributesChecksum(entry)
+	}

 	if _, ok := requestedAttrs["ObjectSize"]; ok {
 		var size int64
@@ -0,0 +1,88 @@
+package s3api
+
+import (
+	"encoding/xml"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestObjectAttributesChecksum verifies that GetObjectAttributes reports the
+// checksum that PutObject or CompleteMultipartUpload stored with the object
+func TestObjectAttributesChecksum(t *testing.T) {
+	testCases := []struct {
+		name     string
+		extended map[string][]byte
+		want     *ObjectAttributesChecksum
+	}{
+		{
+			name: "PutObject with SHA256",
+			extended: map[string][]byte{
+				s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumSHA256),
+				s3_constants.ExtChecksumValue:     []byte("arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0="),
+			},
+			want: &ObjectAttributesChecksum{
+				ChecksumResult: ChecksumResult{ChecksumSHA256: "arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0="},
+			},
+		},
+		{
+			name: "multipart upload with a composite CRC32C",
+			extended: map[string][]byte{
+				s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC32C),
+				s3_constants.ExtChecksumValue:     []byte("x3Y2bw==-3"),
+				s3_constants.ExtChecksumType:      []byte("COMPOSITE"),
+			},
+			want: &ObjectAttributesChecksum{
+				ChecksumResult: ChecksumResult{ChecksumCRC32C: "x3Y2bw==-3"},
+				ChecksumType:   "COMPOSITE",
+			},
+		},
+		{
+			name: "multipart upload with a full object CRC64NVME",
+			extended: map[string][]byte{
+				s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC64NVME),
+				s3_constants.ExtChecksumValue:     []byte("AAAAAAAAAAA="),
+				s3_constants.ExtChecksumType:      []byte("FULL_OBJECT"),
+			},
+			want: &ObjectAttributesChecksum{
+				ChecksumResult: ChecksumResult{ChecksumCRC64NVME: "AAAAAAAAAAA="},
+				ChecksumType:   "FULL_OBJECT",
+			},
+		},
+		{
+			name:     "no checksum",
+			extended: map[string][]byte{s3_constants.ExtETagKey: []byte("d41d8cd98f00b204e9800998ecf8427e")},
+		},
+		{
+			name: "unknown algorithm",
+			extended: map[string][]byte{
+				s3_constants.ExtChecksumAlgorithm: []byte("X-Amz-Checksum-Md5"),
+				s3_constants.ExtChecksumValue:     []byte("1B2M2Y8AsgTpgAmY7PhCfg=="),
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, objectAttributesChecksum(&filer_pb.Entry{Extended: tc.extended}))
+		})
+	}
+	assert.Nil(t, objectAttributesChecksum(&filer_pb.Entry{}))
+}
+
+// TestGetObjectAttributesChecksumXML verifies the Checksum element's layout
+func TestGetObjectAttributesChecksumXML(t *testing.T) {
+	resp := GetObjectAttributesResponse{
+		Checksum: &ObjectAttributesChecksum{
+			ChecksumResult: ChecksumResult{ChecksumCRC32: "NhCmhg=="},
+			ChecksumType:   "FULL_OBJECT",
+		},
+	}
+	out, err := xml.Marshal(resp)
+	require.NoError(t, err)
+	assert.Equal(t, "<GetObjectAttributesResponse><Checksum><ChecksumCRC32>NhCmhg==</ChecksumCRC32>"+
+		"<ChecksumType>FULL_OBJECT</ChecksumType></Checksum></GetObjectAttributesResponse>", string(out))
+}
@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/base64"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"io"
@@ -792,6 +793,19 @@ type CopyPartResult struct {
 	ChecksumResult
 }

+// MarshalXML writes LastModified in the S3 timestamp format (see xsdDateTime)
+// instead of encoding/xml's RFC 3339 with trimmed fractional seconds.
+func (r CopyPartResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type T CopyPartResult
+	var layout struct {
+		*T
+		LastModified xsdDateTime `xml:"LastModified"`
+	}
+	layout.T = (*T)(&r)
+	layout.LastModified = xsdDateTime(r.LastModified)
+	return e.EncodeElement(layout, start)
+}
+
 func buildCopyPartResult(etag string, lastModified time.Time, metadata SSEResponseMetadata) CopyPartResult {
 	result := CopyPartResult{
 		ETag:         etag,
@@ -1183,7 +1197,11 @@ func processMetadataBytes(reqHeader http.Header, existing map[string][]byte, rep
 			}
 		}
 		for _, h := range copyReplaceSystemHeaders {
-			if v := reqHeader.Get(h); v != "" {
+			v := reqHeader.Get(h)
+			if h == "Content-Encoding" {
+				v = storedContentEncoding(reqHeader.Values(h))
+			}
+			if v != "" {
 				metadata[h] = []byte(v)
 			}
 		}
@@ -119,7 +119,7 @@ func TestListObjectsHandler(t *testing.T) {
 	// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html

 	expected := `<?xml version="1.0" encoding="UTF-8"?>
-<ListBucketResult><Name>test_container</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>1.zip</Key><ETag>&#34;4397da7a7649e8085de9916c240e8166&#34;</ETag><Size>1234567</Size><Owner><ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID></Owner><StorageClass>STANDARD</StorageClass><LastModified>2011-04-09T12:34:49Z</LastModified></Contents><EncodingType></EncodingType></ListBucketResult>`
+<ListBucketResult><Name>test_container</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>1.zip</Key><ETag>&#34;4397da7a7649e8085de9916c240e8166&#34;</ETag><Size>1234567</Size><Owner><ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID></Owner><StorageClass>STANDARD</StorageClass><LastModified>2011-04-09T12:34:49.000Z</LastModified></Contents><EncodingType></EncodingType></ListBucketResult>`

 	response := ListBucketResult{
 		Name:        "test_container",
@@ -833,8 +833,12 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
 				entry.Extended[k] = []byte(v[0])
 			} else {
 				switch k {
-				case "Cache-Control", "Expires", "Content-Disposition", "Content-Encoding", "Content-Language":
+				case "Cache-Control", "Expires", "Content-Disposition", "Content-Language":
 					entry.Extended[k] = []byte(v[0])
+				case "Content-Encoding":
+					if ce := storedContentEncoding(v); ce != "" {
+						entry.Extended[k] = []byte(ce)
+					}
 				}
 			}
 			if k == "Response-Content-Disposition" {
@@ -2738,7 +2742,7 @@ func (s3a *S3ApiServer) deleteOrphanedChunks(chunks []*filer_pb.FileChunk) {
 	}

 	// Attempt deletion using the operation package's batch delete with custom lookup
-	deleteResults := operation.DeleteFileIdsWithLookupVolumeId(s3a.option.GrpcDialOption, fileIds, lookupFunc)
+	deleteResults := operation.DeleteFileIdsWithLookupVolumeId(context.Background(), s3a.option.GrpcDialOption, fileIds, lookupFunc)

 	// Log results - track successes and failures
 	successCount := 0
@@ -501,7 +501,7 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl

 	s3ApiServer.registerRouter(router)

-	go s3ApiServer.subscribeMetaEvents("s3", startTsNs, filer.DirectoryEtcRoot, []string{
+	watched := []string{
 		option.BucketsPath,
 		filer.IamConfigDirectory,
 		filer.IamConfigDirectory + "/identities",
@@ -509,7 +509,13 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
 		filer.IamConfigDirectory + "/service_accounts",
 		filer.IamConfigDirectory + "/groups",
 		filer.IamConfigDirectory + "/oidc-providers",
-	})
+		filer.IamConfigDirectory + "/roles",
+	}
+	// A role store configured with its own basePath is watched there too.
+	if dir := s3ApiServer.roleStoreDir(); !slices.Contains(watched, dir) {
+		watched = append(watched, dir)
+	}
+	go s3ApiServer.subscribeMetaEvents("s3", startTsNs, filer.DirectoryEtcRoot, watched)

 	// Start bucket size metrics collection in background
 	go s3ApiServer.startBucketSizeMetricsLoop(context.Background())
@@ -1168,6 +1174,12 @@ func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() str
 		Policy    *policy.PolicyEngineConfig    `json:"policy"`
 		Providers []map[string]interface{}      `json:"providers"`
 		Roles     []*integration.RoleDefinition `json:"roles"`
+		// OIDCProviderStore selects where IAM-managed OIDC providers persist.
+		// Absent, they live in memory and are lost on restart.
+		OIDCProviderStore *integration.OIDCProviderStoreConfig `json:"oidcProviderStore"`
+		// RoleStore selects where roles persist. Absent, they live in memory and
+		// only the roles in this file exist.
+		RoleStore *integration.RoleStoreConfig `json:"roleStore"`
 		Policies  []struct {
 			Name     string                 `json:"name"`
 			Document *policy.PolicyDocument `json:"document"`
@@ -1213,13 +1225,30 @@ func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() str
 		glog.V(1).Infof("Using policy defaults: DefaultEffect=%s, StoreType=%s", configRoot.Policy.DefaultEffect, configRoot.Policy.StoreType)
 	}

+	// With no IAM config file there is nothing static for a persisted
+	// provider or role to shadow or outlive, so those created at runtime
+	// default to the filer, where restarts and peer S3 servers see them. A
+	// config file keeps the in-memory defaults unless it sets
+	// oidcProviderStore / roleStore.
+	persistByDefault := configPath == "" && filerAddressProvider != nil
+	oidcProviderStore := configRoot.OIDCProviderStore
+	if oidcProviderStore == nil && persistByDefault {
+		oidcProviderStore = &integration.OIDCProviderStoreConfig{StoreType: "filer"}
+	}
+	roleStore := configRoot.RoleStore
+	if roleStore == nil {
+		roleStore = &integration.RoleStoreConfig{StoreType: sts.StoreTypeMemory}
+		if persistByDefault {
+			roleStore = &integration.RoleStoreConfig{StoreType: "filer"}
+		}
+	}
+
 	// Create IAM configuration
 	iamConfig := &integration.IAMConfig{
-		STS:    configRoot.STS,
-		Policy: configRoot.Policy,
-		Roles: &integration.RoleStoreConfig{
-			StoreType: sts.StoreTypeMemory, // Use memory store for JSON config-based setup
-		},
+		STS:           configRoot.STS,
+		Policy:        configRoot.Policy,
+		Roles:         roleStore,
+		OIDCProviders: oidcProviderStore,
 	}

 	// Apply default signing key if not present in config
@@ -1302,11 +1331,7 @@ func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() str
 	}

 	// Load roles
-	for _, roleDef := range configRoot.Roles {
-		if err := iamManager.CreateRole(context.Background(), "", roleDef.RoleName, roleDef); err != nil {
-			glog.Warningf("Failed to create role %s: %v", roleDef.RoleName, err)
-		}
-	}
+	iamManager.LoadStaticRoles(context.Background(), configRoot.Roles)

 	glog.V(1).Infof("Loaded %d providers, %d policies and %d roles from config", len(configRoot.Providers), len(configRoot.Policies), len(configRoot.Roles))

@@ -400,6 +400,9 @@ func (h *STSHandlers) handleAssumeRole(w http.ResponseWriter, r *http.Request) {
 		identity.Name, roleArn, roleSessionName)

 	assumesSelf := roleArn == ""
+	// trustedPrincipal is the caller a named role's trust policy admitted;
+	// issuance evaluates that trust again on the definition it binds.
+	var trustedPrincipal string

 	// A named role is authorized by its trust policy, which declares which
 	// principals may assume it, so no separate identity-side sts:AssumeRole allow
@@ -415,6 +418,7 @@ func (h *STSHandlers) handleAssumeRole(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		callerArn := h.callerPrincipalArn(identity)
+		trustedPrincipal = callerArn
 		if err := h.iam.ValidateTrustPolicyForPrincipal(r.Context(), roleArn, callerArn); err != nil {
 			glog.V(2).Infof("AssumeRole: %s not authorized to assume %s: %v", identity.Name, roleArn, err)
 			h.writeSTSErrorResponse(w, r, STSErrAccessDenied,
@@ -461,7 +465,13 @@ func (h *STSHandlers) handleAssumeRole(w http.ResponseWriter, r *http.Request) {
 	}

 	// Generate common STS components
-	stsCreds, assumedUser, err := h.prepareSTSCredentials(r.Context(), roleArn, roleSessionName, durationSeconds, sessionPolicyJSON, modifyClaims)
+	stsCreds, assumedUser, err := h.prepareSTSCredentials(r.Context(), roleArn, trustedPrincipal, roleSessionName, durationSeconds, sessionPolicyJSON, modifyClaims)
+	if errors.Is(err, integration.ErrTrustPolicyDenied) {
+		// The role was replaced after its trust check by one that does not
+		// trust the caller.
+		h.writeSTSErrorResponse(w, r, STSErrAccessDenied, fmt.Errorf("trust policy denies access"))
+		return
+	}
 	if err != nil {
 		h.writeSTSErrorResponse(w, r, STSErrInternalError, err)
 		return
@@ -584,7 +594,8 @@ func (h *STSHandlers) handleAssumeRoleWithLDAPIdentity(w http.ResponseWriter, r

 	// Verify that the identity is allowed to assume the role by checking the Trust Policy
 	// The LDAP user doesn't have identity policies, so we strictly check if the Role trusts this principal.
-	if err := h.iam.ValidateTrustPolicyForPrincipal(r.Context(), roleArn, ldapUserIdentity.PrincipalArn); err != nil {
+	trustedPrincipal := ldapUserIdentity.PrincipalArn
+	if err := h.iam.ValidateTrustPolicyForPrincipal(r.Context(), roleArn, trustedPrincipal); err != nil {
 		glog.V(2).Infof("AssumeRoleWithLDAPIdentity: trust policy validation failed for %s to assume %s: %v", ldapUsername, roleArn, err)
 		h.writeSTSErrorResponse(w, r, STSErrAccessDenied, fmt.Errorf("trust policy denies access"))
 		return
@@ -602,7 +613,13 @@ func (h *STSHandlers) handleAssumeRoleWithLDAPIdentity(w http.ResponseWriter, r
 		claims.WithIdentityProvider("ldap", identity.UserID, identity.Provider)
 	}

-	stsCreds, assumedUser, err := h.prepareSTSCredentials(r.Context(), roleArn, roleSessionName, durationSeconds, sessionPolicyJSON, modifyClaims)
+	stsCreds, assumedUser, err := h.prepareSTSCredentials(r.Context(), roleArn, trustedPrincipal, roleSessionName, durationSeconds, sessionPolicyJSON, modifyClaims)
+	if errors.Is(err, integration.ErrTrustPolicyDenied) {
+		// The role was replaced after its trust check by one that does not
+		// trust the caller.
+		h.writeSTSErrorResponse(w, r, STSErrAccessDenied, fmt.Errorf("trust policy denies access"))
+		return
+	}
 	if err != nil {
 		h.writeSTSErrorResponse(w, r, STSErrInternalError, err)
 		return
@@ -831,7 +848,14 @@ func (h *STSHandlers) handleGetFederationToken(w http.ResponseWriter, r *http.Re
 }

 // prepareSTSCredentials extracts common shared logic for credential generation
-func (h *STSHandlers) prepareSTSCredentials(ctx context.Context, roleArn, roleSessionName string,
+//
+// principalArn is the caller whose assumption of a named role was authorized
+// by its trust policy. The role is resolved once, with that trust evaluated
+// again on the definition resolved, and the session's role ID, duration cap
+// and embedded policies all come from that one definition: a role replaced
+// under the same name between the caller's trust check and here yields no
+// session rather than one for a role whose trust was never checked.
+func (h *STSHandlers) prepareSTSCredentials(ctx context.Context, roleArn, principalArn, roleSessionName string,
 	durationSeconds *int64, sessionPolicy string, modifyClaims func(*sts.STSSessionClaims)) (STSCredentials, *AssumedRoleUser, error) {

 	duration := time.Hour
@@ -843,12 +867,31 @@ func (h *STSHandlers) prepareSTSCredentials(ctx context.Context, roleArn, roleSe

 	// A named role's MaxSessionDuration bounds the resolved duration the same
 	// way capDurationByRole does on the SDK paths; self-assumption has no role
-	// definition to consult.
+	// definition to consult. The role's ID binds the session to this role, so
+	// a named role that cannot be resolved here gets no session: one issued
+	// without the ID would be bound to no role at all.
+	var roleID string
+	var resolvedRole *integration.RoleDefinition
 	if h.iam != nil && h.iam.iamIntegration != nil {
 		if roleName := utils.ExtractRoleNameFromArn(roleArn); roleName != "" {
 			if provider, ok := h.iam.iamIntegration.(IAMManagerProvider); ok {
 				if mgr := provider.GetIAMManager(); mgr != nil {
-					if roleDef, roleErr := mgr.GetRole(ctx, roleName); roleErr == nil && roleDef.MaxSessionDuration > 0 {
+					var roleDef *integration.RoleDefinition
+					var roleErr error
+					if principalArn != "" {
+						roleDef, roleErr = mgr.ResolveRoleForPrincipal(ctx, roleArn, principalArn)
+					} else {
+						roleDef, roleErr = mgr.GetRole(ctx, roleName)
+					}
+					if roleErr != nil {
+						return STSCredentials{}, nil, fmt.Errorf("resolve role %s: %w", roleName, roleErr)
+					}
+					if roleDef == nil {
+						return STSCredentials{}, nil, fmt.Errorf("role %s not found", roleName)
+					}
+					resolvedRole = roleDef
+					roleID = roleDef.RoleId
+					if roleDef.MaxSessionDuration > 0 {
 						if roleMax := time.Duration(roleDef.MaxSessionDuration) * time.Second; duration > roleMax {
 							duration = roleMax
 						}
@@ -894,7 +937,8 @@ func (h *STSHandlers) prepareSTSCredentials(ctx context.Context, roleArn, roleSe
 	// This ensures that subsequent requests using this token are correctly identified as the assumed role.
 	claims := sts.NewSTSSessionClaims(sessionId, h.stsService.Config.Issuer, expiration).
 		WithSessionName(roleSessionName).
-		WithRoleInfo(effectiveRoleArn, fmt.Sprintf("%s:%s", roleName, roleSessionName), assumedRoleArn)
+		WithRoleInfo(effectiveRoleArn, fmt.Sprintf("%s:%s", roleName, roleSessionName), assumedRoleArn).
+		WithRoleId(roleID)

 	// If IAM integration is available, embed the role's attached policies into the session token.
 	// This makes the token self-sufficient for authorization even when role lookup is unavailable.
@@ -912,7 +956,10 @@ func (h *STSHandlers) prepareSTSCredentials(ctx context.Context, roleArn, roleSe
 		}

 		if roleNameForPolicies != "" && len(claims.Policies) == 0 {
-			roleDef, err := policyManager.GetRole(ctx, roleNameForPolicies)
+			roleDef, err := resolvedRole, error(nil)
+			if roleDef == nil || roleDef.RoleName != roleNameForPolicies {
+				roleDef, err = policyManager.GetRole(ctx, roleNameForPolicies)
+			}
 			if err != nil {
 				glog.V(2).Infof("Failed to load role %q for policy embedding: %v", roleNameForPolicies, err)
 			} else if roleDef == nil {
@@ -78,7 +78,7 @@ func TestAssumeRole_CallerIdentityFallback(t *testing.T) {
 			}
 		}

-		stsCreds, assumedUser, err := stsHandlers.prepareSTSCredentials(context.Background(), fallbackRoleArn, "test-session", nil, "", modifyClaims)
+		stsCreds, assumedUser, err := stsHandlers.prepareSTSCredentials(context.Background(), fallbackRoleArn, "", "test-session", nil, "", modifyClaims)
 		require.NoError(t, err)

 		// Assertions
@@ -110,7 +110,7 @@ func TestAssumeRole_CallerIdentityFallback(t *testing.T) {

 		fallbackRoleArn := callerIdentity.PrincipalArn

-		stsCreds, assumedUser, err := stsHandlers.prepareSTSCredentials(context.Background(), fallbackRoleArn, "nested-session", nil, "", nil)
+		stsCreds, assumedUser, err := stsHandlers.prepareSTSCredentials(context.Background(), fallbackRoleArn, "", "nested-session", nil, "", nil)
 		require.NoError(t, err)

 		// The role name should be extracted from the assumed role ARN ("admin")
@@ -127,7 +127,7 @@ func TestAssumeRole_CallerIdentityFallback(t *testing.T) {
 	t.Run("Explicit RoleArn Provided", func(t *testing.T) {
 		explicitRoleArn := "arn:aws:iam::111122223333:role/TargetRole"

-		stsCreds, assumedUser, err := stsHandlers.prepareSTSCredentials(context.Background(), explicitRoleArn, "explicit-session", nil, "", nil)
+		stsCreds, assumedUser, err := stsHandlers.prepareSTSCredentials(context.Background(), explicitRoleArn, "", "explicit-session", nil, "", nil)
 		require.NoError(t, err)

 		// Role name should be "TargetRole"
@@ -143,7 +143,7 @@ func TestAssumeRole_CallerIdentityFallback(t *testing.T) {
 	t.Run("Malformed ARN", func(t *testing.T) {
 		malformedArn := "invalid-arn"

-		stsCreds, assumedUser, err := stsHandlers.prepareSTSCredentials(context.Background(), malformedArn, "bad-session", nil, "", nil)
+		stsCreds, assumedUser, err := stsHandlers.prepareSTSCredentials(context.Background(), malformedArn, "", "bad-session", nil, "", nil)
 		require.NoError(t, err)

 		// Fallback behavior: use full string as role name if extraction fails
@@ -187,7 +187,7 @@ func TestAssumeRole_EmbedsRolePolicies(t *testing.T) {
 		stsHandlers := NewSTSHandlers(manager.GetSTSService(), iam)

 		roleArn := fmt.Sprintf("arn:aws:iam::%s:role/%s", defaultAccountID, roleName)
-		stsCreds, _, err := stsHandlers.prepareSTSCredentials(ctx, roleArn, "test-session", nil, "", nil)
+		stsCreds, _, err := stsHandlers.prepareSTSCredentials(ctx, roleArn, "", "test-session", nil, "", nil)
 		require.NoError(t, err)

 		sessionInfo, err := manager.GetSTSService().ValidateSessionToken(ctx, stsCreds.SessionToken)
@@ -211,7 +211,7 @@ func TestAssumeRole_EmbedsRolePolicies(t *testing.T) {
 		stsHandlers := NewSTSHandlers(manager.GetSTSService(), iam)

 		roleArn := fmt.Sprintf("arn:aws:iam::%s:role/%s", defaultAccountID, roleName)
-		stsCreds, _, err := stsHandlers.prepareSTSCredentials(ctx, roleArn, "test-session", nil, "", nil)
+		stsCreds, _, err := stsHandlers.prepareSTSCredentials(ctx, roleArn, "", "test-session", nil, "", nil)
 		require.NoError(t, err)

 		sessionInfo, err := manager.GetSTSService().ValidateSessionToken(ctx, stsCreds.SessionToken)
@@ -257,7 +257,7 @@ func TestPrepareSTSCredentialsHonorsConfiguredDurations(t *testing.T) {
 	roleArn := fmt.Sprintf("arn:aws:iam::%s:role/test-role", defaultAccountID)

 	expiresIn := func(durationSeconds *int64) time.Duration {
-		stsCreds, _, err := stsHandlers.prepareSTSCredentials(context.Background(), roleArn, "test-session", durationSeconds, "", nil)
+		stsCreds, _, err := stsHandlers.prepareSTSCredentials(context.Background(), roleArn, "", "test-session", durationSeconds, "", nil)
 		require.NoError(t, err)
 		exp, err := time.Parse(time.RFC3339, stsCreds.Expiration)
 		require.NoError(t, err)
@@ -291,7 +291,7 @@ func TestPrepareSTSCredentialsCapsAtRoleMaxDuration(t *testing.T) {
 	roleArn := fmt.Sprintf("arn:aws:iam::%s:role/ShortLivedRole", defaultAccountID)

 	expiresIn := func(durationSeconds *int64) time.Duration {
-		stsCreds, _, err := stsHandlers.prepareSTSCredentials(ctx, roleArn, "test-session", durationSeconds, "", nil)
+		stsCreds, _, err := stsHandlers.prepareSTSCredentials(ctx, roleArn, "", "test-session", durationSeconds, "", nil)
 		require.NoError(t, err)
 		exp, err := time.Parse(time.RFC3339, stsCreds.Expiration)
 		require.NoError(t, err)
@@ -302,3 +302,39 @@ func TestPrepareSTSCredentialsCapsAtRoleMaxDuration(t *testing.T) {
 	assert.InDelta(t, float64(3600), expiresIn(nil).Seconds(), 60, "omitted duration resolves to the 2h default but the role caps it at 1h")
 	assert.InDelta(t, float64(3600), expiresIn(&twoHours).Seconds(), 60, "explicit duration above the role max is capped")
 }
+
+// A session is issued from the role definition whose trust admits the
+// caller. A role replaced after the caller's trust check by one that does
+// not trust the caller yields no session, not one bound to the replacement.
+func TestPrepareSTSCredentialsChecksTrustOnTheRoleItBinds(t *testing.T) {
+	ctx := context.Background()
+	manager := newTestSTSIntegrationManager(t)
+	trusting := func(principal string) *policy.PolicyDocument {
+		return &policy.PolicyDocument{Version: "2012-10-17", Statement: []policy.Statement{{
+			Effect: "Allow", Action: []string{"sts:AssumeRole"},
+			Principal: map[string]interface{}{"AWS": principal},
+		}}}
+	}
+	caller := fmt.Sprintf("arn:aws:iam::%s:user/alice", defaultAccountID)
+	roleName := "ReplacedRole"
+	roleArn := fmt.Sprintf("arn:aws:iam::%s:role/%s", defaultAccountID, roleName)
+	stsHandlers := NewSTSHandlers(manager.GetSTSService(), &IdentityAccessManagement{iamIntegration: NewS3IAMIntegration(manager, "")})
+
+	// The caller's trust check passed against a role since replaced by one
+	// trusting someone else.
+	require.NoError(t, manager.CreateRole(ctx, "", roleName, &integration.RoleDefinition{
+		RoleName: roleName, TrustPolicy: trusting(fmt.Sprintf("arn:aws:iam::%s:user/bob", defaultAccountID)),
+	}))
+	_, _, err := stsHandlers.prepareSTSCredentials(ctx, roleArn, caller, "s", nil, "", nil)
+	require.ErrorIs(t, err, integration.ErrTrustPolicyDenied, "a session was issued for a role that does not trust the caller")
+
+	// A replacement that does trust the caller binds its own ID.
+	require.NoError(t, manager.CreateRole(ctx, "", roleName, &integration.RoleDefinition{RoleName: roleName, TrustPolicy: trusting(caller)}))
+	role, err := manager.GetRole(ctx, roleName)
+	require.NoError(t, err)
+	creds, _, err := stsHandlers.prepareSTSCredentials(ctx, roleArn, caller, "s", nil, "", nil)
+	require.NoError(t, err)
+	session, err := manager.GetSTSService().ValidateSessionToken(ctx, creds.SessionToken)
+	require.NoError(t, err)
+	assert.Equal(t, role.RoleId, session.RoleId)
+}
@@ -0,0 +1,56 @@
+package s3api
+
+import (
+	"encoding/xml"
+	"strings"
+	"testing"
+	"time"
+)
+
+// S3 clients such as minio-java parse LastModified with a fixed-width
+// "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" pattern, so trailing zeros in the
+// fractional seconds must not be trimmed.
+func TestXMLTimestampsHaveFixedMilliseconds(t *testing.T) {
+	cases := []struct {
+		in   time.Time
+		want string
+	}{
+		{time.Date(2026, 9, 29, 20, 30, 4, 560_000_000, time.UTC), "2026-09-29T20:30:04.560Z"},
+		{time.Date(2026, 9, 29, 20, 30, 4, 500_000_000, time.UTC), "2026-09-29T20:30:04.500Z"},
+		{time.Date(2026, 9, 29, 20, 30, 4, 0, time.UTC), "2026-09-29T20:30:04.000Z"},
+		{time.Date(2026, 9, 29, 20, 30, 4, 123_456_789, time.UTC), "2026-09-29T20:30:04.123Z"},
+		{time.Date(2026, 9, 29, 14, 30, 4, 560_000_000, time.FixedZone("MDT", -6*3600)), "2026-09-29T20:30:04.560Z"},
+	}
+	for _, c := range cases {
+		for name, v := range map[string]any{
+			// by value, the way the handlers pass them to writeSuccessResponseXML
+			"CopyObjectResult":       CopyObjectResult{ETag: "e", LastModified: c.in},
+			"CopyPartResult":         CopyPartResult{ETag: "e", LastModified: c.in},
+			"CopyObjectResult (ptr)": &CopyObjectResult{ETag: "e", LastModified: c.in},
+			"CopyPartResult (ptr)":   &CopyPartResult{ETag: "e", LastModified: c.in},
+		} {
+			out, err := xml.Marshal(v)
+			if err != nil {
+				t.Fatalf("%s: %v", name, err)
+			}
+			if !strings.Contains(string(out), "<LastModified>"+c.want+"</LastModified>") {
+				t.Errorf("%s(%v): got %s, want LastModified %s", name, c.in, out, c.want)
+			}
+		}
+	}
+}
+
+func TestXMLTimestampRoundTrip(t *testing.T) {
+	in := CopyObjectResult{ETag: "e", LastModified: time.Date(2026, 9, 29, 20, 30, 4, 560_000_000, time.UTC)}
+	out, err := xml.Marshal(&in)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var back CopyObjectResult
+	if err := xml.Unmarshal(out, &back); err != nil {
+		t.Fatal(err)
+	}
+	if !back.LastModified.Equal(in.LastModified) {
+		t.Fatalf("round trip: got %v, want %v", back.LastModified, in.LastModified)
+	}
+}
@@ -623,13 +623,16 @@ type CopyObjectResult struct {
 	ETag         string    `xml:"ETag"`
 }

-func (t *CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+// MarshalXML has a value receiver: handlers pass CopyObjectResult by value, and
+// encoding/xml does not call a pointer-receiver MarshalXML on a non-addressable
+// value, which silently fell back to time.Time's RFC 3339 encoding.
+func (t CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type T CopyObjectResult
 	var layout struct {
 		*T
 		LastModified *xsdDateTime `xml:"LastModified"`
 	}
-	layout.T = (*T)(t)
+	layout.T = (*T)(&t)
 	layout.LastModified = (*xsdDateTime)(&layout.T.LastModified)
 	return e.EncodeElement(layout, start)
 }
@@ -1512,8 +1515,15 @@ type xsdDateTime time.Time
 func (t *xsdDateTime) UnmarshalText(text []byte) error {
 	return _unmarshalTime(text, (*time.Time)(t), "2006-01-02T15:04:05.999999999")
 }
+
+// s3TimestampFormat is the timestamp layout AWS S3 uses in XML responses:
+// UTC with exactly three fractional digits. Trimming trailing zeros (".56Z",
+// or no fraction at all) breaks clients that parse with a fixed-width pattern,
+// e.g. minio-java's "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'".
+const s3TimestampFormat = "2006-01-02T15:04:05.000Z"
+
 func (t xsdDateTime) MarshalText() ([]byte, error) {
-	return _marshalTime((time.Time)(t), "2006-01-02T15:04:05.999999999")
+	return []byte((time.Time)(t).UTC().Format(s3TimestampFormat)), nil
 }
 func (t xsdDateTime) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	if (time.Time)(t).IsZero() {
@@ -518,7 +518,7 @@ func (fs *FilerServer) applyObjectMutation(ctx context.Context, m *filer_pb.Obje
 		if m.TouchMtime {
 			newEntry.Attr.Mtime = time.Now()
 		}
-		if err := fs.filer.UpdateEntry(ctx, oldEntry, newEntry); err != nil {
+		if err := fs.filer.UpdateEntry(ctx, oldEntry, newEntry, fromOtherCluster); err != nil {
 			return err
 		}
 		// Emit the metadata event so the update replicates and subscribers see it,
@@ -623,7 +623,7 @@ func (fs *FilerServer) applyRecomputeLatest(ctx context.Context, m *filer_pb.Obj
 		}
 	}

-	if err := fs.filer.UpdateEntry(ctx, oldPointer, pointer); err != nil {
+	if err := fs.filer.UpdateEntry(ctx, oldPointer, pointer, fromOtherCluster); err != nil {
 		return err
 	}
 	// Replicate the recomputed pointer to peer filers and subscribers. Without
@@ -653,7 +653,7 @@ func (fs *FilerServer) applyRecomputeLatest(ctx context.Context, m *filer_pb.Obj
 			priorEntry.Extended = make(map[string][]byte)
 		}
 		priorEntry.Extended[rc.DemoteKey] = rc.DemoteValue
-		if err := fs.filer.UpdateEntry(ctx, oldPrior, priorEntry); err != nil {
+		if err := fs.filer.UpdateEntry(ctx, oldPrior, priorEntry, fromOtherCluster); err != nil {
 			return err
 		}
 		fs.filer.NotifyUpdateEvent(ctx, oldPrior, priorEntry, false, fromOtherCluster, signatures)
@@ -746,7 +746,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr

 	ctx, eventSink := filer.WithMetadataEventSink(ctx)
 	resp := &filer_pb.UpdateEntryResponse{LogTsNs: logTsNs, LogSignature: fs.filer.Signature}
-	if err = fs.filer.UpdateEntry(ctx, entry, newEntry); err == nil {
+	if err = fs.filer.UpdateEntry(ctx, entry, newEntry, req.IsFromOtherCluster); err == nil {
 		fs.filer.DeleteChunksNotRecursive(garbage)

 		fs.filer.NotifyUpdateEvent(ctx, entry, newEntry, true, req.IsFromOtherCluster, req.Signatures)
@@ -87,10 +87,14 @@ func clauseSatisfied(c *filer_pb.WriteCondition_Clause, current *filer.Entry) bo
 		if !exists || c.ExpectedEntry == nil {
 			return !exists && c.ExpectedEntry == nil
 		}
-		// Normalize the expected entry the way FindEntry normalizes the stored
-		// one (e.g. FileSize grows to the chunk extent), or an unchanged entry
-		// can compare unequal.
-		return proto.Equal(current.ToProtoEntry(), filer.FromPbEntry("", c.ExpectedEntry).ToProtoEntry())
+		// Compare both sides in serialized form on clones: chunks are matched
+		// by their fid only — the stored entry may carry the restored file_id
+		// while an expected one built from a metadata event does not.
+		expected := proto.Clone(c.ExpectedEntry).(*filer_pb.Entry)
+		filer_pb.BeforeEntrySerialization(expected.Chunks)
+		actual := proto.Clone(current.ToProtoEntry()).(*filer_pb.Entry)
+		filer_pb.BeforeEntrySerialization(actual.Chunks)
+		return proto.Equal(actual, filer.FromPbEntry("", expected).ToProtoEntry())
 	default:
 		// An unrecognized clause kind (e.g. from a newer client) must not be
 		// treated as satisfied, which would silently bypass the guard. Fail
@@ -12,6 +12,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 )

 func entryWithETag(etag string, mtime time.Time) *filer.Entry {
@@ -221,6 +222,30 @@ func TestIfEntryEqualNormalizesExpected(t *testing.T) {
 	}
 }

+// A stamp built from the metadata-log event carries chunks in serialized
+// form (file_id moved into fid), while the stored entry came through
+// FindEntry which restores file_id. The comparison must still match.
+func TestIfEntryEqualSerializedExpected(t *testing.T) {
+	serialized := &filer_pb.Entry{
+		Name:       "f",
+		Attributes: &filer_pb.FuseAttributes{Mtime: 42},
+		Chunks: []*filer_pb.FileChunk{
+			{Fid: &filer_pb.FileId{VolumeId: 3, FileKey: 1, Cookie: 2}, Size: 100},
+		},
+	}
+	storedProto := proto.Clone(serialized).(*filer_pb.Entry)
+	filer_pb.AfterEntryDeserialization(storedProto.Chunks)
+	stored := filer.FromPbEntry("/d", storedProto)
+
+	cond := one(&filer_pb.WriteCondition_Clause{
+		Kind:          filer_pb.WriteCondition_IF_ENTRY_EQUAL,
+		ExpectedEntry: serialized,
+	})
+	if !writeConditionSatisfied(cond, stored) {
+		t.Error("serialized expected entry must equal the deserialized stored entry")
+	}
+}
+
 // storedEntryETag prefers the stored Seaweed ETag attribute and falls back to
 // the Md5-derived ETag, matching the S3 gateway.
 func TestStoredEntryETag(t *testing.T) {
@@ -269,6 +269,14 @@ func (fs *FilerServer) doCacheRemoteObjectToLocalCluster(ctx context.Context, re
 		if len(chunks) > 0 {
 			fs.filer.DeleteUncommittedChunks(ctx, chunks)
 		}
+		if fs.option.RemoteCacheEvictThreshold > 0 && isRemoteCacheCapacityError(err) {
+			fileIds := make([]string, 0, len(chunks))
+			for _, chunk := range chunks {
+				fileIds = append(fileIds, chunk.GetFileIdString())
+			}
+			fs.notePendingRemoteCacheVids(fileIds)
+			go fs.reclaimRemoteCacheSpace(fs.evictCtx(), entry.Remote.RemoteSize, nil)
+		}
 		return nil, err
 	}

@@ -92,6 +92,9 @@ type FilerOption struct {
 	// AllowUntrustedRemoteEndpoints lets a read of a remote-only entry dial a
 	// mounted endpoint that resolves to a loopback / private / metadata host.
 	AllowUntrustedRemoteEndpoints bool
+	// RemoteCacheEvictThreshold is the disk usage fraction at which the filer
+	// evicts remote-mounted cached chunks; 0 disables eviction.
+	RemoteCacheEvictThreshold float64
 }

 type FilerServer struct {
@@ -121,6 +124,15 @@ type FilerServer struct {
 	// deduplicates concurrent remote object caching operations
 	remoteCacheGroup singleflight.Group

+	// serializes remote-cache eviction passes; lastVacuum rate-limits the
+	// compaction trigger that reclaims evicted chunks.
+	remoteCacheEvictMu       sync.Mutex
+	remoteCacheLastVacuum    atomic.Pointer[time.Time]
+	remoteCacheEvictCtx      context.Context
+	remoteCacheEvictCancel   context.CancelFunc
+	remoteCachePendingVidsMu sync.Mutex
+	remoteCachePendingVids   map[uint32]int
+
 	recentCopyRequestsMu sync.Mutex
 	recentCopyRequests   map[string]recentCopyRequest

@@ -209,6 +221,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 	fs.startPosixLockSweeper()
 	fs.mountPeerRegistry = filer.NewMountPeerRegistry()
 	go fs.runMountPeerRegistrySweeper()
+	fs.remoteCacheEvictCtx, fs.remoteCacheEvictCancel = context.WithCancel(context.Background())

 	option.Masters.RefreshBySrvIfAvailable()
 	if len(option.Masters.GetInstances()) == 0 {
@@ -239,6 +252,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 	fs.filer.RemoteStorage.SetConfValidator(func(ctx context.Context, conf *remote_pb.RemoteConf) error {
 		return ValidateRemoteConfForLoad(ctx, conf, option.AllowUntrustedRemoteEndpoints)
 	})
+	go fs.runRemoteCacheEviction()
 	// we do not support IP whitelist right now https://github.com/seaweedfs/seaweedfs/issues/7094
 	if v.GetString("guard.white_list") != "" {
 		glog.Warningf("filer: guard.white_list is configured but the IP whitelist feature is currently disabled. See https://github.com/seaweedfs/seaweedfs/issues/7094")
@@ -363,6 +377,9 @@ func (fs *FilerServer) Shutdown() {
 	if fs.posixLockSweeperStop != nil {
 		close(fs.posixLockSweeperStop)
 	}
+	if fs.remoteCacheEvictCancel != nil {
+		fs.remoteCacheEvictCancel()
+	}
 	fs.filer.Shutdown()
 }

@@ -0,0 +1,300 @@
+package weed_server
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+const (
+	remoteCacheEvictInterval      = 30 * time.Second
+	remoteCacheEvictMinAge        = time.Minute
+	remoteCacheVacuumCooldown     = time.Minute
+	remoteCacheMasterRpcTime      = 30 * time.Second
+	remoteCachePendingVidAttempts = 10
+)
+
+// uncacheRemoteEntry drops the local chunks of one remote-mounted entry, the
+// same state transition remote.uncache applies through UpdateEntry. Cleared
+// chunks go to the deletion queue and are reclaimed by the next compaction.
+// When vids is set, only entries holding chunks on those volumes count toward
+// the freed bytes, and entries contributing nothing are left untouched.
+func (fs *FilerServer) uncacheRemoteEntry(ctx context.Context, fullPath util.FullPath, minCacheAge time.Duration, vids map[uint32]struct{}) (freedBytes int64, err error) {
+	pathLock := fs.entryLockTable.AcquireLock("uncacheRemoteEntry", fullPath, util.ExclusiveLock)
+	defer fs.entryLockTable.ReleaseLock(fullPath, pathLock)
+
+	current, err := fs.filer.FindEntry(ctx, fullPath)
+	if err != nil {
+		return 0, err
+	}
+	if !filer.IsEvictableRemoteEntry(current) {
+		return 0, nil
+	}
+	if time.Since(time.Unix(0, current.Remote.LastLocalSyncTsNs)) < minCacheAge {
+		return 0, nil
+	}
+
+	freedBytes = remoteEntryBytesOnVids(current, vids)
+	if freedBytes == 0 {
+		return 0, nil
+	}
+
+	newEntry := current.ShallowClone()
+	newEntry.Chunks = nil
+	newEntry.Remote = proto.Clone(current.Remote).(*filer_pb.RemoteEntry)
+	newEntry.Remote.LastLocalSyncTsNs = 0
+
+	if err := fs.filer.CreateEntry(ctx, newEntry, current, false, false, nil, true, fs.filer.MaxFilenameLength); err != nil {
+		return 0, err
+	}
+	fileIds := make([]string, 0, len(current.Chunks))
+	for _, chunk := range current.Chunks {
+		fileIds = append(fileIds, chunk.GetFileIdString())
+	}
+	fs.notePendingRemoteCacheVids(fileIds)
+	stats.RemoteCacheEvictedCounter.Inc()
+	glog.V(1).InfofCtx(ctx, "uncacheRemoteEntry %s freed %d bytes", fullPath, freedBytes)
+	return freedBytes, nil
+}
+
+// remoteEntryBytesOnVids sums the entry's chunk bytes on the given volumes; a
+// nil set counts the whole object.
+func remoteEntryBytesOnVids(entry *filer.Entry, vids map[uint32]struct{}) int64 {
+	if vids == nil {
+		return int64(entry.Size())
+	}
+	var bytes int64
+	for _, chunk := range entry.Chunks {
+		fid, err := needle.ParseFileIdFromString(chunk.GetFileIdString())
+		if err != nil {
+			continue
+		}
+		if _, ok := vids[uint32(fid.VolumeId)]; ok {
+			bytes += int64(chunk.Size)
+		}
+	}
+	return bytes
+}
+
+// evictRemoteCachedEntries drops local chunks of remote-mounted entries
+// oldest-cached first until bytesNeeded is met or candidates run out. The
+// first pass honors a minimum cache age so a just-fetched hot object is not
+// dropped under a reader; when aged candidates cannot cover the request a
+// second pass accepts any synchronized cached entry. When pressuredVids is
+// set, only bytes on those volumes count and entries elsewhere are skipped.
+func (fs *FilerServer) evictRemoteCachedEntries(ctx context.Context, bytesNeeded int64, pressuredVids map[uint32]struct{}) (freed int64) {
+	if fs.filer.RemoteStorage == nil {
+		return 0
+	}
+	mounts := fs.filer.RemoteStorage.MountedDirectories()
+	for _, minCacheAge := range []time.Duration{remoteCacheEvictMinAge, 0} {
+		for _, entry := range fs.filer.ListEvictableRemoteEntries(ctx, mounts, minCacheAge) {
+			if bytesNeeded > 0 && freed >= bytesNeeded {
+				return freed
+			}
+			n, err := fs.uncacheRemoteEntry(ctx, entry.FullPath, minCacheAge, pressuredVids)
+			if err != nil {
+				glog.WarningfCtx(ctx, "evict remote cache %s: %v", entry.FullPath, err)
+				continue
+			}
+			freed += n
+		}
+	}
+	return freed
+}
+
+// remoteCacheDiskPressure reports per-disk usage across the cluster: the total
+// bytes to reclaim and the volumes hosted on disks over the eviction threshold.
+func (fs *FilerServer) remoteCacheDiskPressure(ctx context.Context) (bytesToFree int64, pressuredVids map[uint32]struct{}, over bool) {
+	threshold := fs.option.RemoteCacheEvictThreshold
+	if threshold <= 0 {
+		return 0, nil, false
+	}
+	pressuredVids = make(map[uint32]struct{})
+	rpcCtx, cancel := context.WithTimeout(ctx, remoteCacheMasterRpcTime)
+	defer cancel()
+	err := fs.filer.MasterClient.WithClient(rpcCtx, false, func(client master_pb.SeaweedClient) error {
+		resp, err := client.VolumeList(rpcCtx, &master_pb.VolumeListRequest{})
+		if err != nil {
+			return err
+		}
+		for _, dc := range resp.TopologyInfo.DataCenterInfos {
+			for _, rack := range dc.RackInfos {
+				for _, dn := range rack.DataNodeInfos {
+					for _, disk := range dn.DiskInfos {
+						for _, pd := range disk.SplitByPhysicalDisk() {
+							if pd.DiskTotalBytes == 0 {
+								continue
+							}
+							used := pd.DiskTotalBytes - pd.DiskFreeBytes
+							if float64(used) < float64(pd.DiskTotalBytes)*threshold {
+								continue
+							}
+							over = true
+							bytesToFree += int64(used) - int64(float64(pd.DiskTotalBytes)*threshold*0.95)
+							for _, vi := range pd.VolumeInfos {
+								pressuredVids[vi.Id] = struct{}{}
+							}
+						}
+					}
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		glog.WarningfCtx(ctx, "remote cache disk pressure check: %v", err)
+		return 0, nil, false
+	}
+	return bytesToFree, pressuredVids, over
+}
+
+// flushAndVacuumRemoteCacheVolumes forces the deletion queue down to the volume
+// servers so fresh tombstones land, then compacts the volumes carrying them.
+// The flush survives shutdown cancellation: an interrupted delete would be
+// requeued into the in-memory retry queue that exits with the process,
+// stranding bytes whose metadata the eviction already dropped.
+func (fs *FilerServer) flushAndVacuumRemoteCacheVolumes(ctx context.Context) {
+	if len(fs.pendingRemoteCacheVids()) == 0 {
+		return
+	}
+	flushCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), remoteCacheMasterRpcTime)
+	fs.filer.FlushFileIdDeletionQueue(flushCtx, filer.LookupByMasterClientFn(fs.filer.MasterClient))
+	cancel()
+	fs.vacuumPendingRemoteCacheVids(ctx)
+}
+
+// vacuumPendingRemoteCacheVids compacts every volume still owed a vacuum. The
+// master does not report whether a request actually compacted, so a vid keeps
+// roughly ten minutes of attempts: tombstones that land late or compaction
+// losses to another vacuum are retried by later passes instead of stranded.
+func (fs *FilerServer) vacuumPendingRemoteCacheVids(ctx context.Context) {
+	pending := fs.pendingRemoteCacheVids()
+	if len(pending) == 0 {
+		return
+	}
+	if last := fs.remoteCacheLastVacuum.Load(); last != nil && time.Since(*last) < remoteCacheVacuumCooldown {
+		return
+	}
+	now := time.Now()
+	fs.remoteCacheLastVacuum.Store(&now)
+	if err := fs.filer.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
+		for vid := range pending {
+			vCtx, cancel := context.WithTimeout(ctx, remoteCacheMasterRpcTime)
+			_, err := client.VacuumVolume(vCtx, &master_pb.VacuumVolumeRequest{VolumeId: vid})
+			cancel()
+			if err != nil {
+				glog.WarningfCtx(ctx, "remote cache vacuum volume %d: %v", vid, err)
+				continue
+			}
+			fs.completePendingRemoteCacheVid(vid)
+		}
+		return nil
+	}); err != nil {
+		glog.WarningfCtx(ctx, "remote cache vacuum: %v", err)
+	}
+}
+
+func (fs *FilerServer) notePendingRemoteCacheVids(fileIds []string) {
+	fs.remoteCachePendingVidsMu.Lock()
+	defer fs.remoteCachePendingVidsMu.Unlock()
+	if fs.remoteCachePendingVids == nil {
+		fs.remoteCachePendingVids = make(map[uint32]int)
+	}
+	for _, fid := range fileIds {
+		if parsed, err := needle.ParseFileIdFromString(fid); err == nil {
+			fs.remoteCachePendingVids[uint32(parsed.VolumeId)] = remoteCachePendingVidAttempts
+		}
+	}
+}
+
+func (fs *FilerServer) pendingRemoteCacheVids() map[uint32]struct{} {
+	fs.remoteCachePendingVidsMu.Lock()
+	defer fs.remoteCachePendingVidsMu.Unlock()
+	out := make(map[uint32]struct{}, len(fs.remoteCachePendingVids))
+	for vid := range fs.remoteCachePendingVids {
+		out[vid] = struct{}{}
+	}
+	return out
+}
+
+func (fs *FilerServer) completePendingRemoteCacheVid(vid uint32) {
+	fs.remoteCachePendingVidsMu.Lock()
+	defer fs.remoteCachePendingVidsMu.Unlock()
+	if fs.remoteCachePendingVids[vid] <= 1 {
+		delete(fs.remoteCachePendingVids, vid)
+	} else {
+		fs.remoteCachePendingVids[vid]--
+	}
+}
+
+// reclaimRemoteCacheSpace evicts remote-cached content and compacts volumes to
+// release disk space under capacity pressure. A pass already in flight is
+// enough; callers that would queue behind it just fall back to remote reads.
+func (fs *FilerServer) reclaimRemoteCacheSpace(ctx context.Context, bytesNeeded int64, pressuredVids map[uint32]struct{}) {
+	if !fs.remoteCacheEvictMu.TryLock() {
+		return
+	}
+	defer fs.remoteCacheEvictMu.Unlock()
+	freed := fs.evictRemoteCachedEntries(ctx, bytesNeeded, pressuredVids)
+	if freed > 0 {
+		glog.V(0).InfofCtx(ctx, "remote cache eviction freed %d bytes", freed)
+	}
+	fs.flushAndVacuumRemoteCacheVolumes(ctx)
+}
+
+func isRemoteCacheCapacityError(err error) bool {
+	msg := err.Error()
+	return strings.Contains(msg, "writable volumes") ||
+		strings.Contains(msg, "free volumes") ||
+		strings.Contains(msg, "no space left") ||
+		strings.Contains(msg, "out of space")
+}
+
+// runRemoteCacheEviction periodically evicts remote-cached entries once any
+// disk crosses the configured usage threshold, with a vacuum pass to reclaim
+// the deleted chunks.
+func (fs *FilerServer) evictCtx() context.Context {
+	if fs.remoteCacheEvictCtx == nil {
+		return context.Background()
+	}
+	return fs.remoteCacheEvictCtx
+}
+
+func (fs *FilerServer) runRemoteCacheEviction() {
+	if fs.option.RemoteCacheEvictThreshold <= 0 || fs.remoteCacheEvictCtx == nil {
+		return
+	}
+	ctx := fs.remoteCacheEvictCtx
+	ticker := time.NewTicker(remoteCacheEvictInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+		if fs.remoteCacheEvictMu.TryLock() {
+			fs.vacuumPendingRemoteCacheVids(ctx)
+			fs.remoteCacheEvictMu.Unlock()
+		}
+		if fs.filer.RemoteStorage == nil || len(fs.filer.RemoteStorage.MountedDirectories()) == 0 {
+			continue
+		}
+		bytesToFree, pressuredVids, over := fs.remoteCacheDiskPressure(ctx)
+		if !over {
+			continue
+		}
+		glog.V(0).Infof("remote cache: disk usage over %.0f%%, evicting %d bytes", fs.option.RemoteCacheEvictThreshold*100, bytesToFree)
+		fs.reclaimRemoteCacheSpace(ctx, bytesToFree, pressuredVids)
+	}
+}
@@ -22,9 +22,10 @@ func (vs *VolumeServer) VacuumVolumeCheck(ctx context.Context, req *volume_serve

 	resp := &volume_server_pb.VacuumVolumeCheckResponse{}

-	garbageRatio, err := vs.store.CheckCompactVolume(needle.VolumeId(req.VolumeId))
+	garbageRatio, diskSpaceLow, err := vs.store.CheckCompactVolume(needle.VolumeId(req.VolumeId))

 	resp.GarbageRatio = garbageRatio
+	resp.DiskSpaceLow = diskSpaceLow

 	if err != nil {
 		glog.V(3).Infof("check volume %d: %v", req.VolumeId, err)
@@ -339,7 +339,7 @@ func deleteOrphanedNeedles(commandEnv *CommandEnv, entryPath util.FullPath, need
 			continue
 		}
 		for _, loc := range locations {
-			results := operation.DeleteFileIdsAtOneVolumeServer(loc.ServerAddress(), commandEnv.option.GrpcDialOption, fids, includeCookie)
+			results := operation.DeleteFileIdsAtOneVolumeServer(context.Background(), loc.ServerAddress(), commandEnv.option.GrpcDialOption, fids, includeCookie)
 			// Summarize per server: an unreachable volume server returns one
 			// error per needle, which for manifest-heavy files can mean
 			// hundreds of near-identical lines. Keep the first error as the
@@ -102,7 +102,7 @@ func (c *commandRemoteUncache) uncacheContentData(commandEnv *CommandEnv, writer
 			return true
 		}

-		if entry.RemoteEntry.LastLocalSyncTsNs/1e9 < entry.Attributes.Mtime {
+		if entry.RemoteEntry.LastLocalSyncTsNs < entry.Attributes.Mtime*1e9+int64(entry.Attributes.MtimeNs) {
 			return true // should not uncache an entry that is not synchronized with remote
 		}

@@ -759,6 +759,7 @@ func (vcd *volumeCheckDisk) doVolumeCheckDisk(minuend, subtrahend *needle_map.Me
 			vcd.writeVerbose("delete %s %s => %s", needleValue.Key.FileId(source.info.Id), source.location.dataNode.Id, target.location.dataNode.Id)
 		}
 		deleteResults := operation.DeleteFileIdsAtOneVolumeServer(
+			context.Background(),
 			pb.NewServerAddressFromDataNode(target.location.dataNode),
 			vcd.grpcDialOption(), fidList, false)

@@ -939,7 +939,7 @@ func (c *commandVolumeFsck) purgeFileIdsForOneVolume(volumeId uint32, fileIds []
 		go func(server pb.ServerAddress, fidList []string) {
 			defer wg.Done()

-			deleteResults := operation.DeleteFileIdsAtOneVolumeServer(server, c.env.option.GrpcDialOption, fidList, false)
+			deleteResults := operation.DeleteFileIdsAtOneVolumeServer(context.Background(), server, c.env.option.GrpcDialOption, fidList, false)
 			if deleteResults != nil {
 				resultChan <- deleteResults
 			}
@@ -28,6 +28,9 @@ func (c *commandVacuum) Help() string {

 	volume.vacuum [-garbageThreshold=0.3] [-collection=<collection name>] [-volumeId=<volume id>]

+	Without -volumeId this runs the same sweep as the automatic vacuum, which skips
+	read-only volumes. Name a read-only volume with -volumeId to vacuum it anyway.
+
 `
 }

@@ -63,15 +66,20 @@ func (c *commandVacuum) Do(args []string, commandEnv *CommandEnv, writer io.Writ
 		volumeIdInts = append(volumeIdInts, 0)
 	}

+	topo, _, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		if *volumeIds != "" {
+			return fmt.Errorf("collect topology: %w", err)
+		}
+		// The hint below is a courtesy; a sweep must not depend on it.
+		fmt.Fprintf(writer, "could not list volumes to check for read-only ones: %v\n", err)
+	}
+
 	// Reject unknown ids up front. The master's VacuumVolume RPC silently
 	// iterates matching volumes, so a typo or an already-deleted volume just
 	// returns success — making this command look like it worked when nothing
 	// happened.
 	if *volumeIds != "" {
-		topo, _, err := collectTopologyInfo(commandEnv, 0)
-		if err != nil {
-			return fmt.Errorf("collect topology: %w", err)
-		}
 		known := make(map[uint32]bool)
 		eachDataNode(topo, func(_ DataCenterId, _ RackId, dn *master_pb.DataNodeInfo) {
 			for _, disk := range dn.DiskInfos {
@@ -96,6 +104,13 @@ func (c *commandVacuum) Do(args []string, commandEnv *CommandEnv, writer io.Writ
 			sort.Slice(missing, func(i, j int) bool { return missing[i] < missing[j] })
 			return fmt.Errorf("volume(s) not found on master: %v", missing)
 		}
+	} else if topo != nil {
+		// The sweep says nothing about the volumes it leaves alone, so an
+		// operator on a full disk sees the command return and nothing change.
+		if skipped := readOnlyVolumesAboveThreshold(topo, *collection, *garbageThreshold); len(skipped) > 0 {
+			fmt.Fprintf(writer, "%d read-only volume(s) hold garbage above %g and are skipped by the sweep: %v\n", len(skipped), *garbageThreshold, skipped)
+			fmt.Fprintf(writer, "vacuum them explicitly with -volumeId\n")
+		}
 	}

 	for _, volumeId := range volumeIdInts {
@@ -114,3 +129,42 @@ func (c *commandVacuum) Do(args []string, commandEnv *CommandEnv, writer io.Writ

 	return nil
 }
+
+// readOnlyVolumesAboveThreshold lists the volumes a sweep leaves alone: any
+// replica read-only (that is what the sweep checks), in the collection when one
+// is given, and some replica with a garbage ratio at or above the threshold.
+// The ratio uses the sizes the master reports, which is deleted bytes over the
+// .dat size rather than over the content size the volume server divides by, so
+// it can only understate, and a converted index that reports deletes without
+// their sizes is left out because its ratio is not knowable here. This is a
+// hint; the volume server's own check decides.
+func readOnlyVolumesAboveThreshold(topo *master_pb.TopologyInfo, collection string, garbageThreshold float64) []uint32 {
+	readOnly := make(map[uint32]bool)
+	garbage := make(map[uint32]float64) // the highest ratio any replica reports
+	eachDataNode(topo, func(_ DataCenterId, _ RackId, dn *master_pb.DataNodeInfo) {
+		for _, disk := range dn.DiskInfos {
+			for _, v := range disk.VolumeInfos {
+				if collection != "" && v.Collection != collection {
+					continue
+				}
+				if v.ReadOnly {
+					readOnly[v.Id] = true
+				}
+				if v.Size == 0 {
+					continue
+				}
+				if ratio := float64(v.DeletedByteCount) / float64(v.Size); ratio > garbage[v.Id] {
+					garbage[v.Id] = ratio
+				}
+			}
+		}
+	})
+	vids := make([]uint32, 0, len(readOnly))
+	for vid := range readOnly {
+		if garbage[vid] >= garbageThreshold {
+			vids = append(vids, vid)
+		}
+	}
+	sort.Slice(vids, func(i, j int) bool { return vids[i] < vids[j] })
+	return vids
+}
@@ -0,0 +1,58 @@
+package shell
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func vacuumTestTopology(nodes ...*master_pb.DataNodeInfo) *master_pb.TopologyInfo {
+	return &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{{
+			RackInfos: []*master_pb.RackInfo{{DataNodeInfos: nodes}},
+		}},
+	}
+}
+
+func vacuumTestNode(volumes ...*master_pb.VolumeInformationMessage) *master_pb.DataNodeInfo {
+	return &master_pb.DataNodeInfo{
+		DiskInfos: map[string]*master_pb.DiskInfo{"": {VolumeInfos: volumes}},
+	}
+}
+
+func TestReadOnlyVolumesAboveThreshold(t *testing.T) {
+	readOnlyHalfGarbage := &master_pb.VolumeInformationMessage{Id: 1, Collection: "a", Size: 100, DeletedByteCount: 50, ReadOnly: true}
+	writableMostlyGarbage := &master_pb.VolumeInformationMessage{Id: 2, Collection: "a", Size: 100, DeletedByteCount: 90}
+	readOnlyLittleGarbage := &master_pb.VolumeInformationMessage{Id: 3, Collection: "a", Size: 100, DeletedByteCount: 10, ReadOnly: true}
+	readOnlyOtherCollection := &master_pb.VolumeInformationMessage{Id: 4, Collection: "b", Size: 100, DeletedByteCount: 100, ReadOnly: true}
+	readOnlyEmpty := &master_pb.VolumeInformationMessage{Id: 5, Collection: "a", Size: 0, DeletedByteCount: 0, ReadOnly: true}
+	readOnlyAtThreshold := &master_pb.VolumeInformationMessage{Id: 6, Collection: "a", Size: 100, DeletedByteCount: 30, ReadOnly: true}
+	// the sweep skips a volume when any replica is read-only, and the garbage
+	// may sit on the other replica
+	mixedReadOnlyReplica := &master_pb.VolumeInformationMessage{Id: 7, Collection: "a", Size: 100, DeletedByteCount: 5, ReadOnly: true}
+	mixedWritableReplica := &master_pb.VolumeInformationMessage{Id: 7, Collection: "a", Size: 100, DeletedByteCount: 60}
+	// a converted index reports deletes without their sizes; the ratio is not
+	// knowable from here, so it is not listed
+	readOnlyConvertedIndex := &master_pb.VolumeInformationMessage{Id: 8, Collection: "a", Size: 100, DeleteCount: 2, DeletedByteCount: 0, ReadOnly: true}
+
+	topo := vacuumTestTopology(
+		vacuumTestNode(readOnlyHalfGarbage, writableMostlyGarbage, readOnlyLittleGarbage, readOnlyEmpty, readOnlyAtThreshold, mixedReadOnlyReplica, readOnlyConvertedIndex),
+		// the second replica of volume 1 must not list it twice
+		vacuumTestNode(readOnlyHalfGarbage, readOnlyOtherCollection, mixedWritableReplica),
+	)
+
+	got := readOnlyVolumesAboveThreshold(topo, "a", 0.3)
+	if want := []uint32{1, 6, 7}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("collection a, threshold 0.3: got %v, want %v", got, want)
+	}
+
+	got = readOnlyVolumesAboveThreshold(topo, "", 0.3)
+	if want := []uint32{1, 4, 6, 7}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("all collections, threshold 0.3: got %v, want %v", got, want)
+	}
+
+	if got := readOnlyVolumesAboveThreshold(topo, "a", 0.95); len(got) != 0 {
+		t.Fatalf("threshold 0.95: got %v, want none", got)
+	}
+}
@@ -789,6 +789,14 @@ var (
 			Help:      "Remote-mount object read attempts by source, bucket and cache result. A cold object retried before caching completes records a miss per attempt; paths outside the buckets folder use bucket \"_other\".",
 		}, []string{"source", "bucket", "result"})

+	RemoteCacheEvictedCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: subsystemRemote,
+			Name:      "cache_evicted_total",
+			Help:      "Remote-mounted objects whose local chunks were evicted under disk pressure.",
+		})
+
 	UploadErrorCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: Namespace,
@@ -1072,6 +1080,7 @@ func init() {
 	Gather.MustRegister(S3BucketReadOnlyGauge)

 	Gather.MustRegister(RemoteCacheReadCounter)
+	Gather.MustRegister(RemoteCacheEvictedCounter)

 	Gather.MustRegister(S3LifecycleDispatchCounter)
 	Gather.MustRegister(S3LifecycleScheduleDepthGauge)
@@ -7,16 +7,21 @@ import (

 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 )

 var ErrInsufficientSpace = fmt.Errorf("insufficient free space")

-func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (float64, error) {
+func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (garbageRatio float64, diskSpaceLow bool, err error) {
 	if v := s.findVolume(volumeId); v != nil {
 		glog.V(3).Infof("volume %d garbage level: %f", volumeId, v.garbageLevel())
-		return v.garbageLevel(), nil
+		// diskSpaceLow only counts when it is the sole read-only cause — an
+		// operator mark or I/O quarantine still shields the volume.
+		_, noWriteOrDelete, noWriteCanDelete, isLow := v.ReadOnlyReasons()
+		return v.garbageLevel(), isLow && !noWriteOrDelete && !noWriteCanDelete, nil
 	}
-	return 0, fmt.Errorf("volume id %d is not found during check compact: %w", volumeId, ErrVolumeNotFound)
+	return 0, false, fmt.Errorf("volume id %d is not found during check compact: %w", volumeId, ErrVolumeNotFound)
 }

 func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
@@ -56,18 +61,39 @@ func (s *Store) CommitCleanupVolume(vid needle.VolumeId) error {
 	return fmt.Errorf("volume id %d is not found during cleaning up: %w", vid, ErrVolumeNotFound)
 }

+// estimatedCompactedSize is what compaction writes: a superblock, the live
+// needles with their on-disk framing, and an index with live entries only.
+// Deleted bytes do not carry over, so a mostly-garbage volume needs far less
+// space than it occupies.
+func estimatedCompactedSize(v *Volume) int64 {
+	liveCount := v.FileCount()
+	if deleted := v.DeletedCount(); deleted < liveCount {
+		liveCount -= deleted
+	} else {
+		liveCount = 0
+	}
+	liveBytes := v.ContentSize()
+	if deleted := v.DeletedSize(); deleted < liveBytes {
+		liveBytes -= deleted
+	} else {
+		liveBytes = 0
+	}
+	perNeedle := needle.GetActualSize(0, v.Version()) + types.NeedlePaddingSize + types.NeedleMapEntrySize
+	return super_block.SuperBlockSize + int64(liveCount)*perNeedle + int64(liveBytes)
+}
+
 func ensureCompactVolumeSpace(v *Volume, preallocate int64) error {
-	// Get current volume size for space calculation
 	volumeSize, indexSize, _ := v.FileStat()

-	// Calculate space needed for compaction:
-	// 1. Space for the new compacted volume (approximately same as current volume size)
-	// 2. Use the larger of preallocate or estimated volume size
-	estimatedCompactSize := int64(volumeSize + indexSize)
+	// The compacted output holds live needles only, so measure against the
+	// estimated compacted size — otherwise a disk full of garbage can never
+	// reclaim itself.
+	estimatedCompactSize := estimatedCompactedSize(v)
 	spaceNeeded := preallocate
 	if estimatedCompactSize > preallocate {
 		spaceNeeded = estimatedCompactSize
 	}
+	spaceNeeded += spaceNeeded / 10

 	diskStatus := stats.NewDiskStatus(v.dir)
 	if int64(diskStatus.Free) < spaceNeeded {
@@ -2,6 +2,12 @@ package storage

 import (
 	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 )

 func TestSpaceCalculation(t *testing.T) {
@@ -49,3 +55,104 @@ func TestSpaceCalculation(t *testing.T) {
 		})
 	}
 }
+
+// Compaction writes live needles only, so the space check must be measured
+// against the live size, not the .dat the garbage occupies — a full disk
+// needs the estimate to shrink or it can never reclaim.
+func TestEstimatedCompactedSizeCountsLiveNeedles(t *testing.T) {
+	dir := t.TempDir()
+
+	v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
+	if err != nil {
+		t.Fatalf("volume creation: %v", err)
+	}
+	defer v.Close()
+
+	const count = 20
+	for i := 1; i <= count; i++ {
+		if _, _, _, err := v.writeNeedle2(newRandomNeedle(uint64(i)), true, false, false); err != nil {
+			t.Fatalf("write needle %d: %v", i, err)
+		}
+	}
+	datSize, _, _ := v.FileStat()
+
+	fullEstimate := estimatedCompactedSize(v)
+	if fullEstimate <= super_block.SuperBlockSize {
+		t.Fatalf("estimate for all-live volume = %d, want > superblock", fullEstimate)
+	}
+
+	for i := 1; i < count; i++ {
+		if _, err := v.doDeleteRequest(newEmptyNeedle(uint64(i))); err != nil {
+			t.Fatalf("delete needle %d: %v", i, err)
+		}
+	}
+
+	estimate := estimatedCompactedSize(v)
+	if estimate >= int64(datSize) {
+		t.Fatalf("estimate %d not below .dat size %d with 19/20 needles deleted", estimate, datSize)
+	}
+	if estimate <= super_block.SuperBlockSize {
+		t.Fatalf("estimate %d lost the one live needle", estimate)
+	}
+	live := int64(v.FileCount()-v.DeletedCount())*types.NeedleMapEntrySize + super_block.SuperBlockSize
+	if estimate < live {
+		t.Fatalf("estimate %d below superblock + live index entries %d", estimate, live)
+	}
+
+	if _, err := v.doDeleteRequest(newEmptyNeedle(uint64(count))); err != nil {
+		t.Fatalf("delete last needle: %v", err)
+	}
+	if estimate := estimatedCompactedSize(v); estimate != super_block.SuperBlockSize {
+		t.Fatalf("all-deleted estimate = %d, want superblock only (%d)", estimate, super_block.SuperBlockSize)
+	}
+}
+
+// The estimate must cover what compaction writes on disk: each live needle's
+// content plus its header, checksum, timestamp and padding. An all-live
+// volume's compacted .dat is byte-for-byte its current one, so the estimate
+// may not fall below the current file.
+func TestEstimatedCompactedSizeCoversNeedleFraming(t *testing.T) {
+	dir := t.TempDir()
+
+	v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
+	if err != nil {
+		t.Fatalf("volume creation: %v", err)
+	}
+	defer v.Close()
+
+	for i := 1; i <= 100; i++ {
+		if _, _, _, err := v.writeNeedle2(newRandomNeedle(uint64(i)), true, false, false); err != nil {
+			t.Fatalf("write needle %d: %v", i, err)
+		}
+	}
+	datSize, _, _ := v.FileStat()
+
+	if estimate := estimatedCompactedSize(v); estimate < int64(datSize) {
+		t.Fatalf("estimate %d below .dat size %d for an all-live volume: missing per-needle framing", estimate, datSize)
+	}
+}
+
+// disk_space_low is only reported when low space is the sole read-only cause,
+// so a volume also marked read-only by an operator or quarantined by failed
+// I/O stays out of the sweep.
+func TestCheckCompactVolumeDiskLowSoleCauseOnly(t *testing.T) {
+	dir := t.TempDir()
+	store := newSingleDirStore(t, dir)
+	defer store.Close()
+	const vid = needle.VolumeId(7)
+	require.NoError(t, store.AddVolume(vid, "", NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0))
+
+	_, low, err := store.CheckCompactVolume(vid)
+	require.NoError(t, err)
+	require.False(t, low)
+
+	store.Locations[0].isDiskSpaceLow.Store(true)
+	_, low, err = store.CheckCompactVolume(vid)
+	require.NoError(t, err)
+	require.True(t, low)
+
+	require.NoError(t, store.MarkVolumeReadonly(vid, false, false))
+	_, low, err = store.CheckCompactVolume(vid)
+	require.NoError(t, err)
+	require.False(t, low)
+}
@@ -651,6 +651,12 @@ func (t *Topology) SyncDataNodeRegistration(volumes []*master_pb.VolumeInformati
 	for _, v := range deletedVolumes {
 		t.UnRegisterVolumeLayout(v, dn)
 	}
+	for _, v := range changedVolumes {
+		if v.ReplicaPlacement == nil {
+			continue
+		}
+		t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType)).SetReplicaReadOnlyFlag(dn, v.Id, v.ReadOnly)
+	}
 	// Update effective sizes for all reported volumes (decay pending estimates).
 	// If decay brings a volume eagerly removed by RecordAssign back under the
 	// writable threshold, restore the matching activeVolumeCount.
@@ -780,6 +786,9 @@ func (t *Topology) ApplyVolumeChanges(changed []*master_pb.VolumeInformationMess
 		if isNew || becameServable || tierTransition || isChanged {
 			newVolumes = append(newVolumes, vi)
 		}
+		if isChanged {
+			vl.SetReplicaReadOnlyFlag(dn, vi.Id, vi.ReadOnly)
+		}
 		vl.UpdateOversizedState(&vi, dn)
 		if vl.UpdateVolumeSize(vi.Id, vi.Size, vi.CompactRevision, true) {
 			vl.AdjustActiveVolumeCountAfterRecovery(vi.Id)
@@ -864,3 +864,48 @@ func TestSetVolumeAvailableRepairsMissingVolume(t *testing.T) {
 		t.Fatalf("after SetVolumeAvailable: size tracking for %d not seeded", vid)
 	}
 }
+
+// A read-only change that arrives in the regular heartbeat must reach the
+// layout's per-replica flag, which the vacuum sweep consults. Registration
+// and volume.mark already set it; the heartbeat path did not.
+func TestFullHeartbeatUpdatesLayoutReadOnlyFlag(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
+	report := func(readOnly bool) []*master_pb.VolumeInformationMessage {
+		return []*master_pb.VolumeInformationMessage{{
+			Id: 1, Collection: "c", Size: 1 << 20, Version: uint32(needle.GetCurrentVersion()), ReadOnly: readOnly,
+		}}
+	}
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	flag := func() bool {
+		vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
+		vl.accessLock.RLock()
+		defer vl.accessLock.RUnlock()
+		return vl.vid2location[needle.VolumeId(1)].AnyReadOnly()
+	}
+
+	// the full list, which a server sends first and whenever the digests disagree
+	topo.SyncDataNodeRegistration(report(false), dn)
+	if flag() {
+		t.Fatal("volume registered writable is flagged read-only")
+	}
+	topo.SyncDataNodeRegistration(report(true), dn)
+	if !flag() {
+		t.Fatal("full heartbeat turned the volume read-only but the layout flag did not follow")
+	}
+	topo.SyncDataNodeRegistration(report(false), dn)
+	if flag() {
+		t.Fatal("full heartbeat turned the volume writable again but the layout flag stayed read-only")
+	}
+
+	// the changed-volumes delta, which is what a running server normally sends
+	topo.ApplyVolumeChanges(report(true), dn)
+	if !flag() {
+		t.Fatal("delta heartbeat turned the volume read-only but the layout flag did not follow")
+	}
+	topo.ApplyVolumeChanges(report(false), dn)
+	if flag() {
+		t.Fatal("delta heartbeat turned the volume writable again but the layout flag stayed read-only")
+	}
+}
@@ -23,11 +23,11 @@ import (
 )

 func (t *Topology) batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vid needle.VolumeId,
-	locationlist *VolumeLocationList, garbageThreshold float64) (*VolumeLocationList, bool) {
+	locationlist *VolumeLocationList, garbageThreshold float64, skipReadOnly bool) (*VolumeLocationList, bool) {
 	ch := make(chan int, locationlist.Length())
 	errCount := int32(0)
 	for index, dn := range locationlist.list {
-		go func(index int, url pb.ServerAddress, vid needle.VolumeId) {
+		go func(index int, dn *DataNode, url pb.ServerAddress, vid needle.VolumeId) {
 			err := operation.WithVolumeServerClient(false, url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
 				resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
 					VolumeId: uint32(vid),
@@ -37,6 +37,21 @@ func (t *Topology) batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vid ne
 					ch <- -1
 					return err
 				}
+				// A sweep skips a read-only copy unless low disk space is its
+				// only read-only cause — that is the copy compaction exists for.
+				if skipReadOnly {
+					v, lookErr := dn.GetVolumesById(vid)
+					if lookErr != nil {
+						atomic.AddInt32(&errCount, 1)
+						ch <- -1
+						return lookErr
+					}
+					if v.ReadOnly && !resp.DiskSpaceLow {
+						glog.V(0).Infof("skip vacuuming read-only volume %d on %s", vid, url)
+						ch <- -1
+						return nil
+					}
+				}
 				if resp.GarbageRatio >= garbageThreshold {
 					ch <- index
 				} else {
@@ -47,7 +62,7 @@ func (t *Topology) batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vid ne
 			if err != nil {
 				glog.V(0).Infof("Checking vacuuming %d on %s: %v", vid, url, err)
 			}
-		}(index, dn.ServerAddress(), vid)
+		}(index, dn, dn.ServerAddress(), vid)
 	}
 	vacuumLocationList := NewVolumeLocationList()

@@ -356,18 +371,16 @@ func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeL
 }

 // skipReadOnly is set by the background scan and all-volumes sweep, where a
-// read-only flag usually means an unhealthy disk. An explicit volumeId clears
-// it so a benignly read-only (full/oversized) volume can be reclaimed.
+// read-only flag usually means an unhealthy disk. Even then a copy that is
+// read-only because its disk is low on space stays eligible — compaction is
+// how the space comes back. An explicit volumeId clears the rule entirely.
 func (t *Topology) vacuumOneVolumeId(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, locationList *VolumeLocationList, vid needle.VolumeId, preallocate int64, skipReadOnly bool) {
 	volumeLayout.accessLock.RLock()
 	isReadOnly := volumeLayout.vid2location[vid].AnyReadOnly()
 	isEnoughCopies := volumeLayout.enoughCopies(vid)
 	volumeLayout.accessLock.RUnlock()

-	if isReadOnly {
-		if skipReadOnly {
-			return
-		}
+	if isReadOnly && !skipReadOnly {
 		glog.V(0).Infof("vacuuming read-only volume %d on explicit request", vid)
 	}
 	if !isEnoughCopies {
@@ -377,7 +390,7 @@ func (t *Topology) vacuumOneVolumeId(grpcDialOption grpc.DialOption, volumeLayou

 	glog.V(1).Infof("check vacuum on collection:%s volume:%d", c.Name, vid)
 	if vacuumLocationList, needVacuum := t.batchVacuumVolumeCheck(
-		grpcDialOption, vid, locationList, garbageThreshold); needVacuum {
+		grpcDialOption, vid, locationList, garbageThreshold, skipReadOnly); needVacuum {
 		if t.batchVacuumVolumeCompact(grpcDialOption, volumeLayout, vid, vacuumLocationList, preallocate) {
 			t.batchVacuumVolumeCommit(grpcDialOption, volumeLayout, vid, vacuumLocationList, locationList)
 		} else {
--- a/Show More
+++ b/Show More