volume: fix ReadOnly match arm missed by the tuple-variant change

#11544 made VolumeError::ReadOnly carry the volume id; the batch-write
fallback in write_volume_needles still matched and built it as a unit
variant, breaking the build (E0532/E0308).

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.github/workflows/codeql.yml

@@ -27,7 +27,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v4.38.1
+        uses: github/codeql-action/init@v4.38.2
         # Override language selection by uncommenting this and choosing your languages
         with:
           languages: go
@@ -35,7 +35,7 @@ jobs:
       # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below).
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v4.38.1
+        uses: github/codeql-action/autobuild@v4.38.2
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -49,4 +49,4 @@ jobs:
       #     make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v4.38.1
+        uses: github/codeql-action/analyze@v4.38.2

.github/workflows/container_latest.yml

@@ -156,7 +156,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        platform: [amd64, arm64, arm, 386]
+        platform: [amd64, arm64, arm, 386, ppc64le, s390x]
         variant: ${{ fromJSON(needs.setup.outputs.variants) }}
 
     steps:
@@ -405,7 +405,7 @@ jobs:
           output: trivy-results.sarif
           exit-code: '0'
       - name: Upload Trivy scan results to GitHub Security
-        uses: github/codeql-action/upload-sarif@v4.38.1
+        uses: github/codeql-action/upload-sarif@v4.38.2
         if: always()
         with:
           sarif_file: trivy-results.sarif
@@ -505,7 +505,9 @@ jobs:
             ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
             ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
             ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
-            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
+            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 \
+            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-ppc64le \
+            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-s390x
           # The copy and the signature below use the digest this run pushed, not whatever the tag points at by then.
           DIGEST=$(jq -er '."containerimage.descriptor".digest' /tmp/manifest.json)
           echo "digest=${DIGEST}" >> "$GITHUB_OUTPUT"
@@ -549,13 +551,15 @@ jobs:
             echo "Using skopeo to copy..."
             retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs@${DIGEST} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
           else
-            echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
+            echo "Using docker buildx imagetools (pulling 6 images from Docker Hub)..."
             # Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
             retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
               ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
               ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
               ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
-              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
+              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 \
+              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-ppc64le \
+              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-s390x
           fi
 
       - name: Sign

.github/workflows/container_release_unified.yml

@@ -149,12 +149,16 @@ jobs:
           - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/arm64,  arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
           - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest,    qemu: true }
           - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/386,    arch: i386,  runner: ubuntu-latest,    qemu: false }
+          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/ppc64le, arch: ppc64le, runner: ubuntu-latest,  qemu: true }
+          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/s390x,   arch: s390x, runner: ubuntu-latest,    qemu: true }
 
           # Large disk - multi-arch
           - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64,  arch: amd64, runner: ubuntu-latest,    qemu: false }
           - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64,  arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
           - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest,    qemu: true }
           - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386,    arch: i386,  runner: ubuntu-latest,    qemu: false }
+          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/ppc64le, arch: ppc64le, runner: ubuntu-latest,  qemu: true }
+          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/s390x,   arch: s390x, runner: ubuntu-latest,    qemu: true }
 
           # Full tags - multi-arch
           - { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest,    qemu: false }
@@ -456,7 +460,7 @@ jobs:
 
       - name: Upload Trivy scan results to GitHub Security
         if: always()
-        uses: github/codeql-action/upload-sarif@v4.38.1
+        uses: github/codeql-action/upload-sarif@v4.38.2
         with:
           sarif_file: trivy-results.sarif
           category: trivy-${{ matrix.variant }}

.github/workflows/s3-tables-tests.yml

@@ -439,6 +439,117 @@ jobs:
           path: test/s3tables/catalog_clickhouse/test-output.log
           retention-days: 3
 
+  olake-iceberg-catalog-tests:
+    name: OLake Iceberg Catalog Integration Tests (${{ matrix.tag }})
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Pinned baseline, and latest so new OLake releases are exercised
+          # without a code change. OLake's Iceberg writer is a Java sidecar
+          # whose Iceberg version moves independently of the Go release, so
+          # the latest leg is the one that catches library drift.
+          - olake-image: olakego/source-postgres:v0.10.1
+            tag: "v0.10.1"
+          - olake-image: olakego/source-postgres:latest
+            tag: latest
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v7
+
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Configure Docker Hub mirror
+        run: |
+          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
+          sudo systemctl restart docker
+
+      - name: Pre-pull images
+        run: |
+          pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
+          pull ${{ matrix.olake-image }}
+          pull postgres:16
+          pull python:3.11-slim
+
+      - name: Run go mod tidy
+        run: go mod tidy
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false ./weed
+
+      - name: Run OLake Iceberg Catalog Integration Tests
+        timeout-minutes: 25
+        working-directory: test/s3tables/catalog_olake
+        env:
+          OLAKE_IMAGE: ${{ matrix.olake-image }}
+        run: |
+          set -x
+          set -o pipefail
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          df -h
+          docker info
+          echo "=== Starting OLake Iceberg Catalog Tests ==="
+
+          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
+            echo "OLake Iceberg catalog integration tests failed"
+            exit 1
+          }
+
+      # The suite skips itself when Docker is unavailable, so a green job is not
+      # by itself evidence that anything ran. Assert execution explicitly.
+      - name: Assert the suite actually ran
+        working-directory: test/s3tables/catalog_olake
+        run: |
+          log=test-output.log
+          if [ ! -f "$log" ]; then
+            echo "::error::no test-output.log; the suite did not run"
+            exit 1
+          fi
+          passes=$(grep -c '^--- PASS' "$log" || true)
+          skips=$(grep -c '^--- SKIP' "$log" || true)
+          echo "top-level PASS=$passes SKIP=$skips"
+          if [ "$skips" -gt 0 ]; then
+            echo "::error::the OLake suite skipped $skips top-level test(s); the environment it needs was not provisioned, so this job proves nothing"
+            grep '^--- SKIP' "$log" | head -20
+            exit 1
+          fi
+          if [ "$passes" -lt 1 ]; then
+            echo "::error::the OLake suite recorded no passing top-level test"
+            exit 1
+          fi
+
+      - name: Show test output on failure
+        if: failure()
+        working-directory: test/s3tables/catalog_olake
+        run: |
+          echo "=== Test Output ==="
+          if [ -f test-output.log ]; then
+            tail -200 test-output.log
+          fi
+
+          echo "=== Process information ==="
+          ps aux | grep -E "(weed|test|docker|olake|postgres)" || true
+          echo "=== Containers ==="
+          docker ps -a | head -30 || true
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: olake-iceberg-catalog-test-logs-${{ matrix.tag }}
+          path: test/s3tables/catalog_olake/test-output.log
+          retention-days: 3
+
   polaris-integration-tests:
     name: Polaris Integration Tests
     runs-on: ubuntu-22.04

.github/workflows/s3tests.yml

@@ -289,6 +289,7 @@ jobs:
           s3tests/functional/test_s3.py::test_object_write_check_etag \
           s3tests/functional/test_s3.py::test_object_write_cache_control \
           s3tests/functional/test_s3.py::test_object_write_expires \
+          s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \
           s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
           s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
           s3tests/functional/test_s3.py::test_object_write_file \
@@ -311,6 +312,7 @@ jobs:
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
           s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+          s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
           s3tests/functional/test_s3.py::test_bucket_head \
           s3tests/functional/test_s3.py::test_bucket_head_notexist \
           s3tests/functional/test_s3.py::test_object_raw_authenticated \
@@ -1149,6 +1151,7 @@ jobs:
           s3tests/functional/test_s3.py::test_object_write_check_etag \
           s3tests/functional/test_s3.py::test_object_write_cache_control \
           s3tests/functional/test_s3.py::test_object_write_expires \
+          s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \
           s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
           s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
           s3tests/functional/test_s3.py::test_object_write_file \
@@ -1171,6 +1174,7 @@ jobs:
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
           s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+          s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
           s3tests/functional/test_s3.py::test_bucket_head \
           s3tests/functional/test_s3.py::test_bucket_head_notexist \
           s3tests/functional/test_s3.py::test_object_raw_authenticated \

README.md

@@ -21,6 +21,7 @@ One `weed` binary serves an S3 object store, a POSIX file system, and a lakehous
 
 - [Download Binaries for different platforms](https://github.com/seaweedfs/seaweedfs/releases/latest)
 - [Wiki Documentation](https://github.com/seaweedfs/seaweedfs/wiki)
+- [HTTP REST API](REST_API.md) for the filer, master, and volume servers
 - Community: [Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY), [Twitter](https://twitter.com/SeaweedFS), [Telegram](https://t.me/Seaweedfs), [Reddit](https://www.reddit.com/r/SeaweedFS/), [Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [SeaweedFS White Paper](https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS_Architecture.pdf) and introduction slides: [2025.5](https://docs.google.com/presentation/d/1tdkp45J01oRV68dIm4yoTXKJDof-EhainlA0LMXexQE/edit?usp=sharing), [2021.5](https://docs.google.com/presentation/d/1DcxKWlINc-HNCjhYeERkpGXXm6nTCES8mi2W5G0Z4Ts/edit?usp=sharing), [2019.3](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
 

REST_API.md

@@ -0,0 +1,344 @@
+# SeaweedFS HTTP REST API
+
+SeaweedFS exposes three HTTP surfaces:
+
+| Service | Default port | Addressing |
+|---------|--------------|------------|
+| Filer | 8888 | File system paths (`/dir/name`) |
+| Master | 9333 | File id assignment and cluster topology |
+| Volume server | 8080 | File content by file id (`vid,fid`) |
+
+Most clients only need the filer API (paths) or the S3 API. The master and
+volume APIs are the lower-level blob store interface.
+
+Conventions applying to all three:
+
+- Responses are JSON unless noted otherwise. Append `&pretty=y` to pretty-print.
+- A file id (`fid`) has the form `volumeId,fileKeyCookie`, e.g. `3,01637037d6`.
+  An optional suffix selects a reserved id from a `count` assignment
+  (`3,01637037d6_1`, `_2`, ...), and an optional extension
+  (`3,01637037d6.jpg`) sets the content type on reads.
+- `replication` is a 3-digit replica placement `xyz`: `x` copies in other
+  data centers, `y` on other racks in the same data center, `z` on other
+  volume servers on the same rack. `000` = no replication, `001` = one copy
+  on the same rack, `010` = one copy on a different rack, `100` = one copy in
+  another data center, `200` = two copies in two other data centers, `110` =
+  one copy in another data center plus one on another rack.
+- `ttl` units: `m` minute, `h` hour, `d` day, `w` week, `M` month, `y` year.
+
+## Filer API (port 8888)
+
+The filer presents a POSIX-like namespace over the volume servers.
+
+### Upload a file
+
+```bash
+# PUT the raw body to the target path
+curl -T /home/chris/myphoto.jpg "http://localhost:8888/dir/myphoto.jpg"
+
+# or POST as multipart form (the part filename becomes the entry name)
+curl -F file=@/home/chris/myphoto.jpg "http://localhost:8888/dir/"
+```
+
+Response `201 Created`:
+
+```json
+{"name":"myphoto.jpg","size":43234,"eTag":"0x6c656...","mtime":"...","chunks":[...]}
+```
+
+Query parameters:
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `collection` | collection name | empty |
+| `replication` | replica placement code | filer default |
+| `ttl` | file expiration, e.g. `3d` | never |
+| `disk` | disk type to store on | filer default |
+| `fsync` | `true` fsyncs on the volume server | false |
+| `dataCenter` | preferred data center | empty |
+| `rack` | preferred rack | empty |
+| `dataNode` | preferred volume server | empty |
+| `saveInside` | store small content inside the metadata instead of a volume | false |
+| `maxMB` | split the upload into chunks of this many MB | filer `-maxMB` |
+| `mode` | unix permission bits, e.g. `0644` | `0660` |
+| `op` | `append` appends to an existing file | overwrite |
+| `skipCheckParentDir` | `true` skips the parent-directory existence check | false |
+
+### Create a directory
+
+```bash
+curl -X POST "http://localhost:8888/dir/newdir/"
+```
+
+A POST to a path ending in `/` with no content creates the directory,
+including missing parents.
+
+### Read a file
+
+```bash
+curl "http://localhost:8888/dir/myphoto.jpg"
+```
+
+Supports `Range` requests (`Accept-Ranges: bytes`), `ETag`, and the
+`If-None-Match` / `If-Modified-Since` conditional headers. `HEAD` returns
+headers only. Entry headers stored as extended attributes are echoed back,
+minus internal `Seaweed-` and `xattr-` keys.
+
+Entry metadata instead of content:
+
+```bash
+curl "http://localhost:8888/dir/myphoto.jpg?metadata=true"
+```
+
+`metadata=true&resolveManifest=true` additionally resolves chunked-manifest
+entries into their real chunk list.
+
+### List a directory
+
+```bash
+curl -H "Accept: application/json" "http://localhost:8888/dir/?limit=10&lastFileName=a.jpg"
+```
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `limit` | max entries per page | filer `-dirListLimit` |
+| `lastFileName` | resume listing after this entry name | empty |
+| `namePattern` | include only names matching the wildcard | empty |
+| `namePatternExclude` | exclude names matching the wildcard | empty |
+
+The JSON response carries `Path`, `Entries`, `Limit`, `LastFileName`,
+`ShouldDisplayLoadMore`, and `EmptyFolder`. Without the `Accept` header the
+filer renders its HTML browser.
+
+### Move and copy
+
+```bash
+curl -X POST "http://localhost:8888/dir/newname.jpg?mv.from=/dir/myphoto.jpg"
+curl -X POST "http://localhost:8888/dir/copy.jpg?cp.from=/dir/myphoto.jpg"
+```
+
+`mv.from` renames or moves the source to the request path (`204 No Content`).
+`cp.from` copies it.
+
+### Append
+
+```bash
+curl -T chunk2.bin "http://localhost:8888/dir/file.bin?op=append"
+```
+
+### Delete
+
+```bash
+curl -X DELETE "http://localhost:8888/dir/myphoto.jpg"
+curl -X DELETE "http://localhost:8888/dir/?recursive=true"
+```
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `recursive` | delete a non-empty directory tree | false; when the filer runs with `filer.options.recursive_delete=true`, deletes are recursive unless `recursive=false` |
+| `ignoreRecursiveError` | keep deleting remaining entries after an error | false |
+| `skipChunkDeletion` | remove only the metadata, keep volume data | false |
+
+### Tagging
+
+Tags are carried as `Seaweed-`-prefixed request headers, not query
+parameters; `?tagging` selects the tagging handler and `?tagging=K1,K2`
+lists the keys to remove. Header names are canonicalized on write
+(`Seaweed-k1` is stored as `Seaweed-K1`), and the delete list is matched
+case-sensitively against the stored names.
+
+```bash
+curl -X PUT -H "Seaweed-k1: v1" -H "Seaweed-k2: v2" "http://localhost:8888/dir/file.jpg?tagging"
+curl -X DELETE "http://localhost:8888/dir/file.jpg?tagging=K1,K2"
+```
+
+### Read by file id
+
+```bash
+curl "http://localhost:8888/?proxyChunkId=3,01637037d6"
+```
+
+The filer proxies the chunk read to the right volume server, so only the
+filer port needs to be exposed.
+
+### Resumable uploads
+
+The filer serves the [TUS protocol](https://tus.io/) for resumable uploads
+(`POST`, `PATCH`, `HEAD` on upload URLs). It is enabled by default at
+`/.tus`; `-tusBasePath` changes the endpoint base path.
+
+### Health
+
+`GET /healthz` and `GET /readyz` return `200 OK`.
+
+## Master API (port 9333)
+
+Write-affecting endpoints are automatically proxied to the current leader, so
+any master in the quorum can serve them.
+
+### Assign a file id
+
+```bash
+curl "http://localhost:9333/dir/assign?count=1&replication=001&collection=turbo&dataCenter=dc1&ttl=3d&disk=ssd"
+{"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
+```
+
+Upload the file content to `http://<url>/<fid>` afterwards. With `count>1`,
+use `<fid>_1`, `<fid>_2`, ... for the additional ids.
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `count` | file ids to reserve | 1 |
+| `collection` | collection name | empty |
+| `dataCenter` | preferred data center | empty |
+| `rack` | preferred rack | empty |
+| `dataNode` | preferred volume server | empty |
+| `replication` | replica placement | master `-defaultReplication` |
+| `ttl` | file expiration, e.g. `3d` | never |
+| `disk` | disk type | empty |
+| `dataSize` | expected file size in bytes | 0 |
+| `preallocate` | bytes to preallocate for new volumes | master `-volumePreallocate` |
+| `writableVolumeCount` | grow this many volumes when none are writable | master default |
+| `memoryMapMaxSizeMb` | memory-mapped file size (Windows) | 0 |
+
+### Look up a volume or file id
+
+```bash
+curl "http://localhost:9333/dir/lookup?volumeId=3"
+{"locations":[{"url":"localhost:8080","publicUrl":"localhost:8080"}]}
+```
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `volumeId` | volume id; a full `vid,fid` is accepted too | required |
+| `fileId` | like `volumeId`, but also returns a write JWT when security is on | empty |
+| `collection` | speeds up the lookup | empty |
+| `read` | `yes` generates a read JWT instead of a write JWT | empty |
+
+### Store a file in one call
+
+```bash
+curl -F file=@/home/chris/report.pdf "http://localhost:9333/submit?collection=turbo&replication=001"
+{"fileName":"report.pdf","fid":"3,01637037d6","fileUrl":"localhost:8080/3,01637037d6","size":43234,"eTag":"0x6c656..."}
+```
+
+`POST /submit` accepts multipart file data plus the `dir/assign` placement
+parameters (`count`, `collection`, `dataCenter`, `rack`, `replication`,
+`ttl`, `disk`), assigns a file id, uploads to the volume server, and returns
+the result.
+
+### Redirect to a file
+
+```bash
+curl -v "http://localhost:9333/3,01637037d6"
+```
+
+`GET /{fileId}` answers `308 Permanent Redirect` to a volume server holding
+the file, preserving the query string (e.g. image-resize parameters).
+
+### Cluster status
+
+```bash
+curl "http://localhost:9333/dir/status?pretty=y"   # full topology tree
+curl "http://localhost:9333/vol/status?pretty=y"   # every volume on every node
+curl "http://localhost:9333/collection/info?collection=turbo"
+curl "http://localhost:9333/collection/info?collection=turbo&detail=true"
+```
+
+`collection/info` returns aggregated `TotalSize`, `FileCount`, `UsedSize`,
+`VolumeCount`; `detail=true` splits them per volume layout.
+
+### Grow volumes
+
+```bash
+curl "http://localhost:9333/vol/grow?count=4&replication=001&collection=turbo&ttl=5d&disk=ssd&dataCenter=dc1&rack=rack1"
+{"count":4}
+```
+
+`count` is required; the placement parameters match `dir/assign`. One volume
+serves one write at a time, so pre-allocated volumes raise write concurrency.
+
+### Vacuum deleted space
+
+```bash
+curl "http://localhost:9333/vol/vacuum?garbageThreshold=0.4"
+```
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `garbageThreshold` | minimum deleted-bytes ratio before a volume is compacted | master `-garbageThreshold` (0.3) |
+
+Vacuuming makes a volume read-only, copies live needles to a new volume, and
+swaps it in.
+
+### Delete a collection
+
+```bash
+curl "http://localhost:9333/col/delete?collection=benchmark"
+```
+
+Deletes all volumes of the collection, including erasure-coded shards.
+`204 No Content` on success.
+
+### Health
+
+```bash
+curl -I "http://localhost:9333/healthz"     # liveness
+curl -I "http://localhost:9333/readyz"      # readiness
+curl "http://localhost:9333/"               # web UI
+```
+
+## Volume server API (port 8080)
+
+The volume server stores file content by file id. Clients normally get the
+volume URL from `dir/assign` or `dir/lookup`.
+
+### Upload
+
+```bash
+curl -F file=@/home/chris/myphoto.jpg "http://127.0.0.1:8080/3,01637037d6"
+{"name":"myphoto.jpg","size":43234,"eTag":"0x6c656...","mime":"image/jpeg","contentMd5":"..."}
+```
+
+PUT or POST the body (or a multipart `file` part) to `/{vid},{fid}`.
+`204 No Content` is returned when the content is unchanged. `?ts=<unix>`
+sets the stored modification time.
+
+### Read
+
+```bash
+curl "http://127.0.0.1:8080/3,01637037d6"
+curl "http://127.0.0.1:8080/3,01637037d6.jpg"        # sets Content-Type from the extension
+```
+
+Supports `Range` and `HEAD`. Image files can be resized server-side:
+
+| Parameter | Description |
+|-----------|-------------|
+| `width`, `height` | resize bounds in pixels |
+| `mode` | `fit` (contain) or `fill` (cover); omitted resizes to `width`/`height` |
+| `crop_x1`, `crop_y1`, `crop_x2`, `crop_y2` | explicit crop rectangle |
+| `cm` | `false` returns the chunk-manifest blob instead of resolving it |
+| `readDeleted` | `true` reads soft-deleted needles |
+| `collection` | passed through redirects for the right volume |
+
+### Delete
+
+```bash
+curl -X DELETE "http://127.0.0.1:8080/3,01637037d6"
+{"size":43234}
+```
+
+`?ts=<unix>` sets the deletion timestamp. Replicated volumes propagate the
+delete to every replica.
+
+### Status
+
+```bash
+curl "http://localhost:8080/status?pretty=y"  # disk and volume inventory
+curl -I "http://localhost:8080/healthz"       # liveness/readiness
+```
+
+`OPTIONS` preflights answer CORS headers. When `-port.public` differs from
+`-port`, the volume server opens a separate read-only public listener on
+that port; `-publicUrl` sets the address it advertises to clients.

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dustin/go-humanize v1.0.1
+	github.com/dustin/go-humanize v1.1.0
 	github.com/eapache/go-resiliency v1.6.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
@@ -94,7 +94,7 @@ require (
 	golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597
 	golang.org/x/image v0.46.0
 	golang.org/x/net v0.58.0
-	golang.org/x/oauth2 v0.36.0
+	golang.org/x/oauth2 v0.37.0
 	golang.org/x/sys v0.48.0
 	golang.org/x/text v0.42.0 // indirect
 	golang.org/x/tools v0.49.0 // indirect
@@ -111,7 +111,7 @@ require (
 )
 
 require (
-	cloud.google.com/go/kms v1.34.0
+	cloud.google.com/go/kms v1.35.0
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/Jille/raft-grpc-transport v1.6.1
@@ -154,7 +154,7 @@ require (
 	github.com/tarantool/go-tarantool/v3 v3.0.2
 	github.com/testcontainers/testcontainers-go v0.44.0
 	github.com/tikv/client-go/v2 v2.0.7
-	github.com/twmb/avro v1.8.0
+	github.com/twmb/avro v1.9.0
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1
@@ -302,13 +302,13 @@ require (
 	cloud.google.com/go/iam v1.12.0 // indirect
 	cloud.google.com/go/monitoring v1.30.0 // indirect
 	filippo.io/edwards25519 v1.2.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
 	github.com/Azure/go-ntlmssp v0.1.1 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect
 	github.com/Files-com/files-sdk-go/v3 v3.3.194 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect

go.sum

@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
 cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
 cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
 cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
-cloud.google.com/go/kms v1.34.0 h1:mxWcXEiyjxwFH5gclulLx+B8Y2OEpKJRZ5FOF78c2XE=
-cloud.google.com/go/kms v1.34.0/go.mod h1:FbxZWUiihmyjxlaBha84OK5+fmJHPrS6F5/mBFdJk6A=
+cloud.google.com/go/kms v1.35.0 h1:nJ/ktaqspx1nPM9vIcO0SHbhqCAm8nvAxL1siuVgKm0=
+cloud.google.com/go/kms v1.35.0/go.mod h1:0++71pIHvJL+GmMa8K4jOWFq7gNOX3jm2PRMSJwTKJw=
 cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
 cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
 cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
@@ -553,10 +553,10 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum
 git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4=
@@ -577,8 +577,8 @@ github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDP
 github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d h1:iG9B49Q218F/XxXNRM7k/vWf7MKmLIS8AcJV9cGN4nA=
@@ -953,8 +953,8 @@ github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.4.0/go.mod h1:gDXhl0OElhzYoDs
 github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
 github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
+github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
 github.com/eapache/go-resiliency v1.6.0 h1:CqGDTLtpwuWKn6Nj3uNUdflaq+/kIPsg0gfNzHton30=
 github.com/eapache/go-resiliency v1.6.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
 github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws=
@@ -1965,8 +1965,8 @@ github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365/go.mod h1:zj0GJHGvyf
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
-github.com/twmb/avro v1.8.0 h1:UMWLg+nH4P3yad5Om7yFSohYLy2RG1s7BcFFiOvmK9Q=
-github.com/twmb/avro v1.8.0/go.mod h1:X0fT1dY2xcbV4YuCE4mYro+qljHl4kUF5uA/2z1rgSk=
+github.com/twmb/avro v1.9.0 h1:JSiqewo3AANj7vlVCQXBsIZTA3LiiBble7Xp8T4lbtA=
+github.com/twmb/avro v1.9.0/go.mod h1:X0fT1dY2xcbV4YuCE4mYro+qljHl4kUF5uA/2z1rgSk=
 github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=
 github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4=
@@ -2354,8 +2354,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
 golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
 golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
-golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
-golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
+golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
+golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.47"
+appVersion: "4.48"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.47.1
+version: 4.48.0

k8s/charts/seaweedfs/templates/cert/admin-cert.yaml

@@ -25,6 +25,10 @@ spec:
     organizations:
     - "SeaweedFS CA"
   dnsNames:
+    - '{{ include "seaweedfs.fullname" . }}-admin'
+    - '{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}'
+    - '{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}.svc'
+    - '{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}.svc.cluster.local'
     - '*.{{ include "seaweedfs.fullname" . }}-admin'
     - '*.{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}'
     - '*.{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}.svc'

seaweed-volume/Cargo.lock

@@ -4555,6 +4555,7 @@ dependencies = [
  "aws-config",
  "aws-credential-types",
  "aws-sdk-s3",
+ "aws-smithy-runtime-api",
  "aws-types",
  "axum",
  "base64",

seaweed-volume/Cargo.toml

@@ -157,6 +157,9 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] }
 
 [dev-dependencies]
 tempfile = "3"
+# Already a transitive dependency of aws-sdk-s3 at a single locked version;
+# needed directly only for the canned HttpClient in remote_storage::s3 tests.
+aws-smithy-runtime-api = "1"
 
 [build-dependencies]
 tonic-prost-build = "0.14"

seaweed-volume/proto/volume_server.proto

@@ -168,6 +168,7 @@ message VacuumVolumeCheckRequest {
 }
 message VacuumVolumeCheckResponse {
     double garbage_ratio = 1;
+    bool disk_space_low = 4; // the volume is read-only solely because its disk is low on space — a cause compaction itself reclaims
 }
 
 message VacuumVolumeCompactRequest {
@@ -259,6 +260,10 @@ message VolumeDeleteRequest {
     // when true, do not remove the cloud-tier object backing the volume.
     // used for moves where another server is taking over the same .vif.
     bool keep_remote_data = 3;
+    // when true, delete only if every needle is deleted: the volume held
+    // data once but nothing is live anymore. Passing either check,
+    // only_empty or this one, is enough to delete.
+    bool only_garbage = 4;
 }
 message VolumeDeleteResponse {
 }
@@ -471,6 +476,7 @@ message VolumeEcShardsDeleteRequest {
     repeated uint32 shard_ids = 3;
     bool full_teardown = 4; // pre-encode cleanup: wipe every EC artifact + generation for this volume, not just shard_ids
     int64 encode_ts_ns = 5; // full_teardown generation fence: delete only a disk whose .vif generation is strictly OLDER than this; preserve same-or-newer, generation 0, and an unreadable .vif. 0 => wipe-all (shell pre-encode / pre-upgrade)
+    uint32 delete_generations_older_than = 6; // post-commit cleanup: delete only staged <base>.*.v<N> artifacts with N strictly below this; 0 disables
 }
 message VolumeEcShardsDeleteResponse {
     bool full_teardown_done = 1; // set by a new server that performed full_teardown; absent from an old server lets the caller detect the silent no-op

seaweed-volume/src/config.rs

@@ -178,12 +178,22 @@ pub struct Cli {
     pub min_free_space: String,
 
     /// Inflight upload data wait timeout of volume servers.
-    #[arg(long = "inflightUploadDataTimeout", default_value = "60s")]
-    pub inflight_upload_data_timeout: String,
+    /// Go `time.Duration` syntax: `300ms`, `1.5h`, `2h45m`; a bare number other than `0` is rejected.
+    #[arg(
+        long = "inflightUploadDataTimeout",
+        default_value = "60s",
+        value_parser = parse_go_duration
+    )]
+    pub inflight_upload_data_timeout: std::time::Duration,
 
     /// Inflight download data wait timeout of volume servers.
-    #[arg(long = "inflightDownloadDataTimeout", default_value = "60s")]
-    pub inflight_download_data_timeout: String,
+    /// Go `time.Duration` syntax: `300ms`, `1.5h`, `2h45m`; a bare number other than `0` is rejected.
+    #[arg(
+        long = "inflightDownloadDataTimeout",
+        default_value = "60s",
+        value_parser = parse_go_duration
+    )]
+    pub inflight_download_data_timeout: std::time::Duration,
 
     /// <experimental> if true, prevents slow reads from blocking other requests,
     /// but large file read P99 latency will increase.
@@ -471,99 +481,214 @@ fn find_options_arg(args: &[String]) -> String {
     String::new()
 }
 
-/// Parse a duration string like "60s", "5m", "1h" into a std::time::Duration.
-fn parse_duration(s: &str) -> std::time::Duration {
-    let s = s.trim();
+/// Parse a duration with Go's `time.ParseDuration` grammar, as Go's
+/// `flag.Duration` flags do: `[0-9]*(\.[0-9]*)?<unit>` repeated, units
+/// `ns`/`us`/`µs`/`μs`/`ms`/`s`/`m`/`h`. A bare `0` is the only unit-less
+/// value; a leading sign parses in Go but a negative has no
+/// `std::time::Duration`, so it is an error, not a clamp.
+fn parse_go_duration(s: &str) -> Result<std::time::Duration, String> {
+    let orig = s;
+    let invalid = || format!("time: invalid duration {:?}", orig);
+
+    let mut s = s;
+    if let Some(rest) = s.strip_prefix('-') {
+        if rest.is_empty() {
+            return Err(invalid());
+        }
+        return Err(format!("negative duration {:?} is not supported", orig));
+    }
+    if let Some(rest) = s.strip_prefix('+') {
+        s = rest;
+    }
+    if s == "0" {
+        return Ok(std::time::Duration::ZERO);
+    }
     if s.is_empty() {
-        return std::time::Duration::from_secs(60);
+        return Err(invalid());
     }
-    if let Some(secs) = s.strip_suffix('s')
-        && let Ok(v) = secs.parse::<u64>()
-    {
-        return std::time::Duration::from_secs(v);
+
+    // Go bounds the total by 1<<63 (int64 nanoseconds): overflow is an error.
+    const MAX: u64 = 1 << 63;
+    let mut total: u64 = 0;
+    let mut bytes = s.as_bytes();
+    while !bytes.is_empty() {
+        if !(bytes[0] == b'.' || bytes[0].is_ascii_digit()) {
+            return Err(invalid());
+        }
+
+        // Consume [0-9]*
+        let mut i = 0;
+        let mut v: u64 = 0;
+        while i < bytes.len() && bytes[i].is_ascii_digit() {
+            v = v
+                .checked_mul(10)
+                .and_then(|v| v.checked_add(u64::from(bytes[i] - b'0')))
+                .filter(|v| *v <= MAX)
+                .ok_or_else(invalid)?;
+            i += 1;
+        }
+        let pre = i != 0;
+        bytes = &bytes[i..];
+
+        // Consume (\.[0-9]*)?
+        let mut f: u64 = 0;
+        let mut scale: f64 = 1.0;
+        let mut post = false;
+        if let Some((b'.', rest)) = bytes.split_first() {
+            bytes = rest;
+            let mut i = 0;
+            let mut overflow = false;
+            while i < bytes.len() && bytes[i].is_ascii_digit() {
+                if !overflow {
+                    // Like Go's leadingFraction, digits past ~19 are dropped.
+                    match f
+                        .checked_mul(10)
+                        .and_then(|f| f.checked_add(u64::from(bytes[i] - b'0')))
+                        .filter(|f| *f <= MAX)
+                    {
+                        Some(next) => {
+                            f = next;
+                            scale *= 10.0;
+                        }
+                        None => overflow = true,
+                    }
+                }
+                i += 1;
+            }
+            post = i != 0;
+            bytes = &bytes[i..];
+        }
+        if !pre && !post {
+            return Err(invalid());
+        }
+
+        // Consume the unit: everything up to the next digit or '.'.
+        let unit_len = bytes
+            .iter()
+            .position(|c| *c == b'.' || c.is_ascii_digit())
+            .unwrap_or(bytes.len());
+        if unit_len == 0 {
+            return Err(format!("time: missing unit in duration {:?}", orig));
+        }
+        // The boundary is an ASCII byte or the end, so it is a char boundary.
+        let unit = std::str::from_utf8(&bytes[..unit_len]).map_err(|_| invalid())?;
+        bytes = &bytes[unit_len..];
+        let unit_nanos: u64 = match unit {
+            "ns" => 1,
+            "us" | "\u{00b5}s" | "\u{03bc}s" => 1_000,
+            "ms" => 1_000_000,
+            "s" => 1_000_000_000,
+            "m" => 60 * 1_000_000_000,
+            "h" => 3600 * 1_000_000_000,
+            _ => {
+                return Err(format!(
+                    "time: unknown unit {:?} in duration {:?}",
+                    unit, orig
+                ));
+            }
+        };
+
+        if v > MAX / unit_nanos {
+            return Err(invalid());
+        }
+        v *= unit_nanos;
+        if f > 0 {
+            // Go's float math, so `1.5h` rounds the same way on both sides.
+            v += (f as f64 * (unit_nanos as f64 / scale)) as u64;
+            if v > MAX {
+                return Err(invalid());
+            }
+        }
+        total = total
+            .checked_add(v)
+            .filter(|t| *t <= MAX)
+            .ok_or_else(invalid)?;
     }
-    if let Some(mins) = s.strip_suffix('m')
-        && let Ok(v) = mins.parse::<u64>()
-        && let Some(seconds) = v.checked_mul(60)
-    {
-        return std::time::Duration::from_secs(seconds);
+    if total > MAX - 1 {
+        return Err(invalid());
     }
-    if let Some(hours) = s.strip_suffix('h')
-        && let Ok(v) = hours.parse::<u64>()
-        && let Some(seconds) = v.checked_mul(3600)
-    {
-        return std::time::Duration::from_secs(seconds);
-    }
-    // Fallback: try parsing as raw seconds
-    if let Ok(v) = s.parse::<u64>() {
-        return std::time::Duration::from_secs(v);
-    }
-    std::time::Duration::from_secs(60)
+    Ok(std::time::Duration::from_nanos(total))
 }
 
-/// Parse minFreeSpace / minFreeSpacePercent into MinFreeSpace values.
-/// Mirrors Go's `util.MustParseMinFreeSpace()`.
-fn parse_min_free_spaces(min_free_space: &str, min_free_space_percent: &str) -> Vec<MinFreeSpace> {
-    // If --minFreeSpace is provided, use it (takes precedence).
+/// Byte-size suffixes of Go's `util.ParseBytes`, lower-cased: SI (`kb`, `k`)
+/// and IEC (`kib`, `ki`) forms, `b`, and no suffix.
+fn byte_unit_multiplier(unit: &str) -> Option<u64> {
+    const K: u64 = 1_000;
+    const KI: u64 = 1 << 10;
+    Some(match unit {
+        "" | "b" => 1,
+        "kb" | "k" => K,
+        "kib" | "ki" => KI,
+        "mb" | "m" => K * K,
+        "mib" | "mi" => KI * KI,
+        "gb" | "g" => K * K * K,
+        "gib" | "gi" => KI * KI * KI,
+        "tb" | "t" => K * K * K * K,
+        "tib" | "ti" => KI * KI * KI * KI,
+        "pb" | "p" => K * K * K * K * K,
+        "pib" | "pi" => KI * KI * KI * KI * KI,
+        "eb" | "e" => K * K * K * K * K * K,
+        "eib" | "ei" => KI * KI * KI * KI * KI * KI,
+        _ => return None,
+    })
+}
+
+/// Parse a human-readable byte count with Go's `util.ParseBytes` grammar:
+/// decimal number, optional whitespace, then a [`byte_unit_multiplier`]
+/// suffix, case-insensitive. `10GiB` → 10737418240, `1.5TB` → 1500000000000.
+///
+/// Thousands commas parse as in Go but never arrive from `-minFreeSpace`,
+/// which is split on commas into per-directory entries first.
+fn parse_bytes(s: &str) -> Result<u64, String> {
+    let digits_end = s
+        .find(|c: char| !(c.is_ascii_digit() || c == '.' || c == ','))
+        .unwrap_or(s.len());
+    let num = s[..digits_end].replace(',', "");
+    let value: f64 = num
+        .parse()
+        .map_err(|e| format!("invalid byte size {:?}: {}", s, e))?;
+    let unit = s[digits_end..].trim().to_lowercase();
+    let multiplier =
+        byte_unit_multiplier(&unit).ok_or_else(|| format!("unhandled size name: {}", unit))?;
+    let bytes = value * multiplier as f64;
+    if bytes >= u64::MAX as f64 {
+        return Err(format!("too large: {}", s));
+    }
+    Ok(bytes as u64)
+}
+
+/// Parse one `-minFreeSpace` entry like Go's `util.ParseMinFreeSpace()`: a
+/// plain number is a percentage in `0..=100`, anything else a byte size
+/// above 100 bytes. `150` errors rather than reading as bytes.
+fn parse_min_free_space(s: &str) -> Result<MinFreeSpace, String> {
+    let s = s.trim();
+    if let Ok(percent) = s.parse::<f64>() {
+        // Go's range check lets NaN through; rejecting it is the one
+        // intentional difference.
+        if !(0.0..=100.0).contains(&percent) {
+            return Err(format!("minFreeSpace is invalid: {:?}", s));
+        }
+        return Ok(MinFreeSpace::Percent(percent));
+    }
+    match parse_bytes(s) {
+        Ok(bytes) if bytes > 100 => Ok(MinFreeSpace::Bytes(bytes)),
+        _ => Err(format!("minFreeSpace is invalid: {:?}", s)),
+    }
+}
+
+/// Parse minFreeSpace / minFreeSpacePercent like Go's
+/// `util.MustParseMinFreeSpace()`: `--minFreeSpace` wins when set, and any
+/// invalid comma-separated entry fails the whole list.
+fn parse_min_free_spaces(
+    min_free_space: &str,
+    min_free_space_percent: &str,
+) -> Result<Vec<MinFreeSpace>, String> {
     let source = if !min_free_space.is_empty() {
         min_free_space
     } else {
         min_free_space_percent
     };
-
-    source
-        .split(',')
-        .map(|s| {
-            let s = s.trim();
-            // Try parsing as a percentage (value <= 100)
-            if let Ok(v) = s.parse::<f64>() {
-                if v <= 100.0 {
-                    return MinFreeSpace::Percent(v);
-                }
-                // Treat as bytes if > 100
-                return MinFreeSpace::Bytes(v as u64);
-            }
-            // Try parsing human-readable bytes: e.g. "10GiB", "500MiB", "1TiB"
-            let s_upper = s.to_uppercase();
-            if let Some(rest) = s_upper.strip_suffix("TIB")
-                && let Ok(v) = rest.trim().parse::<f64>()
-            {
-                return MinFreeSpace::Bytes((v * 1024.0 * 1024.0 * 1024.0 * 1024.0) as u64);
-            }
-            if let Some(rest) = s_upper.strip_suffix("GIB")
-                && let Ok(v) = rest.trim().parse::<f64>()
-            {
-                return MinFreeSpace::Bytes((v * 1024.0 * 1024.0 * 1024.0) as u64);
-            }
-            if let Some(rest) = s_upper.strip_suffix("MIB")
-                && let Ok(v) = rest.trim().parse::<f64>()
-            {
-                return MinFreeSpace::Bytes((v * 1024.0 * 1024.0) as u64);
-            }
-            if let Some(rest) = s_upper.strip_suffix("KIB")
-                && let Ok(v) = rest.trim().parse::<f64>()
-            {
-                return MinFreeSpace::Bytes((v * 1024.0) as u64);
-            }
-            if let Some(rest) = s_upper.strip_suffix("TB")
-                && let Ok(v) = rest.trim().parse::<f64>()
-            {
-                return MinFreeSpace::Bytes((v * 1_000_000_000_000.0) as u64);
-            }
-            if let Some(rest) = s_upper.strip_suffix("GB")
-                && let Ok(v) = rest.trim().parse::<f64>()
-            {
-                return MinFreeSpace::Bytes((v * 1_000_000_000.0) as u64);
-            }
-            if let Some(rest) = s_upper.strip_suffix("MB")
-                && let Ok(v) = rest.trim().parse::<f64>()
-            {
-                return MinFreeSpace::Bytes((v * 1_000_000.0) as u64);
-            }
-            // Default: 1%
-            MinFreeSpace::Percent(1.0)
-        })
-        .collect()
+    source.split(',').map(parse_min_free_space).collect()
 }
 
 /// Parse comma-separated tag groups like "fast:ssd,archive" into per-folder tag vectors.
@@ -657,7 +782,9 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
 
     // Parse min free spaces
     let mut min_free_spaces =
-        parse_min_free_spaces(&cli.min_free_space, &cli.min_free_space_percent);
+        parse_min_free_spaces(&cli.min_free_space, &cli.min_free_space_percent).unwrap_or_else(
+            |e| panic!("The value specified in -minFreeSpace not a valid value: {e}"),
+        );
     if min_free_spaces.len() == 1 && folder_count > 1 {
         let v = min_free_spaces[0].clone();
         min_free_spaces.resize(folder_count, v);
@@ -772,10 +899,6 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
         .collect();
     white_list.extend(sec.guard_white_list.iter().cloned());
 
-    // Parse durations
-    let inflight_upload_data_timeout = parse_duration(&cli.inflight_upload_data_timeout);
-    let inflight_download_data_timeout = parse_duration(&cli.inflight_download_data_timeout);
-
     VolumeServerConfig {
         port: cli.port,
         grpc_port,
@@ -807,8 +930,8 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
         file_size_limit_bytes: cli.file_size_limit_mb as i64 * 1024 * 1024,
         concurrent_upload_limit: cli.concurrent_upload_limit_mb as i64 * 1024 * 1024,
         concurrent_download_limit: cli.concurrent_download_limit_mb as i64 * 1024 * 1024,
-        inflight_upload_data_timeout,
-        inflight_download_data_timeout,
+        inflight_upload_data_timeout: cli.inflight_upload_data_timeout,
+        inflight_download_data_timeout: cli.inflight_download_data_timeout,
         has_slow_read: cli.has_slow_read,
         read_buffer_size_mb: cli.read_buffer_size_mb,
         ldb_timeout: cli.ldb_timeout,
@@ -1260,25 +1383,224 @@ mod tests {
     }
 
     #[test]
-    fn test_parse_duration() {
-        assert_eq!(parse_duration("60s"), std::time::Duration::from_secs(60));
-        assert_eq!(parse_duration("5m"), std::time::Duration::from_secs(300));
-        assert_eq!(parse_duration("1h"), std::time::Duration::from_secs(3600));
-        assert_eq!(parse_duration("30"), std::time::Duration::from_secs(30));
-        assert_eq!(parse_duration(""), std::time::Duration::from_secs(60));
+    fn test_parse_go_duration_accepts_go_grammar() {
+        use std::time::Duration;
+        let cases = [
+            ("60s", Duration::from_secs(60)),
+            ("5m", Duration::from_secs(300)),
+            ("1h", Duration::from_secs(3600)),
+            ("1.5h", Duration::from_secs(5400)),
+            ("2h45m", Duration::from_secs(2 * 3600 + 45 * 60)),
+            ("300ms", Duration::from_millis(300)),
+            ("1.5s", Duration::from_millis(1500)),
+            ("250us", Duration::from_micros(250)),
+            ("250\u{00b5}s", Duration::from_micros(250)),
+            ("250\u{03bc}s", Duration::from_micros(250)),
+            ("7ns", Duration::from_nanos(7)),
+            (".5s", Duration::from_millis(500)),
+            ("5.s", Duration::from_secs(5)),
+            ("+10s", Duration::from_secs(10)),
+            ("0", Duration::ZERO),
+            ("0s", Duration::ZERO),
+            // Go: fraction digits past the 19th are ignored, not an error.
+            ("1.00000000000000000001s", Duration::from_secs(1)),
+        ];
+        for (input, expected) in cases {
+            assert_eq!(parse_go_duration(input), Ok(expected), "input={:?}", input);
+        }
+    }
+
+    #[test]
+    fn test_parse_go_duration_rejects_what_go_rejects() {
+        // Each of these makes Go's flag.Duration fail flag parsing; the old
+        // Rust parser turned them all into the 60 s default.
+        let rejected = [
+            "30",                    // bare number other than 0
+            "30sec",                 // unknown unit
+            "abc",                   // no number at all
+            "",                      // empty
+            "1h ",                   // trailing space is not a unit
+            "s",                     // unit without number
+            ".s",                    // dot without digits
+            "1.5",                   // fraction without unit
+            "1h-30m",                // sign is only allowed at the front
+            "-",                     // sign alone
+            "5124095576030432h",     // overflows int64 nanoseconds
+            "307445734561825861m",   // overflows int64 nanoseconds
+            "9223372036854775808ns", // 1<<63, one past int64
+        ];
+        for input in rejected {
+            let result = parse_go_duration(input);
+            assert!(result.is_err(), "input={:?} parsed as {:?}", input, result);
+            let message = result.unwrap_err();
+            assert!(
+                message.contains(&format!("{:?}", input)) || input.is_empty(),
+                "error for {:?} does not name the value: {}",
+                input,
+                message
+            );
+        }
+        // Largest value Go accepts: 1<<63 - 1 nanoseconds.
         assert_eq!(
-            parse_duration("307445734561825861m"),
+            parse_go_duration("9223372036854775807ns"),
+            Ok(std::time::Duration::from_nanos(i64::MAX as u64))
+        );
+        // Go accepts negatives; std::time::Duration cannot, so it errors.
+        let negative = parse_go_duration("-1s").unwrap_err();
+        assert!(negative.contains("negative"), "{}", negative);
+    }
+
+    /// Clap refuses the flag so the process exits with a usage error, exactly
+    /// where Go's flag package does.
+    #[test]
+    fn test_cli_rejects_invalid_duration_flags() {
+        for value in ["30sec", "abc", "30"] {
+            let result = Cli::try_parse_from(["bin", "--inflightUploadDataTimeout", value]);
+            assert!(
+                result.is_err(),
+                "--inflightUploadDataTimeout={} was accepted as {:?}",
+                value,
+                result.map(|cli| cli.inflight_upload_data_timeout)
+            );
+            let err = Cli::try_parse_from(["bin", "--inflightDownloadDataTimeout", value])
+                .err()
+                .unwrap_or_else(|| panic!("--inflightDownloadDataTimeout={} was accepted", value));
+            assert_eq!(err.kind(), clap::error::ErrorKind::ValueValidation);
+            assert!(err.to_string().contains(value), "{}", err);
+        }
+        let cli = Cli::parse_from(["bin", "--inflightUploadDataTimeout", "1.5h"]);
+        assert_eq!(
+            cli.inflight_upload_data_timeout,
+            std::time::Duration::from_secs(5400)
+        );
+        // The default still parses through the same grammar.
+        let cli = Cli::parse_from(["bin"]);
+        assert_eq!(
+            cli.inflight_upload_data_timeout,
             std::time::Duration::from_secs(60)
         );
         assert_eq!(
-            parse_duration("5124095576030432h"),
+            cli.inflight_download_data_timeout,
             std::time::Duration::from_secs(60)
         );
     }
 
+    #[test]
+    fn test_parse_bytes_matches_go_parse_bytes() {
+        // Expected values are what Go's util.ParseBytes returns (weed/util/bytes_test.go).
+        let cases: [(&str, u64); 14] = [
+            ("42", 42),
+            ("42MB", 42_000_000),
+            ("42 MB", 42_000_000),
+            ("42 mib", 44_040_192),
+            ("42M", 42_000_000),
+            ("100Ki", 100 * 1024),
+            ("10GiB", 10 * 1024 * 1024 * 1024),
+            ("10gib", 10 * 1024 * 1024 * 1024),
+            ("10GB", 10_000_000_000),
+            ("1.5TB", 1_500_000_000_000),
+            // Go's ParseBytes accepts thousands commas; the -minFreeSpace flag
+            // never delivers one (see test_min_free_space_flag_commas_are_list_separators).
+            ("1,024MB", 1_024_000_000),
+            ("1,024", 1024),
+            ("1PiB", 1 << 50),
+            ("1eb", 1_000_000_000_000_000_000),
+        ];
+        for (input, expected) in cases {
+            assert_eq!(parse_bytes(input), Ok(expected), "input={:?}", input);
+        }
+        for input in [
+            "10GiBx",
+            "GiB",
+            "",
+            "-5GB",
+            "1.2.3GB",
+            "10 Gi B",
+            "99999999999999999999EB",
+        ] {
+            assert!(
+                parse_bytes(input).is_err(),
+                "input={:?} was accepted",
+                input
+            );
+        }
+    }
+
+    fn expect_percent(result: Result<MinFreeSpace, String>, expected: f64, input: &str) {
+        match result {
+            Ok(MinFreeSpace::Percent(v)) => {
+                assert!(
+                    (v - expected).abs() < f64::EPSILON,
+                    "input={:?}: {}",
+                    input,
+                    v
+                )
+            }
+            other => panic!(
+                "input={:?}: expected Percent({}), got {:?}",
+                input, expected, other
+            ),
+        }
+    }
+
+    fn expect_bytes(result: Result<MinFreeSpace, String>, expected: u64, input: &str) {
+        match result {
+            Ok(MinFreeSpace::Bytes(v)) => assert_eq!(v, expected, "input={:?}", input),
+            other => panic!(
+                "input={:?}: expected Bytes({}), got {:?}",
+                input, expected, other
+            ),
+        }
+    }
+
+    #[test]
+    fn test_parse_min_free_space_matches_go() {
+        // Fixtures from weed/util/minfreespace_test.go plus the byte-unit table.
+        expect_percent(parse_min_free_space("42"), 42.0, "42");
+        expect_percent(parse_min_free_space(" 42 "), 42.0, " 42 ");
+        expect_percent(parse_min_free_space("50"), 50.0, "50");
+        expect_percent(parse_min_free_space("100"), 100.0, "100");
+        expect_percent(parse_min_free_space("0"), 0.0, "0");
+        expect_percent(parse_min_free_space("2.5"), 2.5, "2.5");
+        expect_bytes(parse_min_free_space("100Ki"), 100 * 1024, "100Ki");
+        expect_bytes(parse_min_free_space("100GiB"), 100 << 30, "100GiB");
+        expect_bytes(parse_min_free_space("42M"), 42_000_000, "42M");
+        expect_bytes(parse_min_free_space(" 42M "), 42_000_000, " 42M ");
+        expect_bytes(parse_min_free_space("10GiB"), 10 << 30, "10GiB");
+        expect_bytes(parse_min_free_space("10gib"), 10 << 30, "10gib");
+        expect_bytes(parse_min_free_space("1.5TB"), 1_500_000_000_000, "1.5TB");
+        // Comma inputs never arrive whole from the flag; see
+        // test_min_free_space_flag_commas_are_list_separators.
+        expect_bytes(parse_min_free_space("1,024MB"), 1_024_000_000, "1,024MB");
+        expect_bytes(parse_min_free_space("1,000"), 1000, "1,000");
+
+        // Go: percent outside 0..=100, or a byte size of at most 100 bytes.
+        for input in [
+            "150", "101", "-1", "1e3", "inf", "nan", "100B", "50B", "0B", "10GiBx", "", "abc",
+            "10 Gi B",
+        ] {
+            let result = parse_min_free_space(input);
+            assert!(
+                result.is_err(),
+                "input={:?} was accepted as {:?}",
+                input,
+                result
+            );
+            assert!(
+                result
+                    .as_ref()
+                    .unwrap_err()
+                    .contains(&format!("{:?}", input.trim())),
+                "error for {:?} does not name the value: {:?}",
+                input,
+                result
+            );
+        }
+    }
+
     #[test]
     fn test_parse_min_free_spaces_percent() {
-        let result = parse_min_free_spaces("", "1");
+        let result = parse_min_free_spaces("", "1").unwrap();
         assert_eq!(result.len(), 1);
         match &result[0] {
             MinFreeSpace::Percent(v) => assert!((v - 1.0).abs() < f64::EPSILON),
@@ -1288,7 +1610,7 @@ mod tests {
 
     #[test]
     fn test_parse_min_free_spaces_bytes() {
-        let result = parse_min_free_spaces("10GiB", "");
+        let result = parse_min_free_spaces("10GiB", "").unwrap();
         assert_eq!(result.len(), 1);
         match &result[0] {
             MinFreeSpace::Bytes(v) => assert_eq!(*v, 10 * 1024 * 1024 * 1024),
@@ -1296,6 +1618,87 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_parse_min_free_spaces_list_and_precedence() {
+        // --minFreeSpace wins over --minFreeSpacePercent, as in Go's EmptyTo().
+        let result = parse_min_free_spaces("5, 10GiB", "1").unwrap();
+        assert_eq!(result.len(), 2);
+        assert!(matches!(result[0], MinFreeSpace::Percent(v) if v == 5.0));
+        assert!(matches!(result[1], MinFreeSpace::Bytes(v) if v == 10 << 30));
+        // One bad entry fails the whole list and names that entry.
+        let err = parse_min_free_spaces("5,10GiBx", "1").unwrap_err();
+        assert!(err.contains("\"10GiBx\""), "{}", err);
+        // Both flags empty: Go fatals (ParseMinFreeSpace("") fails) rather than
+        // assuming 1%.
+        assert!(parse_min_free_spaces("", "").is_err());
+    }
+
+    /// A comma in `-minFreeSpace` separates per-directory entries, as in Go's
+    /// `MustParseMinFreeSpace`: `1,024MB` is `1` (percent) and `024MB`, and
+    /// one `-dir` fails the count check.
+    #[test]
+    fn test_min_free_space_flag_commas_are_list_separators() {
+        let result = parse_min_free_spaces("1,024MB", "").unwrap();
+        assert_eq!(result.len(), 2, "{result:?}");
+        assert!(
+            matches!(result[0], MinFreeSpace::Percent(v) if v == 1.0),
+            "{result:?}"
+        );
+        assert!(
+            matches!(result[1], MinFreeSpace::Bytes(v) if v == 24_000_000),
+            "{result:?}"
+        );
+
+        let _guard = process_state_lock();
+        let cli = Cli::parse_from(["bin", "--dir", "/tmp/a", "--minFreeSpace", "1,024MB"]);
+        let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            resolve_config_with_env(cli, &|_| None).min_free_spaces
+        }));
+        assert!(
+            outcome.is_err(),
+            "one -dir with two minFreeSpace entries must abort"
+        );
+    }
+
+    /// Negative control at the startup layer: resolving the config with an
+    /// invalid --minFreeSpace must abort, as Go's MustParseMinFreeSpace does.
+    #[test]
+    fn test_resolve_config_rejects_invalid_min_free_space() {
+        // The security-config search reads the working directory.
+        let _guard = process_state_lock();
+        for (flag, value) in [
+            ("--minFreeSpace", "10GiBx"),
+            ("--minFreeSpace", "150"),
+            ("--minFreeSpace", "50B"),
+            ("--minFreeSpacePercent", "150"),
+            ("--minFreeSpacePercent", "abc"),
+        ] {
+            let cli = Cli::parse_from(["bin", flag, value]);
+            let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                resolve_config_with_env(cli, &|_| None).min_free_spaces
+            }));
+            let payload = match outcome {
+                Ok(spaces) => panic!("{}={} was accepted as {:?}", flag, value, spaces),
+                Err(payload) => payload,
+            };
+            let message = payload
+                .downcast_ref::<String>()
+                .cloned()
+                .or_else(|| payload.downcast_ref::<&str>().map(|s| s.to_string()))
+                .unwrap_or_default();
+            assert!(
+                message.contains("-minFreeSpace") && message.contains(value),
+                "{}={}: unexpected panic message {:?}",
+                flag,
+                value,
+                message
+            );
+        }
+        let cfg =
+            resolve_config_with_env(Cli::parse_from(["bin", "--minFreeSpace", "100"]), &|_| None);
+        assert!(matches!(cfg.min_free_spaces[0], MinFreeSpace::Percent(v) if v == 100.0));
+    }
+
     #[test]
     fn test_parse_volume_tags_single() {
         let tags = parse_volume_tags("fast:ssd", 3);

seaweed-volume/src/remote_storage/s3.rs

@@ -4,6 +4,7 @@
 
 use aws_sdk_s3::Client;
 use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
+use aws_sdk_s3::error::{DisplayErrorContext, SdkError};
 use aws_sdk_s3::primitives::ByteStream;
 
 use super::{RemoteEntry, RemoteStorageClient, RemoteStorageError};
@@ -25,6 +26,23 @@ impl S3RemoteStorageClient {
         endpoint: &str,
         force_path_style: bool,
     ) -> Self {
+        let client = Client::from_conf(
+            Self::config_builder(access_key, secret_key, region, endpoint, force_path_style)
+                .build(),
+        );
+
+        S3RemoteStorageClient { client, conf }
+    }
+
+    /// Build the SDK config for the given credentials and endpoint. Split out so
+    /// tests can attach a canned HTTP client before building the [`Client`].
+    fn config_builder(
+        access_key: &str,
+        secret_key: &str,
+        region: &str,
+        endpoint: &str,
+        force_path_style: bool,
+    ) -> aws_sdk_s3::config::Builder {
         let region = if region.is_empty() {
             "us-east-1"
         } else {
@@ -49,9 +67,7 @@ impl S3RemoteStorageClient {
             s3_config = s3_config.endpoint_url(endpoint);
         }
 
-        let client = Client::from_conf(s3_config.build());
-
-        S3RemoteStorageClient { client, conf }
+        s3_config
     }
 }
 
@@ -75,13 +91,14 @@ impl RemoteStorageClient for S3RemoteStorageClient {
             req = req.range(format!("bytes={}-", offset));
         }
 
-        let resp = req.send().await.map_err(|e| {
-            let msg = format!("{}", e);
-            if msg.contains("NoSuchKey") || msg.contains("404") {
+        let resp = req.send().await.map_err(|e| match e {
+            // Go compares `aerr.Code()` to NoSuchKey on GET
+            // (s3_storage_client.go:436): a bare 404 maps to "NotFound"
+            // and stays a generic error, as it does here.
+            SdkError::ServiceError(ref se) if se.err().is_no_such_key() => {
                 RemoteStorageError::ObjectNotFound(format!("{}/{}", loc.bucket, key))
-            } else {
-                RemoteStorageError::Other(format!("s3 get object: {}", e))
             }
+            e => RemoteStorageError::Other(format!("s3 get object: {}", DisplayErrorContext(&e))),
         })?;
 
         let data = resp
@@ -108,7 +125,9 @@ impl RemoteStorageClient for S3RemoteStorageClient {
             .body(ByteStream::from(data.to_vec()))
             .send()
             .await
-            .map_err(|e| RemoteStorageError::Other(format!("s3 put object: {}", e)))?;
+            .map_err(|e| {
+                RemoteStorageError::Other(format!("s3 put object: {}", DisplayErrorContext(&e)))
+            })?;
 
         Ok(RemoteEntry {
             size: data.len() as i64,
@@ -134,13 +153,18 @@ impl RemoteStorageClient for S3RemoteStorageClient {
             .key(key)
             .send()
             .await
-            .map_err(|e| {
-                let msg = format!("{}", e);
-                if msg.contains("404") || msg.contains("NotFound") {
+            .map_err(|e| match e {
+                // Go checks only the raw HTTP status on HEAD
+                // (s3_storage_client.go:373): a HEAD response carries no
+                // error body, so a 404 is not-found whatever code the SDK
+                // assigns, and a non-404 is not.
+                SdkError::ServiceError(ref se) if se.raw().status().as_u16() == 404 => {
                     RemoteStorageError::ObjectNotFound(format!("{}/{}", loc.bucket, key))
-                } else {
-                    RemoteStorageError::Other(format!("s3 head object: {}", e))
                 }
+                e => RemoteStorageError::Other(format!(
+                    "s3 head object: {}",
+                    DisplayErrorContext(&e)
+                )),
             })?;
 
         Ok(RemoteEntry {
@@ -160,18 +184,17 @@ impl RemoteStorageClient for S3RemoteStorageClient {
             .key(key)
             .send()
             .await
-            .map_err(|e| RemoteStorageError::Other(format!("s3 delete object: {}", e)))?;
+            .map_err(|e| {
+                RemoteStorageError::Other(format!("s3 delete object: {}", DisplayErrorContext(&e)))
+            })?;
 
         Ok(())
     }
 
     async fn list_buckets(&self) -> Result<Vec<String>, RemoteStorageError> {
-        let resp = self
-            .client
-            .list_buckets()
-            .send()
-            .await
-            .map_err(|e| RemoteStorageError::Other(format!("s3 list buckets: {}", e)))?;
+        let resp = self.client.list_buckets().send().await.map_err(|e| {
+            RemoteStorageError::Other(format!("s3 list buckets: {}", DisplayErrorContext(&e)))
+        })?;
 
         Ok(resp
             .buckets()
@@ -184,3 +207,178 @@ impl RemoteStorageClient for S3RemoteStorageClient {
         &self.conf
     }
 }
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+    use aws_sdk_s3::config::http::{HttpRequest, HttpResponse};
+    use aws_sdk_s3::config::retry::RetryConfig;
+    use aws_sdk_s3::config::{HttpClient, RuntimeComponents};
+    use aws_sdk_s3::primitives::SdkBody;
+    use aws_smithy_runtime_api::client::http::{
+        HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector,
+    };
+    use aws_smithy_runtime_api::http::StatusCode;
+
+    /// An SDK HTTP client that answers every request with one canned response,
+    /// so the error-mapping paths can be exercised without a network or a
+    /// running S3 server.
+    #[derive(Debug, Clone)]
+    pub(crate) struct CannedResponse {
+        pub(crate) status: u16,
+        pub(crate) body: &'static str,
+    }
+
+    impl HttpConnector for CannedResponse {
+        fn call(&self, _request: HttpRequest) -> HttpConnectorFuture {
+            let status = StatusCode::try_from(self.status).expect("valid HTTP status");
+            HttpConnectorFuture::ready(Ok(HttpResponse::new(status, SdkBody::from(self.body))))
+        }
+    }
+
+    impl HttpClient for CannedResponse {
+        fn http_connector(
+            &self,
+            _settings: &HttpConnectorSettings,
+            _components: &RuntimeComponents,
+        ) -> SharedHttpConnector {
+            SharedHttpConnector::new(self.clone())
+        }
+    }
+
+    fn client_with(status: u16, body: &'static str) -> S3RemoteStorageClient {
+        let config = S3RemoteStorageClient::config_builder(
+            "AKIATEST",
+            "secret",
+            "us-east-1",
+            "http://127.0.0.1:1",
+            true,
+        )
+        .http_client(CannedResponse { status, body })
+        .retry_config(RetryConfig::disabled())
+        .build();
+        S3RemoteStorageClient {
+            client: Client::from_conf(config),
+            conf: RemoteConf::default(),
+        }
+    }
+
+    fn location() -> RemoteStorageLocation {
+        RemoteStorageLocation {
+            name: "remote".to_string(),
+            bucket: "bucket".to_string(),
+            path: "/dir/missing".to_string(),
+            ..Default::default()
+        }
+    }
+
+    pub(crate) const NO_SUCH_KEY: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
+<Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><Key>dir/missing</Key></Error>"#;
+
+    const NOT_FOUND_BODY: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
+<Error><Code>NotFound</Code><Message>Not Found</Message></Error>"#;
+
+    const ACCESS_DENIED: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
+<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>"#;
+
+    #[tokio::test]
+    async fn get_no_such_key_is_object_not_found() {
+        let err = client_with(404, NO_SUCH_KEY)
+            .read_file(&location(), 0, 0)
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, RemoteStorageError::ObjectNotFound(path) if path == "bucket/dir/missing"),
+            "expected ObjectNotFound, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn get_bare_404_is_not_object_not_found() {
+        // Go compares codes, not statuses, on GET: a body-less 404 stays generic.
+        let err = client_with(404, "")
+            .read_file(&location(), 0, 0)
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(err, RemoteStorageError::Other(_)),
+            "expected Other, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn head_404_is_object_not_found() {
+        let err = client_with(404, "")
+            .stat_file(&location())
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, RemoteStorageError::ObjectNotFound(path) if path == "bucket/dir/missing"),
+            "expected ObjectNotFound, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn head_404_with_foreign_error_body_is_object_not_found() {
+        // The raw status check makes a 404 not-found whatever body it carries.
+        let err = client_with(404, NO_SUCH_KEY)
+            .stat_file(&location())
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(err, RemoteStorageError::ObjectNotFound(_)),
+            "expected ObjectNotFound, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn head_not_found_code_on_a_non_404_status_is_not_object_not_found() {
+        // A NotFound body on a non-404 status stays an error, as in Go.
+        let err = client_with(400, NOT_FOUND_BODY)
+            .stat_file(&location())
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, RemoteStorageError::Other(msg) if msg.contains("NotFound")),
+            "expected Other naming the code, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn get_access_denied_keeps_service_error_code() {
+        let err = client_with(403, ACCESS_DENIED)
+            .read_file(&location(), 0, 0)
+            .await
+            .unwrap_err();
+        let msg = err.to_string();
+        assert!(
+            matches!(err, RemoteStorageError::Other(_)),
+            "expected Other, got {err:?}"
+        );
+        assert!(
+            msg.contains("AccessDenied"),
+            "message should carry the S3 error code, got: {msg}"
+        );
+        assert!(
+            !msg.ends_with("service error"),
+            "message should not be the bare SdkError Display, got: {msg}"
+        );
+    }
+
+    #[tokio::test]
+    async fn head_access_denied_keeps_service_error_code() {
+        let err = client_with(403, ACCESS_DENIED)
+            .stat_file(&location())
+            .await
+            .unwrap_err();
+        let msg = err.to_string();
+        assert!(
+            matches!(err, RemoteStorageError::Other(_)),
+            "expected Other, got {err:?}"
+        );
+        assert!(
+            msg.contains("AccessDenied"),
+            "message should carry the S3 error code, got: {msg}"
+        );
+    }
+}

seaweed-volume/src/remote_storage/s3_tier.rs

@@ -8,7 +8,11 @@ use std::future::Future;
 use std::sync::{Arc, OnceLock, RwLock};
 
 use aws_sdk_s3::Client;
+use aws_sdk_s3::config::http::HttpResponse;
 use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
+use aws_sdk_s3::error::{DisplayErrorContext, SdkError};
+use aws_sdk_s3::operation::get_object::GetObjectError;
+use aws_sdk_s3::operation::head_object::HeadObjectError;
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
 use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
 use tokio::sync::Semaphore;
@@ -16,6 +20,53 @@ use tokio::sync::Semaphore;
 /// Concurrency limit for multipart upload/download (matches Go's s3manager).
 const CONCURRENCY: usize = 5;
 
+/// A tier transfer failure. The variant is what callers match on; the
+/// message is the operator-facing text.
+#[derive(Debug, thiserror::Error)]
+pub enum TierError {
+    /// The remote object does not exist.
+    #[error("{0}")]
+    NotFound(String),
+    /// An S3 request or a local file operation failed.
+    #[error("{0}")]
+    Io(String),
+    /// The tier I/O runtime could not be built or dropped the task.
+    #[error("{0}")]
+    RuntimeUnavailable(String),
+    /// The progress callback asked to stop.
+    #[error("{0}")]
+    Aborted(String),
+}
+
+// Not-found rules as in remote_storage/s3.rs: HEAD by the raw 404 status,
+// GET by the NoSuchKey code only.
+fn head_object_error(key: &str, e: SdkError<HeadObjectError, HttpResponse>) -> TierError {
+    let message = format!("failed to head object {}: {}", key, DisplayErrorContext(&e));
+    match e {
+        SdkError::ServiceError(ref se) if se.raw().status().as_u16() == 404 => {
+            TierError::NotFound(message)
+        }
+        _ => TierError::Io(message),
+    }
+}
+
+fn get_object_error(
+    key: &str,
+    range: &str,
+    e: SdkError<GetObjectError, HttpResponse>,
+) -> TierError {
+    let message = format!(
+        "failed to get object {} range {}: {}",
+        key,
+        range,
+        DisplayErrorContext(&e)
+    );
+    match e {
+        SdkError::ServiceError(ref se) if se.err().is_no_such_key() => TierError::NotFound(message),
+        _ => TierError::Io(message),
+    }
+}
+
 /// Configuration for an S3 tier backend.
 #[derive(Debug, Clone)]
 pub struct S3TierConfig {
@@ -89,7 +140,7 @@ impl S3TierBackend {
         &self,
         file_path: &str,
         progress_fn: F,
-    ) -> Result<(String, u64), String>
+    ) -> Result<(String, u64), TierError>
     where
         F: FnMut(i64, f32) -> Result<(), String> + Send + Sync + 'static,
     {
@@ -97,7 +148,7 @@ impl S3TierBackend {
 
         let metadata = tokio::fs::metadata(file_path)
             .await
-            .map_err(|e| format!("failed to stat file {}: {}", file_path, e))?;
+            .map_err(|e| TierError::Io(format!("failed to stat file {}: {}", file_path, e)))?;
         let file_size = metadata.len();
 
         // Calculate part size: start at 64MB, scale up for very large files (matches Go)
@@ -119,11 +170,16 @@ impl S3TierBackend {
             )
             .send()
             .await
-            .map_err(|e| format!("failed to create multipart upload: {}", e))?;
+            .map_err(|e| {
+                TierError::Io(format!(
+                    "failed to create multipart upload: {}",
+                    DisplayErrorContext(&e)
+                ))
+            })?;
 
         let upload_id = create_resp
             .upload_id()
-            .ok_or_else(|| "no upload_id in multipart upload response".to_string())?
+            .ok_or_else(|| TierError::Io("no upload_id in multipart upload response".to_string()))?
             .to_string();
 
         // Build list of (part_number, offset, size) for all parts
@@ -159,19 +215,21 @@ impl S3TierBackend {
                 let _permit = sem
                     .acquire()
                     .await
-                    .map_err(|e| format!("semaphore error: {}", e))?;
+                    .map_err(|e| TierError::Io(format!("semaphore error: {}", e)))?;
 
                 // Read this part's data from the file at the correct offset
                 let mut file = tokio::fs::File::open(&fp)
                     .await
-                    .map_err(|e| format!("failed to open file {}: {}", fp, e))?;
+                    .map_err(|e| TierError::Io(format!("failed to open file {}: {}", fp, e)))?;
                 file.seek(std::io::SeekFrom::Start(off))
                     .await
-                    .map_err(|e| format!("failed to seek to offset {}: {}", off, e))?;
+                    .map_err(|e| {
+                        TierError::Io(format!("failed to seek to offset {}: {}", off, e))
+                    })?;
                 let mut buf = vec![0u8; size];
-                file.read_exact(&mut buf)
-                    .await
-                    .map_err(|e| format!("failed to read file at offset {}: {}", off, e))?;
+                file.read_exact(&mut buf).await.map_err(|e| {
+                    TierError::Io(format!("failed to read file at offset {}: {}", off, e))
+                })?;
 
                 let upload_part_resp = client
                     .upload_part()
@@ -183,7 +241,12 @@ impl S3TierBackend {
                     .send()
                     .await
                     .map_err(|e| {
-                        format!("failed to upload part {} at offset {}: {}", pn, off, e)
+                        TierError::Io(format!(
+                            "failed to upload part {} at offset {}: {}",
+                            pn,
+                            off,
+                            DisplayErrorContext(&e)
+                        ))
                     })?;
 
                 let e_tag = upload_part_resp.e_tag().unwrap_or_default().to_string();
@@ -202,9 +265,9 @@ impl S3TierBackend {
                     };
                     (guard.1)(uploaded as i64, pct)
                 };
-                progress_result?;
+                progress_result.map_err(TierError::Aborted)?;
 
-                Ok::<_, String>(
+                Ok::<_, TierError>(
                     CompletedPart::builder()
                         .e_tag(e_tag)
                         .part_number(pn)
@@ -219,7 +282,7 @@ impl S3TierBackend {
             for handle in handles {
                 let part = handle
                     .await
-                    .map_err(|e| format!("upload task panicked: {}", e))??;
+                    .map_err(|e| TierError::Io(format!("upload task panicked: {}", e)))??;
                 completed_parts.push(part);
             }
 
@@ -236,9 +299,14 @@ impl S3TierBackend {
                 .multipart_upload(completed_upload)
                 .send()
                 .await
-                .map_err(|e| format!("failed to complete multipart upload: {}", e))?;
+                .map_err(|e| {
+                    TierError::Io(format!(
+                        "failed to complete multipart upload: {}",
+                        DisplayErrorContext(&e)
+                    ))
+                })?;
 
-            Ok::<(), String>(())
+            Ok::<(), TierError>(())
         }
         .await;
 
@@ -281,7 +349,7 @@ impl S3TierBackend {
         dest_path: &str,
         key: &str,
         progress_fn: F,
-    ) -> Result<u64, String>
+    ) -> Result<u64, TierError>
     where
         F: FnMut(i64, f32) -> Result<(), String> + Send + Sync + 'static,
     {
@@ -293,7 +361,7 @@ impl S3TierBackend {
             .key(key)
             .send()
             .await
-            .map_err(|e| format!("failed to head object {}: {}", key, e))?;
+            .map_err(|e| head_object_error(key, e))?;
 
         let file_size = head_resp.content_length().unwrap_or(0) as u64;
 
@@ -305,10 +373,12 @@ impl S3TierBackend {
                 .truncate(true)
                 .open(dest_path)
                 .await
-                .map_err(|e| format!("failed to open dest file {}: {}", dest_path, e))?;
+                .map_err(|e| {
+                    TierError::Io(format!("failed to open dest file {}: {}", dest_path, e))
+                })?;
             file.set_len(file_size)
                 .await
-                .map_err(|e| format!("failed to set file length: {}", e))?;
+                .map_err(|e| TierError::Io(format!("failed to set file length: {}", e)))?;
         }
 
         let part_size: u64 = 64 * 1024 * 1024;
@@ -344,7 +414,7 @@ impl S3TierBackend {
                 let _permit = sem
                     .acquire()
                     .await
-                    .map_err(|e| format!("semaphore error: {}", e))?;
+                    .map_err(|e| TierError::Io(format!("semaphore error: {}", e)))?;
 
                 let end = off + size - 1;
                 let range = format!("bytes={}-{}", off, end);
@@ -356,13 +426,13 @@ impl S3TierBackend {
                     .range(&range)
                     .send()
                     .await
-                    .map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
+                    .map_err(|e| get_object_error(&key, &range, e))?;
 
                 let body = get_resp
                     .body
                     .collect()
                     .await
-                    .map_err(|e| format!("failed to read body: {}", e))?;
+                    .map_err(|e| TierError::Io(format!("failed to read body: {}", e)))?;
                 let bytes = body.into_bytes();
 
                 // Write at the correct offset (like Go's WriteAt)
@@ -370,13 +440,17 @@ impl S3TierBackend {
                     .write(true)
                     .open(&dp)
                     .await
-                    .map_err(|e| format!("failed to open dest file {}: {}", dp, e))?;
+                    .map_err(|e| {
+                        TierError::Io(format!("failed to open dest file {}: {}", dp, e))
+                    })?;
                 file.seek(std::io::SeekFrom::Start(off))
                     .await
-                    .map_err(|e| format!("failed to seek to offset {}: {}", off, e))?;
+                    .map_err(|e| {
+                        TierError::Io(format!("failed to seek to offset {}: {}", off, e))
+                    })?;
                 file.write_all(&bytes)
                     .await
-                    .map_err(|e| format!("failed to write to {}: {}", dp, e))?;
+                    .map_err(|e| TierError::Io(format!("failed to write to {}: {}", dp, e)))?;
 
                 // Report progress. The lock is released before the result is
                 // propagated so an aborting callback cannot poison the mutex
@@ -392,9 +466,9 @@ impl S3TierBackend {
                     };
                     (guard.1)(downloaded as i64, pct)
                 };
-                progress_result?;
+                progress_result.map_err(TierError::Aborted)?;
 
-                Ok::<_, String>(())
+                Ok::<_, TierError>(())
             }));
         }
 
@@ -402,7 +476,7 @@ impl S3TierBackend {
         for handle in handles {
             handle
                 .await
-                .map_err(|e| format!("download task panicked: {}", e))??;
+                .map_err(|e| TierError::Io(format!("download task panicked: {}", e)))??;
         }
 
         // fsync the file so its content is durable before the caller trims the .vif
@@ -411,16 +485,21 @@ impl S3TierBackend {
             .write(true)
             .open(dest_path)
             .await
-            .map_err(|e| format!("failed to open {} for fsync: {}", dest_path, e))?;
+            .map_err(|e| TierError::Io(format!("failed to open {} for fsync: {}", dest_path, e)))?;
         synced
             .sync_all()
             .await
-            .map_err(|e| format!("failed to fsync {}: {}", dest_path, e))?;
+            .map_err(|e| TierError::Io(format!("failed to fsync {}: {}", dest_path, e)))?;
 
         Ok(file_size)
     }
 
-    pub async fn read_range(&self, key: &str, offset: u64, size: usize) -> Result<Vec<u8>, String> {
+    pub async fn read_range(
+        &self,
+        key: &str,
+        offset: u64,
+        size: usize,
+    ) -> Result<Vec<u8>, TierError> {
         let end = offset + (size as u64).saturating_sub(1);
         let range = format!("bytes={}-{}", offset, end);
         let resp = self
@@ -431,29 +510,35 @@ impl S3TierBackend {
             .range(&range)
             .send()
             .await
-            .map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
+            .map_err(|e| get_object_error(key, &range, e))?;
 
         let body = resp
             .body
             .collect()
             .await
-            .map_err(|e| format!("failed to read object {} body: {}", key, e))?;
+            .map_err(|e| TierError::Io(format!("failed to read object {} body: {}", key, e)))?;
         Ok(body.into_bytes().to_vec())
     }
 
     /// Delete a file from S3.
-    pub async fn delete_file(&self, key: &str) -> Result<(), String> {
+    pub async fn delete_file(&self, key: &str) -> Result<(), TierError> {
         self.client
             .delete_object()
             .bucket(&self.bucket)
             .key(key)
             .send()
             .await
-            .map_err(|e| format!("failed to delete object {}: {}", key, e))?;
+            .map_err(|e| {
+                TierError::Io(format!(
+                    "failed to delete object {}: {}",
+                    key,
+                    DisplayErrorContext(&e)
+                ))
+            })?;
         Ok(())
     }
 
-    pub fn delete_file_blocking(&self, key: &str) -> Result<(), String> {
+    pub fn delete_file_blocking(&self, key: &str) -> Result<(), TierError> {
         let client = self.client.clone();
         let bucket = self.bucket.clone();
         let key = key.to_string();
@@ -464,7 +549,13 @@ impl S3TierBackend {
                 .key(&key)
                 .send()
                 .await
-                .map_err(|e| format!("failed to delete object {}: {}", key, e))?;
+                .map_err(|e| {
+                    TierError::Io(format!(
+                        "failed to delete object {}: {}",
+                        key,
+                        DisplayErrorContext(&e)
+                    ))
+                })?;
             Ok(())
         })
     }
@@ -474,7 +565,7 @@ impl S3TierBackend {
         key: &str,
         offset: u64,
         size: usize,
-    ) -> Result<Vec<u8>, String> {
+    ) -> Result<Vec<u8>, TierError> {
         let client = self.client.clone();
         let bucket = self.bucket.clone();
         let key = key.to_string();
@@ -488,13 +579,12 @@ impl S3TierBackend {
                 .range(&range)
                 .send()
                 .await
-                .map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
+                .map_err(|e| get_object_error(&key, &range, e))?;
 
-            let body = resp
-                .body
-                .collect()
-                .await
-                .map_err(|e| format!("failed to read object {} body: {}", key, e))?;
+            let body =
+                resp.body.collect().await.map_err(|e| {
+                    TierError::Io(format!("failed to read object {} body: {}", key, e))
+                })?;
             Ok(body.into_bytes().to_vec())
         })
     }
@@ -555,18 +645,279 @@ pub fn global_s3_tier_registry() -> &'static RwLock<S3TierRegistry> {
     GLOBAL_S3_TIER_REGISTRY.get_or_init(|| RwLock::new(S3TierRegistry::new()))
 }
 
-fn block_on_tier_future<F, T>(future: F) -> Result<T, String>
-where
-    F: Future<Output = Result<T, String>> + Send + 'static,
-    T: Send + 'static,
-{
-    std::thread::spawn(move || {
-        let runtime = tokio::runtime::Builder::new_current_thread()
+/// The one process-wide runtime for tiered-S3 I/O issued from synchronous
+/// storage code. A per-call runtime tore down the SDK's pooled connections
+/// after every 64 KiB chunk, re-dialing TLS per read; a long-lived runtime
+/// keeps the pool warm.
+///
+/// Built on first use. A build failure is returned, not cached or panicked:
+/// callers sit inside `Volume::destroy` and needle reads, whose own error
+/// paths must run, and a later call may succeed.
+static TIER_RUNTIME: std::sync::Mutex<Option<tokio::runtime::Runtime>> =
+    std::sync::Mutex::new(None);
+
+fn tier_handle() -> Result<tokio::runtime::Handle, TierError> {
+    let mut slot = TIER_RUNTIME
+        .lock()
+        .unwrap_or_else(|poisoned| poisoned.into_inner());
+    if slot.is_none() {
+        let runtime = tokio::runtime::Builder::new_multi_thread()
+            .worker_threads(2)
+            .thread_name("tier-io")
             .enable_all()
             .build()
-            .map_err(|e| format!("failed to build tokio runtime: {}", e))?;
-        runtime.block_on(future)
-    })
-    .join()
-    .map_err(|_| "tier runtime thread panicked".to_string())?
+            .map_err(|e| {
+                TierError::RuntimeUnavailable(format!(
+                    "failed to build the tier I/O tokio runtime: {}",
+                    e
+                ))
+            })?;
+        *slot = Some(runtime);
+    }
+    Ok(slot.as_ref().expect("just initialised").handle().clone())
+}
+
+/// Run `future` on the tier runtime and block the calling thread until it
+/// finishes. The caller may be a worker of *another* tokio runtime, so this
+/// waits on a channel rather than `Handle::block_on`, which panics when
+/// called from inside any runtime context.
+fn block_on_tier_future<F, T>(future: F) -> Result<T, TierError>
+where
+    F: Future<Output = Result<T, TierError>> + Send + 'static,
+    T: Send + 'static,
+{
+    let handle = tier_handle()?;
+    let task = handle.spawn(future);
+    let (tx, rx) = std::sync::mpsc::sync_channel(1);
+    handle.spawn(async move {
+        // The receiver only goes away if the caller was unwound; nothing to
+        // report then.
+        let _ = tx.send(task.await);
+    });
+    match rx.recv() {
+        Ok(Ok(result)) => result,
+        Ok(Err(join_error)) => Err(describe_join_error(join_error)),
+        Err(_) => Err(TierError::RuntimeUnavailable(
+            "tier I/O runtime dropped the task before it finished".to_string(),
+        )),
+    }
+}
+
+/// Turn a `JoinError` into a message that keeps the panic payload, so an
+/// SDK panic surfaces as "boom" rather than a fixed "thread panicked".
+fn describe_join_error(join_error: tokio::task::JoinError) -> TierError {
+    if join_error.is_panic() {
+        let payload = join_error.into_panic();
+        let message = if let Some(s) = payload.downcast_ref::<&str>() {
+            (*s).to_string()
+        } else if let Some(s) = payload.downcast_ref::<String>() {
+            s.clone()
+        } else {
+            "non-string panic payload".to_string()
+        };
+        TierError::Io(format!("tier I/O task panicked: {}", message))
+    } else {
+        TierError::RuntimeUnavailable(format!("tier I/O task failed: {}", join_error))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::remote_storage::s3::tests::{CannedResponse, NO_SUCH_KEY};
+    use std::collections::HashSet;
+    use tokio::runtime::Handle;
+
+    fn probe() -> Result<(tokio::runtime::Id, Option<String>), TierError> {
+        block_on_tier_future(async {
+            Ok((
+                Handle::current().id(),
+                std::thread::current().name().map(str::to_string),
+            ))
+        })
+    }
+
+    #[test]
+    fn block_on_tier_future_reuses_one_runtime() {
+        let (first_runtime, first_thread) = probe().expect("first call");
+        let (second_runtime, second_thread) = probe().expect("second call");
+        assert_eq!(
+            first_runtime, second_runtime,
+            "each call must run on the same long-lived tier runtime"
+        );
+        assert_eq!(first_thread.as_deref(), Some("tier-io"));
+        assert_eq!(second_thread.as_deref(), Some("tier-io"));
+
+        let mut runtimes = HashSet::new();
+        for _ in 0..20 {
+            let (id, _) = probe().expect("probe");
+            runtimes.insert(id);
+        }
+        assert_eq!(runtimes.len(), 1);
+    }
+
+    #[test]
+    fn block_on_tier_future_returns_the_value_and_the_error() {
+        assert_eq!(block_on_tier_future(async { Ok(7u32) }).unwrap(), 7);
+        let err = block_on_tier_future::<_, u32>(async { Err(TierError::NotFound("nope".into())) })
+            .unwrap_err();
+        assert!(
+            matches!(&err, TierError::NotFound(m) if m == "nope"),
+            "{err:?}"
+        );
+    }
+
+    #[test]
+    fn block_on_tier_future_works_from_a_std_thread() {
+        let (id, _) = std::thread::spawn(probe)
+            .join()
+            .expect("probe thread")
+            .expect("probe");
+        assert_eq!(id, tier_handle().expect("tier runtime").id());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn block_on_tier_future_works_from_spawn_blocking() {
+        let (id, _) = tokio::task::spawn_blocking(probe)
+            .await
+            .expect("spawn_blocking")
+            .expect("probe");
+        assert_eq!(id, tier_handle().expect("tier runtime").id());
+        assert_ne!(id, Handle::current().id());
+    }
+
+    // Called straight from another runtime's async context: the case that
+    // would panic with `Handle::block_on` ("Cannot start a runtime from
+    // within a runtime").
+    #[tokio::test]
+    async fn block_on_tier_future_works_from_a_current_thread_runtime() {
+        let (id, _) = probe().expect("probe");
+        assert_eq!(id, tier_handle().expect("tier runtime").id());
+        assert_ne!(id, Handle::current().id());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn block_on_tier_future_works_from_a_multi_thread_runtime_worker() {
+        let (id, _) = probe().expect("probe");
+        assert_eq!(id, tier_handle().expect("tier runtime").id());
+        assert_ne!(id, Handle::current().id());
+    }
+
+    #[test]
+    fn block_on_tier_future_reports_the_panic_payload() {
+        let err = block_on_tier_future::<_, ()>(async {
+            if std::hint::black_box(true) {
+                panic!("boom {}", 42);
+            }
+            Ok(())
+        })
+        .expect_err("a panicking future must be an error");
+        assert!(matches!(err, TierError::Io(_)), "got: {err:?}");
+        assert!(err.to_string().contains("boom 42"), "got: {err}");
+        assert!(err.to_string().contains("panicked"), "got: {err}");
+    }
+
+    #[test]
+    fn block_on_tier_future_reports_a_str_panic_payload() {
+        let err = block_on_tier_future::<_, ()>(async {
+            if std::hint::black_box(true) {
+                panic!("static boom");
+            }
+            Ok(())
+        })
+        .expect_err("a panicking future must be an error");
+        assert!(err.to_string().contains("static boom"), "got: {err}");
+    }
+
+    fn backend_answering(status: u16, body: &'static str) -> S3TierBackend {
+        let config = aws_sdk_s3::Config::builder()
+            .behavior_version(BehaviorVersion::latest())
+            .region(Region::new("us-east-1"))
+            .credentials_provider(Credentials::new("AKIATEST", "secret", None, None, "test"))
+            .endpoint_url("http://127.0.0.1:1")
+            .force_path_style(true)
+            .http_client(CannedResponse { status, body })
+            .retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
+            .build();
+        S3TierBackend {
+            client: Client::from_conf(config),
+            bucket: "bucket".to_string(),
+            storage_class: "STANDARD".to_string(),
+        }
+    }
+
+    #[tokio::test]
+    async fn download_head_404_is_not_found() {
+        let tmp = tempfile::tempdir().unwrap();
+        let dest = tmp.path().join("1.dat");
+        let err = backend_answering(404, "")
+            .download_file(dest.to_str().unwrap(), "missing", |_, _| Ok(()))
+            .await
+            .unwrap_err();
+        assert!(matches!(err, TierError::NotFound(_)), "{err:?}");
+        assert!(
+            err.to_string()
+                .starts_with("failed to head object missing: "),
+            "{err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn download_head_403_is_io() {
+        let tmp = tempfile::tempdir().unwrap();
+        let dest = tmp.path().join("1.dat");
+        let err = backend_answering(403, "")
+            .download_file(dest.to_str().unwrap(), "denied", |_, _| Ok(()))
+            .await
+            .unwrap_err();
+        assert!(matches!(err, TierError::Io(_)), "{err:?}");
+    }
+
+    #[tokio::test]
+    async fn read_range_no_such_key_is_not_found() {
+        let err = backend_answering(404, NO_SUCH_KEY)
+            .read_range("missing", 0, 8)
+            .await
+            .unwrap_err();
+        assert!(matches!(err, TierError::NotFound(_)), "{err:?}");
+        assert!(
+            err.to_string()
+                .starts_with("failed to get object missing range bytes=0-7: "),
+            "{err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn read_range_bare_404_is_io() {
+        // As in Go, GET is not-found by the NoSuchKey code, not the status.
+        let err = backend_answering(404, "")
+            .read_range("missing", 0, 8)
+            .await
+            .unwrap_err();
+        assert!(matches!(err, TierError::Io(_)), "{err:?}");
+    }
+
+    #[test]
+    fn read_range_blocking_no_such_key_is_not_found() {
+        let err = backend_answering(404, NO_SUCH_KEY)
+            .read_range_blocking("missing", 0, 8)
+            .unwrap_err();
+        assert!(matches!(err, TierError::NotFound(_)), "{err:?}");
+    }
+
+    #[test]
+    fn backend_name_to_type_id_splits_on_dot() {
+        assert_eq!(
+            backend_name_to_type_id("s3"),
+            ("s3".to_string(), "default".to_string())
+        );
+        assert_eq!(
+            backend_name_to_type_id("s3.eu"),
+            ("s3".to_string(), "eu".to_string())
+        );
+        assert_eq!(
+            backend_name_to_type_id("s3.a.b"),
+            (String::new(), String::new())
+        );
+    }
 }

seaweed-volume/src/server/handlers.rs

@@ -21,6 +21,7 @@ use super::volume_server::{VolumeServerState, normalize_outgoing_http_url, to_ht
 use crate::config::ReadMode;
 use crate::metrics;
 use crate::pb::volume_server_pb;
+use crate::storage::needle::CRC;
 use crate::storage::needle::needle::Needle;
 use crate::storage::types::*;
 
@@ -142,6 +143,78 @@ const STREAMING_THRESHOLD: u32 = 1024 * 1024; // 1 MB
 /// Default chunk size for streaming reads from the dat file.
 const DEFAULT_STREAMING_CHUNK_SIZE: usize = 64 * 1024; // 64 KB
 
+/// Whether a GET/HEAD can be answered from the needle meta and the data file.
+#[derive(Clone, Copy)]
+struct SourceReadRequest {
+    is_head: bool,
+    has_range: bool,
+    has_image_ops: bool,
+    bypass_cm: bool,
+}
+
+impl SourceReadRequest {
+    /// The payload can be served as stored.
+    fn direct(&self, n: &Needle) -> bool {
+        !n.is_compressed() && !(n.is_chunk_manifest() && !self.bypass_cm) && !self.has_image_ops
+    }
+
+    /// HEAD, a range, or a stream.
+    fn served_from_source(&self, n: &Needle) -> bool {
+        self.is_head || (self.direct(n) && (self.has_range || n.data_size > STREAMING_THRESHOLD))
+    }
+
+    /// Meta first only if the needle could be served from source; its data
+    /// size is below its index size.
+    fn try_meta_first(&self, size: Size) -> bool {
+        self.is_head
+            || (!self.has_image_ops && (self.has_range || size.0 > STREAMING_THRESHOLD as i32))
+    }
+}
+
+/// Blocking part of a GET/HEAD: the store guard is dropped before any needle
+/// I/O, so a slow or tiered read cannot park store writers. `Ok(None)` is a
+/// cookie mismatch.
+fn read_needle_for_get(
+    state: &VolumeServerState,
+    vid: VolumeId,
+    needle_id: NeedleId,
+    cookie: Cookie,
+    read_deleted: bool,
+    request: SourceReadRequest,
+) -> Result<
+    Option<(Needle, Option<crate::storage::volume::NeedleStreamInfo>)>,
+    crate::storage::volume::VolumeError,
+> {
+    let mut plan = state
+        .store
+        .read()
+        .unwrap()
+        .needle_read_plan(vid, needle_id, read_deleted)?;
+    let blank = || Needle {
+        id: needle_id,
+        cookie,
+        ..Needle::default()
+    };
+    let mut n = blank();
+    if request.try_meta_first(plan.size()) {
+        plan.read_meta(&mut n)?;
+        if n.cookie != cookie {
+            return Ok(None);
+        }
+        if request.served_from_source(&n) {
+            let info = plan.into_stream_info(&n);
+            return Ok(Some((n, Some(info))));
+        }
+        // Compressed or a chunk manifest: the payload is needed after all.
+        n = blank();
+    }
+    plan.read_full(&mut n)?;
+    if n.cookie != cookie {
+        return Ok(None);
+    }
+    Ok(Some((n, None)))
+}
+
 /// A body that streams needle data from the dat file in chunks using pread,
 /// avoiding loading the entire payload into memory at once.
 struct StreamingBody {
@@ -167,6 +240,10 @@ struct StreamingBody {
     /// Compaction revision at the time of the initial read; if the volume's revision
     /// changes between chunks, the needle may have moved and we must re-lookup its offset.
     compaction_revision: u16,
+    /// Checksum stored in the needle tail.
+    expected_checksum: u32,
+    /// Running checksum over the emitted data.
+    crc: CRC,
 }
 
 impl http_body::Body for StreamingBody {
@@ -187,7 +264,32 @@ impl http_body::Body for StreamingBody {
                         match result {
                             Ok(Ok(chunk)) => {
                                 let len = chunk.len();
+                                self.crc = self.crc.update(&chunk);
                                 self.pos += len;
+                                // The last frame is held back until the checksum verifies
+                                // (Go parity: readNeedleDataInto). On a mismatch it is
+                                // never emitted, so the response ends short of the declared
+                                // Content-Length and the reader sees a failed transfer.
+                                if self.pos >= self.data_size as usize {
+                                    let ok = self.expected_checksum == self.crc.0
+                                        || self.expected_checksum == self.crc.legacy_value();
+                                    if !ok {
+                                        metrics::HANDLER_COUNTER
+                                            .with_label_values(&[metrics::ERROR_CRC])
+                                            .inc();
+                                        return std::task::Poll::Ready(Some(Err(
+                                            std::io::Error::new(
+                                                std::io::ErrorKind::InvalidData,
+                                                format!(
+                                                    "needle data checksum {} expected {} for needle {}",
+                                                    self.crc.0,
+                                                    self.expected_checksum,
+                                                    self.needle_id
+                                                ),
+                                            ),
+                                        )));
+                                    }
+                                }
                                 return std::task::Poll::Ready(Some(Ok(http_body::Frame::data(
                                     chunk,
                                 ))));
@@ -211,6 +313,9 @@ impl http_body::Body for StreamingBody {
             let relookup_result = {
                 let store = self.server_state.store.read().unwrap();
                 if let Some((_, vol)) = store.find_volume(self.volume_id) {
+                    if let Some(e) = vol.unavailable_error() {
+                        return std::task::Poll::Ready(Some(Err(std::io::Error::other(e))));
+                    }
                     if vol.super_block.compaction_revision != self.compaction_revision {
                         // Compaction occurred — re-lookup the needle's data offset
                         Some(vol.re_lookup_needle_data_offset(self.needle_id))
@@ -845,7 +950,7 @@ async fn proxy_request(
         if name.as_str().eq_ignore_ascii_case("server") {
             continue;
         }
-        response_headers.insert(name.clone(), value.clone());
+        response_headers.append(name.clone(), value.clone());
     }
 
     // Stream the proxy response body instead of buffering it entirely
@@ -1079,11 +1184,7 @@ async fn get_or_head_handler_inner(
 
     // Read needle — branching between regular volume and EC volume paths.
     // EC volumes always do a full read (no streaming/meta-only).
-    let mut n = Needle {
-        id: needle_id,
-        cookie,
-        ..Needle::default()
-    };
+    let n: Needle;
 
     let read_deleted = query.read_deleted.as_deref() == Some("true");
     let has_range = headers.contains_key(header::RANGE);
@@ -1160,24 +1261,49 @@ async fn get_or_head_handler_inner(
         can_handle_range_from_source = false;
     } else {
         // ---- Regular volume read path (with streaming support) ----
+        bypass_cm = query.cm.as_deref() == Some("false");
+        track_download = download_guard.is_some();
+        let request_kind = SourceReadRequest {
+            is_head: method == Method::HEAD,
+            has_range,
+            has_image_ops,
+            bypass_cm,
+        };
 
-        // Try meta-only read first for potential streaming
-        let store = state.store.read().unwrap();
-        let si_result = store.read_volume_needle_stream_info(vid, &mut n, read_deleted);
-        stream_info = match si_result {
-            Ok(info) => Some(info),
-            Err(crate::storage::volume::VolumeError::StreamingUnsupported) => None,
-            Err(crate::storage::volume::VolumeError::NotFound) => {
+        let read_state = state.clone();
+        let read = tokio::task::spawn_blocking(move || {
+            read_needle_for_get(
+                &read_state,
+                vid,
+                needle_id,
+                cookie,
+                read_deleted,
+                request_kind,
+            )
+        })
+        .await;
+        (n, stream_info) = match read {
+            Ok(Ok(Some(found))) => found,
+            // Cookie mismatch
+            Ok(Ok(None)) => return StatusCode::NOT_FOUND.into_response(),
+            Ok(Err(
+                crate::storage::volume::VolumeError::NotFound
+                | crate::storage::volume::VolumeError::Deleted,
+            )) => {
                 metrics::HANDLER_COUNTER
                     .with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
                     .inc();
                 return StatusCode::NOT_FOUND.into_response();
             }
-            Err(crate::storage::volume::VolumeError::Deleted) => {
+            Ok(Err(e)) => {
                 metrics::HANDLER_COUNTER
-                    .with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
+                    .with_label_values(&[metrics::ERROR_GET_INTERNAL])
                     .inc();
-                return StatusCode::NOT_FOUND.into_response();
+                return (
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    format!("read error: {}", e),
+                )
+                    .into_response();
             }
             Err(e) => {
                 metrics::HANDLER_COUNTER
@@ -1190,19 +1316,9 @@ async fn get_or_head_handler_inner(
                     .into_response();
             }
         };
-        drop(store);
 
-        // Validate cookie
-        if n.cookie != cookie {
-            return StatusCode::NOT_FOUND.into_response();
-        }
-
-        bypass_cm = query.cm.as_deref() == Some("false");
-        track_download = download_guard.is_some();
-        let can_direct_source_read = stream_info.is_some()
-            && !n.is_compressed()
-            && !(n.is_chunk_manifest() && !bypass_cm)
-            && !has_image_ops;
+        // Stream info is only returned for a reply served from the data file.
+        let can_direct_source_read = stream_info.is_some() && request_kind.direct(&n);
 
         // Determine if we can stream (large, direct-source eligible, no range)
         can_stream = can_direct_source_read
@@ -1213,41 +1329,6 @@ async fn get_or_head_handler_inner(
         // Go uses meta-only reads for all HEAD requests, regardless of compression/chunked files.
         can_handle_head_from_meta = stream_info.is_some() && method == Method::HEAD;
         can_handle_range_from_source = can_direct_source_read && has_range;
-
-        // For chunk manifest or any non-streaming path, we need the full data.
-        // If we can't stream, do a full read now.
-        if !can_stream && !can_handle_head_from_meta && !can_handle_range_from_source {
-            // Re-read with full data
-            let mut n_full = Needle {
-                id: needle_id,
-                cookie,
-                ..Needle::default()
-            };
-            let store = state.store.read().unwrap();
-            match store.read_volume_needle_opt(vid, &mut n_full, read_deleted) {
-                Ok(count) => {
-                    if count < 0 {
-                        return StatusCode::NOT_FOUND.into_response();
-                    }
-                }
-                Err(crate::storage::volume::VolumeError::NotFound) => {
-                    return StatusCode::NOT_FOUND.into_response();
-                }
-                Err(crate::storage::volume::VolumeError::Deleted) => {
-                    return StatusCode::NOT_FOUND.into_response();
-                }
-                Err(e) => {
-                    return (
-                        StatusCode::INTERNAL_SERVER_ERROR,
-                        format!("read error: {}", e),
-                    )
-                        .into_response();
-                }
-            }
-            drop(store);
-            // Use the full needle from here (it has the same metadata + data)
-            n = n_full;
-        }
     }
 
     // Build ETag and Last-Modified BEFORE conditional checks and chunk manifest expansion
@@ -1509,6 +1590,8 @@ async fn get_or_head_handler_inner(
             volume_id: info.volume_id,
             needle_id: info.needle_id,
             compaction_revision: info.compaction_revision,
+            expected_checksum: info.checksum,
+            crc: CRC(0),
         };
 
         let body = Body::new(streaming);
@@ -1530,12 +1613,19 @@ async fn get_or_head_handler_inner(
         && let (Some(range_header), Some(info)) = (headers.get(header::RANGE), stream_info)
         && let Ok(range_str) = range_header.to_str()
     {
-        return handle_range_request_from_source(
-            range_str,
-            info,
-            response_headers,
-            track_download.then(|| state.clone()),
-        );
+        let range_str = range_str.to_string();
+        let tracking = track_download.then(|| state.clone());
+        return tokio::task::spawn_blocking(move || {
+            handle_range_request_from_source(&range_str, info, response_headers, tracking)
+        })
+        .await
+        .unwrap_or_else(|e| {
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                format!("range read error: {}", e),
+            )
+                .into_response()
+        });
     }
 
     // ---- Buffered path: small files, compressed, images, range requests ----
@@ -2661,8 +2751,15 @@ pub async fn post_handler(
     // sees it the same way the primary did.
     let fsync = form_value("fsync").as_deref() == Some("true");
 
+    // Go computes the checksum while building the needle (CreateNeedleFromRequest).
+    // The write queue takes the needle away, so the response fields that come
+    // from it are read here, before the write, rather than from a copy.
+    n.checksum = crate::storage::needle::crc::CRC::new(&n.data);
+    let needle_etag = n.etag();
+    let needle_has_name = n.has_name();
+
     let write_result = if let Some(wq) = state.write_queue.get() {
-        wq.submit(vid, n.clone(), fsync).await
+        wq.submit(vid, n, fsync).await
     } else {
         let mut store = state.store.write().unwrap();
         store.write_volume_needle(vid, &mut n, fsync)
@@ -2713,22 +2810,22 @@ pub async fn post_handler(
     let resp = match write_result {
         Ok((_offset, _size, is_unchanged)) => {
             if is_unchanged {
-                let etag = format!("\"{}\"", n.etag());
+                let etag = format!("\"{}\"", needle_etag);
                 (StatusCode::NO_CONTENT, [(header::ETAG, etag)]).into_response()
             } else {
                 // Go only includes contentMd5 when the client provided Content-MD5
                 let result = UploadResult {
-                    name: if n.has_name() {
+                    name: if needle_has_name {
                         filename.clone()
                     } else {
                         String::new()
                     },
                     size: original_data_size, // H3: use original size, not compressed
-                    etag: n.etag(),
+                    etag: needle_etag.clone(),
                     mime: mime_type.clone(),
                     content_md5: original_content_md5.clone(),
                 };
-                let etag = n.etag();
+                let etag = needle_etag;
                 let etag_header = if etag.starts_with('"') {
                     etag.clone()
                 } else {
@@ -4573,4 +4670,463 @@ mod tests {
 
         server.abort();
     }
+
+    /// Repeated target response headers (e.g. Set-Cookie) must all reach the
+    /// client, as Go's `w.Header().Add` does; only `Server` is dropped.
+    #[tokio::test]
+    async fn test_proxy_request_keeps_repeated_response_headers() {
+        use axum::{Router, routing::get};
+
+        let app = Router::new().route(
+            "/3,01637037d6",
+            get(|| async {
+                (
+                    axum::response::AppendHeaders([
+                        (header::SET_COOKIE, "a=1"),
+                        (header::SET_COOKIE, "b=2"),
+                        (header::SERVER, "target"),
+                    ]),
+                    "payload",
+                )
+            }),
+        );
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = listener.local_addr().unwrap();
+        let server = tokio::spawn(async move {
+            axum::serve(listener, app).await.unwrap();
+        });
+
+        let info = ProxyRequestInfo {
+            original_headers: HeaderMap::new(),
+            original_query: String::new(),
+            path: "/3,01637037d6".to_string(),
+            vid_str: "3".to_string(),
+            fid_str: "01637037d6".to_string(),
+        };
+        let target = VolumeLocation {
+            url: addr.to_string(),
+            public_url: String::new(),
+            grpc_port: 0,
+            read_only: false,
+            read_only_can_delete: false,
+        };
+
+        let response = proxy_request(&streaming_test_state(), &info, &target).await;
+        assert_eq!(response.status(), StatusCode::OK);
+        let cookies: Vec<_> = response
+            .headers()
+            .get_all(header::SET_COOKIE)
+            .iter()
+            .map(|v| v.to_str().unwrap().to_string())
+            .collect();
+        assert_eq!(cookies, vec!["a=1", "b=2"]);
+        assert!(response.headers().get(header::SERVER).is_none());
+        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+            .await
+            .unwrap();
+        assert_eq!(&body[..], b"payload");
+
+        server.abort();
+    }
+
+    fn streaming_test_state() -> Arc<VolumeServerState> {
+        use crate::storage::needle_map::NeedleMapKind;
+        use crate::storage::store::Store;
+        test_state_with_store(Store::new(NeedleMapKind::InMemory))
+    }
+
+    fn test_state_with_store(store: crate::storage::store::Store) -> Arc<VolumeServerState> {
+        use crate::security::{Guard, SigningKey};
+        use crate::server::volume_server::RuntimeMetricsConfig;
+        use std::sync::RwLock;
+        use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU32};
+
+        Arc::new(VolumeServerState {
+            store: RwLock::new(store),
+            guard: RwLock::new(Guard::new(
+                &[],
+                SigningKey(vec![]),
+                0,
+                SigningKey(vec![]),
+                0,
+            )),
+            is_stopping: RwLock::new(false),
+            maintenance: AtomicBool::new(false),
+            state_version: AtomicU32::new(0),
+            concurrent_upload_limit: 0,
+            concurrent_download_limit: 0,
+            inflight_upload_data_timeout: std::time::Duration::ZERO,
+            inflight_download_data_timeout: std::time::Duration::ZERO,
+            inflight_upload_bytes: AtomicI64::new(0),
+            inflight_download_bytes: AtomicI64::new(0),
+            upload_notify: tokio::sync::Notify::new(),
+            download_notify: tokio::sync::Notify::new(),
+            data_center: String::new(),
+            rack: String::new(),
+            file_size_limit_bytes: 0,
+            maintenance_byte_per_second: 0,
+            is_heartbeating: AtomicBool::new(false),
+            has_master: false,
+            pre_stop_seconds: 0,
+            volume_state_notify: tokio::sync::Notify::new(),
+            write_queue: std::sync::OnceLock::new(),
+            read_mode: crate::config::ReadMode::Local,
+            allow_untrusted_remote_endpoints: false,
+            master_url: String::new(),
+            master_urls: Vec::new(),
+            seed_master_set: std::collections::HashSet::new(),
+            current_master_url: tokio::sync::RwLock::new(String::new()),
+            self_url: String::new(),
+            http_client: reqwest::Client::new(),
+            outgoing_http_scheme: "http".to_string(),
+            outgoing_grpc_tls: None,
+            metrics_runtime: RwLock::new(RuntimeMetricsConfig::default()),
+            metrics_notify: tokio::sync::Notify::new(),
+            fix_jpg_orientation: false,
+            has_slow_read: false,
+            read_buffer_size_bytes: 4 * 1024 * 1024,
+            security_file: String::new(),
+            cli_white_list: vec![],
+            state_file_path: String::new(),
+        })
+    }
+
+    /// Go's readNeedleDataInto holds back the last chunk of a whole-needle read
+    /// until the checksum verifies; a streamed needle that fails its CRC must
+    /// likewise end short of the declared length instead of looking complete.
+    #[tokio::test]
+    async fn test_streaming_body_checksum_failure_drops_last_chunk() {
+        use futures::StreamExt;
+        use std::io::Write;
+
+        let data: Vec<u8> = (0..10_000u32).map(|i| (i % 251) as u8).collect();
+        let mut tmp = tempfile::NamedTempFile::new().unwrap();
+        tmp.write_all(&data).unwrap();
+        let file = tmp.reopen().unwrap();
+
+        let control = Arc::new(crate::storage::volume::DataFileAccessControl::default());
+        let mk_body = |expected_checksum: u32, file: &std::fs::File| StreamingBody {
+            source: crate::storage::volume::NeedleStreamSource::Local(file.try_clone().unwrap()),
+            data_offset: 0,
+            data_size: data.len() as u32,
+            pos: 0,
+            chunk_size: 4096,
+            data_file_access_control: control.clone(),
+            hold_read_lock_for_stream: true,
+            _held_read_lease: None,
+            pending: None,
+            state: None,
+            tracked_bytes: 0,
+            server_state: streaming_test_state(),
+            volume_id: VolumeId(1),
+            needle_id: NeedleId(7),
+            compaction_revision: 0,
+            expected_checksum,
+            crc: CRC(0),
+        };
+
+        // Healthy needle: every byte is delivered.
+        let body = Body::new(mk_body(CRC::new(&data).0, &file));
+        let mut stream = body.into_data_stream();
+        let mut got = 0usize;
+        while let Some(item) = stream.next().await {
+            got += item.expect("healthy stream errored").len();
+        }
+        assert_eq!(got, data.len());
+
+        // Corrupted needle: the last chunk fails the checksum and is never
+        // emitted — the body ends short of the data size.
+        let body = Body::new(mk_body(CRC::new(&data).0 ^ 0xff, &file));
+        let mut stream = body.into_data_stream();
+        let mut got = 0usize;
+        let mut saw_err = false;
+        while let Some(item) = stream.next().await {
+            match item {
+                Ok(bytes) => got += bytes.len(),
+                Err(e) => {
+                    assert!(
+                        e.to_string().contains("checksum"),
+                        "unexpected stream error: {e}"
+                    );
+                    saw_err = true;
+                }
+            }
+        }
+        assert!(saw_err, "corrupted stream must fail");
+        assert!(got < data.len(), "delivered {got} of {} bytes", data.len());
+    }
+
+    const TEST_COOKIE: u32 = 0x1234_5678;
+
+    /// State whose store holds volume 1 in `tmp`.
+    fn volume_test_state(tmp: &tempfile::TempDir) -> Arc<VolumeServerState> {
+        use crate::storage::needle_map::NeedleMapKind;
+        use crate::storage::store::Store;
+        use crate::storage::volume::VolumeSpec;
+
+        let dir = tmp.path().to_str().unwrap();
+        let mut store = Store::new(NeedleMapKind::InMemory);
+        store
+            .add_location(
+                dir,
+                dir,
+                10,
+                DiskType::HardDrive,
+                crate::config::MinFreeSpace::Percent(1.0),
+                Vec::new(),
+            )
+            .unwrap();
+        store
+            .add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
+            .unwrap();
+        test_state_with_store(store)
+    }
+
+    /// Write a needle to volume 1 and return its URL path.
+    fn put_test_needle(state: &VolumeServerState, id: u64, data: &[u8]) -> String {
+        put_test_needle_with(state, id, data, |_| {})
+    }
+
+    fn put_test_needle_with(
+        state: &VolumeServerState,
+        id: u64,
+        data: &[u8],
+        prepare: impl FnOnce(&mut Needle),
+    ) -> String {
+        let mut n = Needle {
+            id: NeedleId(id),
+            cookie: Cookie(TEST_COOKIE),
+            data: data.to_vec(),
+            data_size: data.len() as u32,
+            ..Needle::default()
+        };
+        prepare(&mut n);
+        state
+            .store
+            .write()
+            .unwrap()
+            .write_volume_needle(VolumeId(1), &mut n, false)
+            .unwrap();
+        format!("/1,{:x}{:08x}", id, TEST_COOKIE)
+    }
+
+    async fn send_read(
+        state: &Arc<VolumeServerState>,
+        method: Method,
+        path: &str,
+        range: Option<&str>,
+    ) -> (StatusCode, HeaderMap, Vec<u8>) {
+        use tower::ServiceExt;
+        let mut req = Request::builder().method(method).uri(path);
+        if let Some(range) = range {
+            req = req.header(header::RANGE, range);
+        }
+        let resp = super::super::volume_server::build_public_router(state.clone())
+            .oneshot(req.body(Body::empty()).unwrap())
+            .await
+            .unwrap();
+        let status = resp.status();
+        let headers = resp.headers().clone();
+        let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
+            .await
+            .unwrap()
+            .to_vec();
+        (status, headers, body)
+    }
+
+    /// A GET whose needle read is parked must not hold the store lock: a
+    /// writer, here a real append to the same volume, must get through.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_get_needle_read_runs_off_the_store_lock() {
+        use crate::storage::volume::needle_read_hook;
+        use std::sync::Mutex;
+        use std::sync::mpsc;
+        use std::time::Duration;
+
+        const ID: u64 = 0x6e7a_0001;
+        let tmp = tempfile::TempDir::new().unwrap();
+        let state = volume_test_state(&tmp);
+        let data = b"parked needle".to_vec();
+        let path = put_test_needle(&state, ID, &data);
+
+        let (entered_tx, entered_rx) = mpsc::channel::<()>();
+        let (release_tx, release_rx) = mpsc::channel::<()>();
+        let release_rx = Mutex::new(release_rx);
+        let _hook = needle_read_hook::register(NeedleId(ID), move |_| {
+            let _ = entered_tx.send(());
+            // Returns once released, or at once when the sender is gone.
+            let _ = release_rx.lock().unwrap().recv();
+        });
+
+        let get = tokio::spawn({
+            let state = state.clone();
+            async move { send_read(&state, Method::GET, &path, None).await }
+        });
+        tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(10)))
+            .await
+            .unwrap()
+            .expect("the GET never reached its needle read");
+
+        let try_write_ok = state.store.try_write().is_ok();
+        let (wrote_tx, wrote_rx) = mpsc::channel();
+        std::thread::spawn({
+            let state = state.clone();
+            move || {
+                put_test_needle(&state, ID + 1, b"written meanwhile");
+                let _ = wrote_tx.send(());
+            }
+        });
+        let write_done = tokio::task::spawn_blocking(move || {
+            wrote_rx.recv_timeout(Duration::from_secs(5)).is_ok()
+        })
+        .await
+        .unwrap();
+        drop(release_tx);
+
+        assert!(
+            try_write_ok,
+            "a parked GET read must not hold the store lock"
+        );
+        assert!(write_done, "an append must not wait for a parked GET read");
+        let (status, _, body) = get.await.unwrap();
+        assert_eq!(status, StatusCode::OK);
+        assert_eq!(body, data);
+    }
+
+    /// A small needle is read from disk once, not once for its meta and
+    /// again in full; a large one never reads its payload before streaming.
+    #[tokio::test]
+    async fn test_get_reads_small_needle_once_and_large_needle_meta_only() {
+        use crate::storage::volume::needle_read_hook;
+        use std::sync::Mutex;
+
+        const SMALL: u64 = 0x6e7a_0101;
+        const LARGE: u64 = 0x6e7a_0102;
+        let tmp = tempfile::TempDir::new().unwrap();
+        let state = volume_test_state(&tmp);
+        let small = b"small needle".to_vec();
+        let large: Vec<u8> = (0..(STREAMING_THRESHOLD as usize + 4096))
+            .map(|i| (i % 251) as u8)
+            .collect();
+        let small_path = put_test_needle(&state, SMALL, &small);
+        let large_path = put_test_needle(&state, LARGE, &large);
+
+        let small_reads = Arc::new(Mutex::new(Vec::new()));
+        let large_reads = Arc::new(Mutex::new(Vec::new()));
+        let _small_hook = needle_read_hook::register(NeedleId(SMALL), {
+            let reads = small_reads.clone();
+            move |len| reads.lock().unwrap().push(len)
+        });
+        let _large_hook = needle_read_hook::register(NeedleId(LARGE), {
+            let reads = large_reads.clone();
+            move |len| reads.lock().unwrap().push(len)
+        });
+
+        let (status, _, body) = send_read(&state, Method::GET, &small_path, None).await;
+        assert_eq!(status, StatusCode::OK);
+        assert_eq!(body, small);
+        let reads = small_reads.lock().unwrap().clone();
+        assert_eq!(reads.len(), 1, "reads of the small needle: {reads:?}");
+
+        let (status, _, body) = send_read(&state, Method::GET, &large_path, None).await;
+        assert_eq!(status, StatusCode::OK);
+        assert_eq!(body, large);
+        let reads = large_reads.lock().unwrap().clone();
+        assert!(
+            !reads.is_empty() && reads.iter().all(|&len| len < 64 * 1024),
+            "the large needle's payload must not be read before streaming: {reads:?}"
+        );
+    }
+
+    /// HEAD and ranged reads keep answering from the needle meta and the
+    /// data file, and a missing or wrong-cookie needle is still a 404.
+    #[tokio::test]
+    async fn test_get_head_range_and_not_found_after_single_read() {
+        const ID: u64 = 0x6e7a_0201;
+        let tmp = tempfile::TempDir::new().unwrap();
+        let state = volume_test_state(&tmp);
+        let data: Vec<u8> = (0..4096u32).map(|i| (i % 251) as u8).collect();
+        let path = put_test_needle(&state, ID, &data);
+
+        let (status, headers, body) = send_read(&state, Method::HEAD, &path, None).await;
+        assert_eq!(status, StatusCode::OK);
+        assert!(body.is_empty());
+        assert_eq!(headers[header::CONTENT_LENGTH], data.len().to_string());
+        assert!(headers.contains_key(header::ETAG));
+
+        let (status, headers, body) =
+            send_read(&state, Method::GET, &path, Some("bytes=10-19")).await;
+        assert_eq!(status, StatusCode::PARTIAL_CONTENT);
+        assert_eq!(body, &data[10..20]);
+        assert_eq!(
+            headers["Content-Range"],
+            format!("bytes 10-19/{}", data.len())
+        );
+
+        let wrong_cookie = format!("/1,{:x}{:08x}", ID, TEST_COOKIE ^ 1);
+        let (status, _, _) = send_read(&state, Method::GET, &wrong_cookie, None).await;
+        assert_eq!(status, StatusCode::NOT_FOUND);
+        let missing = format!("/1,{:x}{:08x}", ID + 1, TEST_COOKIE);
+        let (status, _, _) = send_read(&state, Method::GET, &missing, None).await;
+        assert_eq!(status, StatusCode::NOT_FOUND);
+    }
+
+    /// A large compressed needle cannot be streamed as stored: its meta is
+    /// read first, then the payload exactly once.
+    #[tokio::test]
+    async fn test_get_large_compressed_needle_reads_payload_once() {
+        use crate::storage::volume::needle_read_hook;
+        use std::sync::Mutex;
+
+        const ID: u64 = 0x6e7a_0301;
+        let tmp = tempfile::TempDir::new().unwrap();
+        let state = volume_test_state(&tmp);
+        // Incompressible, so the stored gzip stays above the stream threshold.
+        let mut x = 0x2545_f491_u32;
+        let plain: Vec<u8> = (0..(STREAMING_THRESHOLD as usize + 64 * 1024))
+            .map(|_| {
+                x ^= x << 13;
+                x ^= x >> 17;
+                x ^= x << 5;
+                x as u8
+            })
+            .collect();
+        let gz = try_gzip_data(&plain).unwrap();
+        assert!(gz.len() > STREAMING_THRESHOLD as usize);
+        let path = put_test_needle_with(&state, ID, &gz, |n| n.set_is_compressed());
+
+        let reads = Arc::new(Mutex::new(Vec::new()));
+        let _hook = needle_read_hook::register(NeedleId(ID), {
+            let reads = reads.clone();
+            move |len| reads.lock().unwrap().push(len)
+        });
+        let (status, _, body) = send_read(&state, Method::GET, &path, None).await;
+        assert_eq!(status, StatusCode::OK);
+        assert_eq!(body, plain);
+        let reads = reads.lock().unwrap().clone();
+        let full = reads.iter().filter(|&&len| len >= gz.len()).count();
+        assert_eq!(full, 1, "reads: {reads:?}");
+    }
+
+    /// The single full read still verifies the needle checksum.
+    #[tokio::test]
+    async fn test_get_small_needle_with_bad_checksum_fails() {
+        const ID: u64 = 0x6e7a_0401;
+        let tmp = tempfile::TempDir::new().unwrap();
+        let state = volume_test_state(&tmp);
+        let data = b"checksummed payload, to be corrupted".to_vec();
+        let path = put_test_needle(&state, ID, &data);
+
+        let dat = tmp.path().join("1.dat");
+        let mut bytes = std::fs::read(&dat).unwrap();
+        let at = bytes
+            .windows(data.len())
+            .position(|w| w == data.as_slice())
+            .unwrap();
+        bytes[at] ^= 0xff;
+        std::fs::write(&dat, bytes).unwrap();
+
+        let (status, _, _) = send_read(&state, Method::GET, &path, None).await;
+        assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
+    }
 }

seaweed-volume/src/server/mod.rs

@@ -1,5 +1,6 @@
 use tonic::Status;
 
+use crate::remote_storage::s3_tier::TierError;
 use crate::storage::volume::VolumeError;
 
 #[cfg(unix)]
@@ -23,8 +24,12 @@ impl From<VolumeError> for Status {
     fn from(err: VolumeError) -> Self {
         let message = err.to_string();
         match err {
-            VolumeError::NotFound | VolumeError::VolumeNotFound(_) => Status::not_found(message),
-            VolumeError::ReadOnly | VolumeError::NotEmpty => Status::failed_precondition(message),
+            VolumeError::NotFound
+            | VolumeError::VolumeNotFound(_)
+            | VolumeError::Tier(TierError::NotFound(_)) => Status::not_found(message),
+            VolumeError::ReadOnly(_) | VolumeError::NotEmpty => {
+                Status::failed_precondition(message)
+            }
             VolumeError::InsufficientSpace { .. } => Status::resource_exhausted(message),
             VolumeError::AlreadyExists => Status::already_exists(message),
             _ => Status::internal(message),
@@ -66,7 +71,14 @@ mod tests {
             Code::NotFound
         );
         assert_eq!(code(VolumeError::NotFound), Code::NotFound);
-        assert_eq!(code(VolumeError::ReadOnly), Code::FailedPrecondition);
+        assert_eq!(
+            code(VolumeError::ReadOnly(VolumeId(7))),
+            Code::FailedPrecondition
+        );
+        assert_eq!(
+            VolumeError::ReadOnly(VolumeId(7)).to_string(),
+            "volume 7 is read only"
+        );
         assert_eq!(
             code(VolumeError::InsufficientSpace {
                 vid: VolumeId(7),
@@ -77,6 +89,27 @@ mod tests {
         );
         assert_eq!(code(VolumeError::AlreadyExists), Code::AlreadyExists);
         assert_eq!(code(VolumeError::NotInitialized), Code::Internal);
+        assert_eq!(
+            code(TierError::NotFound("gone".into()).into()),
+            Code::NotFound
+        );
+        for tier in [
+            TierError::Io("io".into()),
+            TierError::RuntimeUnavailable("rt".into()),
+            TierError::Aborted("bye".into()),
+        ] {
+            assert_eq!(code(tier.into()), Code::Internal);
+        }
+
+        let status = status_with_context(
+            "backend s3.default copy file /data/1.dat",
+            TierError::NotFound("failed to head object k: NotFound".into()).into(),
+        );
+        assert_eq!(status.code(), Code::NotFound);
+        assert_eq!(
+            status.message(),
+            "backend s3.default copy file /data/1.dat: failed to head object k: NotFound"
+        );
 
         let status = status_with_context(
             "compact volume 7",

seaweed-volume/src/server/store_ec.rs

@@ -95,20 +95,41 @@ struct Snapshot {
     encode_ts_ns: i64,
 }
 
-/// Top-level entry point. Returns `Ok(None)` for "not found" (matches
-/// Go's `ReadEcShardNeedle`); errors propagate as `io::Error`.
+/// Why a distributed EC read has no needle to return.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EcMiss {
+    NotFound,
+    /// Tombstoned in the local `.ecx`/`.ecj`, or reported deleted by a peer.
+    Deleted,
+    VolumeNotFound,
+}
+
+/// Top-level entry point. Returns `Ok(None)` for any miss — absent,
+/// deleted, or volume gone; errors propagate as `io::Error`.
 pub async fn read_ec_shard_needle_distributed(
     state: &Arc<VolumeServerState>,
     vid: VolumeId,
     needle_id: NeedleId,
 ) -> io::Result<Option<Needle>> {
+    Ok(read_ec_shard_needle_or_miss(state, vid, needle_id)
+        .await?
+        .ok())
+}
+
+/// Like `read_ec_shard_needle_distributed`, but says why there is no needle,
+/// as Go's `ReadEcShardNeedle` tells `ErrorDeleted` from not-found.
+pub async fn read_ec_shard_needle_or_miss(
+    state: &Arc<VolumeServerState>,
+    vid: VolumeId,
+    needle_id: NeedleId,
+) -> io::Result<Result<Needle, EcMiss>> {
     // Phase A — under the Store read lock, locate the needle, compute
     // intervals, and read any locally-mounted shard intervals. We must
     // not `.await` while holding this guard (std::sync::RwLockReadGuard
     // is !Send).
     let mut snapshot = match snapshot_under_lock(state, vid, needle_id)? {
-        Some(s) => s,
-        None => return Ok(None),
+        Ok(s) => s,
+        Err(miss) => return Ok(Err(miss)),
     };
 
     // Phase B — refresh the shard_locations cache from the master if
@@ -210,17 +231,11 @@ pub async fn read_ec_shard_needle_distributed(
         .collect()
         .await;
 
-    let mut assembled: Vec<Vec<u8>> = Vec::with_capacity(fetched.len());
-    for res in fetched {
-        let (buf, is_deleted) = res?;
-        // A peer reports the needle deleted (a cross-server window where the
-        // local index still shows it live): treat as not-found rather than
-        // serving zeros, mirroring Go's ErrorDeleted.
-        if is_deleted {
-            return Ok(None);
-        }
-        assembled.push(buf);
-    }
+    // A peer reports the needle deleted (a cross-server window where the
+    // local index still shows it live): answer deleted rather than serving zeros.
+    let Some(assembled) = gather_intervals(fetched)? else {
+        return Ok(Err(EcMiss::Deleted));
+    };
 
     // Phase D — assemble and parse the Needle. Mirrors the tail of
     // `EcVolume::read_ec_shard_needle`.
@@ -252,7 +267,20 @@ pub async fn read_ec_shard_needle_distributed(
         snapshot.version,
     )
     .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{}", e)))?;
-    Ok(Some(n))
+    Ok(Ok(n))
+}
+
+/// `None` when any holder reported the needle deleted. That outranks another
+/// interval's error: deletes are never invented, so the needle is gone either way.
+fn gather_intervals(fetched: Vec<io::Result<(Vec<u8>, bool)>>) -> io::Result<Option<Vec<Vec<u8>>>> {
+    if fetched.iter().any(|r| matches!(r, Ok((_, true)))) {
+        return Ok(None);
+    }
+    fetched
+        .into_iter()
+        .map(|r| r.map(|(buf, _)| buf))
+        .collect::<io::Result<_>>()
+        .map(Some)
 }
 
 /// What one EC delete RPC carries — `VolumeEcBlobDeleteRequest` minus tonic.
@@ -864,11 +892,10 @@ fn snapshot_under_lock(
     state: &Arc<VolumeServerState>,
     vid: VolumeId,
     needle_id: NeedleId,
-) -> io::Result<Option<Snapshot>> {
+) -> io::Result<Result<Snapshot, EcMiss>> {
     let store = state.store.read().unwrap();
-    let ecv = match store.find_ec_volume(vid) {
-        Some(v) => v,
-        None => return Ok(None),
+    let Some(ecv) = store.find_ec_volume(vid) else {
+        return Ok(Err(EcMiss::VolumeNotFound));
     };
 
     // Reuse EcVolume::locate_needle for offset/size resolution AND
@@ -876,11 +903,17 @@ fn snapshot_under_lock(
     // local-only read path uses, so we stay byte-identical on the
     // shard-size + interval boundaries. locate_needle applies the runtime
     // delete mask, which is correct for serving reads.
-    let (offset, size, intervals) = match ecv.locate_needle(needle_id)? {
-        Some(v) => v,
-        None => return Ok(None),
+    let Some((offset, size, intervals)) = ecv.locate_needle(needle_id)? else {
+        // locate_needle folds a tombstone into not-found.
+        let deleted =
+            matches!(ecv.find_needle_from_ecx(needle_id)?, Some((_, s)) if s.is_deleted());
+        return Ok(Err(if deleted {
+            EcMiss::Deleted
+        } else {
+            EcMiss::NotFound
+        }));
     };
-    build_snapshot(ecv, offset, size, &intervals).map(Some)
+    build_snapshot(ecv, offset, size, &intervals).map(Ok)
 }
 
 /// Like `snapshot_under_lock`, but locates intervals from the RAW .ecx
@@ -1790,6 +1823,21 @@ async fn drain_copy_stream(
 mod tests {
     use super::*;
 
+    #[test]
+    fn gather_intervals_puts_a_reported_deletion_ahead_of_errors() {
+        let failed = || Err(io::Error::other("shard unreachable"));
+        assert!(
+            gather_intervals(vec![failed(), Ok((Vec::new(), true))])
+                .unwrap()
+                .is_none()
+        );
+        assert!(gather_intervals(vec![failed(), Ok((vec![1], false))]).is_err());
+        assert_eq!(
+            gather_intervals(vec![Ok((vec![1], false)), Ok((vec![2], false))]).unwrap(),
+            Some(vec![vec![1], vec![2]])
+        );
+    }
+
     fn locations(count: usize) -> HashMap<ShardId, Vec<String>> {
         (0..count)
             .map(|sid| (sid as ShardId, vec!["127.0.0.1:8080".to_string()]))

seaweed-volume/src/server/write_queue.rs

@@ -3,8 +3,8 @@
 //! Instead of each upload handler directly calling `write_needle`, writes are
 //! submitted to a queue. A background worker drains the queue in batches (up to
 //! 128 entries), groups them by volume ID, and processes them together under a
-//! single store lock. Requests that asked for `fsync` are flushed by
-//! `write_needle` itself, one flush per durable write.
+//! single store lock. Durable writes to a volume share their .dat and .idx
+//! flushes (see `Volume::write_needles_grouped`).
 
 use std::sync::Arc;
 
@@ -159,8 +159,12 @@ fn process_batch(state: Arc<VolumeServerState>, batch: Vec<WriteRequest>) {
     let mut store = state.store.write().unwrap();
 
     for (vid, entries) in groups {
-        for (mut needle, fsync, response_tx) in entries {
-            let result = store.write_volume_needle(vid, &mut needle, fsync);
+        let (mut writes, senders): (Vec<_>, Vec<_>) = entries
+            .into_iter()
+            .map(|(needle, fsync, response_tx)| ((needle, fsync), response_tx))
+            .unzip();
+        let results = store.write_volume_needles(vid, &mut writes);
+        for (response_tx, result) in senders.into_iter().zip(results) {
             // Send result back; ignore error if receiver dropped.
             let _ = response_tx.send(result);
         }
@@ -312,6 +316,63 @@ mod tests {
         }
     }
 
+    /// The queue hands a volume's batch to the grouped path, so ten durable
+    /// writes cost one .dat sync and one .idx sync, not ten of each.
+    #[test]
+    fn test_process_batch_group_commits_fsync_writes() {
+        use crate::config::MinFreeSpace;
+        use crate::storage::types::{DiskType, NeedleId};
+        use crate::storage::volume::VolumeSpec;
+
+        let tmp = tempfile::TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let state = make_test_state();
+        {
+            let mut store = state.store.write().unwrap();
+            store
+                .add_location(
+                    dir,
+                    dir,
+                    10,
+                    DiskType::HardDrive,
+                    MinFreeSpace::Percent(1.0),
+                    Vec::new(),
+                )
+                .unwrap();
+            store
+                .add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
+                .unwrap();
+        }
+
+        let mut receivers = Vec::new();
+        let batch = (1..=10u64)
+            .map(|id| {
+                let (response_tx, response_rx) = oneshot::channel();
+                receivers.push(response_rx);
+                WriteRequest {
+                    volume_id: VolumeId(1),
+                    needle: Needle {
+                        id: NeedleId(id),
+                        cookie: 0x1111.into(),
+                        data: vec![id as u8; 8],
+                        data_size: 8,
+                        ..Needle::default()
+                    },
+                    fsync: true,
+                    response_tx,
+                }
+            })
+            .collect();
+        process_batch(state.clone(), batch);
+
+        for mut rx in receivers {
+            assert!(matches!(rx.try_recv().unwrap(), Ok((_, _, false))));
+        }
+        let store = state.store.read().unwrap();
+        let (_, vol) = store.find_volume(VolumeId(1)).unwrap();
+        assert_eq!(vol.sync_counts_for_test(), (1, 1));
+    }
+
     #[tokio::test]
     async fn test_write_queue_dropped_sender() {
         // When the queue is dropped, subsequent submits should fail gracefully.

seaweed-volume/src/storage/disk_location.rs

@@ -18,7 +18,7 @@ use crate::storage::erasure_coding::ec_shard::{
     DATA_SHARDS_COUNT, ERASURE_CODING_LARGE_BLOCK_SIZE, ERASURE_CODING_SMALL_BLOCK_SIZE,
     EcVolumeShard, ShardId,
 };
-use crate::storage::erasure_coding::ec_volume::EcVolume;
+use crate::storage::erasure_coding::ec_volume::{EcVolume, is_usable_ecx_file};
 use crate::storage::needle_map::NeedleMapKind;
 use crate::storage::super_block::SUPER_BLOCK_SIZE;
 use crate::storage::types::*;
@@ -479,6 +479,13 @@ impl DiskLocation {
         if self.idx_directory != self.directory {
             remove_bitrot_sidecars(&idx_base)?;
         }
+
+        // Staged 2PC generations (<base>.ecNN.v<N>, versioned .ecx/.ecj/.vif)
+        // belong to this volume's EC state too; leaving them orphans the files.
+        crate::storage::erasure_coding::ec_shard::remove_ec_generation_files(&base, 0)?;
+        if self.idx_directory != self.directory {
+            crate::storage::erasure_coding::ec_shard::remove_ec_generation_files(&idx_base, 0)?;
+        }
         Ok(())
     }
 
@@ -598,13 +605,20 @@ impl DiskLocation {
         &mut self,
         vid: VolumeId,
         only_empty: bool,
+        only_garbage: bool,
         keep_remote_data: bool,
     ) -> Result<(), VolumeError> {
+        // Refuse before removing: a refused destroy must leave it mounted.
+        if let Some(v) = self.volumes.get(&vid)
+            && v.is_compacting()
+        {
+            return Err(v.compacting_error());
+        }
         if let Some(mut v) = self.volumes.remove(&vid) {
             crate::metrics::VOLUME_GAUGE
                 .with_label_values(&[&v.collection, "volume"])
                 .dec();
-            v.destroy(only_empty, keep_remote_data)?;
+            v.destroy(only_empty, only_garbage, keep_remote_data)?;
             Ok(())
         } else {
             Err(VolumeError::NotFound)
@@ -625,7 +639,7 @@ impl DiskLocation {
                 crate::metrics::VOLUME_GAUGE
                     .with_label_values(&[&v.collection, "volume"])
                     .dec();
-                if let Err(e) = v.destroy(false, false) {
+                if let Err(e) = v.destroy(false, false, false) {
                     warn!(volume_id = vid.0, error = %e, "delete collection: failed to destroy volume");
                 }
             }
@@ -779,21 +793,17 @@ impl DiskLocation {
     /// Mirrors `DiskLocation.HasEcxFileOnDisk` in
     /// `weed/storage/disk_location_ec.go`. Skips entries that are
     /// directories so a stray dir named `<collection>_<vid>.ecx` doesn't
-    /// register as a present index file.
+    /// register as a present index file. A 0-byte `.ecx` is a corrupt stub
+    /// left by a failed EC distribute copy; it must not steer placement
+    /// toward this disk, so it counts as absent (Go requires `Size() > 0`).
     pub fn has_ecx_file_on_disk(&self, collection: &str, vid: VolumeId) -> bool {
         let idx_base = volume_file_name(&self.idx_directory, collection, vid);
-        let idx_path = format!("{}.ecx", idx_base);
-        if let Ok(meta) = fs::metadata(&idx_path)
-            && !meta.is_dir()
-        {
+        if is_usable_ecx_file(&format!("{}.ecx", idx_base)) {
             return true;
         }
         if self.idx_directory != self.directory {
             let data_base = volume_file_name(&self.directory, collection, vid);
-            let data_path = format!("{}.ecx", data_base);
-            if let Ok(meta) = fs::metadata(&data_path)
-                && !meta.is_dir()
-            {
+            if is_usable_ecx_file(&format!("{}.ecx", data_base)) {
                 return true;
             }
         }
@@ -805,6 +815,20 @@ impl DiskLocation {
         self.ec_volumes.remove(&vid)
     }
 
+    /// Drop the in-memory EC volume for vid and close its descriptors without
+    /// deleting files, so a following unlink frees the inodes instead of
+    /// leaving open fds serving the old bytes. Mirrors Go's unloadEcVolume.
+    pub fn unload_ec_volume(&mut self, vid: VolumeId) {
+        if let Some(mut ec_vol) = self.ec_volumes.remove(&vid) {
+            for _ in 0..ec_vol.shard_count() {
+                crate::metrics::VOLUME_GAUGE
+                    .with_label_values(&[&ec_vol.collection, "ec_shards"])
+                    .dec();
+            }
+            ec_vol.close();
+        }
+    }
+
     /// Mount EC shards for a volume on this location.
     ///
     /// `source_disk_type` is the source volume's disk type carried on the
@@ -1737,7 +1761,7 @@ mod tests {
             .unwrap();
         assert_eq!(loc.volumes_len(), 2);
 
-        loc.delete_volume(VolumeId(1), false, false).unwrap();
+        loc.delete_volume(VolumeId(1), false, false, false).unwrap();
         assert_eq!(loc.volumes_len(), 1);
         assert!(loc.find_volume(VolumeId(1)).is_none());
     }
@@ -1790,6 +1814,34 @@ mod tests {
         assert!(loc.find_volume(VolumeId(3)).is_some());
     }
 
+    /// A 0-byte `.ecx` is the stub a failed EC distribute copy leaves behind.
+    /// Go's HasEcxFileOnDisk requires Size() > 0 so the stub cannot pin
+    /// placement to a disk that has no usable index.
+    #[test]
+    fn test_has_ecx_file_on_disk_ignores_zero_byte_stub() {
+        let tmp = TempDir::new().unwrap();
+        let data = tmp.path().join("data");
+        let idx = tmp.path().join("idx");
+        fs::create_dir_all(&data).unwrap();
+        fs::create_dir_all(&idx).unwrap();
+        let loc = DiskLocation::new(
+            data.to_str().unwrap(),
+            idx.to_str().unwrap(),
+            10,
+            DiskType::HardDrive,
+            MinFreeSpace::Percent(1.0),
+            Vec::new(),
+        )
+        .unwrap();
+
+        fs::write(idx.join("pics_7.ecx"), b"").unwrap();
+        assert!(!loc.has_ecx_file_on_disk("pics", VolumeId(7)));
+
+        // A real index in the data dir still counts, stub or no stub.
+        fs::write(data.join("pics_7.ecx"), [0u8; 16]).unwrap();
+        assert!(loc.has_ecx_file_on_disk("pics", VolumeId(7)));
+    }
+
     #[test]
     fn test_disk_location_delete_collection_removes_ec_volumes() {
         let tmp = TempDir::new().unwrap();
@@ -1806,6 +1858,8 @@ mod tests {
 
         let shard_path = format!("{}/pics_7.ec00", dir);
         std::fs::write(&shard_path, b"ec-shard").unwrap();
+        // An EC volume needs its .ecx to mount.
+        std::fs::write(format!("{}/pics_7.ecx", dir), [0u8; 16]).unwrap();
 
         loc.mount_ec_shards(VolumeId(7), "pics", &[0], "").unwrap();
         assert!(loc.has_ec_volume(VolumeId(7)));
@@ -1843,6 +1897,7 @@ mod tests {
         // mount_ec_shards with source_disk_type="ssd" — simulating the
         // VolumeEcShardsMount RPC path.
         std::fs::write(format!("{}/pics_7.ec00", dir), b"ec-shard").unwrap();
+        std::fs::write(format!("{}/pics_7.ecx", dir), [0u8; 16]).unwrap();
         loc.mount_ec_shards(VolumeId(7), "pics", &[0], "ssd")
             .unwrap();
         {
@@ -1942,6 +1997,7 @@ mod tests {
         // A collection name unique to this test: the gauge is process-global
         // and sibling tests running in parallel touch other labels.
         std::fs::write(format!("{}/dupmount_11.ec00", dir), b"shard bytes").unwrap();
+        std::fs::write(format!("{}/dupmount_11.ecx", dir), [0u8; 16]).unwrap();
         let gauge = crate::metrics::VOLUME_GAUGE.with_label_values(&["dupmount", "ec_shards"]);
         let before = gauge.get();
 

seaweed-volume/src/storage/erasure_coding/ec_decoder.rs

@@ -3,10 +3,12 @@
 //! Rebuilds the original .dat + .idx files from data shards (.ec00-.ec09)
 //! and the sorted index (.ecx) + deletion journal (.ecj).
 
+use std::collections::HashSet;
 use std::fs::File;
 use std::io::{self, Read, Write};
 
 use crate::storage::erasure_coding::ec_shard::*;
+use crate::storage::erasure_coding::ec_volume::read_ecj_ids;
 use crate::storage::idx;
 use crate::storage::needle::needle::get_actual_size;
 use crate::storage::super_block::SUPER_BLOCK_SIZE;
@@ -20,18 +22,21 @@ use crate::storage::volume::{fsync_dir, volume_file_name};
 /// `dir` is used both for reading `.ec00` and `.ecx`. For split-disk
 /// reconciled volumes call [`find_dat_file_size_with_dirs`] instead.
 pub fn find_dat_file_size(dir: &str, collection: &str, volume_id: VolumeId) -> io::Result<i64> {
-    find_dat_file_size_with_dirs(dir, dir, collection, volume_id)
+    let deleted = read_ecj_deletions(&[dir], collection, volume_id)?;
+    find_dat_file_size_with_dirs(dir, dir, collection, volume_id, &deleted)
 }
 
 /// Like [`find_dat_file_size`] but lets the caller pass separate dirs
 /// for `.ec00` (the data shard) and `.ecx` (the sealed index). This
 /// is the form needed when shards are split across data dirs and the
-/// `.ecx` lives on a sibling disk's idx dir (#9252).
+/// `.ecx` lives on a sibling disk's idx dir (#9252). Needles in `deleted`
+/// count as deleted.
 pub fn find_dat_file_size_with_dirs(
     ec00_dir: &str,
     ecx_dir: &str,
     collection: &str,
     volume_id: VolumeId,
+    deleted: &HashSet<NeedleId>,
 ) -> io::Result<i64> {
     let ec00_base = volume_file_name(ec00_dir, collection, volume_id);
     let ecx_base = volume_file_name(ecx_dir, collection, volume_id);
@@ -53,9 +58,9 @@ pub fn find_dat_file_size_with_dirs(
 
     for i in 0..entry_count {
         let start = i * NEEDLE_MAP_ENTRY_SIZE;
-        let (_, offset, size) =
+        let (key, offset, size) =
             idx_entry_from_bytes(&ecx_data[start..start + NEEDLE_MAP_ENTRY_SIZE]);
-        if size.is_deleted() {
+        if size.is_deleted() || deleted.contains(&key) {
             continue;
         }
         let entry_stop = offset.to_actual_offset() + get_actual_size(size, version);
@@ -67,6 +72,48 @@ pub fn find_dat_file_size_with_dirs(
     Ok(dat_size)
 }
 
+/// Whether the `.ecx` in `ecx_dir` indexes a needle deleted neither there nor
+/// in `deleted`.
+pub fn has_live_needles(
+    ecx_dir: &str,
+    collection: &str,
+    volume_id: VolumeId,
+    deleted: &HashSet<NeedleId>,
+) -> io::Result<bool> {
+    let ecx_base = volume_file_name(ecx_dir, collection, volume_id);
+    let ecx_data = std::fs::read(format!("{}.ecx", ecx_base))?;
+    let (entries, _) = ecx_data.as_chunks::<NEEDLE_MAP_ENTRY_SIZE>();
+    Ok(entries.iter().any(|entry| {
+        let (key, _, size) = idx_entry_from_bytes(entry);
+        !size.is_deleted() && !deleted.contains(&key)
+    }))
+}
+
+/// Distinct needle ids journaled in the `.ecj` of any of `dirs`. Go folds the
+/// journal into the `.ecx` (RebuildEcxFile) before a decode; reading it leaves
+/// the sealed index untouched. Only NotFound means "no journal".
+pub fn read_ecj_deletions(
+    dirs: &[&str],
+    collection: &str,
+    volume_id: VolumeId,
+) -> io::Result<HashSet<NeedleId>> {
+    let mut ids = HashSet::new();
+    for (i, dir) in dirs.iter().enumerate() {
+        if dirs[..i].contains(dir) {
+            continue;
+        }
+        let path = format!("{}.ecj", volume_file_name(dir, collection, volume_id));
+        let file = match File::open(&path) {
+            Ok(file) => file,
+            Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
+            Err(e) => return Err(e),
+        };
+        let len = file.metadata()?.len();
+        read_ecj_ids(&file, len, &mut ids)?;
+    }
+    Ok(ids)
+}
+
 /// What it takes to rebuild a volume's .dat from its EC data shards.
 ///
 /// Mirrors Go's `WriteDatFile(baseFileName, datFileSize,
@@ -267,51 +314,61 @@ pub fn write_dat_file_from_shards(spec: &DatRebuild<'_>) -> io::Result<()> {
 
 /// Write .idx file from .ecx index + .ecj deletion journal.
 ///
-/// Copies sorted .ecx entries to .idx, then appends tombstones for
-/// deleted needles from .ecj.
+/// See [`write_idx_file_from_ec_index_with_dirs`]; everything lives in `dir`.
 pub fn write_idx_file_from_ec_index(
     dir: &str,
     collection: &str,
     volume_id: VolumeId,
 ) -> io::Result<()> {
-    let base = volume_file_name(dir, collection, volume_id);
-    let ecx_path = format!("{}.ecx", base);
-    let ecj_path = format!("{}.ecj", base);
-    let idx_path = format!("{}.idx", base);
+    let deleted = read_ecj_deletions(&[dir], collection, volume_id)?;
+    let dat_file_size = find_dat_file_size_with_dirs(dir, dir, collection, volume_id, &deleted)?;
+    write_idx_file_from_ec_index_with_dirs(dir, dir, collection, volume_id, &deleted, dat_file_size)
+}
+
+/// Write the `.idx` for a `.dat` decoded to `dat_file_size` bytes, from the
+/// `.ecx` in `ecx_dir`, into `idx_dir`.
+///
+/// Copies the `.ecx` rows, then appends one tombstone per row whose needle is
+/// in `deleted`. A deleted needle at or past `dat_file_size` was cut from the
+/// `.dat`, so its row is dropped: a row pointing past the end of the `.dat`
+/// makes the volume load read-only.
+pub fn write_idx_file_from_ec_index_with_dirs(
+    ecx_dir: &str,
+    idx_dir: &str,
+    collection: &str,
+    volume_id: VolumeId,
+    deleted: &HashSet<NeedleId>,
+    dat_file_size: i64,
+) -> io::Result<()> {
+    let ecx_path = format!("{}.ecx", volume_file_name(ecx_dir, collection, volume_id));
+    let idx_path = format!("{}.idx", volume_file_name(idx_dir, collection, volume_id));
     // Write to a temp file and atomically rename into place, so a crash
     // mid-write never leaves a partial .idx at the final name beside the
     // source shards.
     let tmp_path = format!("{}.tmp", idx_path);
 
     let write_result = (|| -> io::Result<()> {
-        // Copy .ecx to the temp .idx
-        std::fs::copy(&ecx_path, &tmp_path)?;
-
-        // Append deletions from .ecj as tombstones. Read the journal directly
-        // and treat only NotFound as "no journal": Path::exists would also
-        // swallow a permission/IO error and silently skip deletions, which
-        // would resurrect deleted needles as live.
-        let mut idx_file = std::fs::OpenOptions::new().append(true).open(&tmp_path)?;
-        match std::fs::read(&ecj_path) {
-            Ok(ecj_data) => {
-                let count = ecj_data.len() / NEEDLE_ID_SIZE;
-                for i in 0..count {
-                    let start = i * NEEDLE_ID_SIZE;
-                    let needle_id = NeedleId::from_bytes(&ecj_data[start..start + NEEDLE_ID_SIZE]);
-                    idx::write_index_entry(
-                        &mut idx_file,
-                        needle_id,
-                        Offset::default(),
-                        TOMBSTONE_FILE_SIZE,
-                    )?;
-                }
+        let mut ecx_file = File::open(&ecx_path)?;
+        let mut idx_file = io::BufWriter::new(File::create(&tmp_path)?);
+        let mut tombstoned = Vec::new();
+        idx::walk_index_file(&mut ecx_file, 0, |key, offset, size| {
+            let is_deleted = size.is_deleted() || deleted.contains(&key);
+            if is_deleted && offset.to_actual_offset() >= dat_file_size {
+                return Ok(());
             }
-            Err(e) if e.kind() == io::ErrorKind::NotFound => {}
-            Err(e) => return Err(e),
+            idx::write_index_entry(&mut idx_file, key, offset, size)?;
+            if !size.is_deleted() && deleted.contains(&key) {
+                tombstoned.push(key);
+            }
+            Ok(())
+        })?;
+        for key in tombstoned {
+            idx::write_index_entry(&mut idx_file, key, Offset::default(), TOMBSTONE_FILE_SIZE)?;
         }
 
         // fsync, rename, then fsync the dir so the decoded .idx is durable and
         // atomically published before the caller deletes the source shards.
+        let idx_file = idx_file.into_inner().map_err(|e| e.into_error())?;
         idx_file.sync_all()?;
         drop(idx_file);
         // Windows rename does not replace an existing file on every version;
@@ -694,4 +751,43 @@ mod tests {
             .is_err()
         );
     }
+
+    /// A journal many chunks long that repeats a few ids reads back as those
+    /// ids, from each dir once, whatever its length.
+    #[test]
+    fn test_read_ecj_deletions_collects_distinct_ids_across_dirs() {
+        let tmp = TempDir::new().unwrap();
+        let data = tmp.path().join("data");
+        let idx = tmp.path().join("idx");
+        let missing = tmp.path().join("missing");
+        std::fs::create_dir_all(&data).unwrap();
+        std::fs::create_dir_all(&idx).unwrap();
+        let (data, idx, missing) = (
+            data.to_str().unwrap(),
+            idx.to_str().unwrap(),
+            missing.to_str().unwrap(),
+        );
+
+        let entry = |id: u64| {
+            let mut buf = [0u8; NEEDLE_ID_SIZE];
+            NeedleId(id).to_bytes(&mut buf);
+            buf
+        };
+        // Past two load chunks of three repeating ids, then an id only in the
+        // last chunk and a torn trailing record.
+        let mut ecj = Vec::new();
+        while ecj.len() <= 2 * (1 << 20) {
+            for id in [1, 2, 3] {
+                ecj.extend_from_slice(&entry(id));
+            }
+        }
+        ecj.extend_from_slice(&entry(7));
+        ecj.extend_from_slice(&entry(8)[..3]);
+        std::fs::write(format!("{idx}/1.ecj"), &ecj).unwrap();
+        std::fs::write(format!("{data}/1.ecj"), entry(9)).unwrap();
+
+        let ids = read_ecj_deletions(&[data, idx, idx, missing], "", VolumeId(1)).unwrap();
+        let expected: HashSet<NeedleId> = [1, 2, 3, 7, 9].into_iter().map(NeedleId).collect();
+        assert_eq!(ids, expected);
+    }
 }

seaweed-volume/src/storage/erasure_coding/ec_encoder.rs

@@ -813,7 +813,7 @@ impl EncodeRun<'_> {
         // explicitly.
         for (i, buf) in self.buffers[..self.data_shards].iter_mut().enumerate() {
             let read_offset = offset + (i * block_size) as u64;
-            let n = read_at_most(self.dat_file, buf, read_offset)?;
+            let n = crate::storage::io::read_full_at(self.dat_file, buf, read_offset)?;
             buf[n..].fill(0);
         }
 
@@ -834,19 +834,6 @@ impl EncodeRun<'_> {
     }
 }
 
-/// Read into `buf` at `offset` until it is full or EOF; returns bytes read.
-fn read_at_most(dat_file: &File, buf: &mut [u8], offset: u64) -> io::Result<usize> {
-    let mut n = 0;
-    while n < buf.len() {
-        let r = crate::storage::io::read_at(dat_file, &mut buf[n..], offset + n as u64)?;
-        if r == 0 {
-            break;
-        }
-        n += r;
-    }
-    Ok(n)
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

seaweed-volume/src/storage/erasure_coding/ec_shard.rs

@@ -87,14 +87,14 @@ impl EcVolumeShard {
         Ok(())
     }
 
-    /// Read data at a specific offset.
+    /// Read data at a specific offset, filling `buf` unless the shard ends first.
     pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize> {
         let file = self
             .ecd_file
             .as_ref()
             .ok_or_else(|| io::Error::other("shard file not open"))?;
 
-        crate::storage::io::read_at(file, buf, offset)
+        crate::storage::io::read_full_at(file, buf, offset)
     }
 
     /// Write data to the shard file (appends).
@@ -195,6 +195,104 @@ impl ShardBits {
     }
 }
 
+/// Parses the generation of a 2PC-staged `<base>.v<N>` file: `None` means the
+/// name is not a generation file of `base`.
+pub fn ec_file_generation(name: &str, base: &str) -> Option<u32> {
+    let suffix = name.strip_prefix(&format!("{}.v", base))?;
+    match suffix.parse::<u32>() {
+        Ok(g) if g > 0 => Some(g),
+        _ => None,
+    }
+}
+
+/// Removes 2PC generation files staged under `base`:
+/// `<base>.ecNN.v<N>`, `<base>.ecx.v<N>`, `<base>.ecj.v<N>`, `<base>.ecsum.v<N>`
+/// and `<base>.vif.v<N>`. `generations_older_than == 0` removes every
+/// generation; otherwise only generations strictly below it. Returns the
+/// first real removal failure. Mirrors Go's `RemoveEcGenerationFiles`.
+pub fn remove_ec_generation_files(base: &str, generations_older_than: u32) -> io::Result<()> {
+    let path = std::path::Path::new(base);
+    let (Some(parent), Some(fname)) = (path.parent(), path.file_name()) else {
+        return Ok(());
+    };
+    let ec_prefix = format!("{}.ec", fname.to_string_lossy());
+    let vif_name = format!("{}.vif", fname.to_string_lossy());
+    let mut first_err: Option<io::Error> = None;
+    let mut record = |res: io::Result<()>| {
+        if let Err(e) = res
+            && first_err.is_none()
+        {
+            first_err = Some(e);
+        }
+    };
+    match fs::read_dir(parent) {
+        Ok(entries) => {
+            for entry in entries {
+                let entry = match entry {
+                    Ok(entry) => entry,
+                    Err(e) => {
+                        // A skipped entry means an incomplete sweep; report it
+                        // instead of pretending the cleanup finished.
+                        record(Err(e));
+                        continue;
+                    }
+                };
+                let name = entry.file_name().to_string_lossy().into_owned();
+                let Some((artifact, _)) = name.rsplit_once(".v") else {
+                    continue;
+                };
+                if artifact != vif_name && !artifact.starts_with(&ec_prefix) {
+                    continue;
+                }
+                let Some(generation) = ec_file_generation(&name, artifact) else {
+                    continue;
+                };
+                if generations_older_than > 0 && generation >= generations_older_than {
+                    continue;
+                }
+                record(match fs::remove_file(entry.path()) {
+                    Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e),
+                    _ => Ok(()),
+                });
+            }
+        }
+        Err(e) if e.kind() != io::ErrorKind::NotFound => record(Err(e)),
+        Err(_) => {}
+    }
+    match first_err {
+        Some(e) => Err(e),
+        None => Ok(()),
+    }
+}
+
+/// Removes every staged generation `<shard_file>.v<N>` of one shard file.
+/// Returns true when at least one generation file was removed.
+pub fn remove_ec_shard_generations(shard_file: &str) -> io::Result<bool> {
+    let path = std::path::Path::new(shard_file);
+    let (Some(parent), Some(fname)) = (path.parent(), path.file_name()) else {
+        return Ok(false);
+    };
+    let fname = fname.to_string_lossy().into_owned();
+    let mut removed = false;
+    match fs::read_dir(parent) {
+        Ok(entries) => {
+            for entry in entries {
+                let entry = entry?;
+                let name = entry.file_name().to_string_lossy().into_owned();
+                if ec_file_generation(&name, &fname).is_some() {
+                    match fs::remove_file(entry.path()) {
+                        Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e),
+                        _ => removed = true,
+                    }
+                }
+            }
+        }
+        Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e),
+        Err(_) => {}
+    }
+    Ok(removed)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

seaweed-volume/src/storage/erasure_coding/ec_volume.rs

@@ -48,6 +48,29 @@ pub(crate) struct ShardLocationCache {
 /// A multiple of `NEEDLE_ID_SIZE`; 1 MiB is 131072 entries per syscall.
 const ECJ_LOAD_CHUNK_BYTES: usize = 1 << 20;
 
+/// Adds every whole needle id in the first `len` bytes of `ecj_file` to `ids`,
+/// reading `ECJ_LOAD_CHUNK_BYTES` at a time; a trailing partial record is
+/// ignored.
+pub(crate) fn read_ecj_ids(
+    ecj_file: &File,
+    len: u64,
+    ids: &mut HashSet<NeedleId>,
+) -> io::Result<()> {
+    let mut buf = vec![0u8; std::cmp::min(ECJ_LOAD_CHUNK_BYTES as u64, len) as usize];
+    let mut off: u64 = 0;
+    while off + NEEDLE_ID_SIZE as u64 <= len {
+        let mut want = std::cmp::min(ECJ_LOAD_CHUNK_BYTES as u64, len - off) as usize;
+        want -= want % NEEDLE_ID_SIZE;
+        // Positional read: the loader's handle is shared with journal appends.
+        read_exact_at(ecj_file, &mut buf[..want], off)?;
+        for entry in buf[..want].as_chunks::<NEEDLE_ID_SIZE>().0 {
+            ids.insert(NeedleId::from_bytes(entry));
+        }
+        off += want as u64;
+    }
+    Ok(())
+}
+
 /// An erasure-coded volume managing its local shards and index.
 pub struct EcVolume {
     pub volume_id: VolumeId,
@@ -383,8 +406,26 @@ pub fn validate_block_size(block_size: i64) -> io::Result<()> {
     Ok(())
 }
 
+/// Size of the `.ecx` at `path`, or `None` when it is absent or a directory.
+/// Mirrors Go's `statEcxSize`.
+pub(crate) fn ecx_file_size(path: &str) -> Option<u64> {
+    match std::fs::metadata(path) {
+        Ok(meta) if !meta.is_dir() => Some(meta.len()),
+        _ => None,
+    }
+}
+
+/// Whether `path` is an `.ecx` that can steer placement, ownership or a copy:
+/// a regular file with content. A 0-byte `.ecx` is what a failed EC distribute
+/// copy leaves behind, and Go treats it as absent at every such decision
+/// (`HasEcxFileOnDisk`, `findEcxIdxDirForVolume`, `indexEcxOwners`) so the scan
+/// moves on to a sibling disk that may hold a valid index.
+pub(crate) fn is_usable_ecx_file(path: &str) -> bool {
+    ecx_file_size(path).is_some_and(|size| size > 0)
+}
+
 impl EcVolume {
-    /// Create a new EcVolume. Loads .ecx index and .ecj journal if present.
+    /// Create a new EcVolume. Opens the .ecx index (required) and the .ecj journal.
     pub fn new(
         dir: &str,
         dir_idx: &str,
@@ -467,27 +508,62 @@ impl EcVolume {
 
         // Open .ecx file (sorted index) in read/write mode for in-place deletion marking.
         // Matches Go which opens ecx for writing via MarkNeedleDeleted.
-        let ecx_path = vol.ecx_file_name();
-        if std::path::Path::new(&ecx_path).exists() {
-            let file = open_volume_file(OpenOptions::new().read(true).write(true), &ecx_path)?;
-            vol.ecx_file_size = file.metadata()?.len() as i64;
-            vol.ecx_file = Some(file);
-        } else if dir_idx != dir {
-            // Fall back to data directory if .ecx was created before -dir.idx was configured
-            let data_base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
-            let fallback_ecx = format!("{}.ecx", data_base);
-            if std::path::Path::new(&fallback_ecx).exists() {
-                tracing::info!(
+        //
+        // Resolve it the way Go's NewEcVolume does: prefer a non-empty copy,
+        // the one co-located with the shard data first, then the caller's
+        // index directory — either the shared -dir.idx dir or a sibling disk
+        // that owns the .ecx when this disk holds only a 0-byte stub left by an
+        // interrupted copy. A 0-byte .ecx is also a legitimate empty index, so
+        // it yields only to a non-empty copy elsewhere, never to a mere
+        // absence. No .ecx at all fails the mount with NotFound (Go wraps
+        // os.ErrNotExist): an EcVolume without an index would advertise shards
+        // that no read can ever serve.
+        let local_ecx = format!(
+            "{}.ecx",
+            crate::storage::volume::volume_file_name(dir, collection, volume_id)
+        );
+        let shared_ecx = vol.ecx_file_name();
+        let local_size = ecx_file_size(&local_ecx);
+        let shared_size = if dir_idx != dir {
+            ecx_file_size(&shared_ecx)
+        } else {
+            None
+        };
+        let use_local = match (local_size, shared_size) {
+            (Some(n), _) if n > 0 => true,
+            (_, Some(n)) if n > 0 => {
+                tracing::debug!(
                     volume_id = volume_id.0,
-                    "ecx file not found in idx dir, falling back to data dir"
+                    "ecx not local at {}, using {}",
+                    local_ecx,
+                    shared_ecx
                 );
-                let file =
-                    open_volume_file(OpenOptions::new().read(true).write(true), &fallback_ecx)?;
-                vol.ecx_file_size = file.metadata()?.len() as i64;
-                vol.ecx_file = Some(file);
-                vol.ecx_actual_dir = dir.to_string();
+                false
             }
-        }
+            // Only 0-byte copies exist: an empty index, local first.
+            (Some(_), _) => true,
+            (None, Some(_)) => false,
+            (None, None) => {
+                let tried = if dir_idx != dir {
+                    format!("{} (or {})", local_ecx, shared_ecx)
+                } else {
+                    local_ecx
+                };
+                return Err(io::Error::new(
+                    io::ErrorKind::NotFound,
+                    format!("cannot open ec volume index {}: not found", tried),
+                ));
+            }
+        };
+        let ecx_path = if use_local {
+            vol.ecx_actual_dir = dir.to_string();
+            local_ecx
+        } else {
+            shared_ecx
+        };
+        let file = open_volume_file(OpenOptions::new().read(true).write(true), &ecx_path)?;
+        vol.ecx_file_size = file.metadata()?.len() as i64;
+        vol.ecx_file = Some(file);
 
         // Open .ecj file (deletion journal) — use ecx_actual_dir for consistency.
         // Note: Go does NOT replay .ecj into .ecx at volume load (RebuildEcxFile
@@ -757,38 +833,7 @@ impl EcVolume {
         // held the `deleted_needles` write lock for the whole scan, which on a
         // bloated journal is the entire (unbounded) startup.
         let mut loaded: HashSet<NeedleId> = HashSet::new();
-        let mut buf = vec![0u8; ECJ_LOAD_CHUNK_BYTES];
-        let end = self.ecj_file_size as u64;
-        let mut off: u64 = 0;
-        while off + NEEDLE_ID_SIZE as u64 <= end {
-            // Whole entries only; a trailing partial record is ignored, as the
-            // per-entry loop did by construction.
-            let mut want = std::cmp::min(ECJ_LOAD_CHUNK_BYTES as u64, end - off) as usize;
-            want -= want % NEEDLE_ID_SIZE;
-            if want == 0 {
-                break;
-            }
-            #[cfg(unix)]
-            {
-                use std::os::unix::fs::FileExt;
-                ecj_file.read_exact_at(&mut buf[..want], off)?;
-            }
-            #[cfg(windows)]
-            {
-                // Positional read so concurrent readers of the shared .ecj
-                // handle can't interleave seek/read. Mirrors the
-                // read_exact_at helper at the bottom of this file.
-                read_exact_at(ecj_file, &mut buf[..want], off)?;
-            }
-            #[cfg(not(any(unix, windows)))]
-            {
-                compile_error!("Platform not supported: only unix and windows are supported");
-            }
-            for entry in buf[..want].chunks_exact(NEEDLE_ID_SIZE) {
-                loaded.insert(NeedleId::from_bytes(entry));
-            }
-            off += want as u64;
-        }
+        read_ecj_ids(ecj_file, self.ecj_file_size as u64, &mut loaded)?;
 
         let mut set = self
             .deleted_needles
@@ -1864,6 +1909,63 @@ mod tests {
     use super::*;
     use tempfile::TempDir;
 
+    /// Go's NewEcVolume fails with os.ErrNotExist when neither directory has an
+    /// `.ecx`. Mounting anyway advertises shards every read then fails on with
+    /// "ecx file not open", and zeroes the size `add_shard`'s 0-byte guard needs.
+    #[test]
+    fn test_new_without_ecx_is_not_found() {
+        let data = TempDir::new().unwrap();
+        let idx = TempDir::new().unwrap();
+        let (dir, dir_idx) = (data.path().to_str().unwrap(), idx.path().to_str().unwrap());
+        std::fs::write(format!("{}/7.ec00", dir), b"shard").unwrap();
+
+        for idx_dir in [dir, dir_idx] {
+            let err = EcVolume::new(dir, idx_dir, "", VolumeId(7))
+                .err()
+                .expect("an EC volume without an .ecx must not mount");
+            assert_eq!(err.kind(), io::ErrorKind::NotFound, "{}", err);
+        }
+    }
+
+    /// A 0-byte `.ecx` stub left by an interrupted copy yields to a non-empty
+    /// copy in the other directory, whichever side the stub is on.
+    #[test]
+    fn test_new_prefers_non_empty_ecx_over_zero_byte_stub() {
+        for stub_in_idx_dir in [true, false] {
+            let data = TempDir::new().unwrap();
+            let idx = TempDir::new().unwrap();
+            let (dir, dir_idx) = (data.path().to_str().unwrap(), idx.path().to_str().unwrap());
+            let (stub_dir, valid_dir) = if stub_in_idx_dir {
+                (dir_idx, dir)
+            } else {
+                (dir, dir_idx)
+            };
+            std::fs::write(format!("{}/7.ecx", stub_dir), b"").unwrap();
+            std::fs::write(
+                format!("{}/7.ecx", valid_dir),
+                vec![0u8; NEEDLE_MAP_ENTRY_SIZE],
+            )
+            .unwrap();
+
+            let vol = EcVolume::new(dir, dir_idx, "", VolumeId(7)).unwrap();
+            assert_eq!(vol.ecx_actual_dir(), valid_dir);
+            assert_eq!(vol.ecx_file_size, NEEDLE_MAP_ENTRY_SIZE as i64);
+        }
+    }
+
+    /// With no other copy a 0-byte `.ecx` is a legitimate empty index (a volume
+    /// whose needles were all deleted before encoding) and still mounts, as in Go.
+    #[test]
+    fn test_new_accepts_lone_zero_byte_ecx_as_empty_index() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        std::fs::write(format!("{}/7.ecx", dir), b"").unwrap();
+
+        let vol = EcVolume::new(dir, dir, "", VolumeId(7)).unwrap();
+        assert_eq!(vol.ecx_file_size, 0);
+        assert!(!is_usable_ecx_file(&vol.ecx_file_name()));
+    }
+
     /// `destroy()` must remove co-located `.ecsum` sidecars (Go Destroy parity).
     /// Without this, `collection.delete` leaves orphaned bitrot files that
     /// inflate EC-health scanners after the shards are gone.
@@ -4558,7 +4660,7 @@ impl EcLocalShard {
             .file
             .as_ref()
             .map_err(|e| io::Error::new(e.kind(), e.to_string()))?;
-        crate::storage::io::read_at(file, buf, offset)
+        crate::storage::io::read_full_at(file, buf, offset)
     }
 }
 

seaweed-volume/src/storage/idx/mod.rs

@@ -21,12 +21,26 @@ where
     let mut buf = vec![0u8; NEEDLE_MAP_ENTRY_SIZE * ROWS_TO_READ];
 
     loop {
-        let count = match reader.read(&mut buf) {
-            Ok(0) => return Ok(()),
-            Ok(n) => n,
-            Err(ref e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(()),
-            Err(e) => return Err(e),
-        };
+        // Fill the batch before decoding: `read` may return a count that is
+        // not a multiple of the entry size, and a split entry would misalign
+        // every later row. Go is immune: `ReadAt` fills or errors.
+        let mut count = 0;
+        let mut eof = false;
+        while count < buf.len() {
+            match reader.read(&mut buf[count..]) {
+                Ok(0) => {
+                    eof = true;
+                    break;
+                }
+                Ok(n) => count += n,
+                Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
+                Err(ref e) if e.kind() == io::ErrorKind::UnexpectedEof => {
+                    eof = true;
+                    break;
+                }
+                Err(e) => return Err(e),
+            }
+        }
 
         let mut i = 0;
         while i + NEEDLE_MAP_ENTRY_SIZE <= count {
@@ -34,6 +48,11 @@ where
             f(key, offset, size)?;
             i += NEEDLE_MAP_ENTRY_SIZE;
         }
+
+        // A trailing partial entry at EOF is ignored, as Go does on `io.EOF`.
+        if eof {
+            return Ok(());
+        }
     }
 }
 
@@ -177,6 +196,111 @@ mod tests {
         data
     }
 
+    /// Reader that hands back at most `chunk` bytes per `read`. 7 is coprime
+    /// with the 17-byte entry size, so nearly every read ends mid-entry. With
+    /// `interrupts`, every other call fails with `ErrorKind::Interrupted`.
+    struct ShortReader {
+        inner: Cursor<Vec<u8>>,
+        chunk: usize,
+        interrupts: bool,
+        interrupt_next: bool,
+    }
+
+    impl ShortReader {
+        fn new(data: Vec<u8>, interrupts: bool) -> Self {
+            ShortReader {
+                inner: Cursor::new(data),
+                chunk: 7,
+                interrupts,
+                interrupt_next: false,
+            }
+        }
+    }
+
+    impl Read for ShortReader {
+        fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+            if self.interrupt_next {
+                self.interrupt_next = false;
+                return Err(io::Error::from(io::ErrorKind::Interrupted));
+            }
+            self.interrupt_next = self.interrupts;
+            let n = buf.len().min(self.chunk);
+            self.inner.read(&mut buf[..n])
+        }
+    }
+
+    impl Seek for ShortReader {
+        fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
+            self.inner.seek(pos)
+        }
+    }
+
+    fn walk_all<R: Read + Seek>(reader: &mut R, start_from: u64) -> Vec<(NeedleId, i64, Size)> {
+        let mut collected = Vec::new();
+        walk_index_file(reader, start_from, |key, offset, size| {
+            collected.push((key, offset.to_actual_offset(), size));
+            Ok(())
+        })
+        .unwrap();
+        collected
+    }
+
+    /// More than one ROWS_TO_READ batch, so the walk crosses a buffer refill.
+    fn many_entries() -> Vec<(NeedleId, Offset, Size)> {
+        (0..(ROWS_TO_READ as u64 * 2 + 37))
+            .map(|i| {
+                (
+                    NeedleId(i * 7 + 1),
+                    Offset::from_actual_offset(i as i64 * 128),
+                    Size(i as i32 + 1),
+                )
+            })
+            .collect()
+    }
+
+    #[test]
+    fn test_walk_index_file_short_reads_keep_alignment() {
+        let data = idx_bytes(&many_entries());
+        let expected = walk_all(&mut Cursor::new(data.clone()), 0);
+        assert_eq!(expected.len(), ROWS_TO_READ * 2 + 37);
+
+        let mut short = ShortReader::new(data, false);
+        assert_eq!(walk_all(&mut short, 0), expected);
+    }
+
+    #[test]
+    fn test_walk_index_file_retries_interrupted_reads() {
+        let data = idx_bytes(&many_entries());
+        let expected = walk_all(&mut Cursor::new(data.clone()), 0);
+
+        let mut short = ShortReader::new(data, true);
+        assert_eq!(walk_all(&mut short, 0), expected);
+    }
+
+    #[test]
+    fn test_walk_index_file_short_reads_start_from() {
+        let data = idx_bytes(&many_entries());
+        let expected = walk_all(&mut Cursor::new(data.clone()), 0);
+
+        let start = ROWS_TO_READ as u64 + 5;
+        let mut short = ShortReader::new(data, false);
+        assert_eq!(walk_all(&mut short, start), expected[start as usize..]);
+    }
+
+    #[test]
+    fn test_walk_index_file_ignores_trailing_partial_entry() {
+        // A torn final entry is dropped without an error, as Go does on io.EOF.
+        let entries = many_entries();
+        let mut data = idx_bytes(&entries);
+        data.extend_from_slice(&[0xAB; NEEDLE_MAP_ENTRY_SIZE - 1]);
+
+        let expected = walk_all(&mut Cursor::new(idx_bytes(&entries)), 0);
+        assert_eq!(walk_all(&mut Cursor::new(data.clone()), 0), expected);
+
+        let mut short = ShortReader::new(data, false);
+        assert_eq!(walk_all(&mut short, 0), expected);
+    }
+
     #[test]
     fn test_check_index_file_clean() {
         let data = idx_bytes(&[

seaweed-volume/src/storage/io.rs

@@ -36,23 +36,11 @@ pub(crate) fn read_exact_at(file: &File, buf: &mut [u8], offset: u64) -> io::Res
     }
     #[cfg(windows)]
     {
-        use std::os::windows::fs::FileExt;
-        let mut filled = 0;
-        let mut at = offset;
-        while filled < buf.len() {
-            let n = match file.seek_read(&mut buf[filled..], at) {
-                Ok(n) => n,
-                Err(err) if err.kind() == io::ErrorKind::Interrupted => continue,
-                Err(err) => return Err(err),
-            };
-            if n == 0 {
-                return Err(io::Error::new(
-                    io::ErrorKind::UnexpectedEof,
-                    "unexpected EOF in seek_read",
-                ));
-            }
-            filled += n;
-            at += n as u64;
+        if read_full_at(file, buf, offset)? < buf.len() {
+            return Err(io::Error::new(
+                io::ErrorKind::UnexpectedEof,
+                "unexpected EOF in seek_read",
+            ));
         }
     }
     #[cfg(not(any(unix, windows)))]
@@ -84,9 +72,34 @@ pub(crate) fn read_at(file: &File, buf: &mut [u8], offset: u64) -> io::Result<us
     }
 }
 
+/// Reads into `buf` at `offset` until it is full or the file ends, retrying
+/// interrupted reads; returns how many bytes were read.
+///
+/// Unlike [`read_at`], a count below `buf.len()` always means end of file.
+pub(crate) fn read_full_at(file: &File, buf: &mut [u8], offset: u64) -> io::Result<usize> {
+    fill_at(|b, at| read_at(file, b, at), buf, offset)
+}
+
+fn fill_at(
+    mut read: impl FnMut(&mut [u8], u64) -> io::Result<usize>,
+    buf: &mut [u8],
+    offset: u64,
+) -> io::Result<usize> {
+    let mut filled = 0;
+    while filled < buf.len() {
+        match read(&mut buf[filled..], offset + filled as u64) {
+            Ok(0) => break,
+            Ok(n) => filled += n,
+            Err(err) if err.kind() == io::ErrorKind::Interrupted => {}
+            Err(err) => return Err(err),
+        }
+    }
+    Ok(filled)
+}
+
 #[cfg(test)]
 mod tests {
-    use super::{read_at, read_exact_at};
+    use super::{fill_at, read_at, read_exact_at, read_full_at};
     use std::io::{ErrorKind, Write};
 
     fn temp_file(bytes: &[u8]) -> tempfile::NamedTempFile {
@@ -139,4 +152,57 @@ mod tests {
         let n = read_at(f.as_file(), &mut buf, 10).expect("read");
         assert_eq!(n, 0);
     }
+
+    /// A source that returns at most `chunk` bytes per call and fails with
+    /// `Interrupted` on its first call, like a network mount under a signal.
+    fn chunked(src: &[u8], chunk: usize) -> impl FnMut(&mut [u8], u64) -> std::io::Result<usize> {
+        let mut interrupted = false;
+        move |buf, at| {
+            if !interrupted {
+                interrupted = true;
+                return Err(ErrorKind::Interrupted.into());
+            }
+            let at = (at as usize).min(src.len());
+            let n = buf.len().min(chunk).min(src.len() - at);
+            buf[..n].copy_from_slice(&src[at..at + n]);
+            Ok(n)
+        }
+    }
+
+    #[test]
+    fn fill_at_fills_across_short_and_interrupted_reads() {
+        let src: Vec<u8> = (0..=255).collect();
+        let mut buf = [0u8; 100];
+        let n = fill_at(chunked(&src, 7), &mut buf, 50).expect("read");
+        assert_eq!(n, buf.len());
+        assert_eq!(&buf[..], &src[50..150]);
+    }
+
+    #[test]
+    fn fill_at_stops_at_end_of_source() {
+        let src: Vec<u8> = (0..=255).collect();
+        let mut buf = [0u8; 100];
+        let n = fill_at(chunked(&src, 7), &mut buf, 200).expect("read");
+        assert_eq!(n, 56);
+        assert_eq!(&buf[..n], &src[200..]);
+    }
+
+    #[test]
+    fn fill_at_propagates_other_errors() {
+        let mut buf = [0u8; 8];
+        let err = fill_at(|_, _| Err(ErrorKind::PermissionDenied.into()), &mut buf, 0)
+            .expect_err("error");
+        assert_eq!(err.kind(), ErrorKind::PermissionDenied);
+    }
+
+    #[test]
+    fn read_full_at_returns_the_short_count_only_at_eof() {
+        let f = temp_file(b"0123456789");
+        let mut buf = [0u8; 8];
+        assert_eq!(read_full_at(f.as_file(), &mut buf, 0).expect("read"), 8);
+        assert_eq!(&buf, b"01234567");
+        assert_eq!(read_full_at(f.as_file(), &mut buf, 6).expect("read"), 4);
+        assert_eq!(&buf[..4], b"6789");
+        assert_eq!(read_full_at(f.as_file(), &mut buf, 10).expect("read"), 0);
+    }
 }

seaweed-volume/src/storage/io_error.rs

@@ -3,7 +3,7 @@
 
 use std::io;
 use std::sync::Mutex;
-use std::sync::atomic::{AtomicBool, AtomicI32, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 
 /// Consecutive storage-media errors allowed before the volume is quarantined.
 pub(crate) const IO_ERROR_TOLERANCE: i32 = 3;
@@ -34,10 +34,30 @@ pub(crate) fn is_storage_io_error(e: &io::Error) -> bool {
 #[derive(Default)]
 pub(crate) struct IoErrorTracker {
     last: Mutex<Option<String>>,
-    count: AtomicI32,
+    /// The consecutive error count in the low 32 bits and, in the high 32,
+    /// how many times it has been cleared. They share one word so that
+    /// `record_success_at` updates both in one step: reads record their
+    /// outcomes here without the volume's write lock.
+    streak: AtomicU64,
     quarantined: AtomicBool,
 }
 
+const STREAK_COUNT_BITS: u64 = 0xffff_ffff;
+
+fn streak_count(streak: u64) -> i32 {
+    (streak & STREAK_COUNT_BITS) as i32
+}
+
+/// `streak` with its count cleared and one more clear on record.
+fn streak_cleared(streak: u64) -> u64 {
+    (streak >> 32).wrapping_add(1) << 32
+}
+
+/// A point in the error streak, taken where a write landed whose success
+/// is only recorded later. See `IoErrorTracker::record_success_at`.
+#[derive(Clone, Copy)]
+pub(crate) struct StreakMark(u64);
+
 impl IoErrorTracker {
     /// `Some(e)` records a failure, `None` a success. Only storage-media
     /// failures count; every other outcome clears the count and last error.
@@ -45,14 +65,22 @@ impl IoErrorTracker {
         if let Some(e) = err
             && is_storage_io_error(e)
         {
-            self.count.fetch_add(1, Ordering::Relaxed);
+            self.streak.fetch_add(1, Ordering::Relaxed);
             if let Ok(mut guard) = self.last.lock() {
                 *guard = Some(e.to_string());
             }
             crate::metrics::STORAGE_IO_ERROR_COUNTER.inc();
             return;
         }
-        self.count.store(0, Ordering::Relaxed);
+        self.clear_count();
+        self.clear_last();
+    }
+
+    fn clear_count(&self) {
+        self.update_streak(|streak| Some(streak_cleared(streak)));
+    }
+
+    fn clear_last(&self) {
         if let Ok(mut guard) = self.last.lock()
             && guard.is_some()
         {
@@ -60,17 +88,58 @@ impl IoErrorTracker {
         }
     }
 
+    /// Apply `f` to the streak atomically; `None` leaves it as it is.
+    /// Returns the streak `f` produced, if any.
+    fn update_streak(&self, mut f: impl FnMut(u64) -> Option<u64>) -> Option<u64> {
+        let mut updated = None;
+        let _ = self
+            .streak
+            .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |streak| {
+                updated = f(streak);
+                updated
+            });
+        updated
+    }
+
+    pub(crate) fn mark(&self) -> StreakMark {
+        StreakMark(self.streak.load(Ordering::Relaxed))
+    }
+
+    /// Record a success as if it had come at `mark`: the errors counted
+    /// before the mark are cleared and the ones counted since still stand,
+    /// as they would had each outcome been recorded in order. A streak
+    /// cleared since the mark is left as it is.
+    pub(crate) fn record_success_at(&self, mark: StreakMark) {
+        let before = streak_count(mark.0);
+        let updated = self.update_streak(|streak| {
+            if streak >> 32 != mark.0 >> 32 {
+                return None;
+            }
+            if streak_count(streak) <= before {
+                return Some(streak_cleared(streak));
+            }
+            Some(streak - before as u64)
+        });
+        // The last error stays when one counted since the mark is left.
+        if updated.is_some_and(|streak| streak_count(streak) == 0) {
+            self.clear_last();
+        }
+    }
+
     /// The last recorded error, the consecutive count, and the quarantine flag.
     pub(crate) fn get_io_error_state(&self) -> (Option<String>, i32, bool) {
         let err = self.last.lock().ok().and_then(|g| g.clone());
-        let count = self.count.load(Ordering::Relaxed);
+        let count = self.count();
         let quarantined = self.quarantined.load(Ordering::Relaxed);
         (err, count, quarantined)
     }
 
+    fn count(&self) -> i32 {
+        streak_count(self.streak.load(Ordering::Relaxed))
+    }
+
     pub(crate) fn should_quarantine(&self) -> bool {
-        self.quarantined.load(Ordering::Relaxed)
-            || self.count.load(Ordering::Relaxed) >= IO_ERROR_TOLERANCE
+        self.quarantined.load(Ordering::Relaxed) || self.count() >= IO_ERROR_TOLERANCE
     }
 
     pub(crate) fn mark_io_quarantined(&self) {
@@ -78,7 +147,7 @@ impl IoErrorTracker {
     }
 
     pub(crate) fn reset_io_error_state(&self) {
-        self.count.store(0, Ordering::Relaxed);
+        self.clear_count();
         self.quarantined.store(false, Ordering::Relaxed);
         if let Ok(mut guard) = self.last.lock() {
             *guard = None;
@@ -91,9 +160,11 @@ impl IoErrorTracker {
             *guard = err.map(|value| value.to_string());
         }
         if err.is_some() {
-            self.count.store(IO_ERROR_TOLERANCE, Ordering::Relaxed);
+            self.update_streak(|streak| {
+                Some((streak & !STREAK_COUNT_BITS) | IO_ERROR_TOLERANCE as u64)
+            });
         } else {
-            self.count.store(0, Ordering::Relaxed);
+            self.clear_count();
         }
     }
 }
@@ -171,6 +242,87 @@ mod tests {
         assert!(tracker.should_quarantine());
     }
 
+    #[test]
+    fn success_at_a_mark_keeps_only_the_errors_after_it() {
+        let tracker = IoErrorTracker::default();
+        tracker.check_read_write_error(Some(&media_error()));
+        tracker.check_read_write_error(Some(&media_error()));
+        let mark = tracker.mark();
+        tracker.check_read_write_error(Some(&media_error()));
+        tracker.record_success_at(mark);
+
+        assert_eq!(
+            tracker.get_io_error_state(),
+            (Some(media_error().to_string()), 1, false)
+        );
+    }
+
+    #[test]
+    fn success_at_a_mark_with_nothing_after_it_clears_the_streak() {
+        let tracker = IoErrorTracker::default();
+        tracker.check_read_write_error(Some(&media_error()));
+        let mark = tracker.mark();
+        tracker.record_success_at(mark);
+
+        assert_eq!(tracker.get_io_error_state(), (None, 0, false));
+    }
+
+    #[test]
+    fn success_at_a_mark_leaves_a_streak_cleared_since() {
+        let tracker = IoErrorTracker::default();
+        tracker.check_read_write_error(Some(&media_error()));
+        tracker.check_read_write_error(Some(&media_error()));
+        let mark = tracker.mark();
+        tracker.check_read_write_error(None);
+        tracker.check_read_write_error(Some(&media_error()));
+        tracker.record_success_at(mark);
+
+        assert_eq!(
+            tracker.get_io_error_state(),
+            (Some(media_error().to_string()), 1, false)
+        );
+    }
+
+    /// Reads update the tracker without the volume's write lock, so a
+    /// success replayed at a mark must not lose the errors they record
+    /// while it runs.
+    #[test]
+    fn success_at_a_mark_keeps_concurrent_errors() {
+        use std::sync::{Arc, Barrier};
+        const READERS: i32 = 4;
+        const ERRORS: i32 = 200;
+
+        for _ in 0..500 {
+            let tracker = Arc::new(IoErrorTracker::default());
+            tracker.check_read_write_error(Some(&media_error()));
+            tracker.check_read_write_error(Some(&media_error()));
+            let mark = tracker.mark();
+            let start = Arc::new(Barrier::new(READERS as usize + 1));
+            let readers: Vec<_> = (0..READERS)
+                .map(|_| {
+                    let (tracker, start) = (tracker.clone(), start.clone());
+                    std::thread::spawn(move || {
+                        start.wait();
+                        for _ in 0..ERRORS {
+                            tracker.check_read_write_error(Some(&media_error()));
+                        }
+                    })
+                })
+                .collect();
+            start.wait();
+            while tracker.get_io_error_state().1 < 2 + READERS * ERRORS / 2 {
+                std::hint::spin_loop();
+            }
+            tracker.record_success_at(mark);
+            for reader in readers {
+                reader.join().unwrap();
+            }
+            // The two errors before the mark are cleared; every error the
+            // readers recorded after it stands.
+            assert_eq!(tracker.get_io_error_state().1, READERS * ERRORS);
+        }
+    }
+
     #[test]
     fn reset_io_error_state_lifts_the_quarantine() {
         let tracker = IoErrorTracker::default();

seaweed-volume/src/storage/needle_map.rs

@@ -247,6 +247,8 @@ impl CompactNeedleMap {
     pub fn load_from_idx<R: Read + Seek>(reader: &mut R, version: Version) -> io::Result<Self> {
         let mut nm = CompactNeedleMap::new();
         idx::walk_index_file(reader, 0, |key, offset, size| {
+            // A read-only load attaches no writer, so this is its only size.
+            nm.idx_file_offset += NEEDLE_MAP_ENTRY_SIZE as u64;
             nm.metric.maybe_set_max_needle_end(offset, size, version);
             if offset.is_zero() || size.is_deleted() {
                 nm.delete_from_map(key);
@@ -1386,6 +1388,18 @@ impl NeedleMap {
         }
     }
 
+    /// Skew the live file count away from what the `.idx` holds, so tests
+    /// can build a volume whose reported count disagrees with a reload.
+    #[cfg(test)]
+    pub(crate) fn add_file_count_for_test(&self, delta: i64) {
+        let metric = match self {
+            NeedleMap::InMemory(nm) => &nm.metric,
+            NeedleMap::Redb(nm) => &nm.metric,
+            NeedleMap::SortedFile(_) => panic!("sorted-file needle maps are read-only"),
+        };
+        metric.file_count.fetch_add(delta, Ordering::Relaxed);
+    }
+
     /// Largest (offset + actual size) seen during the load walk; 0 if the
     /// map is empty. Used at volume load to detect .idx entries that
     /// reference past the end of .dat (issue #8928) without a second scan.

seaweed-volume/src/storage/store_ec_mirror.rs

@@ -131,6 +131,12 @@ impl Store {
                     let Some(base) = name.strip_suffix(".ecx") else {
                         continue;
                     };
+                    // A 0-byte .ecx is a corrupt stub from a failed copy, not a
+                    // credible owner — skip it so the scan keeps looking for a
+                    // real index on a sibling disk (Go's indexEcxOwners).
+                    if !ent.metadata().is_ok_and(|m| m.len() > 0) {
+                        continue;
+                    }
                     let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
                         continue;
                     };
@@ -417,4 +423,33 @@ mod tests {
         let post = fs::read(dir0.join(format!("{}_{}.ecx", collection, vid))).unwrap();
         assert_eq!(post, ecx_local, "mirror overwrote dir0's existing .ecx");
     }
+
+    /// The mirror shares Go's indexEcxOwners, which skips a 0-byte `.ecx`:
+    /// a stub must not be chosen as the source to mirror from.
+    #[test]
+    fn mirror_owner_index_skips_zero_byte_ecx() {
+        let tmp = TempDir::new().unwrap();
+        let dir0 = tmp.path().join("data0");
+        let dir1 = tmp.path().join("data1");
+        fs::create_dir_all(&dir0).unwrap();
+        fs::create_dir_all(&dir1).unwrap();
+
+        let collection = "video-recordings";
+        let vid = 4123u32;
+        plant_ecx(&dir0, collection, vid, b"");
+        plant_ecx(&dir1, collection, vid, &[0xA1u8; 20]);
+
+        let mut store = Store::new(NeedleMapKind::InMemory);
+        add_loc(&mut store, &dir0);
+        add_loc(&mut store, &dir1);
+
+        let owners = store.index_ecx_owners_for_mirror();
+        let owner = owners
+            .get(&EcKey {
+                collection: collection.to_string(),
+                vid: VolumeId(vid),
+            })
+            .expect("the valid .ecx on disk 1 must be indexed");
+        assert_eq!(owner.location, 1);
+    }
 }

seaweed-volume/src/storage/store_ec_reconcile.rs

@@ -37,6 +37,7 @@ pub(crate) struct EcVolumeMissingIndex {
     pub data_dir: String,
 }
 
+#[cfg(test)]
 pub(crate) fn ec_local_ecx_path(dir: &str, collection: &str, vid: VolumeId) -> String {
     if collection.is_empty() {
         format!("{}/{}.ecx", dir, vid.0)
@@ -117,10 +118,9 @@ impl Store {
                     );
                     continue;
                 };
-                let local_ecx = ec_local_ecx_path(&loc.idx_directory, &key.collection, key.vid);
-                let local_ecx_in_data = ec_local_ecx_path(&loc.directory, &key.collection, key.vid);
-                let use_local_idx = std::path::Path::new(&local_ecx).exists()
-                    || std::path::Path::new(&local_ecx_in_data).exists();
+                // A 0-byte local stub is not a mirrored index (Go gates this fast
+                // path on HasEcxFileOnDisk); mount against the owner instead.
+                let use_local_idx = loc.has_ecx_file_on_disk(&key.collection, key.vid);
 
                 if !use_local_idx && owner.location == loc_idx && owner.idx_dir == loc.idx_directory
                 {
@@ -422,6 +422,12 @@ impl Store {
                     let Some(base) = name.strip_suffix(".ecx") else {
                         continue;
                     };
+                    // A 0-byte .ecx is a corrupt stub from a failed copy, not a
+                    // credible owner — skip it so the scan keeps looking for a
+                    // real index on a sibling disk (Go's indexEcxOwners).
+                    if !ent.metadata().is_ok_and(|m| m.len() > 0) {
+                        continue;
+                    }
                     let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
                         continue;
                     };
@@ -639,6 +645,34 @@ mod tests {
         .unwrap();
     }
 
+    /// A 0-byte `.ecx` is not a credible owner (Go's indexEcxOwners skips
+    /// it): picking the stub would hide the valid index on the sibling disk.
+    #[test]
+    fn test_index_ecx_owners_skips_zero_byte_stub() {
+        let (store, _tmp) = make_test_store(2, None);
+        let d0 = store.locations[0].directory.clone();
+        let d1 = store.locations[1].directory.clone();
+        std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(7)), b"").unwrap();
+        write_index_files(&d1, "pics", 7, 10, 4);
+
+        let owners = store.index_ecx_owners();
+        let owner = owners
+            .get(&EcKey {
+                collection: "pics".to_string(),
+                vid: VolumeId(7),
+            })
+            .expect("the valid .ecx on disk 1 must be indexed");
+        assert_eq!(owner.location, 1);
+        assert_eq!(owner.idx_dir, d1);
+
+        // A stub with no real index anywhere owns nothing.
+        std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(8)), b"").unwrap();
+        assert!(!store.index_ecx_owners().contains_key(&EcKey {
+            collection: "pics".to_string(),
+            vid: VolumeId(8),
+        }));
+    }
+
     /// An empty `.dat` (<= a superblock, i.e. zero needles) for an EC volume
     /// is a leftover stub from the pre-fix loader. It must be swept on startup,
     /// not loaded as a phantom empty volume. With the same vid's stub on two
@@ -971,6 +1005,56 @@ mod tests {
         );
     }
 
+    /// dir0 holds orphan shards next to a 0-byte `.ecx` stub from a failed
+    /// copy; the real index is on dir1. The stub must not count as a
+    /// locally-mirrored index (Go gates that fast path on HasEcxFileOnDisk),
+    /// or the shards get registered against an empty index.
+    #[test]
+    fn test_reconcile_ignores_zero_byte_local_ecx_stub() {
+        let tmp = TempDir::new().unwrap();
+        let dir0 = tmp.path().join("data0");
+        let dir1 = tmp.path().join("data1");
+        std::fs::create_dir_all(&dir0).unwrap();
+        std::fs::create_dir_all(&dir1).unwrap();
+
+        let collection = "grafana-loki";
+        let vid = 1094u32;
+
+        write_shard(dir0.to_str().unwrap(), collection, vid, 0);
+        write_shard(dir1.to_str().unwrap(), collection, vid, 1);
+        write_index_files(dir1.to_str().unwrap(), collection, vid, 10, 4);
+
+        let mut store = Store::new(NeedleMapKind::InMemory);
+        for dir in [&dir0, &dir1] {
+            store
+                .add_location(
+                    dir.to_str().unwrap(),
+                    dir.to_str().unwrap(),
+                    100,
+                    DiskType::HardDrive,
+                    MinFreeSpace::Percent(0.0),
+                    Vec::new(),
+                )
+                .unwrap();
+        }
+        // Plant the stub after the startup scan so only the reconcile decision
+        // is under test, then drop dir0's mount and reconcile again.
+        store.locations[0].remove_ec_volume(VolumeId(vid));
+        std::fs::write(
+            ec_local_ecx_path(dir0.to_str().unwrap(), collection, VolumeId(vid)),
+            b"",
+        )
+        .unwrap();
+
+        store.reconcile_ec_shards_across_disks();
+
+        let ev0 = store.locations[0]
+            .find_ec_volume(VolumeId(vid))
+            .expect("dir0's shard must be mounted against the owner's index");
+        assert!(ev0.has_shard(0));
+        assert_eq!(ev0.ecx_actual_dir(), dir1.to_str().unwrap());
+    }
+
     /// PR 9244 review case: idx_directory is configured but the
     /// owner's .ecx / .ecj / .vif live in the owner's data dir
     /// (the legacy "written before -dir.idx was set" layout). The
@@ -1392,7 +1476,7 @@ mod tests {
         let vid = VolumeId(7004);
         let collection = "grafana-loki";
 
-        store.delete_ec_shards(vid, collection, &[1]);
+        store.delete_ec_shards(vid, collection, &[1]).unwrap();
 
         // Shard 1 file is gone on disk 1.
         let p1 = format!(

seaweed-volume/tests/http_integration.rs

@@ -477,6 +477,74 @@ async fn delete_then_get_returns_404() {
     );
 }
 
+// Go answers both with 500 and an error containing "volume N is read only"
+// (the write behind "failed to write to local disk: ").
+#[tokio::test]
+async fn write_and_delete_on_read_only_volume_say_is_read_only() {
+    let (state, _tmp) = test_state();
+    let uri = "/1,01637037d6";
+
+    let request = |method: &str, body: &[u8]| {
+        Request::builder()
+            .method(method)
+            .uri(uri)
+            .body(Body::from(body.to_vec()))
+            .unwrap()
+    };
+    let error_of = |body: Vec<u8>| -> String {
+        let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
+        json["error"].as_str().unwrap_or_default().to_string()
+    };
+
+    let response = build_admin_router(state.clone())
+        .oneshot(request("POST", b"written before read-only"))
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::CREATED);
+
+    state
+        .store
+        .write()
+        .unwrap()
+        .find_volume_mut(VolumeId(1))
+        .unwrap()
+        .1
+        .set_no_write_or_delete(true);
+
+    let response = build_admin_router(state.clone())
+        .oneshot(request("POST", b"refused"))
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
+    let error = error_of(body_bytes(response).await);
+    assert!(error.contains("volume 1 is read only"), "{error}");
+
+    let response = build_admin_router(state.clone())
+        .oneshot(request("DELETE", b""))
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
+    assert_eq!(
+        error_of(body_bytes(response).await),
+        "Deletion Failed: volume 1 is read only"
+    );
+
+    // The needle is still there to delete once the volume is writable again.
+    state
+        .store
+        .write()
+        .unwrap()
+        .find_volume_mut(VolumeId(1))
+        .unwrap()
+        .1
+        .set_no_write_or_delete(false);
+    let response = build_admin_router(state.clone())
+        .oneshot(request("DELETE", b""))
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::ACCEPTED);
+}
+
 // ============================================================================
 // 6. HEAD returns headers without body
 // ============================================================================
@@ -1255,3 +1323,102 @@ async fn non_ascii_fid_and_ttl_are_rejected_not_panicked() {
         "a non-ASCII ttl must behave like any other invalid ttl, not panic"
     );
 }
+
+// ============================================================================
+// The write queue answers an upload with the needle's real ETag
+//
+// The queue worker computes the CRC on the needle it was handed, so the handler
+// has to know the checksum before it submits. Without that every queued upload
+// came back as "00000000". The direct path is the reference: same payload, same
+// ETag, for a plain body and for one the handler gzips before storing.
+// ============================================================================
+
+#[tokio::test]
+async fn write_queue_upload_returns_same_etag_as_direct_write() {
+    use seaweed_volume::server::write_queue::WriteQueue;
+
+    let (direct_state, _direct_tmp) = test_state();
+    let (queued_state, _queued_tmp) = test_state();
+    let wq = WriteQueue::new(queued_state.clone(), 128);
+    let _ = queued_state.write_queue.set(wq);
+
+    let compressible = "seaweedfs ".repeat(200).into_bytes();
+    let uploads: [(&str, &[u8]); 2] = [
+        ("/1,01637037d6", b"hello, seaweedfs!"),
+        ("/1/02637037d6/notes.txt", &compressible),
+    ];
+
+    for (uri, payload) in uploads {
+        let mut etags = Vec::new();
+        for state in [&direct_state, &queued_state] {
+            let app = build_admin_router(state.clone());
+            let response = app
+                .oneshot(
+                    Request::builder()
+                        .method("POST")
+                        .uri(uri)
+                        .body(Body::from(payload.to_vec()))
+                        .unwrap(),
+                )
+                .await
+                .unwrap();
+            assert_eq!(response.status(), StatusCode::CREATED);
+
+            let header = response
+                .headers()
+                .get("ETag")
+                .expect("upload response has no ETag")
+                .to_str()
+                .unwrap()
+                .to_string();
+            let body = body_bytes(response).await;
+            let json: serde_json::Value =
+                serde_json::from_slice(&body).expect("POST response is not valid JSON");
+            let etag = json["eTag"].as_str().unwrap().to_string();
+            assert_eq!(header, format!("\"{}\"", etag));
+            etags.push(etag);
+        }
+        assert_ne!(etags[0], "00000000", "{}: direct ETag is the zero CRC", uri);
+        assert_eq!(
+            etags[1], etags[0],
+            "{}: queued upload must return the direct path's ETag",
+            uri
+        );
+    }
+
+    // The second upload really was stored gzipped, so its ETag is the CRC of
+    // the compressed bytes on both paths.
+    let app = build_admin_router(queued_state.clone());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri(uploads[1].0)
+                .header("Accept-Encoding", "gzip")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::OK);
+    assert_eq!(response.headers()["Content-Encoding"], "gzip");
+
+    // Re-uploading the same bytes is the unchanged path: 204 with the same ETag.
+    let (uri, payload) = uploads[0];
+    let mut etags = Vec::new();
+    for state in [&direct_state, &queued_state] {
+        let app = build_admin_router(state.clone());
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .method("POST")
+                    .uri(uri)
+                    .body(Body::from(payload.to_vec()))
+                    .unwrap(),
+            )
+            .await
+            .unwrap();
+        assert_eq!(response.status(), StatusCode::NO_CONTENT);
+        etags.push(response.headers()["ETag"].to_str().unwrap().to_string());
+    }
+    assert_eq!(etags[1], etags[0], "unchanged upload ETag differs");
+}

test/java/spark/pom.xml

@@ -21,7 +21,7 @@
         <scala.binary.version>2.12</scala.binary.version>
         <junit.version>4.13.2</junit.version>
         <seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
-        <jackson.version>2.22.1</jackson.version> <!-- Upgraded from 2.15.3 -->
+        <jackson.version>2.22.3</jackson.version> <!-- Upgraded from 2.15.3 -->
         <!-- jackson-annotations dropped the patch digit in 2.20; matches jackson.version's major.minor -->
         <jackson.annotations.version>2.22</jackson.annotations.version>
         <netty.version>4.2.16.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->

test/s3/iam/s3_sts_web_identity_api_test.go

@@ -0,0 +1,258 @@
+package iam
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/xml"
+	"io"
+	"math/big"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/golang-jwt/jwt/v5"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	webIdentityAudience = "web-identity-api-test"
+	webIdentitySubject  = "spiffe://example.org/ns/app/sa/app"
+)
+
+// webIdentityIssuer is an OIDC issuer served by the test: discovery document
+// and JWKS for one RSA key, so the server under test fetches real keys.
+type webIdentityIssuer struct {
+	server *httptest.Server
+	key    *rsa.PrivateKey
+}
+
+func newWebIdentityIssuer(t *testing.T) *webIdentityIssuer {
+	t.Helper()
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	require.NoError(t, err)
+	issuer := &webIdentityIssuer{key: key}
+	mux := http.NewServeMux()
+	mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"issuer":                                issuer.server.URL,
+			"jwks_uri":                              issuer.server.URL + "/keys",
+			"id_token_signing_alg_values_supported": []string{"RS256"},
+		})
+	})
+	mux.HandleFunc("/keys", func(w http.ResponseWriter, _ *http.Request) {
+		b64 := func(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) }
+		_ = json.NewEncoder(w).Encode(map[string]any{"keys": []any{map[string]any{
+			"kty": "RSA", "kid": "k1", "use": "sig", "alg": "RS256",
+			"n": b64(key.PublicKey.N.Bytes()), "e": b64(big.NewInt(int64(key.PublicKey.E)).Bytes()),
+		}}})
+	})
+	issuer.server = httptest.NewServer(mux)
+	t.Cleanup(issuer.server.Close)
+	return issuer
+}
+
+func (i *webIdentityIssuer) claims(sub string) jwt.MapClaims {
+	now := time.Now()
+	return jwt.MapClaims{"iss": i.server.URL, "sub": sub, "aud": webIdentityAudience,
+		"iat": now.Unix(), "exp": now.Add(10 * time.Minute).Unix()}
+}
+
+func (i *webIdentityIssuer) token(t *testing.T, sub string, key *rsa.PrivateKey) string {
+	t.Helper()
+	tok := jwt.NewWithClaims(jwt.SigningMethodRS256, i.claims(sub))
+	tok.Header["kid"] = "k1"
+	signed, err := tok.SignedString(key)
+	require.NoError(t, err)
+	return signed
+}
+
+func (i *webIdentityIssuer) tokenForAudience(t *testing.T, sub, aud string) string {
+	t.Helper()
+	claims := i.claims(sub)
+	claims["aud"] = aud
+	tok := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)
+	tok.Header["kid"] = "k1"
+	signed, err := tok.SignedString(i.key)
+	require.NoError(t, err)
+	return signed
+}
+
+func (i *webIdentityIssuer) unsignedToken(t *testing.T, sub string) string {
+	t.Helper()
+	signed, err := jwt.NewWithClaims(jwt.SigningMethodNone, i.claims(sub)).SignedString(jwt.UnsafeAllowNoneSignatureType)
+	require.NoError(t, err)
+	return signed
+}
+
+func trustPolicyFor(issuerURL, sub string) string {
+	doc, _ := json.Marshal(map[string]any{"Version": "2012-10-17", "Statement": []any{map[string]any{
+		"Effect": "Allow", "Principal": map[string]any{"Federated": issuerURL},
+		"Action":    []string{"sts:AssumeRoleWithWebIdentity"},
+		"Condition": map[string]any{"StringEquals": map[string]any{"oidc:sub": sub}},
+	}}})
+	return string(doc)
+}
+
+// assumeWithWebIdentity returns session credentials, or nil and the error body.
+func assumeWithWebIdentity(t *testing.T, roleArn, token string) (*credentials.Credentials, string) {
+	t.Helper()
+	resp, err := callSTSAPI(t, url.Values{
+		"Action": {"AssumeRoleWithWebIdentity"}, "Version": {"2011-06-15"},
+		"RoleArn": {roleArn}, "RoleSessionName": {"web-identity-api"}, "WebIdentityToken": {token},
+	})
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	if resp.StatusCode != http.StatusOK {
+		return nil, string(body)
+	}
+	var out AssumeRoleWithWebIdentityTestResponse
+	require.NoError(t, xml.Unmarshal(body, &out), "body: %s", body)
+	c := out.Result.Credentials
+	return credentials.NewStaticCredentials(c.AccessKeyId, c.SecretAccessKey, c.SessionToken), ""
+}
+
+func s3ClientWith(t *testing.T, creds *credentials.Credentials) *s3.S3 {
+	t.Helper()
+	sess, err := session.NewSession(&aws.Config{
+		Region: aws.String(TestRegion), Endpoint: aws.String(TestS3Endpoint),
+		Credentials: creds, S3ForcePathStyle: aws.Bool(true), DisableSSL: aws.Bool(true),
+	})
+	require.NoError(t, err)
+	return s3.New(sess)
+}
+
+// TestWebIdentityWithProviderAndRoleManagedThroughIAMAPI configures STS
+// federation entirely at runtime — an OIDC provider, a managed policy and a role
+// created through the IAM API, with no static configuration — and checks that
+// the role admits exactly the subject its trust policy names, with exactly the
+// permissions of its attached policy.
+func TestWebIdentityWithProviderAndRoleManagedThroughIAMAPI(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	if !isSTSEndpointRunning(t) {
+		t.Fatal("SeaweedFS STS endpoint is not running at", TestSTSEndpoint, "- please run 'make setup-all-tests' first")
+	}
+
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+	admin, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+	adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	issuer := newWebIdentityIssuer(t)
+	bucket := framework.GenerateUniqueBucketName("web-identity")
+	require.NoError(t, framework.CreateBucketWithCleanup(adminS3, bucket))
+
+	provider, err := admin.CreateOpenIDConnectProvider(&iam.CreateOpenIDConnectProviderInput{
+		Url:          aws.String(issuer.server.URL),
+		ClientIDList: []*string{aws.String(webIdentityAudience)},
+		// Required by this SDK version; pinning only applies to a TLS issuer,
+		// and the test issuer is plain HTTP.
+		ThumbprintList: []*string{aws.String("0000000000000000000000000000000000000000")},
+	})
+	require.NoError(t, err)
+	defer admin.DeleteOpenIDConnectProvider(&iam.DeleteOpenIDConnectProviderInput{OpenIDConnectProviderArn: provider.OpenIDConnectProviderArn})
+
+	policyDoc, _ := json.Marshal(map[string]any{"Version": "2012-10-17", "Statement": []any{map[string]any{
+		"Effect": "Allow", "Action": []string{"s3:*"},
+		"Resource": []string{"arn:aws:s3:::" + bucket, "arn:aws:s3:::" + bucket + "/*"},
+	}}})
+	policy, err := admin.CreatePolicy(&iam.CreatePolicyInput{
+		PolicyName: aws.String(bucket + "-rw"), PolicyDocument: aws.String(string(policyDoc)),
+	})
+	require.NoError(t, err)
+	defer admin.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policy.Policy.Arn})
+
+	// Role names are at most 64 characters; the bucket name is longer.
+	roleName := "web-identity-" + strconv.FormatInt(time.Now().UnixNano(), 36)
+	role, err := admin.CreateRole(&iam.CreateRoleInput{
+		RoleName:                 aws.String(roleName),
+		AssumeRolePolicyDocument: aws.String(trustPolicyFor(issuer.server.URL, webIdentitySubject)),
+	})
+	require.NoError(t, err)
+	defer admin.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
+	_, err = admin.AttachRolePolicy(&iam.AttachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
+	require.NoError(t, err)
+	defer admin.DetachRolePolicy(&iam.DetachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
+	roleArn := aws.StringValue(role.Role.Arn)
+
+	t.Run("the trusted subject gets credentials scoped to the attached policy", func(t *testing.T) {
+		creds, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
+		require.NotNil(t, creds, "AssumeRoleWithWebIdentity refused the trusted subject: %s", failure)
+		client := s3ClientWith(t, creds)
+		_, err := client.PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("federated.txt")})
+		assert.NoError(t, err, "the session cannot write the bucket its policy grants")
+		_, err = client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket + "-other")})
+		assert.Error(t, err, "the session created a bucket its policy does not grant")
+	})
+
+	refusals := map[string]string{
+		"another subject": issuer.token(t, "spiffe://example.org/ns/other/sa/other", issuer.key),
+		"a token signed by another key": func() string {
+			other, err := rsa.GenerateKey(rand.Reader, 2048)
+			require.NoError(t, err)
+			return issuer.token(t, webIdentitySubject, other)
+		}(),
+		"an unsigned token": issuer.unsignedToken(t, webIdentitySubject),
+		// The provider accepts only its registered client IDs as audiences.
+		"a token issued for another audience": issuer.tokenForAudience(t, webIdentitySubject, "some-other-service"),
+	}
+	for name, token := range refusals {
+		t.Run("refuses "+name, func(t *testing.T) {
+			creds, _ := assumeWithWebIdentity(t, roleArn, token)
+			assert.Nil(t, creds, "AssumeRoleWithWebIdentity issued credentials for %s", name)
+		})
+	}
+
+	t.Run("an updated trust policy takes effect", func(t *testing.T) {
+		moved := "spiffe://example.org/ns/app/sa/moved"
+		_, err := admin.UpdateAssumeRolePolicy(&iam.UpdateAssumeRolePolicyInput{
+			RoleName: aws.String(roleName), PolicyDocument: aws.String(trustPolicyFor(issuer.server.URL, moved)),
+		})
+		require.NoError(t, err)
+		creds, _ := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
+		assert.Nil(t, creds, "the subject removed from the trust policy still assumes the role")
+		creds, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, moved, issuer.key))
+		assert.NotNil(t, creds, "the subject added to the trust policy was refused: %s", failure)
+	})
+
+	t.Run("a session does not survive its role being deleted and created again", func(t *testing.T) {
+		trust := trustPolicyFor(issuer.server.URL, webIdentitySubject)
+		_, err := admin.UpdateAssumeRolePolicy(&iam.UpdateAssumeRolePolicyInput{RoleName: aws.String(roleName), PolicyDocument: aws.String(trust)})
+		require.NoError(t, err)
+		before, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
+		require.NotNil(t, before, "precondition: %s", failure)
+
+		_, err = admin.DetachRolePolicy(&iam.DetachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
+		require.NoError(t, err)
+		_, err = admin.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
+		require.NoError(t, err)
+		recreated, err := admin.CreateRole(&iam.CreateRoleInput{RoleName: aws.String(roleName), AssumeRolePolicyDocument: aws.String(trust)})
+		require.NoError(t, err)
+		_, err = admin.AttachRolePolicy(&iam.AttachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
+		require.NoError(t, err)
+		assert.NotEqual(t, aws.StringValue(role.Role.RoleId), aws.StringValue(recreated.Role.RoleId), "the recreated role reuses the deleted role's ID")
+
+		_, err = s3ClientWith(t, before).PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("revived.txt")})
+		assert.Error(t, err, "a session of the deleted role works again under the new role of the same name")
+		after, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
+		require.NotNil(t, after, "the new role refuses its trusted subject: %s", failure)
+		_, err = s3ClientWith(t, after).PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("fresh.txt")})
+		assert.NoError(t, err, "a session of the new role cannot use its policy")
+	})
+}

test/s3/remote_cache/remote_cache_evict_test.go

@@ -0,0 +1,231 @@
+package remote_cache
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	evictRemoteS3      = "http://localhost:28334"
+	evictRemoteMaster  = "30334"
+	evictRemoteFiler   = "28889"
+	evictRemoteVolume  = "30341"
+	evictRemoteWebdav  = "27334"
+	evictRemoteMetrics = "30326"
+
+	evictPrimaryS3      = "http://localhost:28333"
+	evictPrimaryMaster  = "30333"
+	evictPrimaryFiler   = "28887"
+	evictPrimaryVolume  = "30340"
+	evictPrimaryWebdav  = "27333"
+	evictPrimaryMetrics = "30327"
+
+	evictBucket = "evictsrc"
+	evictMount  = "evictmnt"
+)
+
+var miniLogPaths []string
+
+func startMini(t *testing.T, dir string, args ...string) {
+	t.Helper()
+	require.NoError(t, os.MkdirAll(dir, 0755))
+	logPath := filepath.Join(dir, "weed.log")
+	logFile, err := os.Create(logPath)
+	require.NoError(t, err)
+	miniLogPaths = append(miniLogPaths, logPath)
+	cmd := exec.Command(weedBinary, append([]string{"mini",
+		"-dir=" + dir,
+		"-s3.config=s3_config.json",
+		"-s3.allowDeleteBucketNotEmpty=true",
+		"-ip=127.0.0.1", "-ip.bind=127.0.0.1",
+	}, args...)...)
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	require.NoError(t, cmd.Start())
+	t.Cleanup(func() {
+		cmd.Process.Kill()
+		cmd.Wait()
+		logFile.Close()
+	})
+}
+
+func waitForHTTP(t *testing.T, url string) {
+	t.Helper()
+	deadline := time.Now().Add(90 * time.Second)
+	for time.Now().Before(deadline) {
+		if resp, err := http.Get(url); err == nil {
+			resp.Body.Close()
+			return
+		}
+		time.Sleep(time.Second)
+	}
+	for _, p := range miniLogPaths {
+		if data, err := os.ReadFile(p); err == nil {
+			lines := strings.Split(string(data), "\n")
+			if len(lines) > 30 {
+				lines = lines[len(lines)-30:]
+			}
+			t.Logf("last lines of %s:\n%s", p, strings.Join(lines, "\n"))
+		}
+	}
+	t.Fatalf("timed out waiting for %s", url)
+}
+
+func shellOn(t *testing.T, masterPort, command string) string {
+	t.Helper()
+	cmd := exec.Command(weedBinary, "shell", "-master=localhost:"+masterPort)
+	cmd.Stdin = strings.NewReader(command + "\nexit\n")
+	out, err := cmd.CombinedOutput()
+	require.NoErrorf(t, err, "shell %q failed: %s", command, out)
+	return stripLogs(string(out))
+}
+
+func chunkCountOn(t *testing.T, masterPort, path string) string {
+	meta := shellOn(t, masterPort, "fs.meta.cat "+path)
+	idx := strings.LastIndex(meta, "chunks ")
+	require.GreaterOrEqualf(t, idx, 0, "no chunk count in %s", meta)
+	return strings.Fields(meta[idx+len("chunks "):])[0]
+}
+
+func volumeStats(t *testing.T, volumePort string) (size, garbage uint64) {
+	t.Helper()
+	resp, err := http.Get("http://localhost:" + volumePort + "/status")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	var status struct {
+		Volumes []struct {
+			Size             uint64 `json:"Size"`
+			DeletedByteCount uint64 `json:"DeletedByteCount"`
+		} `json:"Volumes"`
+	}
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&status))
+	for _, v := range status.Volumes {
+		size += v.Size
+		garbage += v.DeletedByteCount
+	}
+	return size, garbage
+}
+
+func readViaFiler(t *testing.T, filerPort, path string) []byte {
+	t.Helper()
+	resp, err := http.Get("http://localhost:" + filerPort + path)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	require.Equal(t, http.StatusOK, resp.StatusCode, "read %s", path)
+	data, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	return data
+}
+
+// TestRemoteCacheEvictUnderPressure fills a cache constrained to two small
+// volumes with remote-mounted objects until writes fail, then verifies the
+// filer evicts the oldest synced entry, vacuums the garbage, and a later read
+// caches again.
+func TestRemoteCacheEvictUnderPressure(t *testing.T) {
+	if testing.Short() {
+		t.Skip("spawns two weed mini clusters")
+	}
+	if _, err := os.Stat(weedBinary); err != nil {
+		t.Skipf("weed binary not found at %s; run make build-weed", weedBinary)
+	}
+	if isServerRunning(evictRemoteS3) || isServerRunning(evictPrimaryS3) {
+		t.Skip("eviction test ports are already in use")
+	}
+
+	tmp := t.TempDir()
+	startMini(t, filepath.Join(tmp, "remote"),
+		"-s3.port=28334", "-master.port="+evictRemoteMaster,
+		"-filer.port="+evictRemoteFiler, "-volume.port="+evictRemoteVolume,
+		"-webdav.port="+evictRemoteWebdav, "-metricsPort="+evictRemoteMetrics)
+	waitForHTTP(t, evictRemoteS3)
+	startMini(t, filepath.Join(tmp, "primary"),
+		"-s3.port=28333", "-master.port="+evictPrimaryMaster,
+		"-filer.port="+evictPrimaryFiler, "-volume.port="+evictPrimaryVolume,
+		"-webdav.port="+evictPrimaryWebdav, "-metricsPort="+evictPrimaryMetrics,
+		"-volume.allowUntrustedRemoteEndpoints", "-filer.allowUntrustedRemoteEndpoints",
+		"-s3.allowUntrustedRemoteEndpoints",
+		"-master.volumeSizeLimitMB=32", "-volume.max=2",
+		"-filer.remoteCacheEvictThreshold=0.99")
+	waitForHTTP(t, evictPrimaryS3)
+
+	remote := createS3Client(evictRemoteS3)
+	_, err := remote.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(evictBucket)})
+	require.NoError(t, err)
+	for i := 0; i < 5; i++ {
+		data := make([]byte, 16*1024*1024)
+		for j := range data {
+			data[j] = byte(i + j%251)
+		}
+		_, err = remote.PutObject(&s3.PutObjectInput{
+			Bucket: aws.String(evictBucket),
+			Key:    aws.String(fmt.Sprintf("obj%d.bin", i)),
+			Body:   bytes.NewReader(data),
+		})
+		require.NoError(t, err)
+	}
+
+	shellOn(t, evictPrimaryMaster, fmt.Sprintf(
+		"remote.configure -name=evictremote -type=s3 -s3.access_key=%s -s3.secret_key=%s -s3.endpoint=%s -s3.region=us-east-1",
+		accessKey, secretKey, evictRemoteS3))
+	shellOn(t, evictPrimaryMaster, fmt.Sprintf(
+		"remote.mount -dir=/buckets/%s -remote=evictremote/%s -nonempty", evictMount, evictBucket))
+	shellOn(t, evictPrimaryMaster, fmt.Sprintf("remote.meta.sync -dir=/buckets/%s", evictMount))
+	time.Sleep(2 * time.Second)
+
+	mount := "/buckets/" + evictMount
+
+	// obj0 caches alone so it is the oldest evictable entry.
+	first := readViaFiler(t, evictPrimaryFiler, mount+"/obj0.bin")
+	require.Len(t, first, 16*1024*1024)
+	require.NotEqual(t, "0", chunkCountOn(t, evictPrimaryMaster, mount+"/obj0.bin"), "obj0 should be cached")
+
+	// 4 x 16MB against ~64MB of capacity: the fills overrun, hit the
+	// capacity error path, and trigger eviction + vacuum.
+	var wg sync.WaitGroup
+	for i := 1; i <= 4; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			data := readViaFiler(t, evictPrimaryFiler, fmt.Sprintf("%s/obj%d.bin", mount, i))
+			assert.Len(t, data, 16*1024*1024)
+		}(i)
+	}
+	wg.Wait()
+
+	// Eviction must have dropped obj0's local chunks and vacuumed the
+	// tombstoned bytes, leaving only live data within the two volumes.
+	require.True(t, waitForCondition(t, func() bool {
+		size, garbage := volumeStats(t, evictPrimaryVolume)
+		return garbage == 0 && size <= 72*1024*1024
+	}, 2*time.Minute, "volumes to be reclaimed by eviction+vacuum"),
+		"evicted chunks were not reclaimed")
+
+	assert.Equal(t, "0", chunkCountOn(t, evictPrimaryMaster, mount+"/obj0.bin"),
+		"oldest cached entry should be evicted back to remote-only")
+
+	// The cache self-heals: reading the evicted object re-caches it. Fills
+	// still running from the concurrent wave may evict it again, so retry
+	// the read until a commit sticks.
+	var again []byte
+	require.True(t, waitForCondition(t, func() bool {
+		again = readViaFiler(t, evictPrimaryFiler, mount+"/obj0.bin")
+		return chunkCountOn(t, evictPrimaryMaster, mount+"/obj0.bin") != "0"
+	}, 2*time.Minute, "obj0 to re-cache"),
+		"evicted object did not re-cache on read")
+	assert.Equal(t, first, again)
+}

test/s3tables/catalog_olake/Dockerfile.reader

@@ -0,0 +1,14 @@
+# Strict Iceberg reader used by the OLake integration test. PyIceberg is
+# deliberate here: it rejects manifests that omit the spec's field ids and
+# parquet without either field ids or a name mapping, so a passing read proves
+# the catalog served something every engine can consume -- not just something
+# OLake itself can read back.
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN pip install --no-cache-dir "pyiceberg[s3fs]==0.11.1" "pyarrow==25.0.0"
+
+COPY inspect_table.py /app/
+
+ENTRYPOINT ["python3", "/app/inspect_table.py"]

test/s3tables/catalog_olake/README.md

@@ -0,0 +1,104 @@
+# OLake Iceberg Catalog Integration Test
+
+An integration test for [OLake](https://github.com/datazip-inc/olake) against
+SeaweedFS's Iceberg REST Catalog, in the same shape as `catalog_clickhouse`.
+
+## Why OLake, given we already test five engines
+
+Two things here are covered by nothing else in this directory.
+
+**It is a strict Java Iceberg client.** OLake does not write Iceberg from Go —
+its Go process spawns a Java sidecar over gRPC and writes through the official
+Apache Iceberg library, because the Go library has no equality deletes and CDC
+needs them. So this test exercises the client class that
+`weed/s3api/iceberg/metadata_compliance.go` exists to serve: the one
+that fails with *"Cannot parse missing long current-snapshot-id"* when the
+catalog omits spec-required keys that `iceberg-go` strips via `omitempty`.
+
+**It produces equality deletes.** OLake is a CDC tool. Its upsert path commits
+`operation=overwrite` with an equality-delete file, and a delete manifest
+alongside the data manifests. ClickHouse, StarRocks, Doris and DuckDB all only
+append.
+
+## What it asserts
+
+`TestOLakeIcebergCatalog` runs six subtests against a `weed mini` cluster with
+a pre-created table bucket and a Postgres source:
+
+| Subtest | What a failure means |
+|---|---|
+| `CheckDestination` | `olake check` did not reach `SUCCEEDED`, or it passed without ever loading `org.apache.iceberg.rest.RESTSessionCatalog` — the second case means the destination was never actually contacted. |
+| `Discover` | OLake could not enumerate the source table, or wrote no `streams.json`. |
+| `FullSyncAppendsRows` | The sync read fewer than the three seeded rows, or committed no Iceberg snapshot. |
+| `StrictReaderSeesRows` | PyIceberg could not read back what the Java writer committed, or the values differ. This is the data path, not just metadata. |
+| `UpsertProducesEqualityDelete` | After an `UPDATE` and a re-sync, no snapshot recorded an overwrite carrying equality deletes, or the current snapshot has no delete manifest. |
+| `CompliantWriterNeedsNoRepair` | The catalog rewrote a manifest the official Iceberg Java writer produced. That is a regression in the repair gate, not a problem with OLake. |
+
+## The one thing this test deliberately does not check
+
+It does **not** assert that a reader sees the updated row and no duplicate.
+
+PyIceberg refuses to scan a table carrying equality deletes
+([apache/iceberg#6568](https://github.com/apache/iceberg/issues/6568)) while
+reading its metadata perfectly well — so a rows-mode read after the upsert would
+raise, not pass. The alternative is an engine that applies equality deletes,
+which for StarRocks means a 3 GB image and 12 GB of RAM in CI.
+
+The line drawn instead: **recording the commit correctly is the catalog's
+contract; applying deletes on read is the query engine's.** The metadata
+assertions cover our half.
+
+This was verified once by hand outside CI, on 2026-09-23, with StarRocks 4.1.4
+attached to the same catalog: after the upsert it read 3 rows / 3 distinct ids
+with `id=1` showing the updated value and `_op_type=u`. If someone later wants
+that inside the gate, add a reader that supports equality deletes — **do not**
+"upgrade" this test to a PyIceberg rows-mode read after the upsert. It would not
+pass; and if PyIceberg ever starts silently skipping deletes instead of raising,
+it would pass by not looking.
+
+## What the config proves
+
+Nothing in the destination config is SeaweedFS-specific:
+
+```json
+{
+  "catalog_type": "rest",
+  "rest_catalog_url": "http://HOST:ICEBERG_PORT",
+  "iceberg_s3_path": "s3://olake-tables",
+  "s3_endpoint": "http://HOST:S3_PORT",
+  "rest_auth_type": "oauth2",
+  "oauth2_uri": "http://HOST:ICEBERG_PORT/v1/oauth/tokens",
+  "credential": "ACCESS_KEY:SECRET_KEY"
+}
+```
+
+`catalog_type` is the generic `rest`, auth is the standard OAuth2
+client-credentials flow, and `s3_path_style` does not even need setting —
+OLake turns it on by itself whenever `s3_endpoint` is non-empty. As of
+OLake v0.10.1 this works with no change on either side.
+
+## Running it
+
+```sh
+go test ./test/s3tables/catalog_olake/ -run TestOLakeIcebergCatalog -v -timeout 25m
+```
+
+Needs Docker and a `weed` binary (at `weed/weed` under the repo root, or on
+`PATH`). Takes about 35 seconds. It skips rather than fails when Docker is
+absent, and `SEAWEEDFS_SKIP_OLAKE_TESTS=1` skips it outright.
+
+Overrides: `OLAKE_IMAGE` (default `olakego/source-postgres:latest`),
+`POSTGRES_IMAGE` (default `postgres:16`).
+
+## In CI
+
+Runs as `olake-iceberg-catalog-tests` in `.github/workflows/s3-tables-tests.yml`,
+on a matrix of a pinned image plus `latest` — the same shape the ClickHouse job
+uses, and for the same reason: OLake's Iceberg writer is a Java sidecar whose
+library version moves independently of the Go release, so the `latest` leg is
+what catches drift in the client rather than in OLake itself.
+
+The job asserts the suite actually ran — at least one top-level `--- PASS` and
+zero `--- SKIP` — rather than trusting a green exit. This suite skips itself
+when Docker is unavailable, and a skipped suite reporting success is how a gate
+quietly stops being one.

test/s3tables/catalog_olake/inspect_table.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Inspect an Iceberg table written by OLake, through the SeaweedFS REST catalog.
+
+Two modes, because a strict reader cannot do both:
+
+  rows      -- scan the table and print "id,region,amount" per row, ordered by
+               id. Only valid while the table has no equality deletes.
+  snapshots -- print one line per snapshot with its operation and delete-file
+               counters, plus the manifest content kinds of the current
+               snapshot.
+
+The split exists because PyIceberg refuses to scan a table carrying equality
+deletes (apache/iceberg#6568) while reading its metadata perfectly well. OLake
+is a CDC tool, so its upsert path produces exactly those deletes -- asserting
+the commit landed is the catalog's concern, and applying deletes on read is the
+query engine's.
+"""
+
+import argparse
+import sys
+
+from pyiceberg.catalog import load_catalog
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("mode", choices=["rows", "snapshots"])
+    p.add_argument("--catalog-url", required=True)
+    p.add_argument("--warehouse", required=True)
+    p.add_argument("--prefix", required=True)
+    p.add_argument("--s3-endpoint", required=True)
+    p.add_argument("--access-key", required=True)
+    p.add_argument("--secret-key", required=True)
+    p.add_argument("--region", default="us-east-1")
+    p.add_argument("--namespace", action="append", required=True)
+    p.add_argument("--table", required=True)
+    args = p.parse_args()
+
+    catalog = load_catalog(
+        "rest",
+        **{
+            "type": "rest",
+            "uri": args.catalog_url,
+            "warehouse": args.warehouse,
+            "prefix": args.prefix,
+            "credential": f"{args.access_key}:{args.secret_key}",
+            "s3.access-key-id": args.access_key,
+            "s3.secret-access-key": args.secret_key,
+            "s3.endpoint": args.s3_endpoint,
+            "s3.region": args.region,
+            "s3.path-style-access": "true",
+        },
+    )
+
+    table = catalog.load_table(tuple(args.namespace) + (args.table,))
+
+    if args.mode == "rows":
+        data = table.scan().to_arrow().to_pydict()
+        # amount is decimal(10,2) in Postgres but arrives here as a float, whose
+        # repr drops trailing zeros (120.5, not 120.50). Format it to the source
+        # scale so the expected values in the Go test stay readable.
+        for row_id, region, amount in sorted(
+            zip(data["id"], data["region"], data["amount"])
+        ):
+            print("%s,%s,%.2f" % (row_id, region, float(amount)))
+        return 0
+
+    print("format-version=%d" % table.metadata.format_version)
+    ids = table.metadata.schemas[-1].identifier_field_ids
+    print("identifier-field-ids=%s" % ",".join(str(i) for i in ids))
+    for snap in table.metadata.snapshots:
+        s = snap.summary
+        print(
+            "snapshot operation=%s total-delete-files=%s added-delete-files=%s "
+            "added-equality-deletes=%s total-records=%s"
+            % (
+                s.operation,
+                s.get("total-delete-files", "0"),
+                s.get("added-delete-files", "0"),
+                s.get("added-equality-deletes", "0"),
+                s.get("total-records", "0"),
+            )
+        )
+    current = table.current_snapshot()
+    kinds = [str(m.content).rsplit(".", 1)[-1] for m in current.manifests(table.io)]
+    print("current-manifest-kinds=%s" % ",".join(kinds))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

test/s3tables/catalog_olake/olake_catalog_test.go

@@ -0,0 +1,623 @@
+// Package catalog_olake provides an integration test for OLake
+// (github.com/datazip-inc/olake) against the SeaweedFS Iceberg REST Catalog.
+//
+// OLake matters here for two reasons that no other engine in this directory
+// covers. First, it does not write Iceberg from Go: it spawns a Java sidecar
+// over gRPC and writes through the official Apache Iceberg library, so this is
+// a strict Java client -- the class that iceberg/metadata_compliance.go exists
+// to serve. Second, it is a CDC tool, so its upsert path emits equality
+// deletes, which neither the ClickHouse nor the Doris test exercises.
+package catalog_olake
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+)
+
+const (
+	olakeDefaultImage    = "olakego/source-postgres:latest"
+	postgresDefaultImage = "postgres:16"
+	readerImage          = "seaweedfs-olake-reader:test"
+
+	tableBucketName = "olake-tables"
+	sourceDatabase  = "olakedb"
+	sourceUser      = "olake"
+	sourcePassword  = "olakepw"
+
+	// OLake derives the destination namespace from the source: it joins the
+	// connector name, database and schema. Asserting the derived name rather
+	// than configuring one keeps the test honest about what OLake actually
+	// does with our catalog.
+	expectedNamespace = "postgres_olakedb_public"
+	expectedTable     = "orders"
+
+	postgresStartTimeout = 90 * time.Second
+	olakeRunTimeout      = 6 * time.Minute
+)
+
+type TestEnvironment struct {
+	seaweedDir string
+	weedBinary string
+	dataDir    string
+	configDir  string
+	bindIP     string
+
+	masterPort     int
+	masterGrpcPort int
+	volumePort     int
+	volumeGrpcPort int
+	filerPort      int
+	filerGrpcPort  int
+	s3Port         int
+	s3GrpcPort     int
+	icebergPort    int
+	postgresPort   int
+
+	accessKey string
+	secretKey string
+
+	weedProcess       *exec.Cmd
+	weedCancel        func()
+	postgresContainer string
+}
+
+func TestOLakeIcebergCatalog(t *testing.T) {
+	requireOLakeRuntime(t)
+
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	env.StartSeaweedFS(t)
+	env.startPostgres(t)
+	env.seedSource(t)
+
+	buildReaderImage(t)
+	env.writeOLakeConfigs(t)
+
+	t.Run("CheckDestination", func(t *testing.T) {
+		out := env.runOLake(t, "check",
+			"--config", "/mnt/config/source.json",
+			"--destination", "/mnt/config/destination.json")
+		if !strings.Contains(out, `"status":"SUCCEEDED"`) {
+			t.Fatalf("olake check did not report SUCCEEDED.\n%s", tailLines(out, 40))
+		}
+		// The Java sidecar, not the Go process, is what talks to our catalog.
+		// If this line is missing the check passed without exercising the
+		// Iceberg REST path at all.
+		if !strings.Contains(out, "org.apache.iceberg.rest.RESTSessionCatalog") {
+			t.Errorf("check passed but never loaded the Iceberg REST catalog; "+
+				"the destination may not have been contacted.\n%s", tailLines(out, 40))
+		}
+	})
+
+	t.Run("Discover", func(t *testing.T) {
+		out := env.runOLake(t, "discover",
+			"--config", "/mnt/config/source.json",
+			"--destination", "/mnt/config/destination.json")
+		if !strings.Contains(out, `"stream_name":"`+expectedTable+`"`) {
+			t.Fatalf("discover did not report the %s stream.\n%s", expectedTable, tailLines(out, 20))
+		}
+		if _, err := os.Stat(filepath.Join(env.configDir, "streams.json")); err != nil {
+			t.Fatalf("discover did not write streams.json: %v", err)
+		}
+	})
+
+	t.Run("FullSyncAppendsRows", func(t *testing.T) {
+		out := env.runOLake(t, "sync",
+			"--config", "/mnt/config/source.json",
+			"--destination", "/mnt/config/destination.json",
+			"--streams", "/mnt/config/streams.json",
+			"--state", "/mnt/config/state.json")
+		if !strings.Contains(out, "Total records read: 3") {
+			t.Fatalf("sync did not read the three seeded rows.\n%s", tailLines(out, 30))
+		}
+		if !strings.Contains(out, "Committed snapshot") {
+			t.Fatalf("sync never committed an Iceberg snapshot.\n%s", tailLines(out, 30))
+		}
+	})
+
+	t.Run("StrictReaderSeesRows", func(t *testing.T) {
+		rows := env.readTable(t, "rows")
+		want := []string{"1,us-east,120.50", "2,us-west,87.20", "3,eu-west,210.00"}
+		got := nonEmptyLines(rows)
+		if len(got) != len(want) {
+			t.Fatalf("PyIceberg read %d rows, want %d.\n%s", len(got), len(want), rows)
+		}
+		for i := range want {
+			if got[i] != want[i] {
+				t.Errorf("row %d = %q, want %q", i, got[i], want[i])
+			}
+		}
+	})
+
+	t.Run("UpsertProducesEqualityDelete", func(t *testing.T) {
+		// Push the row past the incremental cursor so the next sync re-emits it
+		// as an update rather than skipping it.
+		env.execSQL(t, "UPDATE orders SET amount=999.99, region='ap-south' WHERE id=1")
+
+		out := env.runOLake(t, "sync",
+			"--config", "/mnt/config/source.json",
+			"--destination", "/mnt/config/destination.json",
+			"--streams", "/mnt/config/streams.json",
+			"--state", "/mnt/config/state.json")
+		if !strings.Contains(out, "delete files") {
+			t.Fatalf("re-sync committed no delete files.\n%s", tailLines(out, 30))
+		}
+
+		meta := env.readTable(t, "snapshots")
+		if !strings.Contains(meta, "format-version=2") {
+			t.Errorf("expected Iceberg format-version 2, got:\n%s", meta)
+		}
+		if !strings.Contains(meta, "identifier-field-ids=") ||
+			strings.Contains(meta, "identifier-field-ids=\n") {
+			t.Errorf("table carries no identifier fields, so OLake could not "+
+				"have upserted:\n%s", meta)
+		}
+
+		var sawOverwrite bool
+		for _, line := range nonEmptyLines(meta) {
+			if !strings.HasPrefix(line, "snapshot operation=") {
+				continue
+			}
+			if strings.Contains(line, "operation=Operation.OVERWRITE") &&
+				!strings.Contains(line, "added-equality-deletes=0") {
+				sawOverwrite = true
+			}
+		}
+		if !sawOverwrite {
+			t.Errorf("no snapshot recorded an overwrite carrying equality "+
+				"deletes:\n%s", meta)
+		}
+		if !strings.Contains(meta, "current-manifest-kinds=") ||
+			!strings.Contains(meta, "DELETES") {
+			t.Errorf("current snapshot has no delete manifest:\n%s", meta)
+		}
+		// Deliberately NOT asserted here: that a reader sees the updated value
+		// and no duplicate row. PyIceberg refuses to scan a table carrying
+		// equality deletes (apache/iceberg#6568), so a rows-mode read would
+		// fail rather than pass, and swapping in an engine that can apply them
+		// costs this test a multi-gigabyte image. Applying deletes on read is
+		// the engine's contract; recording the commit correctly is ours, and
+		// that is what the assertions above cover. See the README.
+	})
+
+	t.Run("CompliantWriterNeedsNoRepair", func(t *testing.T) {
+		names := env.listTableMetadata(t)
+		if len(names) == 0 {
+			t.Fatalf("no metadata files found for %s.%s", expectedNamespace, expectedTable)
+		}
+		for _, n := range names {
+			if strings.HasPrefix(n, "repaired-") {
+				t.Errorf("catalog repaired a manifest written by OLake (%s); "+
+					"the official Iceberg Java writer is expected to be "+
+					"spec-compliant, so this is a regression in the repair "+
+					"gate rather than in OLake", n)
+			}
+		}
+	})
+}
+
+func NewTestEnvironment(t *testing.T) *TestEnvironment {
+	t.Helper()
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("Failed to get working directory: %v", err)
+	}
+
+	seaweedDir := wd
+	for i := 0; i < 6; i++ {
+		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
+			break
+		}
+		seaweedDir = filepath.Dir(seaweedDir)
+	}
+
+	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
+	if info, err := os.Stat(weedBinary); err != nil || info.IsDir() {
+		weedBinary = filepath.Join(seaweedDir, "weed", "weed", "weed")
+		if info, err := os.Stat(weedBinary); err != nil || info.IsDir() {
+			weedBinary = "weed"
+			if _, err := exec.LookPath(weedBinary); err != nil {
+				t.Skip("weed binary not found, skipping integration test")
+			}
+		}
+	}
+
+	dataDir, err := os.MkdirTemp("", "seaweed-olake-test-*")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %v", err)
+	}
+	configDir := filepath.Join(dataDir, "olake")
+	if err := os.MkdirAll(configDir, 0755); err != nil {
+		t.Fatalf("Failed to create config dir: %v", err)
+	}
+
+	// 9 for the mini cluster, 1 for the Postgres source mapped on the host.
+	ports := testutil.MustAllocatePorts(t, 10)
+
+	return &TestEnvironment{
+		seaweedDir:     seaweedDir,
+		weedBinary:     weedBinary,
+		dataDir:        dataDir,
+		configDir:      configDir,
+		bindIP:         testutil.FindBindIP(),
+		masterPort:     ports[0],
+		masterGrpcPort: ports[1],
+		volumePort:     ports[2],
+		volumeGrpcPort: ports[3],
+		filerPort:      ports[4],
+		filerGrpcPort:  ports[5],
+		s3Port:         ports[6],
+		s3GrpcPort:     ports[7],
+		icebergPort:    ports[8],
+		postgresPort:   ports[9],
+		accessKey:      "AKIAIOSFODNN7EXAMPLE",
+		secretKey:      "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+	}
+}
+
+func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
+	t.Helper()
+
+	iamConfigPath, err := testutil.WriteIAMConfig(env.dataDir, env.accessKey, env.secretKey)
+	if err != nil {
+		t.Fatalf("Failed to create IAM config: %v", err)
+	}
+
+	cmd := exec.Command(env.weedBinary, "mini",
+		"-master.port", fmt.Sprintf("%d", env.masterPort),
+		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
+		"-volume.port", fmt.Sprintf("%d", env.volumePort),
+		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
+		"-filer.port", fmt.Sprintf("%d", env.filerPort),
+		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
+		"-s3.port", fmt.Sprintf("%d", env.s3Port),
+		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
+		"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergPort),
+		"-s3.config", iamConfigPath,
+		// Pre-create the table bucket the way an operator would, rather than
+		// reaching for the S3 Tables control plane from the test.
+		"-tableBucket", tableBucketName,
+		"-ip", env.bindIP,
+		"-ip.bind", "0.0.0.0",
+		"-dir", env.dataDir,
+	)
+	cmd.Dir = env.dataDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = append(os.Environ(),
+		"AWS_ACCESS_KEY_ID="+env.accessKey,
+		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
+	)
+
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("Failed to start SeaweedFS: %v", err)
+	}
+	env.weedProcess = cmd
+	env.weedCancel = func() {
+		if cmd.Process != nil {
+			_ = cmd.Process.Kill()
+		}
+	}
+
+	url := fmt.Sprintf("http://%s:%d/v1/config", env.bindIP, env.icebergPort)
+	if !waitForService(url, 45*time.Second) {
+		t.Fatalf("Iceberg REST API did not become ready at %s", url)
+	}
+}
+
+func (env *TestEnvironment) startPostgres(t *testing.T) {
+	t.Helper()
+
+	name := fmt.Sprintf("olake-pg-%d", env.postgresPort)
+	_ = exec.Command("docker", "rm", "-f", name).Run()
+
+	cmd := exec.Command("docker", "run", "-d", "--name", name,
+		"-e", "POSTGRES_USER="+sourceUser,
+		"-e", "POSTGRES_PASSWORD="+sourcePassword,
+		"-e", "POSTGRES_DB="+sourceDatabase,
+		"-p", fmt.Sprintf("%d:5432", env.postgresPort),
+		postgresImage(),
+	)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("Failed to start Postgres: %v\n%s", err, out)
+	}
+	env.postgresContainer = name
+
+	deadline := time.Now().Add(postgresStartTimeout)
+	for time.Now().Before(deadline) {
+		probe := exec.Command("docker", "exec", name,
+			"pg_isready", "-U", sourceUser, "-d", sourceDatabase)
+		if err := probe.Run(); err == nil {
+			return
+		}
+		time.Sleep(2 * time.Second)
+	}
+	t.Fatalf("Postgres did not become ready within %s", postgresStartTimeout)
+}
+
+func (env *TestEnvironment) seedSource(t *testing.T) {
+	t.Helper()
+	env.execSQL(t, `
+CREATE TABLE orders (
+    id       int PRIMARY KEY,
+    region   text,
+    amount   numeric(10,2),
+    order_ts timestamp
+);
+INSERT INTO orders VALUES
+    (1,'us-east',120.50,'2026-07-27 09:15'),
+    (2,'us-west', 87.20,'2026-07-27 09:20'),
+    (3,'eu-west',210.00,'2026-07-27 09:31');
+ALTER TABLE orders REPLICA IDENTITY FULL;`)
+}
+
+func (env *TestEnvironment) execSQL(t *testing.T, sqlText string) {
+	t.Helper()
+	cmd := exec.Command("docker", "exec", env.postgresContainer,
+		"psql", "-v", "ON_ERROR_STOP=1", "-U", sourceUser, "-d", sourceDatabase,
+		"-c", sqlText)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("psql failed: %v\n%s", err, out)
+	}
+}
+
+// writeOLakeConfigs writes the source and destination configs. The destination
+// is the point of this test: nothing in it is SeaweedFS-specific. catalog_type
+// is the generic "rest", auth is the standard OAuth2 client-credentials flow,
+// and path-style access is not even set here because OLake turns it on by
+// itself whenever s3_endpoint is non-empty.
+func (env *TestEnvironment) writeOLakeConfigs(t *testing.T) {
+	t.Helper()
+
+	source := map[string]any{
+		"host":            env.bindIP,
+		"port":            env.postgresPort,
+		"database":        sourceDatabase,
+		"username":        sourceUser,
+		"password":        sourcePassword,
+		"jdbc_url_params": map[string]any{},
+		"ssl":             map[string]any{"mode": "disable"},
+		"update_method":   map[string]any{"type": "Standalone"},
+		"max_threads":     2,
+		"retry_count":     0,
+	}
+
+	catalogURL := fmt.Sprintf("http://%s:%d", env.bindIP, env.icebergPort)
+	destination := map[string]any{
+		"type": "ICEBERG",
+		"writer": map[string]any{
+			"catalog_type":     "rest",
+			"rest_catalog_url": catalogURL,
+			"catalog_name":     "olake",
+			"iceberg_s3_path":  "s3://" + tableBucketName,
+			"s3_endpoint":      fmt.Sprintf("http://%s:%d", env.bindIP, env.s3Port),
+			"s3_use_ssl":       false,
+			"s3_path_style":    true,
+			"aws_region":       "us-east-1",
+			"aws_access_key":   env.accessKey,
+			"aws_secret_key":   env.secretKey,
+			"rest_auth_type":   "oauth2",
+			"oauth2_uri":       catalogURL + "/v1/oauth/tokens",
+			"credential":       env.accessKey + ":" + env.secretKey,
+		},
+	}
+
+	writeJSON(t, filepath.Join(env.configDir, "source.json"), source)
+	writeJSON(t, filepath.Join(env.configDir, "destination.json"), destination)
+	if err := os.WriteFile(filepath.Join(env.configDir, "state.json"), []byte("{}\n"), 0644); err != nil {
+		t.Fatalf("Failed to write state.json: %v", err)
+	}
+}
+
+func (env *TestEnvironment) runOLake(t *testing.T, args ...string) string {
+	t.Helper()
+
+	full := append([]string{
+		"run", "--rm",
+		"-v", dockerMount(env.configDir) + ":/mnt/config",
+		olakeImage(),
+	}, args...)
+
+	cmd := exec.Command("docker", full...)
+	done := make(chan struct{})
+	var out []byte
+	var err error
+	go func() {
+		out, err = cmd.CombinedOutput()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(olakeRunTimeout):
+		if cmd.Process != nil {
+			_ = cmd.Process.Kill()
+		}
+		t.Fatalf("olake %s did not finish within %s", args[0], olakeRunTimeout)
+	}
+	if err != nil {
+		t.Fatalf("olake %s failed: %v\n%s", args[0], err, tailLines(string(out), 40))
+	}
+	return string(out)
+}
+
+func (env *TestEnvironment) readTable(t *testing.T, mode string) string {
+	t.Helper()
+
+	cmd := exec.Command("docker", "run", "--rm", readerImage, mode,
+		"--catalog-url", fmt.Sprintf("http://%s:%d", env.bindIP, env.icebergPort),
+		"--warehouse", "s3://"+tableBucketName,
+		"--prefix", tableBucketName,
+		"--s3-endpoint", fmt.Sprintf("http://%s:%d", env.bindIP, env.s3Port),
+		"--access-key", env.accessKey,
+		"--secret-key", env.secretKey,
+		"--namespace", expectedNamespace,
+		"--table", expectedTable,
+	)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("reader (%s) failed: %v\n%s", mode, err, tailLines(string(out), 30))
+	}
+	return string(out)
+}
+
+// listTableMetadata lists the table's metadata directory through the filer so
+// the repair assertion looks at real objects rather than at what the catalog
+// reports about itself.
+func (env *TestEnvironment) listTableMetadata(t *testing.T) []string {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s:%d/buckets/%s/%s/%s/metadata/?limit=200",
+		env.bindIP, env.filerPort, tableBucketName, expectedNamespace, expectedTable)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		t.Fatalf("Failed to build filer request: %v", err)
+	}
+	req.Header.Set("Accept", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("filer listing failed: %v", err)
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("filer listing returned %d: %s", resp.StatusCode, body)
+	}
+
+	var parsed struct {
+		Entries []struct {
+			FullPath string `json:"FullPath"`
+		} `json:"Entries"`
+	}
+	if err := json.Unmarshal(body, &parsed); err != nil {
+		t.Fatalf("Failed to parse filer listing: %v\n%s", err, body)
+	}
+
+	names := make([]string, 0, len(parsed.Entries))
+	for _, e := range parsed.Entries {
+		names = append(names, e.FullPath[strings.LastIndex(e.FullPath, "/")+1:])
+	}
+	return names
+}
+
+func (env *TestEnvironment) Cleanup(t *testing.T) {
+	t.Helper()
+
+	if env.postgresContainer != "" {
+		_ = exec.Command("docker", "rm", "-f", env.postgresContainer).Run()
+	}
+	if env.weedCancel != nil {
+		env.weedCancel()
+	}
+	if env.weedProcess != nil {
+		_ = env.weedProcess.Wait()
+	}
+	if env.dataDir != "" {
+		_ = os.RemoveAll(env.dataDir)
+	}
+}
+
+func buildReaderImage(t *testing.T) {
+	t.Helper()
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("Failed to get working directory: %v", err)
+	}
+	cmd := exec.Command("docker", "build",
+		"-f", filepath.Join(wd, "Dockerfile.reader"),
+		"-t", readerImage, wd)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("Failed to build reader image: %v\n%s", err, tailLines(string(out), 30))
+	}
+}
+
+func olakeImage() string {
+	if v := os.Getenv("OLAKE_IMAGE"); v != "" {
+		return v
+	}
+	return olakeDefaultImage
+}
+
+func postgresImage() string {
+	if v := os.Getenv("POSTGRES_IMAGE"); v != "" {
+		return v
+	}
+	return postgresDefaultImage
+}
+
+func requireOLakeRuntime(t *testing.T) {
+	t.Helper()
+	if os.Getenv("SEAWEEDFS_SKIP_OLAKE_TESTS") != "" {
+		t.Skip("SEAWEEDFS_SKIP_OLAKE_TESTS set")
+	}
+	if _, err := exec.LookPath("docker"); err != nil {
+		t.Skip("docker not available, skipping OLake integration test")
+	}
+	if err := exec.Command("docker", "info").Run(); err != nil {
+		t.Skip("docker daemon not reachable, skipping OLake integration test")
+	}
+}
+
+func waitForService(url string, timeout time.Duration) bool {
+	deadline := time.Now().Add(timeout)
+	client := &http.Client{Timeout: 3 * time.Second}
+	for time.Now().Before(deadline) {
+		resp, err := client.Get(url)
+		if err == nil {
+			resp.Body.Close()
+			if resp.StatusCode < 500 {
+				return true
+			}
+		}
+		time.Sleep(time.Second)
+	}
+	return false
+}
+
+func writeJSON(t *testing.T, path string, v any) {
+	t.Helper()
+	body, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		t.Fatalf("Failed to marshal %s: %v", path, err)
+	}
+	if err := os.WriteFile(path, append(body, '\n'), 0644); err != nil {
+		t.Fatalf("Failed to write %s: %v", path, err)
+	}
+}
+
+// dockerMount normalises a host path for a -v bind mount. Docker Desktop
+// accepts forward slashes on Windows; the native separator it does not.
+func dockerMount(path string) string {
+	return strings.ReplaceAll(path, `\`, "/")
+}
+
+func nonEmptyLines(s string) []string {
+	var out []string
+	for _, line := range strings.Split(s, "\n") {
+		if trimmed := strings.TrimSpace(line); trimmed != "" {
+			out = append(out, trimmed)
+		}
+	}
+	return out
+}
+
+func tailLines(s string, n int) string {
+	lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
+	if len(lines) > n {
+		lines = lines[len(lines)-n:]
+	}
+	return strings.Join(lines, "\n")
+}

weed/cluster/lock_client.go

@@ -3,6 +3,7 @@ package cluster
 import (
 	"context"
 	"fmt"
+	"slices"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -27,6 +28,7 @@ type LockClient struct {
 	// correct: the filer forwards to the real primary as a fallback.
 	ringMu      sync.RWMutex
 	ring        *lock_manager.HashRing
+	ringServers []pb.ServerAddress
 	ringVersion int64
 
 	// priorRing is the ring before the most recent change, kept for priorWindow so a
@@ -35,15 +37,24 @@ type LockClient struct {
 	priorRing     *lock_manager.HashRing
 	ringChangedAt time.Time
 	priorWindow   time.Duration
+
+	// noLockServerRetryPeriod bounds retries when every filer reports "no
+	// lock server found": an empty lock ring is a systemic fault that waiting
+	// on a lock holder cannot resolve, unlike ordinary contention. The bound
+	// is long enough to ride out a master leader change (ring reset + fresh
+	// snapshot) but short enough that a write fails instead of hanging
+	// forever.
+	noLockServerRetryPeriod time.Duration
 }
 
 func NewLockClient(grpcDialOption grpc.DialOption, seedFiler pb.ServerAddress) *LockClient {
 	return &LockClient{
-		grpcDialOption:  grpcDialOption,
-		maxLockDuration: 5 * time.Second,
-		sleepDuration:   2473 * time.Millisecond,
-		seedFiler:       seedFiler,
-		priorWindow:     5 * time.Second,
+		grpcDialOption:          grpcDialOption,
+		maxLockDuration:         5 * time.Second,
+		sleepDuration:           2473 * time.Millisecond,
+		seedFiler:               seedFiler,
+		priorWindow:             5 * time.Second,
+		noLockServerRetryPeriod: 15 * time.Second,
 	}
 }
 
@@ -58,10 +69,16 @@ func (lc *LockClient) SetRing(servers []pb.ServerAddress, version int64) {
 		return
 	}
 	lc.ringVersion = version
+	sorted := slices.Clone(servers)
+	slices.Sort(sorted)
+	if slices.Equal(sorted, lc.ringServers) {
+		return
+	}
+	lc.ringServers = sorted
 	// Build a fresh ring (not an in-place mutation) so the outgoing ring survives as
 	// priorRing with its own servers for the cooling-off window.
 	newRing := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
-	newRing.SetServers(servers)
+	newRing.SetServers(sorted)
 	if lc.ring != nil {
 		lc.priorRing = lc.ring
 		lc.ringChangedAt = time.Now()
@@ -69,6 +86,18 @@ func (lc *LockClient) SetRing(servers []pb.ServerAddress, version int64) {
 	lc.ring = newRing
 }
 
+// ResetRing clears only the version gate so the first update from a
+// different master applies unconditionally: ring versions are per-master
+// monotonic and a high version accepted from a former leader must not
+// reject the new leader's snapshot. The last ring keeps routing during the
+// gap rather than falling back to the seed filer, and the arriving ring
+// becomes the prior ring for the cooling-off window.
+func (lc *LockClient) ResetRing() {
+	lc.ringMu.Lock()
+	defer lc.ringMu.Unlock()
+	lc.ringVersion = 0
+}
+
 // hostForKey returns the filer that should own key per the current ring view,
 // falling back to the seed filer when no view has been received yet.
 func (lc *LockClient) hostForKey(key string) pb.ServerAddress {
@@ -133,7 +162,9 @@ type LiveLock struct {
 	consecutiveFailures int // Track connection failures to trigger fallback
 }
 
-// NewShortLivedLock creates a lock with a 5-second duration
+// NewShortLivedLock creates a lock with a 5-second duration.
+// It returns nil when the lock cannot be acquired because no lock server
+// exists; ordinary contention is still waited out.
 func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLock) {
 	lock = &LiveLock{
 		key:            key,
@@ -144,7 +175,10 @@ func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLoc
 		self:           owner,
 		lc:             lc,
 	}
-	lock.retryUntilLocked(5 * time.Second)
+	if err := lock.retryUntilLocked(5 * time.Second); err != nil {
+		glog.Warningf("create lock %s: %v", key, err)
+		return nil
+	}
 	return
 }
 
@@ -167,7 +201,10 @@ func (lc *LockClient) NewBlockingLongLivedLock(key, owner string, lockTTL time.D
 		lockTTL:        lockTTL,
 	}
 	// Block until acquired
-	lock.retryUntilLocked(lockTTL)
+	if err := lock.retryUntilLocked(lockTTL); err != nil {
+		glog.Warningf("create lock %s: %v", key, err)
+		return nil
+	}
 	// Start renewal goroutine using a ticker for interruptible sleep
 	lock.renewalDone = make(chan struct{})
 	go func() {
@@ -266,12 +303,31 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
 // several seconds): when a holder on another mount releases the lock, the
 // waiter must pick it up promptly, otherwise cross-mount write handoff stalls
 // long enough to time out clients.
-func (lock *LiveLock) retryUntilLocked(lockDuration time.Duration) {
+func (lock *LiveLock) retryUntilLocked(lockDuration time.Duration) error {
+	var unavailableSince time.Time
 	for lock.renewToken == "" {
-		if err := lock.AttemptToLock(lockDuration); err != nil {
-			glog.V(1).Infof("create lock %s: %v", lock.key, err)
+		err := lock.AttemptToLock(lockDuration)
+		if err == nil {
+			unavailableSince = time.Time{}
+			continue
+		}
+		glog.V(1).Infof("create lock %s: %v", lock.key, err)
+		if strings.Contains(err.Error(), "lock already owned") {
+			// Ordinary contention: a reachable server holds the lock, so
+			// waiting is the point and has no bound.
+			unavailableSince = time.Time{}
+			continue
+		}
+		// Anything else — "no lock server found", a dead ring member refusing
+		// connections — is a systemic fault waiting cannot fix; give up once
+		// it persists past the retry period.
+		if unavailableSince.IsZero() {
+			unavailableSince = time.Now()
+		} else if time.Since(unavailableSince) > lock.lc.noLockServerRetryPeriod {
+			return err
 		}
 	}
+	return nil
 }
 
 func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {

weed/cluster/lock_client_test.go

@@ -1,13 +1,18 @@
 package cluster
 
 import (
+	"context"
 	"fmt"
+	"net"
 	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 // The gateway must resolve a lock key to the same primary the filers do,
@@ -152,6 +157,135 @@ func TestLockClientPriorOwnerForKeyExpires(t *testing.T) {
 	}
 }
 
+// A master change clears the version gate so the new leader's (lower-versioned)
+// snapshot applies — versions are only comparable within one master's stream.
+func TestLockClientResetRing(t *testing.T) {
+	lc := NewLockClient(nil, "seed:8888")
+
+	lc.SetRing([]pb.ServerAddress{"filer-a:8888", "filer-b:8888"}, 100)
+	lc.ResetRing()
+
+	// The last ring keeps routing during the gap; only version acceptance
+	// is reset so the new leader's (lower-versioned) snapshot applies.
+	if got := lc.hostForKey("k"); got == "seed:8888" {
+		t.Fatal("expected the previous ring to keep routing after reset")
+	}
+	lc.SetRing([]pb.ServerAddress{"filer-z:8888"}, 50)
+	if got := lc.hostForKey("k"); got != "filer-z:8888" {
+		t.Fatalf("lower version from new master not applied, got %q", got)
+	}
+}
+
+type noLockServerFiler struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+}
+
+func (s *noLockServerFiler) DistributedLock(ctx context.Context, req *filer_pb.LockRequest) (*filer_pb.LockResponse, error) {
+	return &filer_pb.LockResponse{Error: lock_manager.NoLockServerError.Error()}, nil
+}
+
+// When every filer reports an empty lock ring, lock acquisition must fail
+// after a bounded period instead of hanging the write forever.
+func TestNewShortLivedLockFailsFastOnNoLockServer(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	grpcServer := grpc.NewServer()
+	filer_pb.RegisterSeaweedFilerServer(grpcServer, &noLockServerFiler{})
+	go grpcServer.Serve(listener)
+	defer grpcServer.Stop()
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	host, port, err := net.SplitHostPort(listener.Addr().String())
+	if err != nil {
+		t.Fatalf("split host port: %v", err)
+	}
+	// "host:httpPort.grpcPort" dials the fake filer's port directly.
+	lc := NewLockClient(dialOption, pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
+	lc.noLockServerRetryPeriod = 200 * time.Millisecond
+
+	start := time.Now()
+	lock := lc.NewShortLivedLock("test-key", "test-owner")
+	elapsed := time.Since(start)
+
+	if lock != nil {
+		t.Fatal("expected nil lock when no lock server exists")
+	}
+	if elapsed > 10*time.Second {
+		t.Fatalf("lock acquisition took %v, expected fail-fast", elapsed)
+	}
+}
+
+type contendedLockFiler struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+}
+
+func (s *contendedLockFiler) DistributedLock(ctx context.Context, req *filer_pb.LockRequest) (*filer_pb.LockResponse, error) {
+	return &filer_pb.LockResponse{Error: "lock already owned by someone"}, nil
+}
+
+// A lock held by another owner is ordinary contention: acquisition waits it
+// out rather than failing on the unavailability bound.
+func TestNewShortLivedLockWaitsOutContention(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	grpcServer := grpc.NewServer()
+	filer_pb.RegisterSeaweedFilerServer(grpcServer, &contendedLockFiler{})
+	go grpcServer.Serve(listener)
+	defer grpcServer.Stop()
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	host, port, err := net.SplitHostPort(listener.Addr().String())
+	if err != nil {
+		t.Fatalf("split host port: %v", err)
+	}
+	lc := NewLockClient(dialOption, pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
+	lc.noLockServerRetryPeriod = 200 * time.Millisecond
+
+	done := make(chan *LiveLock, 1)
+	go func() {
+		done <- lc.NewShortLivedLock("test-key", "test-owner")
+	}()
+	select {
+	case <-done:
+		t.Fatal("ordinary lock contention must not hit the unavailability bound")
+	case <-time.After(3 * lc.noLockServerRetryPeriod):
+	}
+}
+
+// A ring member that refuses connections is unavailability, not contention:
+// acquisition fails on the same bound as "no lock server found".
+func TestNewShortLivedLockFailsFastOnUnreachableFiler(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	addr := listener.Addr().String()
+	listener.Close()
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		t.Fatalf("split host port: %v", err)
+	}
+	lc := NewLockClient(dialOption, pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
+	lc.noLockServerRetryPeriod = 200 * time.Millisecond
+
+	start := time.Now()
+	lock := lc.NewShortLivedLock("test-key", "test-owner")
+	elapsed := time.Since(start)
+
+	if lock != nil {
+		t.Fatal("expected nil lock when the ring member is unreachable")
+	}
+	if elapsed > 10*time.Second {
+		t.Fatalf("lock acquisition took %v, expected fail-fast", elapsed)
+	}
+}
+
 // LiveLock.generation is a fencing token written and read with 64-bit atomic
 // operations. On 32-bit platforms (GOARCH=386 and GOARCH=arm) a 64-bit atomic
 // op requires an 8-byte-aligned address, which Go only guarantees for the

weed/cluster/lock_manager/lock_ring.go

@@ -1,6 +1,7 @@
 package lock_manager
 
 import (
+	"slices"
 	"sort"
 	"sync"
 	"time"
@@ -54,6 +55,14 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress, version int64) bool {
 		r.Unlock()
 		return false
 	}
+	// An unchanged member list is only a version refresh: installing it as a
+	// new snapshot would run the topology-change callback and restart the
+	// prior-owner window on every periodic rebroadcast.
+	if len(r.snapshots) > 0 && slices.Equal(servers, r.snapshots[0].servers) {
+		r.version = version
+		r.Unlock()
+		return true
+	}
 	r.version = version
 	// Update the ring while holding the lock so version and ring state
 	// are always consistent — prevents a concurrent SetSnapshot from
@@ -74,6 +83,19 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress, version int64) bool {
 	return true
 }
 
+// Reset clears only the version gate so the first update from a different
+// master always applies: ring versions are per-master monotonic, and a high
+// version accepted from a former leader must not reject the new leader's
+// view. The ring itself stays installed — writes keep routing to the last
+// known owner during the gap instead of every filer treating itself as the
+// owner, and the arriving snapshot transitions off it with the usual
+// prior-owner window.
+func (r *LockRing) Reset() {
+	r.Lock()
+	defer r.Unlock()
+	r.version = 0
+}
+
 // Version returns the current ring version.
 func (r *LockRing) Version() int64 {
 	r.RLock()

weed/cluster/lock_manager/lock_ring_test.go

@@ -96,3 +96,42 @@ func TestLockRing_VersionRejectsStale(t *testing.T) {
 	assert.True(t, ok)
 	assert.Equal(t, 1, len(r.GetSnapshot()))
 }
+
+func TestLockRing_SetSnapshotUnchangedOnlyBumpsVersion(t *testing.T) {
+	r := NewLockRing(100 * time.Millisecond)
+	callbacks := 0
+	r.SetTakeSnapshotCallback(func(snapshot []pb.ServerAddress) { callbacks++ })
+
+	assert.True(t, r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2"}, 100))
+	assert.Equal(t, 1, callbacks)
+
+	// A periodic rebroadcast with the same members refreshes the version
+	// without a new snapshot or another topology-change callback.
+	assert.True(t, r.SetSnapshot([]pb.ServerAddress{"b:2", "a:1"}, 200))
+	assert.Equal(t, int64(200), r.Version())
+	assert.Equal(t, 1, r.GetSnapshotCount())
+	assert.Equal(t, 1, callbacks, "unchanged ring must not fire the topology callback")
+
+	assert.True(t, r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2", "c:3"}, 300))
+	assert.Equal(t, 2, callbacks)
+}
+
+func TestLockRing_Reset(t *testing.T) {
+	r := NewLockRing(100 * time.Millisecond)
+
+	// A high version accepted from a former leader must not reject the new
+	// leader's view once the client has moved masters.
+	ok := r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2"}, 100)
+	assert.True(t, ok)
+
+	r.Reset()
+	assert.Equal(t, int64(0), r.Version())
+	// The operational ring survives the reset: writes keep routing to the
+	// last known owner until the new leader's snapshot arrives.
+	assert.Equal(t, 2, len(r.GetSnapshot()))
+	assert.NotEqual(t, "", string(r.GetPrimary("key")))
+
+	ok = r.SetSnapshot([]pb.ServerAddress{"c:1"}, 50)
+	assert.True(t, ok, "lower version from a different master must apply after reset")
+	assert.Equal(t, 1, len(r.GetSnapshot()))
+}

weed/cluster/lock_ring_manager.go

@@ -11,29 +11,38 @@ import (
 
 const LockRingStabilizationInterval = 1 * time.Second
 
+// LockRingRebroadcastInterval is how often the ring is re-sent even when
+// membership has not changed. Broadcasts are otherwise purely event-driven,
+// so a single lost or poisoned update would be permanent without this.
+const LockRingRebroadcastInterval = 30 * time.Second
+
 // LockRingManager tracks filer membership for the distributed lock ring.
 // It batches rapid topology changes (e.g., node drop + join) with a
 // stabilization timer, then broadcasts the complete member list atomically
 // so filers receive a single consistent ring update instead of multiple
 // intermediate states.
 type LockRingManager struct {
-	mu             sync.Mutex
-	members        map[FilerGroupName]map[pb.ServerAddress]struct{}
-	version        map[FilerGroupName]int64
-	lastBroadcast  map[FilerGroupName]*master_pb.LockRingUpdate
-	pendingTimer   map[FilerGroupName]*time.Timer
-	broadcastFn    func(resp *master_pb.KeepConnectedResponse)
-	stabilizeDelay time.Duration
+	mu                  sync.Mutex
+	members             map[FilerGroupName]map[pb.ServerAddress]struct{}
+	version             map[FilerGroupName]int64
+	lastBroadcast       map[FilerGroupName]*master_pb.LockRingUpdate
+	pendingTimer        map[FilerGroupName]*time.Timer
+	rebroadcastTimer    map[FilerGroupName]*time.Timer
+	broadcastFn         func(resp *master_pb.KeepConnectedResponse)
+	stabilizeDelay      time.Duration
+	rebroadcastInterval time.Duration
 }
 
 func NewLockRingManager(broadcastFn func(resp *master_pb.KeepConnectedResponse)) *LockRingManager {
 	return &LockRingManager{
-		members:        make(map[FilerGroupName]map[pb.ServerAddress]struct{}),
-		version:        make(map[FilerGroupName]int64),
-		lastBroadcast:  make(map[FilerGroupName]*master_pb.LockRingUpdate),
-		pendingTimer:   make(map[FilerGroupName]*time.Timer),
-		broadcastFn:    broadcastFn,
-		stabilizeDelay: LockRingStabilizationInterval,
+		members:             make(map[FilerGroupName]map[pb.ServerAddress]struct{}),
+		version:             make(map[FilerGroupName]int64),
+		lastBroadcast:       make(map[FilerGroupName]*master_pb.LockRingUpdate),
+		pendingTimer:        make(map[FilerGroupName]*time.Timer),
+		rebroadcastTimer:    make(map[FilerGroupName]*time.Timer),
+		broadcastFn:         broadcastFn,
+		stabilizeDelay:      LockRingStabilizationInterval,
+		rebroadcastInterval: LockRingRebroadcastInterval,
 	}
 }
 
@@ -116,16 +125,64 @@ func (lrm *LockRingManager) scheduleBroadcast(filerGroup FilerGroupName) {
 
 func (lrm *LockRingManager) doBroadcast(filerGroup FilerGroupName) {
 	lrm.mu.Lock()
+	delete(lrm.pendingTimer, filerGroup)
+	lrm.mu.Unlock()
+	lrm.emit(filerGroup)
+}
+
+// rebroadcast re-sends the current ring unless a membership broadcast is
+// still stabilizing — emitting mid-window would publish an intermediate
+// topology that the pending timer immediately replaces. The check and the
+// update must sit in one critical section or a membership change can slip
+// a pending timer in between.
+func (lrm *LockRingManager) rebroadcast(filerGroup FilerGroupName) {
+	lrm.mu.Lock()
+	var update *master_pb.LockRingUpdate
+	if _, pending := lrm.pendingTimer[filerGroup]; !pending {
+		update = lrm.nextBroadcastUpdate(filerGroup)
+	}
+	lrm.mu.Unlock()
+	lrm.sendUpdate(filerGroup, update)
+}
+
+func (lrm *LockRingManager) emit(filerGroup FilerGroupName) {
+	lrm.mu.Lock()
+	update := lrm.nextBroadcastUpdate(filerGroup)
+	lrm.mu.Unlock()
+	lrm.sendUpdate(filerGroup, update)
+}
+
+func (lrm *LockRingManager) sendUpdate(filerGroup FilerGroupName, update *master_pb.LockRingUpdate) {
+	if update == nil {
+		return
+	}
+	glog.V(0).Infof("LockRing: broadcasting ring update for group %q version %d: %v", filerGroup, update.Version, update.Servers)
+	if lrm.broadcastFn != nil {
+		lrm.broadcastFn(&master_pb.KeepConnectedResponse{
+			LockRingUpdate: update,
+		})
+	}
+}
+
+// nextBroadcastUpdate stamps the current members into an update and re-arms
+// the periodic rebroadcast. It returns nil for an empty member list: an empty
+// lock ring is never usable, so a late "last member removed" event from a
+// former leader must not propagate and wedge every lock client. The last
+// non-empty broadcast stays in lastBroadcast for reconnecting clients.
+// Caller must hold lrm.mu.
+func (lrm *LockRingManager) nextBroadcastUpdate(filerGroup FilerGroupName) *master_pb.LockRingUpdate {
+	members := lrm.members[filerGroup]
+	if len(members) == 0 {
+		return nil
+	}
 	// Use wall-clock nanoseconds so the version survives master restarts
 	// without persistence — a restarted master produces a version greater
 	// than any pre-restart value (assuming clocks don't jump backward).
 	version := time.Now().UnixNano()
 	lrm.version[filerGroup] = version
-	servers := make([]string, 0)
-	if members, ok := lrm.members[filerGroup]; ok {
-		for addr := range members {
-			servers = append(servers, string(addr))
-		}
+	servers := make([]string, 0, len(members))
+	for addr := range members {
+		servers = append(servers, string(addr))
 	}
 	update := &master_pb.LockRingUpdate{
 		FilerGroup: string(filerGroup),
@@ -133,16 +190,13 @@ func (lrm *LockRingManager) doBroadcast(filerGroup FilerGroupName) {
 		Version:    version,
 	}
 	lrm.lastBroadcast[filerGroup] = update
-	delete(lrm.pendingTimer, filerGroup)
-	lrm.mu.Unlock()
-
-	glog.V(0).Infof("LockRing: broadcasting ring update for group %q version %d: %v", filerGroup, version, servers)
-
-	if lrm.broadcastFn != nil {
-		lrm.broadcastFn(&master_pb.KeepConnectedResponse{
-			LockRingUpdate: update,
-		})
+	if timer, ok := lrm.rebroadcastTimer[filerGroup]; ok {
+		timer.Stop()
 	}
+	lrm.rebroadcastTimer[filerGroup] = time.AfterFunc(lrm.rebroadcastInterval, func() {
+		lrm.rebroadcast(filerGroup)
+	})
+	return update
 }
 
 // FlushPending fires any pending timer immediately (for testing or shutdown).

weed/cluster/lock_ring_manager_test.go

@@ -213,6 +213,111 @@ func TestLockRingManager_NoBroadcastWithoutFn(t *testing.T) {
 	time.Sleep(50 * time.Millisecond) // should not panic
 }
 
+func TestLockRingManager_EmptyRingNotBroadcast(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 50 * time.Millisecond
+
+	group := FilerGroupName("default")
+
+	lrm.AddServer(group, "filer1:8888")
+	lrm.FlushPending(group)
+
+	mu.Lock()
+	broadcasts = nil
+	mu.Unlock()
+
+	// Removing the last member must not propagate an empty ring.
+	lrm.RemoveServer(group, "filer1:8888")
+	time.Sleep(100 * time.Millisecond)
+
+	mu.Lock()
+	assert.Equal(t, 0, len(broadcasts), "empty ring must not be broadcast")
+	mu.Unlock()
+
+	// The last non-empty snapshot is still served to reconnecting clients.
+	update := lrm.GetLastUpdate(group)
+	require.NotNil(t, update)
+	assert.Equal(t, []string{"filer1:8888"}, update.Servers)
+}
+
+func TestLockRingManager_PeriodicRebroadcast(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 20 * time.Millisecond
+	lrm.rebroadcastInterval = 60 * time.Millisecond
+
+	group := FilerGroupName("default")
+	lrm.AddServer(group, "filer1:8888")
+
+	// Without any further membership change, the ring keeps being re-sent so
+	// a lost or poisoned update cannot be permanent.
+	time.Sleep(200 * time.Millisecond)
+
+	mu.Lock()
+	require.GreaterOrEqual(t, len(broadcasts), 2, "ring should rebroadcast periodically")
+	for i := 1; i < len(broadcasts); i++ {
+		assert.Greater(t, broadcasts[i].Version, broadcasts[i-1].Version)
+	}
+	mu.Unlock()
+}
+
+func TestLockRingManager_RebroadcastDefersToPendingStabilization(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 100 * time.Millisecond
+	lrm.rebroadcastInterval = 30 * time.Millisecond
+
+	group := FilerGroupName("default")
+	lrm.AddServer(group, "filer1:8888")
+	lrm.FlushPending(group)
+
+	mu.Lock()
+	require.Len(t, broadcasts, 1)
+	mu.Unlock()
+
+	// A membership change just before the periodic tick: the rebroadcast must
+	// not publish the unsettled ring ahead of the stabilization timer.
+	lrm.RemoveServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+	time.Sleep(2 * lrm.rebroadcastInterval)
+
+	mu.Lock()
+	assert.Len(t, broadcasts, 1, "rebroadcast during stabilization should be deferred")
+	mu.Unlock()
+
+	time.Sleep(2 * lrm.stabilizeDelay)
+
+	mu.Lock()
+	require.GreaterOrEqual(t, len(broadcasts), 2)
+	assert.Equal(t, []string{"filer2:8888"}, broadcasts[1].Servers)
+	mu.Unlock()
+}
+
 func TestLockRingManager_GetLastUpdateReturnsBroadcastState(t *testing.T) {
 	lrm := NewLockRingManager(nil)
 

weed/command/backup.go

@@ -156,7 +156,7 @@ func backupFromLocation(volumeServer pb.ServerAddress, grpcDialOption grpc.DialO
 
 	// If local volume is larger than remote, recreate it
 	if datSize > stats.TailOffset {
-		if err := v.Destroy(false, false); err != nil {
+		if err := v.Destroy(false, false, false); err != nil {
 			v.Close()
 			return fmt.Errorf("destroying volume: %w", err), false
 		}

weed/command/filer.go

@@ -89,6 +89,7 @@ type FilerOptions struct {
 	s3ConfigFile              *string // optional path to static S3 identity config
 
 	allowUntrustedRemoteEndpoints *bool
+	remoteCacheEvictThreshold     *float64
 	// shutdownCtx, when non-nil, tells startFiler to gracefully shut down its
 	// HTTP/gRPC servers once the ctx is cancelled. Used by integration tests
 	// and by weed mini; nil for standalone weed filer.
@@ -134,6 +135,7 @@ func init() {
 	f.tusMaxSizeMB = cmdFiler.Flag.Int("tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
 	f.tusSessionExpiry = cmdFiler.Flag.Duration("tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"")
 	f.allowUntrustedRemoteEndpoints = cmdFiler.Flag.Bool("allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
+	f.remoteCacheEvictThreshold = cmdFiler.Flag.Float64("remoteCacheEvictThreshold", 0.9, "evict remote-cached objects (oldest first) when any volume disk exceeds this usage fraction; 0 disables")
 
 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -407,6 +409,7 @@ func (fo *FilerOptions) startFiler() {
 		CredentialManager:         credentialManager,
 
 		AllowUntrustedRemoteEndpoints: *fo.allowUntrustedRemoteEndpoints,
+		RemoteCacheEvictThreshold:     *fo.remoteCacheEvictThreshold,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)

weed/command/filer_remote_sync_dir.go

@@ -20,6 +20,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/replication/source"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -41,14 +43,6 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
 
 	var lastLogTsNs = time.Now().UnixNano()
 	processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
-		if resp.EventNotification.NewEntry != nil {
-			if *option.storageClass == "" {
-				delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
-			} else {
-				resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
-			}
-		}
-
 		processor.AddSyncJob(resp)
 		return nil
 	}, 3*time.Second, func(counter int64, lastTsNs int64) error {
@@ -175,10 +169,10 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(parentPath, entryName), remoteStorageMountLocation)
 			if message.NewEntry.IsDirectory {
 				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
-				return client.WriteDirectory(dest, message.NewEntry)
+				return client.WriteDirectory(dest, remoteWriteEntry(message.NewEntry, *option.storageClass))
 			}
 			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
-			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, message.NewEntry, dest)
+			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, remoteWriteEntry(message.NewEntry, *option.storageClass), dest)
 			if errors.Is(writeErr, errSuperseded) {
 				glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr)
 				return nil
@@ -213,7 +207,7 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 			return client.DeleteFile(dest)
 		}
 		if message.OldEntry != nil && message.NewEntry != nil {
-			return processUpdateEvent(option, filerSource, client, mountedDir, remoteStorageMountLocation, resp)
+			return processUpdateEvent(option, filerSource, *option.storageClass, client, mountedDir, remoteStorageMountLocation, resp)
 		}
 
 		return nil
@@ -224,6 +218,7 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 func processUpdateEvent(
 	filerClient filer_pb.FilerClient,
 	filerSource filer_pb.FilerClient,
+	storageClass string,
 	client remote_storage.RemoteStorageClient,
 	mountedDir string,
 	remoteStorageMountLocation *remote_pb.RemoteStorageLocation,
@@ -244,7 +239,7 @@ func processUpdateEvent(
 		return nil
 	}
 	if message.NewEntry.IsDirectory {
-		return client.WriteDirectory(dest, message.NewEntry)
+		return client.WriteDirectory(dest, remoteWriteEntry(message.NewEntry, storageClass))
 	}
 	if isMetadataOnlyUpdate(resp.Directory, message) {
 		remoteEntry, err := liveRemoteEntry(filerClient, message.NewParentPath, message.NewEntry)
@@ -257,7 +252,7 @@ func processUpdateEvent(
 		}
 		if remoteEntry != nil {
 			glog.V(2).Infof("update meta: %+v", resp)
-			return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry)
+			return client.UpdateFileMetadata(dest, message.OldEntry, remoteWriteEntry(message.NewEntry, storageClass))
 		}
 		glog.V(0).Infof("never replicated, uploading %s", remote_storage.FormatLocation(dest))
 	}
@@ -277,7 +272,7 @@ func processUpdateEvent(
 			}
 		}
 	}
-	remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, message.NewEntry, dest)
+	remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, remoteWriteEntry(message.NewEntry, storageClass), dest)
 	if errors.Is(writeErr, errSuperseded) {
 		glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr)
 		return nil
@@ -417,16 +412,66 @@ func shouldSendToRemote(entry *filer_pb.Entry) bool {
 	return false
 }
 
+// remoteWriteEntry returns the entry as remote storage should see it: the
+// storage class attribute is dropped, or overridden by -storageClass. The
+// event entry is left untouched so updateLocalEntry still compares the entry
+// the filer stored.
+func remoteWriteEntry(entry *filer_pb.Entry, storageClass string) *filer_pb.Entry {
+	clone := proto.Clone(entry).(*filer_pb.Entry)
+	if storageClass == "" {
+		delete(clone.Extended, s3_constants.AmzStorageClass)
+	} else {
+		if clone.Extended == nil {
+			clone.Extended = map[string][]byte{}
+		}
+		clone.Extended[s3_constants.AmzStorageClass] = []byte(storageClass)
+	}
+	return clone
+}
+
+// updateLocalEntry stamps the entry an event described with its RemoteEntry.
+// The write carries IF_ENTRY_EQUAL over the event's entry: the filer deletes
+// every stored chunk absent from an updated entry, so a snapshot older than
+// the live entry (the file was rewritten while its upload was in flight, or
+// the event is a replay) would delete the live chunks. A failed precondition
+// means the filer moved past this event; the event that superseded it follows
+// in the log and stamps the current entry, so the stale stamp is skipped the
+// same way a superseded upload is.
 func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
 	remoteEntry.LastLocalSyncTsNs = time.Now().UnixNano()
+	expected := proto.Clone(entry).(*filer_pb.Entry)
 	entry.RemoteEntry = remoteEntry
-	return filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+	err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		_, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
 			Directory: dir,
 			Entry:     entry,
+			Condition: ifEntryEqual(expected),
 		})
 		return err
 	})
+	if isFailedPrecondition(err) {
+		glog.Errorf("skipping stale stamp of %s: %v", util.NewFullPath(dir, entry.Name), err)
+		return nil
+	}
+	return err
+}
+
+// ifEntryEqual builds the precondition that the stored entry still equals the
+// one the event described: chunk fids, inline content, and metadata alike.
+func ifEntryEqual(entry *filer_pb.Entry) *filer_pb.WriteCondition {
+	return &filer_pb.WriteCondition{
+		Clauses: []*filer_pb.WriteCondition_Clause{{Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, ExpectedEntry: entry}},
+	}
+}
+
+// isFailedPrecondition reports a write condition the filer refused, through
+// any wrapping WithFilerClient added.
+func isFailedPrecondition(err error) bool {
+	if err == nil {
+		return false
+	}
+	st, ok := status.FromError(err)
+	return ok && st.Code() == codes.FailedPrecondition
 }
 
 func isMultipartUploadFile(dir string, name string) bool {

weed/command/filer_remote_sync_dir_test.go

@@ -16,6 +16,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -684,7 +686,7 @@ func TestRenameWithInheritedRemoteEntryWritesNewKey(t *testing.T) {
 
 	remote := &recordingRemote{}
 	filerClient := &stubFilerClient{}
-	if err := processUpdateEvent(filerClient, filerClient, remote, mountedDir, mountLoc, resp); err != nil {
+	if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp); err != nil {
 		t.Fatal(err)
 	}
 
@@ -731,7 +733,7 @@ func TestRenameRemoteOnlyEntrySkipsEmptyUpload(t *testing.T) {
 
 	remote := &recordingRemote{}
 	filerClient := &stubFilerClient{}
-	if err := processUpdateEvent(filerClient, filerClient, remote, mountedDir, mountLoc, resp); err != nil {
+	if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp); err != nil {
 		t.Fatal(err)
 	}
 	if len(remote.writes) != 0 {
@@ -764,7 +766,7 @@ func TestRenameDeleteOldKeyFailureReturnsError(t *testing.T) {
 	deleteErr := errors.New("AccessDenied: Access Denied")
 	remote := &recordingRemote{deleteErr: deleteErr}
 	filerClient := &stubFilerClient{}
-	err := processUpdateEvent(filerClient, filerClient, remote, mountedDir, mountLoc, resp)
+	err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp)
 	if !errors.Is(err, deleteErr) {
 		t.Errorf("err = %v, want the delete failure returned so the event is retried", err)
 	}
@@ -800,7 +802,7 @@ func TestRenameDeleteOldKeyNotFoundStillWrites(t *testing.T) {
 
 	remote := &recordingRemote{deleteErr: remote_storage.ErrRemoteObjectNotFound}
 	filerClient := &stubFilerClient{}
-	if err := processUpdateEvent(filerClient, filerClient, remote, mountedDir, mountLoc, resp); err != nil {
+	if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp); err != nil {
 		t.Fatalf("err = %v, want nil: an already-deleted old key must not block the write", err)
 	}
 	wantWrite := &remote_pb.RemoteStorageLocation{Name: "gcs", Bucket: "bucket", Path: "/b/dst/probe.bin"}
@@ -808,3 +810,73 @@ func TestRenameDeleteOldKeyNotFoundStillWrites(t *testing.T) {
 		t.Errorf("writes = %+v, want the new key %s written", remote.writes, remote_storage.FormatLocation(wantWrite))
 	}
 }
+
+func TestIfEntryEqualCarriesTheEventEntry(t *testing.T) {
+	entry := &filer_pb.Entry{
+		Name:    "f",
+		Content: []byte("inline"),
+		Chunks: []*filer_pb.FileChunk{
+			{FileId: "3,01a"},
+			{Fid: &filer_pb.FileId{VolumeId: 4, FileKey: 0x2b, Cookie: 0x0c}},
+		},
+	}
+	cond := ifEntryEqual(entry)
+	if len(cond.Clauses) != 1 || cond.Clauses[0].Kind != filer_pb.WriteCondition_IF_ENTRY_EQUAL {
+		t.Fatalf("condition = %v, want one IF_ENTRY_EQUAL clause", cond)
+	}
+	if got := cond.Clauses[0].ExpectedEntry; !proto.Equal(got, entry) {
+		t.Fatalf("expected_entry = %v, want %v", got, entry)
+	}
+}
+
+// The remote-bound entry loses (or gains) the storage class attribute while
+// the event entry keeps it, so IF_ENTRY_EQUAL still sees the entry the filer
+// stored — otherwise every stamp on an S3-written object reads as stale.
+func TestRemoteWriteEntryLeavesEventEntryUntouched(t *testing.T) {
+	entry := &filer_pb.Entry{
+		Name:       "f",
+		Attributes: &filer_pb.FuseAttributes{Mtime: 1},
+		Extended:   map[string][]byte{s3_constants.AmzStorageClass: []byte("STANDARD")},
+	}
+
+	stripped := remoteWriteEntry(entry, "")
+	if _, ok := stripped.Extended[s3_constants.AmzStorageClass]; ok {
+		t.Fatalf("remote entry still carries %s", s3_constants.AmzStorageClass)
+	}
+	if string(entry.Extended[s3_constants.AmzStorageClass]) != "STANDARD" {
+		t.Fatalf("event entry Extended mutated: %v", entry.Extended)
+	}
+
+	overridden := remoteWriteEntry(entry, "GLACIER")
+	if got := string(overridden.Extended[s3_constants.AmzStorageClass]); got != "GLACIER" {
+		t.Fatalf("override = %q, want GLACIER", got)
+	}
+	if string(entry.Extended[s3_constants.AmzStorageClass]) != "STANDARD" {
+		t.Fatalf("event entry Extended mutated: %v", entry.Extended)
+	}
+
+	bare := &filer_pb.Entry{Name: "g", Attributes: &filer_pb.FuseAttributes{Mtime: 1}}
+	if got := string(remoteWriteEntry(bare, "GLACIER").Extended[s3_constants.AmzStorageClass]); got != "GLACIER" {
+		t.Fatalf("override on nil Extended = %q, want GLACIER", got)
+	}
+}
+
+func TestIsFailedPrecondition(t *testing.T) {
+	refused := status.Error(codes.FailedPrecondition, "precondition failed: /buckets/b/f")
+	cases := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{"nil", nil, false},
+		{"status", refused, true},
+		{"wrapped status", fmt.Errorf("update entry: %w", refused), true},
+		{"message only", errors.New("rpc error: code = FailedPrecondition desc = precondition failed: /f"), false},
+		{"other", status.Error(codes.Unavailable, "filer down"), false},
+	}
+	for _, c := range cases {
+		if got := isFailedPrecondition(c.err); got != c.want {
+			t.Errorf("%s: isFailedPrecondition = %v, want %v", c.name, got, c.want)
+		}
+	}
+}

weed/command/master.go

@@ -46,17 +46,19 @@ const (
 )
 
 type MasterOptions struct {
-	port                       *int
-	portGrpc                   *int
-	ip                         *string
-	ipBind                     *string
-	metaFolder                 *string
-	peers                      *string
-	mastersDeprecated          *string // deprecated, for backward compatibility in master.follower
-	volumeSizeLimitMB          *uint
-	fileSizeLimitMB            *int
-	volumePreallocate          *bool
-	maxParallelVacuumPerServer *int
+	port                          *int
+	portGrpc                      *int
+	ip                            *string
+	ipBind                        *string
+	metaFolder                    *string
+	peers                         *string
+	mastersDeprecated             *string // deprecated, for backward compatibility in master.follower
+	volumeSizeLimitMB             *uint
+	fileSizeLimitMB               *int
+	volumePreallocate             *bool
+	maxParallelVacuumPerServer    *int
+	vacuumIntervalSeconds         *int
+	vacuumDeleteEmptyAfterSeconds *int
 	// pulseSeconds       *int
 	defaultReplication *string
 	garbageThreshold   *float64
@@ -93,6 +95,8 @@ func init() {
 	m.fileSizeLimitMB = cmdMaster.Flag.Int("fileSizeLimitMB", 256, "limit the file size accepted by /submit, should match the volume servers' -fileSizeLimitMB (-volume.fileSizeLimitMB under weed server or weed mini, which set this for you)")
 	m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
 	m.maxParallelVacuumPerServer = cmdMaster.Flag.Int("maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel per volume server")
+	m.vacuumIntervalSeconds = cmdMaster.Flag.Int("vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps")
+	m.vacuumDeleteEmptyAfterSeconds = cmdMaster.Flag.Int("vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables")
 	// m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
 	m.defaultReplication = cmdMaster.Flag.String("defaultReplication", "", "Default replication type if not specified.")
 	m.garbageThreshold = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
@@ -467,12 +471,14 @@ func peerIndex(self pb.ServerAddress, peers []pb.ServerAddress) int {
 func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOption {
 	masterAddress := pb.NewServerAddress(*m.ip, *m.port, *m.portGrpc)
 	return &weed_server.MasterOption{
-		Master:                     masterAddress,
-		MetaFolder:                 *m.metaFolder,
-		VolumeSizeLimitMB:          uint32(*m.volumeSizeLimitMB),
-		FileSizeLimitMB:            *m.fileSizeLimitMB,
-		VolumePreallocate:          *m.volumePreallocate,
-		MaxParallelVacuumPerServer: *m.maxParallelVacuumPerServer,
+		Master:                        masterAddress,
+		MetaFolder:                    *m.metaFolder,
+		VolumeSizeLimitMB:             uint32(*m.volumeSizeLimitMB),
+		FileSizeLimitMB:               *m.fileSizeLimitMB,
+		VolumePreallocate:             *m.volumePreallocate,
+		MaxParallelVacuumPerServer:    *m.maxParallelVacuumPerServer,
+		VacuumIntervalSeconds:         *m.vacuumIntervalSeconds,
+		VacuumDeleteEmptyAfterSeconds: *m.vacuumDeleteEmptyAfterSeconds,
 		// PulseSeconds:            *m.pulseSeconds,
 		DefaultReplicaPlacement: *m.defaultReplication,
 		GarbageThreshold:        *m.garbageThreshold,

weed/command/master_follower.go

@@ -47,6 +47,8 @@ func init() {
 	mf.metricsIntervalSec = aws.Int(0)
 	mf.raftResumeState = aws.Bool(false)
 	mf.maxParallelVacuumPerServer = aws.Int(1)
+	mf.vacuumIntervalSeconds = aws.Int(840)
+	mf.vacuumDeleteEmptyAfterSeconds = aws.Int(0)
 	mf.telemetryUrl = aws.String("https://telemetry.seaweedfs.com/api/collect")
 	mf.telemetryEnabled = aws.Bool(false)
 }

weed/command/mini.go

@@ -427,6 +427,8 @@ func initMiniMasterFlags() {
 	miniMasterOptions.volumeSizeLimitMB = cmdMini.Flag.Uint("master.volumeSizeLimitMB", defaultMiniVolumeSizeMB, "Master stops directing writes to oversized volumes (default: 128MB for mini)")
 	miniMasterOptions.volumePreallocate = cmdMini.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
 	miniMasterOptions.maxParallelVacuumPerServer = cmdMini.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server")
+	miniMasterOptions.vacuumIntervalSeconds = cmdMini.Flag.Int("master.vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps")
+	miniMasterOptions.vacuumDeleteEmptyAfterSeconds = cmdMini.Flag.Int("master.vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables")
 	miniMasterOptions.defaultReplication = cmdMini.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
 	miniMasterOptions.garbageThreshold = cmdMini.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	miniMasterOptions.metricsAddress = cmdMini.Flag.String("master.metrics.address", "", "Prometheus gateway address")
@@ -465,6 +467,7 @@ func initMiniFilerFlags() {
 	miniFilerOptions.tusMaxSizeMB = cmdMini.Flag.Int("filer.tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
 	miniFilerOptions.tusSessionExpiry = cmdMini.Flag.Duration("filer.tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration")
 	miniFilerOptions.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("filer.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
+	miniFilerOptions.remoteCacheEvictThreshold = cmdMini.Flag.Float64("filer.remoteCacheEvictThreshold", 0.9, "evict remote-cached objects (oldest first) when any volume disk exceeds this usage fraction; 0 disables")
 }
 
 // initMiniVolumeFlags initializes Volume server flag options

weed/command/s3.go

@@ -618,6 +618,7 @@ func (s3opt *S3Options) startLanceServer(s3ApiServer *s3api.S3ApiServer) {
 	lanceRouter.Use(util_http.EscapeSemicolonsInQuery)
 
 	lanceServer := lance.NewServer(s3ApiServer, s3ApiServer)
+	lanceServer.SetCredentialValidator(s3ApiServer)
 	if s3opt.icebergCredentialRole != nil && *s3opt.icebergCredentialRole != "" {
 		lanceServer.SetCredentialVendor(lanceCredentialVendor{s3ApiServer})
 	}

weed/command/scaffold/master.toml

@@ -41,6 +41,7 @@ copy_2 = 6                # create 2 x 6 = 12 actual volumes
 copy_3 = 3                # create 3 x 3 = 9 actual volumes
 copy_other = 1            # create n x 1 = n actual volumes
 threshold = 0.9           # create threshold
+reservation_timeout = "5m"# capacity reservation timeout before unreleased reservations expire
 disable = false           # disables volume growth if true
 
 # configuration flags for replication

weed/command/server.go

@@ -98,6 +98,8 @@ func init() {
 	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", util.DefaultVolumeSizeLimitMB, "Master stops directing writes to oversized volumes.")
 	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
 	masterOptions.maxParallelVacuumPerServer = cmdServer.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server")
+	masterOptions.vacuumIntervalSeconds = cmdServer.Flag.Int("master.vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps")
+	masterOptions.vacuumDeleteEmptyAfterSeconds = cmdServer.Flag.Int("master.vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables")
 	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
 	masterOptions.garbageThreshold = cmdServer.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterOptions.metricsAddress = cmdServer.Flag.String("master.metrics.address", "", "Prometheus gateway address")
@@ -134,6 +136,7 @@ func init() {
 	filerOptions.tusMaxSizeMB = cmdServer.Flag.Int("filer.tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
 	filerOptions.tusSessionExpiry = cmdServer.Flag.Duration("filer.tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"")
 	filerOptions.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("filer.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
+	filerOptions.remoteCacheEvictThreshold = cmdServer.Flag.Float64("filer.remoteCacheEvictThreshold", 0.9, "evict remote-cached objects (oldest first) when any volume disk exceeds this usage fraction; 0 disables")
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")

weed/ec/ec_encode.go

@@ -43,7 +43,7 @@ func markVolumeReplicaWritable(ctx context.Context, grpcDialOption grpc.DialOpti
 // deleteVolume removes the volume from sourceVolumeServer via the canonical
 // volume_move helper.
 func deleteVolume(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer pb.ServerAddress, onlyEmpty bool, keepRemoteData bool) (err error) {
-	return volume_move.NewMover(grpcDialOption).DeleteVolume(ctx, volumeId, sourceVolumeServer, onlyEmpty, keepRemoteData)
+	return volume_move.NewMover(grpcDialOption).DeleteVolume(ctx, volumeId, sourceVolumeServer, onlyEmpty, false, keepRemoteData)
 }
 
 func ChunkVolumeIds(volumeIds []needle.VolumeId, batchSize int) [][]needle.VolumeId {

weed/filer/filer.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
@@ -22,6 +23,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
 
 	"google.golang.org/grpc"
+	"google.golang.org/protobuf/proto"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -73,6 +75,11 @@ type Filer struct {
 	EmptyFolderCleanupDelay       time.Duration
 	persistedLogCache             *persistedLogCache
 	metaLogInflight               metaLogInflight
+	remoteTombstones              *remoteDeletionTombstones
+	// remoteTombstonesDone, when non-nil, is closed once the startup tombstone
+	// rebuild finishes; lazy remote reads wait on it so a pending delete
+	// cannot resurrect in the gap.
+	remoteTombstonesDone atomic.Pointer[chan struct{}]
 }
 
 func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer {
@@ -88,6 +95,7 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH
 		deletionQuit:        make(chan struct{}),
 		DeletionRetryQueue:  NewDeletionRetryQueue(),
 		persistedLogCache:   newPersistedLogCache(persistedLogCacheMaxBytes),
+		remoteTombstones:    newRemoteDeletionTombstones(),
 	}
 	if f.UniqueFilerId < 0 {
 		f.UniqueFilerId = -f.UniqueFilerId
@@ -174,6 +182,10 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
 		glog.V(0).Infof("LockRing: applying master ring update v%d: %v", update.Version, servers)
 		f.Dlm.LockRing.SetSnapshot(servers, update.Version)
 	})
+	f.MasterClient.SetOnMasterChangeFn(func(previous, current pb.ServerAddress) {
+		glog.V(0).Infof("LockRing: master changed %s -> %s, resetting ring", previous, current)
+		f.Dlm.LockRing.Reset()
+	})
 
 	// Subscribe to the local filer first: its events reach the aggregated
 	// buffer only through this subscription, and the peer watermarks must
@@ -265,7 +277,13 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
 
 	oldEntry := existing
 	if oldEntry == nil {
-		oldEntry, _ = f.FindEntry(ctx, entry.FullPath)
+		var findErr error
+		oldEntry, findErr = f.FindEntry(ctx, entry.FullPath)
+		if o_excl && findErr != nil && !errors.Is(findErr, filer_pb.ErrNotFound) {
+			// An exclusive create cannot decide whether the path exists when
+			// the lookup itself failed; proceeding would upsert over it.
+			return fmt.Errorf("find entry %s: %w", entry.FullPath, findErr)
+		}
 	}
 
 	/*
@@ -310,7 +328,7 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
 			return fmt.Errorf("%s: %w", entry.FullPath, filer_pb.ErrEntryAlreadyExists)
 		}
 		glog.V(4).InfofCtx(ctx, "UpdateEntry %s: old entry: %v", entry.FullPath, oldEntry.Name())
-		if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
+		if err := f.UpdateEntry(ctx, oldEntry, entry, isFromOtherCluster); err != nil {
 			if errors.Is(err, filer_pb.ErrExistingIsDirectory) || errors.Is(err, filer_pb.ErrExistingIsFile) {
 				glog.V(2).InfofCtx(ctx, "update entry %s: %v", entry.FullPath, err)
 			} else {
@@ -465,7 +483,7 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
 		narrowed := existing.ShallowClone()
 		narrowed.Mode = existing.Mode&^restorableModeBits | kept
 		glog.V(1).InfofCtx(ctx, "restore directory %s: narrowing %v to %v", dirPath, existing.Mode, narrowed.Mode)
-		if err := f.UpdateEntry(ctx, existing, narrowed); err != nil {
+		if err := f.UpdateEntry(ctx, existing, narrowed, false); err != nil {
 			return err
 		}
 		f.NotifyUpdateEvent(ctx, existing, narrowed, false, false, nil)
@@ -498,7 +516,7 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
 	return nil
 }
 
-func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
+func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromOtherCluster bool) (err error) {
 	if oldEntry != nil {
 		entry.Attr.Crtime = oldEntry.Attr.Crtime
 		if oldEntry.Attr.Inode != 0 {
@@ -518,6 +536,17 @@ func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err er
 			glog.V(2).InfofCtx(ctx, "existing %s is a file", oldEntry.FullPath)
 			return fmt.Errorf("%s: %w", oldEntry.FullPath, filer_pb.ErrExistingIsFile)
 		}
+		// A local write to a remote-backed entry leaves the copy on remote
+		// stale until a sync re-uploads it. Content changes that arrive
+		// without a fresh sync stamp are unsynced; clear the stamp so nothing
+		// mistakes the still-local chunks for a re-fetchable cache copy.
+		// Replicated updates carry the writer's authoritative stamp.
+		if !isFromOtherCluster && oldEntry.Remote != nil && entry.Remote != nil &&
+			oldEntry.Remote.LastLocalSyncTsNs == entry.Remote.LastLocalSyncTsNs &&
+			!chunksEqual(oldEntry.Chunks, entry.Chunks) {
+			entry.Remote = proto.Clone(entry.Remote).(*filer_pb.RemoteEntry)
+			entry.Remote.LastLocalSyncTsNs = 0
+		}
 	}
 	if entry.Attr.Atime.IsZero() {
 		entry.Attr.Atime = entryInitialAtime(entry.Attr)
@@ -589,7 +618,7 @@ func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, sta
 	lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) (bool, error) {
 		select {
 		case <-ctx.Done():
-			glog.Errorf("Context is done.")
+			glog.V(1).InfofCtx(ctx, "listing %q canceled: %v", p, ctx.Err())
 			return false, fmt.Errorf("context canceled: %w", ctx.Err())
 		default:
 			if entry.TtlSec > 0 && !entry.IsDirectory() {

weed/filer/filer_delete_collection_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
@@ -96,7 +97,7 @@ func newFilerWithFakeMaster(t *testing.T) (*Filer, *hookedStore, *collectionDele
 
 	mc := wdclient.NewMasterClient(
 		grpc.WithTransportCredentials(insecure.NewCredentials()),
-		"test", cluster.FilerType, pb.ServerAddress("localhost:0"), "", "",
+		"", cluster.FilerType, pb.ServerAddress("localhost:0"), "", "",
 		*pb.NewServiceDiscoveryFromMap(map[string]pb.ServerAddress{"m": masterAddress}),
 	)
 
@@ -181,3 +182,206 @@ func TestDeleteEntryMetaAndDataDeletesCollectionWhenTheRequestIsCancelledMidDele
 		t.Error("the bucket entry survived the delete")
 	}
 }
+
+func seedBucket(t *testing.T, store *hookedStore, path util.FullPath) {
+	t.Helper()
+	if err := store.InsertEntry(context.Background(), &Entry{
+		FullPath: path,
+		Attr:     Attr{Mode: os.ModeDir | 0755},
+	}); err != nil {
+		t.Fatalf("seed bucket %s: %v", path, err)
+	}
+}
+
+// Two buckets resolving to one collection: deleting either must leave the
+// collection for the other. Previously the delete dropped the collection
+// named after the bucket regardless of where its data actually lived.
+func TestDeleteBucketKeepsSharedCollection(t *testing.T) {
+	f, store, master := newFilerWithFakeMaster(t)
+	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets",
+		Collection:     "shared",
+	})
+	seedBucket(t, store, util.FullPath("/buckets/a"))
+	seedBucket(t, store, util.FullPath("/buckets/b"))
+
+	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/a", true, false, true, false, nil, 0); err != nil {
+		t.Fatalf("DeleteEntryMetaAndData: %v", err)
+	}
+
+	select {
+	case call := <-master.calls:
+		t.Fatalf("shared collection was deleted: %q", call.name)
+	default:
+	}
+	if store.getEntry("/buckets/b") == nil {
+		t.Error("the surviving bucket's entry is gone")
+	}
+}
+
+// A bucket named after a collection other buckets resolve to is still just a
+// bucket: deleting it must not take the shared collection down with it.
+func TestDeleteBucketNamedAfterSharedCollection(t *testing.T) {
+	f, store, master := newFilerWithFakeMaster(t)
+	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets",
+		Collection:     "shared",
+	})
+	seedBucket(t, store, util.FullPath("/buckets/shared"))
+	seedBucket(t, store, util.FullPath("/buckets/other"))
+
+	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/shared", true, false, true, false, nil, 0); err != nil {
+		t.Fatalf("DeleteEntryMetaAndData: %v", err)
+	}
+
+	select {
+	case call := <-master.calls:
+		t.Fatalf("collection backing other buckets was deleted: %q", call.name)
+	default:
+	}
+}
+
+// A rule pointing a non-bucket path at the same collection keeps it: the
+// collection serves files the bucket delete must not orphan.
+func TestDeleteBucketKeepsCollectionUsedByNonBucketPath(t *testing.T) {
+	f, store, master := newFilerWithFakeMaster(t)
+	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets/a",
+		Collection:     "cold",
+	})
+	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/archives",
+		Collection:     "cold",
+	})
+	seedBucket(t, store, util.FullPath("/buckets/a"))
+
+	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/a", true, false, true, false, nil, 0); err != nil {
+		t.Fatalf("DeleteEntryMetaAndData: %v", err)
+	}
+
+	select {
+	case call := <-master.calls:
+		t.Fatalf("collection used by /archives was deleted: %q", call.name)
+	default:
+	}
+}
+
+// A broad prefix rule covering the whole tree keeps the collection even for a
+// lone bucket: the same collection backs non-bucket paths too.
+func TestDeleteBucketKeepsCollectionFromBroadRule(t *testing.T) {
+	f, store, master := newFilerWithFakeMaster(t)
+	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/",
+		Collection:     "everything",
+	})
+	seedBucket(t, store, util.FullPath("/buckets/a"))
+
+	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/a", true, false, true, false, nil, 0); err != nil {
+		t.Fatalf("DeleteEntryMetaAndData: %v", err)
+	}
+
+	select {
+	case call := <-master.calls:
+		t.Fatalf("collection from a / rule was deleted: %q", call.name)
+	default:
+	}
+}
+
+// A bucket resolving to the filer's default collection keeps it: rule-less
+// writes outside buckets land there too, so it is never one bucket's alone.
+func TestDeleteBucketKeepsDefaultCollection(t *testing.T) {
+	f, store, master := newFilerWithFakeMaster(t)
+	f.metaLogCollection = "everything"
+	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets/a",
+		Collection:     "everything",
+	})
+	seedBucket(t, store, util.FullPath("/buckets/a"))
+
+	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/a", true, false, true, false, nil, 0); err != nil {
+		t.Fatalf("DeleteEntryMetaAndData: %v", err)
+	}
+
+	select {
+	case call := <-master.calls:
+		t.Fatalf("the filer's default collection was deleted: %q", call.name)
+	default:
+	}
+}
+
+// A rule nested under a surviving bucket keeps the collection: the other
+// bucket resolves elsewhere at its root, but objects deeper inside it still
+// land in the shared collection.
+func TestDeleteBucketKeepsCollectionFromNestedSiblingRule(t *testing.T) {
+	f, store, master := newFilerWithFakeMaster(t)
+	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets/a",
+		Collection:     "shared",
+	})
+	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets/b/deep",
+		Collection:     "shared",
+	})
+	seedBucket(t, store, util.FullPath("/buckets/a"))
+	seedBucket(t, store, util.FullPath("/buckets/b"))
+
+	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/a", true, false, true, false, nil, 0); err != nil {
+		t.Fatalf("DeleteEntryMetaAndData: %v", err)
+	}
+
+	select {
+	case call := <-master.calls:
+		t.Fatalf("collection used under /buckets/b/deep was deleted: %q", call.name)
+	default:
+	}
+}
+
+// A grouped gateway writes to <group>_<bucket> regardless of the storage
+// rules, so that is the collection the delete must drop -- and a rule-named
+// collection the bucket never used must survive.
+func TestDeleteBucketUnderFilerGroup(t *testing.T) {
+	f, store, master := newFilerWithFakeMaster(t)
+	f.MasterClient.FilerGroup = "tenant1"
+	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets/photos",
+		Collection:     "archive",
+	})
+	seedBucket(t, store, util.FullPath("/buckets/photos"))
+
+	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/photos", true, false, true, false, nil, 0); err != nil {
+		t.Fatalf("DeleteEntryMetaAndData: %v", err)
+	}
+
+	select {
+	case call := <-master.calls:
+		if call.name != "tenant1_photos" {
+			t.Fatalf("CollectionDelete = %q, want %q", call.name, "tenant1_photos")
+		}
+	case <-time.After(20 * time.Second):
+		t.Fatal("CollectionDelete never reached the master")
+	}
+}
+
+// A collection only the deleted bucket resolves to is dropped under its real
+// name, so a dedicated custom collection does not leak its volumes.
+func TestDeleteBucketDeletesResolvedCollection(t *testing.T) {
+	f, store, master := newFilerWithFakeMaster(t)
+	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets/only",
+		Collection:     "custom",
+	})
+	seedBucket(t, store, util.FullPath("/buckets/only"))
+
+	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/only", true, false, true, false, nil, 0); err != nil {
+		t.Fatalf("DeleteEntryMetaAndData: %v", err)
+	}
+
+	select {
+	case call := <-master.calls:
+		if call.name != "custom" {
+			t.Fatalf("CollectionDelete = %q, want %q", call.name, "custom")
+		}
+	case <-time.After(20 * time.Second):
+		t.Fatal("CollectionDelete never reached the master")
+	}
+}

weed/filer/filer_delete_entry.go

@@ -68,6 +68,10 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 		return nil
 	}
 	isDeleteCollection := f.IsBucket(entry)
+	collectionName := ""
+	if isDeleteCollection {
+		collectionName = f.bucketCollection(ctx, entry.Name())
+	}
 	if entry.IsDirectory() {
 		// delete the folder children, not including the folder itself
 		err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures, func(hardLinkIds []HardLinkId) error {
@@ -102,17 +106,18 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 	}
 
 	if isDeleteCollection {
-		collectionName := entry.Name()
-		// the entry is already gone: a caller that hung up must not leave the
-		// collection behind, so this cleanup outlives the request -- bounded all
-		// the same, or a master that is down parks this handler indefinitely and
-		// every client retry behind it parks another
-		collectionCtx, cancelCollection := context.WithTimeout(context.WithoutCancel(ctx), collectionDeleteTimeout)
-		f.DoDeleteCollection(collectionCtx, collectionName)
-		cancelCollection()
+		if collectionName != "" {
+			// the entry is already gone: a caller that hung up must not leave the
+			// collection behind, so this cleanup outlives the request -- bounded all
+			// the same, or a master that is down parks this handler indefinitely and
+			// every client retry behind it parks another
+			collectionCtx, cancelCollection := context.WithTimeout(context.WithoutCancel(ctx), collectionDeleteTimeout)
+			f.DoDeleteCollection(collectionCtx, collectionName)
+			cancelCollection()
+		}
 		// drop bucket-labeled series held by this process; the S3 gateway
 		// only cleans its own registry
-		stats.DeleteBucketMetrics(collectionName)
+		stats.DeleteBucketMetrics(entry.Name())
 	}
 
 	return nil
@@ -120,6 +125,15 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 
 func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32, onHardLinkIdsFn OnHardLinkIdsFunc) (err error) {
 
+	var dirTombstoneTs int64
+	if isRecursive {
+		// Tombstone the directory before its children: when the store drops
+		// the subtree without listing it, or a child error aborts the walk,
+		// the ancestor tombstone still covers every descendant.
+		dirTombstoneTs = time.Now().UnixNano()
+		f.noteRemoteDeletion(entry.FullPath, true, dirTombstoneTs)
+	}
+
 	//collect all the chunks of this layer and delete them together at the end
 	var chunksToDelete []*filer_pb.FileChunk
 	lastFileName := ""
@@ -129,6 +143,9 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 		for {
 			entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "", "")
 			if err != nil {
+				// nothing was deleted; a leftover tombstone would hide the
+				// still-existing remote children
+				f.unnoteRemoteDeletion(entry.FullPath, dirTombstoneTs)
 				glog.ErrorfCtx(ctx, "list folder %s: %v", entry.FullPath, err)
 				return fmt.Errorf("list folder %s: %v", entry.FullPath, err)
 			}
@@ -140,6 +157,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 
 			for _, sub := range entries {
 				lastFileName = sub.Name()
+				f.noteRemoteDeletion(sub.FullPath, sub.IsDirectory(), time.Now().UnixNano())
 				if sub.IsDirectory() {
 					subIsDeletingBucket := f.IsBucket(sub)
 					err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, isFromOtherCluster, nil, onHardLinkIdsFn)
@@ -202,6 +220,8 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
 		}
 	}
 
+	f.noteRemoteDeletion(entry.FullPath, entry.IsDirectory(), time.Now().UnixNano())
+
 	if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil {
 		return fmt.Errorf("filer store delete: %w", storeDeletionErr)
 	}
@@ -221,6 +241,75 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
 // DeleteCollection, still has retry budget left.
 const collectionDeleteTimeout = 15 * time.Second
 
+// bucketCollection resolves the collection a bucket's objects land in
+// through the same chain the write path uses -- a grouped gateway's explicit
+// collection, then the storage rules, then the bucket name -- and reports it
+// only when nothing outside the bucket can still route into it. A shared
+// collection must survive the bucket delete: dropping it removes volumes
+// other paths still write to. A listing failure keeps the collection, the
+// safe side of an unknown.
+func (f *Filer) bucketCollection(ctx context.Context, bucket string) (collection string) {
+	bucketDir := f.DirBucketsPath + "/" + bucket + "/"
+	resolve := func(dir, name string) string {
+		if f.MasterClient != nil {
+			if group := f.MasterClient.FilerGroup; group != "" {
+				return group + "_" + name
+			}
+		}
+		return util.Nvl(f.FilerConf.MatchStorageRule(dir).Collection, name)
+	}
+	collection = resolve(bucketDir, bucket)
+
+	// Rule-less writes outside buckets fall back to the filer's default
+	// collection, so a bucket resolving there shares it with them.
+	if collection == f.metaLogCollection {
+		return ""
+	}
+
+	// A rule whose prefix escapes the bucket can route other paths into the
+	// same collection, including prefixes nested under surviving buckets.
+	for _, rule := range f.FilerConf.ToProto().Locations {
+		prefix := strings.TrimSuffix(rule.LocationPrefix, "/") + "/"
+		if strings.HasPrefix(prefix, bucketDir) {
+			continue
+		}
+		if f.FilerConf.MatchStorageRule(prefix).Collection == collection {
+			return ""
+		}
+	}
+
+	siblings, err := f.listBuckets(ctx)
+	if err != nil {
+		glog.ErrorfCtx(ctx, "list buckets for collection check: %v", err)
+		return ""
+	}
+	for _, sibling := range siblings {
+		if sibling != bucket && resolve(f.DirBucketsPath+"/"+sibling+"/", sibling) == collection {
+			return ""
+		}
+	}
+	return collection
+}
+
+func (f *Filer) listBuckets(ctx context.Context) (buckets []string, err error) {
+	lastFileName := ""
+	for {
+		entries, _, listErr := f.ListDirectoryEntries(ctx, util.FullPath(f.DirBucketsPath), lastFileName, false, PaginationSize, "", "", "")
+		if listErr != nil {
+			return nil, listErr
+		}
+		for _, entry := range entries {
+			lastFileName = entry.Name()
+			if f.IsBucket(entry) {
+				buckets = append(buckets, entry.Name())
+			}
+		}
+		if len(entries) < PaginationSize {
+			return buckets, nil
+		}
+	}
+}
+
 func (f *Filer) DoDeleteCollection(ctx context.Context, collectionName string) (err error) {
 
 	return f.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {

weed/filer/filer_deletion.go

@@ -306,24 +306,29 @@ func (f *Filer) loopProcessingDeletion() {
 			glog.V(0).Infof("deletion processor shutting down")
 			return
 		case <-ticker.C:
-			f.FileIdDeletionQueue.Consume(func(fileIds []string) {
-				for i := 0; i < len(fileIds); i += DeletionBatchSize {
-					end := i + DeletionBatchSize
-					if end > len(fileIds) {
-						end = len(fileIds)
-					}
-					toDeleteFileIds := fileIds[i:end]
-					f.processDeletionBatch(toDeleteFileIds, lookupFunc)
-				}
-			})
+			f.FlushFileIdDeletionQueue(context.Background(), lookupFunc)
 		}
 	}
 }
 
+func (f *Filer) FlushFileIdDeletionQueue(ctx context.Context, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) (consumed []string) {
+	f.FileIdDeletionQueue.Consume(func(fileIds []string) {
+		consumed = fileIds
+		for i := 0; i < len(fileIds); i += DeletionBatchSize {
+			end := i + DeletionBatchSize
+			if end > len(fileIds) {
+				end = len(fileIds)
+			}
+			f.processDeletionBatch(ctx, fileIds[i:end], lookupFunc)
+		}
+	})
+	return consumed
+}
+
 // processDeletionBatch handles deletion of a batch of file IDs and processes results.
 // It classifies errors into retryable and permanent categories, adds retryable failures
 // to the retry queue, and logs appropriate messages.
-func (f *Filer) processDeletionBatch(toDeleteFileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) {
+func (f *Filer) processDeletionBatch(ctx context.Context, toDeleteFileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) {
 	// Deduplicate file IDs to prevent incorrect retry count increments for the same file ID within a single batch.
 	uniqueFileIdsSlice := make([]string, 0, len(toDeleteFileIds))
 	processed := make(map[string]struct{}, len(toDeleteFileIds))
@@ -339,7 +344,7 @@ func (f *Filer) processDeletionBatch(toDeleteFileIds []string, lookupFunc func([
 	}
 
 	// Delete files and classify outcomes
-	outcomes := deleteFilesAndClassify(f.GrpcDialOption, uniqueFileIdsSlice, lookupFunc)
+	outcomes := deleteFilesAndClassify(ctx, f.GrpcDialOption, uniqueFileIdsSlice, lookupFunc)
 
 	// Process outcomes
 	var successCount, notFoundCount, retryableErrorCount, permanentErrorCount int
@@ -405,9 +410,9 @@ type deletionOutcome struct {
 }
 
 // deleteFilesAndClassify performs deletion and classifies outcomes for a list of file IDs
-func deleteFilesAndClassify(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) map[string]deletionOutcome {
+func deleteFilesAndClassify(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) map[string]deletionOutcome {
 	// Perform deletion
-	results := operation.DeleteFileIdsWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
+	results := operation.DeleteFileIdsWithLookupVolumeId(ctx, grpcDialOption, fileIds, lookupFunc)
 
 	// Group results by file ID to handle multiple results for replicated volumes
 	resultsByFileId := make(map[string][]*volume_server_pb.DeleteResult)
@@ -545,7 +550,7 @@ func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc fu
 	}
 
 	// Delete files and classify outcomes
-	outcomes := deleteFilesAndClassify(f.GrpcDialOption, fileIds, lookupFunc)
+	outcomes := deleteFilesAndClassify(context.Background(), f.GrpcDialOption, fileIds, lookupFunc)
 
 	// Process outcomes - iterate over readyItems to ensure all items are accounted for
 	var successCount, notFoundCount, retryCount, permanentErrorCount int

weed/filer/filer_inode_test.go

@@ -2,11 +2,13 @@ package filer
 
 import (
 	"context"
+	"errors"
 	"os"
 	"testing"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -101,6 +103,57 @@ func TestCreateEntryAssignsInodesToAutoCreatedParents(t *testing.T) {
 	}
 }
 
+func TestCreateEntryOExclPreservesExistingEntry(t *testing.T) {
+	f, store := newTestFilerWithStubStore()
+
+	original := &Entry{
+		FullPath: util.FullPath("/buckets/my-bucket"),
+		Attr:     Attr{Mode: os.ModeDir | 0o777},
+		Extended: map[string][]byte{
+			"lifecycle": []byte(`<LifecycleConfiguration/>`),
+			"owner":     []byte("alice"),
+		},
+	}
+	require.NoError(t, store.InsertEntry(context.Background(), original))
+
+	replacement := &Entry{
+		FullPath: util.FullPath("/buckets/my-bucket"),
+		Attr:     Attr{Mode: os.ModeDir | 0o777},
+	}
+	err := f.CreateEntry(context.Background(), replacement, original, true, false, nil, false, f.MaxFilenameLength)
+	require.ErrorIs(t, err, filer_pb.ErrEntryAlreadyExists)
+
+	stored, findErr := store.FindEntry(context.Background(), original.FullPath)
+	require.NoError(t, findErr)
+	assert.Equal(t, original.Extended, stored.Extended)
+}
+
+func TestCreateEntryOExclFailsOnLookupError(t *testing.T) {
+	f, store := newTestFilerWithStubStore()
+
+	original := &Entry{
+		FullPath: util.FullPath("/buckets/my-bucket"),
+		Attr:     Attr{Mode: os.ModeDir | 0o777},
+		Extended: map[string][]byte{"owner": []byte("alice")},
+	}
+	require.NoError(t, store.InsertEntry(context.Background(), original))
+
+	// a failed lookup must not masquerade as "not found": without the check
+	// the insert path would upsert over the stored bucket entry
+	store.findErr = errors.New("transient store failure")
+	err := f.CreateEntry(context.Background(), &Entry{
+		FullPath: util.FullPath("/buckets/my-bucket"),
+		Attr:     Attr{Mode: os.ModeDir | 0o777},
+	}, nil, true, false, nil, false, f.MaxFilenameLength)
+	require.Error(t, err)
+	assert.NotErrorIs(t, err, filer_pb.ErrEntryAlreadyExists)
+
+	store.findErr = nil
+	stored, findErr := store.FindEntry(context.Background(), original.FullPath)
+	require.NoError(t, findErr)
+	assert.Equal(t, original.Extended, stored.Extended)
+}
+
 func TestUpdateEntryPreservesExistingInode(t *testing.T) {
 	f, store := newTestFilerWithStubStore()
 
@@ -120,7 +173,7 @@ func TestUpdateEntryPreservesExistingInode(t *testing.T) {
 		},
 	}
 
-	err := f.UpdateEntry(context.Background(), original, updated)
+	err := f.UpdateEntry(context.Background(), original, updated, false)
 	require.Error(t, err)
 
 	updated = &Entry{
@@ -129,7 +182,7 @@ func TestUpdateEntryPreservesExistingInode(t *testing.T) {
 			Mode: 0o600,
 		},
 	}
-	err = f.UpdateEntry(context.Background(), original, updated)
+	err = f.UpdateEntry(context.Background(), original, updated, false)
 	require.NoError(t, err)
 
 	stored, findErr := store.FindEntry(context.Background(), original.FullPath)
@@ -155,7 +208,7 @@ func TestUpdateEntryBackfillsMissingLegacyInode(t *testing.T) {
 			Mode: 0o640,
 		},
 	}
-	err := f.UpdateEntry(context.Background(), original, updated)
+	err := f.UpdateEntry(context.Background(), original, updated, false)
 	require.NoError(t, err)
 
 	stored, findErr := store.FindEntry(context.Background(), original.FullPath)

weed/filer/filer_lazy_remote.go

@@ -7,7 +7,10 @@ import (
 	"strings"
 	"time"
 
+	"google.golang.org/protobuf/proto"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
 	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -43,6 +46,21 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (
 		return nil, nil
 	}
 
+	// A startup tombstone rebuild may still be replaying the meta log; wait
+	// for it so a pending delete cannot resurrect here.
+	if done := f.remoteTombstonesDone.Load(); done != nil {
+		select {
+		case <-*done:
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+
+	if f.isRemoteDeletionPending(ctx, p, mountDir) {
+		glog.V(2).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s deleted locally, remote delete pending", p)
+		return nil, nil
+	}
+
 	remoteConf, found := f.RemoteStorage.FindRemoteStorageConf(p)
 	if !found {
 		return nil, nil
@@ -103,10 +121,27 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (
 		persistBaseCtx, cancelPersist := context.WithTimeout(context.Background(), 30*time.Second)
 		defer cancelPersist()
 		persistCtx := context.WithValue(persistBaseCtx, lazyFetchContextKey{}, true)
+		// A delete may have landed while StatFile was in flight; re-check so
+		// the fetched object cannot resurrect a path whose delete is pending.
+		if f.isRemoteDeletionPending(persistCtx, p, mountDir) {
+			glog.V(2).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s deleted during remote stat", p)
+			return lazyFetchResult{nil}, nil
+		}
 		saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, true, f.MaxFilenameLength)
 		if saveErr != nil {
 			glog.Warningf("maybeLazyFetchFromRemote: failed to persist filer entry for %s: %v", p, saveErr)
 			f.lazyFetchGroup.Forget(key)
+			return lazyFetchResult{entry}, nil
+		}
+
+		// A delete records its tombstone before removing the entry, so a
+		// tombstone visible now means the insert raced a delete that already
+		// ran: retract the persisted entry so the path stays deleted.
+		if f.isRemoteDeletionPending(persistCtx, p, mountDir) {
+			glog.V(2).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s deleted while persisting", p)
+			f.lazyFetchGroup.Forget(key)
+			f.retractLazyRemoteEntry(persistCtx, entry)
+			return lazyFetchResult{nil}, nil
 		}
 
 		return lazyFetchResult{entry}, nil
@@ -122,6 +157,25 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (
 	return result.entry, nil
 }
 
+// retractLazyRemoteEntry deletes the entry at entry.FullPath only when it is
+// still the entry a lazy remote read just materialized — a concurrent write
+// may have replaced it, and deleting by path alone would take that write down.
+func (f *Filer) retractLazyRemoteEntry(ctx context.Context, entry *Entry) {
+	existing, findErr := f.FindEntry(ctx, entry.FullPath)
+	if findErr != nil || existing == nil {
+		return
+	}
+	// The stored entry must still be exactly what this read materialized —
+	// an intervening write (appended chunks, touched attributes) means a
+	// real update owns the path now.
+	if !proto.Equal(existing.ToProtoEntry(), entry.ToProtoEntry()) {
+		return
+	}
+	if err := f.doDeleteEntryMetaAndData(ctx, existing, false, false, nil); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		glog.Warningf("retractLazyRemoteEntry %s: %v", entry.FullPath, err)
+	}
+}
+
 func (f *Filer) maybeDeleteFromRemote(ctx context.Context, entry *Entry) (bool, error) {
 	if entry == nil || f.RemoteStorage == nil {
 		return false, nil

weed/filer/filer_lazy_remote_listing.go

@@ -56,6 +56,14 @@ func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) {
 		}
 	}
 
+	if done := f.remoteTombstonesDone.Load(); done != nil {
+		select {
+		case <-*done:
+		case <-ctx.Done():
+			return
+		}
+	}
+
 	// Lazy listing is opt-in: disabled when TTL is 0
 	if remoteLoc.ListingCacheTtlSeconds <= 0 {
 		return
@@ -109,6 +117,10 @@ func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) {
 				return nil
 			}
 
+			if existingEntry == nil && f.isRemoteDeletionPending(persistCtx, childPath, mountDir) {
+				return nil
+			}
+
 			if existingEntry != nil {
 				// Merge: update remote metadata while preserving local state
 				// (Chunks, Extended, Uid/Gid/Mode, etc.)
@@ -161,6 +173,9 @@ func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) {
 				}
 				if saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, true, f.MaxFilenameLength); saveErr != nil {
 					glog.Warningf("maybeLazyListFromRemote: persist %s: %v", childPath, saveErr)
+				} else if f.isRemoteDeletionPending(persistCtx, childPath, mountDir) {
+					// a delete landed between the check above and the insert
+					f.retractLazyRemoteEntry(persistCtx, entry)
 				}
 			}
 			return nil

weed/filer/filer_lazy_remote_test.go

@@ -34,6 +34,7 @@ type stubFilerStore struct {
 	entries         map[string]*Entry
 	kv              map[string][]byte
 	insertErr       error
+	findErr         error
 	deleteErrByPath map[string]error
 }
 
@@ -174,6 +175,9 @@ func (s *stubFilerStore) UpdateEntry(_ context.Context, entry *Entry) error {
 func (s *stubFilerStore) FindEntry(_ context.Context, p util.FullPath) (*Entry, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
+	if s.findErr != nil {
+		return nil, s.findErr
+	}
 	if e, ok := s.entries[string(p)]; ok {
 		return e, nil
 	}
@@ -279,6 +283,7 @@ func newTestFiler(t *testing.T, store *stubFilerStore, rs *FilerRemoteStorage) *
 		MasterClient:        mc,
 		FileIdDeletionQueue: util.NewUnboundedQueue(),
 		deletionQuit:        make(chan struct{}),
+		remoteTombstones:    newRemoteDeletionTombstones(),
 		LocalMetaLogBuffer: log_buffer.NewLogBuffer("test", time.Minute,
 			func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, func() {}),
 	}

weed/filer/filer_on_meta_event.go

@@ -15,6 +15,7 @@ func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse)
 	f.maybeReloadRemoteStorageConfigurationAndMapping(event)
 	f.onBucketEvents(event)
 	f.onEmptyFolderCleanupEvents(event)
+	f.onRemoteDeletionEvents(event)
 }
 
 func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {

weed/filer/filer_prefix_object_test.go

@@ -104,3 +104,23 @@ func TestExpiredFileIsDeletedOnRead(t *testing.T) {
 	require.ErrorIs(t, err, filer_pb.ErrNotFound,
 		"native TTL lookup should remove the expired metadata row")
 }
+
+func TestListDirectoryEntriesContextCanceled(t *testing.T) {
+	f, store := newTestFilerWithStubStore()
+	ctx := context.Background()
+
+	require.NoError(t, store.InsertEntry(ctx, &Entry{
+		FullPath: util.FullPath("/dir/file1.txt"),
+		Attr:     Attr{Mode: 0o644},
+	}))
+
+	canceledCtx, cancel := context.WithCancel(ctx)
+	cancel()
+
+	_, _, err := f.doListDirectoryEntries(canceledCtx, util.FullPath("/dir"), "", false, 10, "", func(entry *Entry) (bool, error) {
+		return true, nil
+	})
+	require.Error(t, err)
+	assert.ErrorContains(t, err, "context canceled")
+}
+

weed/filer/filer_remote_evict.go

@@ -0,0 +1,89 @@
+package filer
+
+import (
+	"context"
+	"sort"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func chunksEqual(a, b []*filer_pb.FileChunk) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i].GetFileIdString() != b[i].GetFileIdString() ||
+			a[i].Size != b[i].Size ||
+			a[i].Offset != b[i].Offset ||
+			a[i].ModifiedTsNs != b[i].ModifiedTsNs {
+			return false
+		}
+	}
+	return true
+}
+
+// IsEvictableRemoteEntry reports whether an entry's local chunks are backed by
+// a synchronized remote copy, mirroring the checks remote.uncache applies:
+// remote-backed, chunks present, and not ahead of the remote version.
+func IsEvictableRemoteEntry(entry *Entry) bool {
+	if entry.IsDirectory() || entry.Remote == nil {
+		return false
+	}
+	if entry.Remote.LastLocalSyncTsNs <= 0 || len(entry.GetChunks()) == 0 {
+		return false
+	}
+	if entry.Remote.LastLocalSyncTsNs < entry.Mtime.UnixNano() {
+		return false
+	}
+	return true
+}
+
+// ListEvictableRemoteEntries walks every mounted directory and returns
+// synchronized remote entries holding local chunks, oldest cached first.
+func (f *Filer) ListEvictableRemoteEntries(ctx context.Context, mounts []util.FullPath, minCacheAge time.Duration) (out []*Entry) {
+	cutoffNs := time.Now().UnixNano() - minCacheAge.Nanoseconds()
+	for _, dir := range mounts {
+		if err := f.collectEvictableRemoteEntries(ctx, dir, cutoffNs, &out); err != nil {
+			glog.WarningfCtx(ctx, "list evictable remote entries under %s: %v", dir, err)
+		}
+	}
+	sort.Slice(out, func(i, j int) bool {
+		return out[i].Remote.LastLocalSyncTsNs < out[j].Remote.LastLocalSyncTsNs
+	})
+	return out
+}
+
+func (f *Filer) collectEvictableRemoteEntries(ctx context.Context, dir util.FullPath, cutoffNs int64, out *[]*Entry) error {
+	startFileName := ""
+	for {
+		var subDirs []util.FullPath
+		lastFileName, err := f.Store.ListDirectoryEntries(ctx, dir, startFileName, false, 1024, func(entry *Entry) (bool, error) {
+			if err := ctx.Err(); err != nil {
+				return false, err
+			}
+			if entry.IsDirectory() {
+				subDirs = append(subDirs, entry.FullPath)
+				return true, nil
+			}
+			if entry.Remote != nil && entry.Remote.LastLocalSyncTsNs > 0 && entry.Remote.LastLocalSyncTsNs <= cutoffNs && IsEvictableRemoteEntry(entry) {
+				*out = append(*out, entry)
+			}
+			return true, nil
+		})
+		if err != nil {
+			return err
+		}
+		for _, subDir := range subDirs {
+			if err := f.collectEvictableRemoteEntries(ctx, subDir, cutoffNs, out); err != nil {
+				return err
+			}
+		}
+		if lastFileName == "" {
+			return nil
+		}
+		startFileName = lastFileName
+	}
+}

weed/filer/filer_remote_evict_test.go

@@ -0,0 +1,164 @@
+package filer
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func remoteCachedEntry(path string, syncTsNs int64, mtime time.Time, chunks int) *Entry {
+	entry := &Entry{
+		FullPath: util.FullPath(path),
+		Attr: Attr{
+			Mtime:    mtime,
+			Crtime:   mtime,
+			Mode:     0644,
+			Uid:      1,
+			Gid:      1,
+			Mime:     "application/octet-stream",
+			Md5:      nil,
+			FileSize: 0,
+		},
+		Remote: &filer_pb.RemoteEntry{
+			RemoteMtime:       mtime.Unix(),
+			LastLocalSyncTsNs: syncTsNs,
+			RemoteETag:        "etag",
+			RemoteSize:        100,
+		},
+	}
+	for i := 0; i < chunks; i++ {
+		entry.Chunks = append(entry.Chunks, &filer_pb.FileChunk{FileId: "1,01637037d6", Size: 100})
+	}
+	return entry
+}
+
+func TestIsEvictableRemoteEntry(t *testing.T) {
+	now := time.Now()
+	synced := now.Add(-time.Hour).UnixNano()
+	mtime := now.Add(-time.Hour)
+
+	tests := []struct {
+		name     string
+		entry    *Entry
+		expected bool
+	}{
+		{"directory", &Entry{FullPath: "/d", Attr: Attr{Mode: os.ModeDir | 0755}}, false},
+		{"local only, no remote", &Entry{FullPath: "/f", Attr: Attr{Mtime: mtime, Mode: 0644}}, false},
+		{"remote only, never cached", remoteCachedEntry("/f", 0, mtime, 0), false},
+		{"remote, chunks cleared", remoteCachedEntry("/f", synced, mtime, 0), false},
+		{"dirty, newer than remote", remoteCachedEntry("/f", synced, now, 1), false},
+		{"same-second write after sync", remoteCachedEntry("/f",
+			now.Truncate(time.Second).Add(100*time.Millisecond).UnixNano(),
+			now.Truncate(time.Second).Add(900*time.Millisecond), 1), false},
+		{"same-second sync after write", remoteCachedEntry("/f",
+			now.Truncate(time.Second).Add(900*time.Millisecond).UnixNano(),
+			now.Truncate(time.Second).Add(100*time.Millisecond), 1), true},
+		{"synced and cached", remoteCachedEntry("/f", synced, mtime, 1), true},
+	}
+	for _, tt := range tests {
+		assert.Equal(t, tt.expected, IsEvictableRemoteEntry(tt.entry), tt.name)
+	}
+}
+
+func TestListEvictableRemoteEntries(t *testing.T) {
+	now := time.Now()
+	mtime := now.Add(-2 * time.Hour)
+	oldSync := now.Add(-time.Hour).UnixNano()
+	newSync := now.Add(-time.Second).UnixNano()
+
+	store := newStubFilerStore()
+	f := newTestFiler(t, store, NewFilerRemoteStorage())
+	mounts := []util.FullPath{"/buckets/mybucket"}
+
+	seed := func(e *Entry) {
+		require.NoError(t, f.CreateEntry(context.Background(), e, nil, false, false, nil, false, 255))
+	}
+	seed(remoteCachedEntry("/buckets/mybucket/old.bin", oldSync, mtime, 1))
+	seed(remoteCachedEntry("/buckets/mybucket/fresh.bin", newSync, mtime, 1))
+	seed(remoteCachedEntry("/buckets/mybucket/remoteonly.bin", 0, mtime, 0))
+	seed(remoteCachedEntry("/buckets/mybucket/dirty.bin", oldSync, now, 1))
+	seed(&Entry{FullPath: "/buckets/mybucket/plain.bin", Attr: Attr{Mtime: mtime, Mode: 0644}})
+
+	store.entries["/buckets/mybucket/sub"] = &Entry{
+		FullPath: "/buckets/mybucket/sub",
+		Attr:     Attr{Mode: os.ModeDir | 0755, Mtime: mtime},
+	}
+	seed(remoteCachedEntry("/buckets/mybucket/sub/nested.bin", oldSync-1000, mtime, 1))
+
+	got := f.ListEvictableRemoteEntries(context.Background(), mounts, 0)
+	var paths []string
+	for _, e := range got {
+		paths = append(paths, string(e.FullPath))
+	}
+	assert.Equal(t, []string{
+		"/buckets/mybucket/sub/nested.bin",
+		"/buckets/mybucket/old.bin",
+		"/buckets/mybucket/fresh.bin",
+	}, paths, "oldest LastLocalSyncTsNs first; remote-only, dirty, local entries skipped")
+
+	got = f.ListEvictableRemoteEntries(context.Background(), mounts, 30*time.Second)
+	paths = paths[:0]
+	for _, e := range got {
+		paths = append(paths, string(e.FullPath))
+	}
+	assert.Equal(t, []string{
+		"/buckets/mybucket/sub/nested.bin",
+		"/buckets/mybucket/old.bin",
+	}, paths, "minCacheAge excludes freshly cached entries")
+}
+
+func TestUpdateEntryInvalidatesStaleSyncStamp(t *testing.T) {
+	now := time.Now()
+	synced := now.Add(-time.Hour).UnixNano()
+	mtime := now.Add(-time.Hour)
+	path := "/buckets/mybucket/a.bin"
+
+	newFiler := func() *Filer {
+		f := newTestFiler(t, newStubFilerStore(), NewFilerRemoteStorage())
+		require.NoError(t, f.CreateEntry(context.Background(), remoteCachedEntry(path, synced, mtime, 1), nil, false, false, nil, false, 255))
+		return f
+	}
+	syncStamp := func(f *Filer) int64 {
+		entry, err := f.FindEntry(context.Background(), util.FullPath(path))
+		require.NoError(t, err)
+		return entry.Remote.LastLocalSyncTsNs
+	}
+
+	t.Run("local chunk change clears stamp", func(t *testing.T) {
+		f := newFiler()
+		update := remoteCachedEntry(path, synced, mtime, 1)
+		update.Chunks[0].FileId = "2,01637037d6"
+		require.NoError(t, f.CreateEntry(context.Background(), update, nil, false, false, nil, false, 255))
+		assert.Zero(t, syncStamp(f))
+	})
+
+	t.Run("metadata-only update keeps stamp", func(t *testing.T) {
+		f := newFiler()
+		update := remoteCachedEntry(path, synced, mtime, 1)
+		update.Attr.Mime = "text/plain"
+		require.NoError(t, f.CreateEntry(context.Background(), update, nil, false, false, nil, false, 255))
+		assert.Equal(t, synced, syncStamp(f))
+	})
+
+	t.Run("fresh sync stamp survives chunk change", func(t *testing.T) {
+		f := newFiler()
+		update := remoteCachedEntry(path, now.UnixNano(), mtime, 1)
+		update.Chunks[0].FileId = "2,01637037d6"
+		require.NoError(t, f.CreateEntry(context.Background(), update, nil, false, false, nil, false, 255))
+		assert.Equal(t, now.UnixNano(), syncStamp(f))
+	})
+
+	t.Run("replicated update keeps authoritative stamp", func(t *testing.T) {
+		f := newFiler()
+		update := remoteCachedEntry(path, synced, mtime, 1)
+		update.Chunks[0].FileId = "2,01637037d6"
+		require.NoError(t, f.CreateEntry(context.Background(), update, nil, false, true, nil, false, 255))
+		assert.Equal(t, synced, syncStamp(f))
+	})
+}

weed/filer/filer_remote_tombstone.go

@@ -0,0 +1,423 @@
+package filer
+
+import (
+	"context"
+	"math"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+	"google.golang.org/protobuf/proto"
+)
+
+const (
+	// remoteDeletionTombstoneTTL bounds a tombstone when no write-back sync
+	// offset ever confirms the remote delete. A remote object re-created
+	// outside the filer at a deleted path stays hidden for this long.
+	remoteDeletionTombstoneTTL = 24 * time.Hour
+	// remoteDeletionTombstoneLimit bounds tracked paths; past it new
+	// tombstones are dropped after an expired sweep still leaves no room.
+	remoteDeletionTombstoneLimit = 1 << 16
+)
+
+// remoteDeletionTombstones tracks paths deleted under a remote mount whose
+// remote object may still exist because the write-back daemon has not
+// consumed the delete event yet. A lazy remote fetch or listing must not
+// resurrect them. A tombstone lifts when the path is written again, when the
+// mount's persisted sync offset passes the delete event, or on TTL.
+//
+// Tombstones recorded before their metadata event lands (the synchronous
+// delete path) are marked pending: the sync offset orders against event
+// timestamps, which only the event itself knows, so a pending tombstone can
+// only be lifted by the event confirming it or by TTL. Once the event stamps
+// the real timestamp the tombstone is releasable by the offset.
+type remoteDeletionTombstones struct {
+	mu      sync.Mutex
+	files   map[string]int64 // file path -> delete event TsNs
+	dirs    map[string]int64 // deleted directory path -> event TsNs; covers its subtree
+	pending map[string]bool  // tombstone path recorded ahead of its event
+}
+
+func newRemoteDeletionTombstones() *remoteDeletionTombstones {
+	return &remoteDeletionTombstones{
+		files:   make(map[string]int64),
+		dirs:    make(map[string]int64),
+		pending: make(map[string]bool),
+	}
+}
+
+// add records a tombstone ahead of its metadata event — the timestamp is the
+// local delete time, a lower bound the sync offset cannot order against.
+func (t *remoteDeletionTombstones) add(path string, isDir bool, tsNs int64) {
+	t.upsert(path, isDir, tsNs, false)
+}
+
+// addFromEvent records a tombstone stamped by the delete event itself, so the
+// mount's sync offset can release it once the daemon passes that event.
+func (t *remoteDeletionTombstones) addFromEvent(path string, isDir bool, tsNs int64) {
+	t.upsert(path, isDir, tsNs, true)
+}
+
+func (t *remoteDeletionTombstones) upsert(path string, isDir bool, tsNs int64, fromEvent bool) {
+	if t == nil {
+		return
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	// An ancestor directory tombstone at least as new already covers the
+	// path; recording it again only spends capacity.
+	for p := path; ; {
+		i := strings.LastIndexByte(p, '/')
+		if i <= 0 {
+			break
+		}
+		p = p[:i]
+		if ancestorTs, ok := t.dirs[p]; ok && ancestorTs >= tsNs {
+			return
+		}
+	}
+	m := t.files
+	if isDir {
+		m = t.dirs
+	}
+	if cur, ok := m[path]; ok {
+		if tsNs > cur {
+			m[path] = tsNs
+			if !fromEvent {
+				// a newer local delete restamps the tombstone ahead of its
+				// event — the offset cannot vouch for it until the event lands
+				t.pending[path] = true
+			}
+		}
+		if fromEvent && m[path] == tsNs {
+			delete(t.pending, path)
+		}
+		return
+	}
+	if len(t.files)+len(t.dirs) >= remoteDeletionTombstoneLimit {
+		t.evictExpiredLocked(time.Now().UnixNano())
+		if len(t.files)+len(t.dirs) >= remoteDeletionTombstoneLimit {
+			glog.V(0).Infof("remote deletion tombstones full (%d), skipping %s", remoteDeletionTombstoneLimit, path)
+			return
+		}
+	}
+	m[path] = tsNs
+	if fromEvent {
+		delete(t.pending, path)
+	} else {
+		t.pending[path] = true
+	}
+	if isDir {
+		t.dropCoveredLocked(path, tsNs)
+	}
+}
+
+// dropCoveredLocked removes descendant tombstones a new directory tombstone
+// subsumes: their deletes predate it, so the ancestor already hides those
+// remote objects. Descendants deleted later keep their own tombstone.
+// Caller must hold t.mu.
+func (t *remoteDeletionTombstones) dropCoveredLocked(dirPath string, tsNs int64) {
+	prefix := dirPath + "/"
+	for p, ts := range t.files {
+		if ts <= tsNs && strings.HasPrefix(p, prefix) {
+			delete(t.files, p)
+			delete(t.pending, p)
+		}
+	}
+	for p, ts := range t.dirs {
+		if ts <= tsNs && strings.HasPrefix(p, prefix) {
+			delete(t.dirs, p)
+			delete(t.pending, p)
+		}
+	}
+}
+
+// drop removes the exact tombstone recorded for path, e.g. when the delete
+// that recorded it fails before touching anything.
+func (t *remoteDeletionTombstones) drop(path string, tsNs int64) {
+	if t == nil {
+		return
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if cur, ok := t.dirs[path]; ok && cur <= tsNs {
+		delete(t.dirs, path)
+		delete(t.pending, path)
+	}
+	if cur, ok := t.files[path]; ok && cur <= tsNs {
+		delete(t.files, path)
+		delete(t.pending, path)
+	}
+}
+
+// clear drops a file tombstone when a write at the path is at least as new as
+// the delete; a replayed older create must not lift a newer delete.
+func (t *remoteDeletionTombstones) clear(path string, tsNs int64) {
+	if t == nil {
+		return
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if cur, ok := t.files[path]; ok && tsNs >= cur {
+		delete(t.files, path)
+		delete(t.pending, path)
+	}
+}
+
+// blockedSince returns the newest delete timestamp governing path — its own
+// file tombstone or one from a deleted ancestor directory — and whether that
+// tombstone is still waiting for its event. 0 means clear.
+func (t *remoteDeletionTombstones) blockedSince(path string) (tsNs int64, pending bool) {
+	if t == nil {
+		return 0, false
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if ts, ok := t.files[path]; ok {
+		tsNs = ts
+		pending = t.pending[path]
+	}
+	for p := path; ; {
+		if ts, ok := t.dirs[p]; ok && ts > tsNs {
+			tsNs = ts
+			pending = t.pending[p]
+		}
+		i := strings.LastIndexByte(p, '/')
+		if i <= 0 {
+			break
+		}
+		p = p[:i]
+	}
+	return tsNs, pending
+}
+
+// releaseThrough drops the tombstones governing path that are no newer than
+// tsNs, once their remote deletes are confirmed consumed.
+func (t *remoteDeletionTombstones) releaseThrough(path string, tsNs int64) {
+	if t == nil {
+		return
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if cur, ok := t.files[path]; ok && cur <= tsNs {
+		delete(t.files, path)
+		delete(t.pending, path)
+	}
+	for p := path; ; {
+		if cur, ok := t.dirs[p]; ok && cur <= tsNs {
+			delete(t.dirs, p)
+			delete(t.pending, p)
+		}
+		i := strings.LastIndexByte(p, '/')
+		if i <= 0 {
+			break
+		}
+		p = p[:i]
+	}
+}
+
+func (t *remoteDeletionTombstones) evictExpiredLocked(nowNs int64) {
+	for p, ts := range t.files {
+		if nowNs-ts >= int64(remoteDeletionTombstoneTTL) {
+			delete(t.files, p)
+			delete(t.pending, p)
+		}
+	}
+	for p, ts := range t.dirs {
+		if nowNs-ts >= int64(remoteDeletionTombstoneTTL) {
+			delete(t.dirs, p)
+			delete(t.pending, p)
+		}
+	}
+}
+
+// noteRemoteDeletion records a delete of path under a remote mount so lazy
+// remote reads skip it until the remote delete is confirmed.
+func (f *Filer) noteRemoteDeletion(p util.FullPath, isDir bool, tsNs int64) {
+	if f.RemoteStorage == nil || f.remoteTombstones == nil {
+		return
+	}
+	if _, remoteLoc := f.RemoteStorage.FindMountDirectory(p); remoteLoc == nil {
+		return
+	}
+	f.remoteTombstones.add(string(p), isDir, tsNs)
+}
+
+// unnoteRemoteDeletion retracts a tombstone when the delete that recorded it
+// fails before touching anything under path.
+func (f *Filer) unnoteRemoteDeletion(p util.FullPath, tsNs int64) {
+	if f.remoteTombstones == nil {
+		return
+	}
+	f.remoteTombstones.drop(string(p), tsNs)
+}
+
+// isRemoteDeletionPending reports whether a remote write-back delete for p is
+// still owed: p was deleted under mountDir and neither a rewrite, the mount's
+// sync offset, nor the TTL has lifted the tombstone.
+func (f *Filer) isRemoteDeletionPending(ctx context.Context, p util.FullPath, mountDir util.FullPath) bool {
+	if f.remoteTombstones == nil {
+		return false
+	}
+	tsNs, pending := f.remoteTombstones.blockedSince(string(p))
+	if tsNs == 0 {
+		return false
+	}
+	if pending {
+		// Recorded ahead of its delete event — the write-back offset cannot
+		// vouch for it yet; only the TTL lifts it.
+		if f.remoteDeletionExpired(tsNs) {
+			f.remoteTombstones.releaseThrough(string(p), tsNs)
+			return false
+		}
+		return true
+	}
+	if f.remoteDeletionConsumed(ctx, mountDir, tsNs) {
+		f.remoteTombstones.releaseThrough(string(p), tsNs)
+		return false
+	}
+	return true
+}
+
+func (f *Filer) remoteDeletionExpired(tsNs int64) bool {
+	return time.Now().UnixNano()-tsNs >= int64(remoteDeletionTombstoneTTL)
+}
+
+func (f *Filer) remoteDeletionConsumed(ctx context.Context, mountDir util.FullPath, tsNs int64) bool {
+	if f.remoteDeletionExpired(tsNs) {
+		return true
+	}
+	offset, err := f.readRemoteSyncOffset(ctx, mountDir)
+	return err == nil && offset >= tsNs
+}
+
+// readRemoteSyncOffset reads the write-back daemon's persisted watermark for
+// mountDir straight from the local store: every event at or below it has been
+// applied to the remote.
+func (f *Filer) readRemoteSyncOffset(ctx context.Context, mountDir util.FullPath) (int64, error) {
+	value, err := f.Store.KvGet(ctx, remote_storage.SyncOffsetKey(string(mountDir)))
+	if err != nil {
+		return 0, err
+	}
+	if len(value) < 8 {
+		return 0, nil
+	}
+	return int64(util.BytesToUint64(value)), nil
+}
+
+// RebuildRemoteDeletionTombstones gates lazy remote reads and replays the
+// persisted metadata log from the oldest write-back offset across mounts,
+// restoring tombstones for deletes committed before a restart but not yet
+// applied to the remote. Every filer writes its log files under the same
+// directory, so pending peer deletes replay too; only events still inside
+// the unflushed buffer tail are missed. The gate is set synchronously so no
+// lazy read can slip in before replay starts, and it stays closed until a
+// replay succeeds.
+func (f *Filer) RebuildRemoteDeletionTombstones(ctx context.Context) {
+	if f.RemoteStorage == nil || f.remoteTombstones == nil {
+		return
+	}
+	mounts := f.RemoteStorage.MountedDirectories()
+	if len(mounts) == 0 {
+		return
+	}
+	done := make(chan struct{})
+	f.remoteTombstonesDone.Store(&done)
+	go f.rebuildRemoteDeletionTombstones(ctx, mounts, done)
+}
+
+func (f *Filer) rebuildRemoteDeletionTombstones(ctx context.Context, mounts []util.FullPath, done chan struct{}) {
+	// the replay itself lists directories; do not let it wait on its own gate
+	ctx = context.WithValue(ctx, lazyFetchContextKey{}, true)
+	startTsNs := f.remoteDeletionRebuildStartTsNs(ctx, mounts)
+	openGate := func() {
+		close(done)
+		f.remoteTombstonesDone.Store(nil)
+	}
+	// Deletes made after startup are recorded through the live delete and
+	// event paths, so replay can only be missing deletes committed before
+	// the restart — and those have all crossed the tombstone TTL once this
+	// deadline passes. Holding the gate longer protects nothing.
+	replayDeadline := time.Now().Add(remoteDeletionTombstoneTTL)
+	backoff := 2 * time.Second
+	for {
+		_, _, err := f.ReadPersistedLogBuffer(ctx, log_buffer.NewMessagePosition(startTsNs, 0), 0,
+			func(logEntry *filer_pb.LogEntry) (bool, error) {
+				event := &filer_pb.SubscribeMetadataResponse{}
+				if err := proto.Unmarshal(logEntry.Data, event); err != nil {
+					return false, nil
+				}
+				f.onRemoteDeletionEvents(event)
+				return false, nil
+			})
+		if err == nil {
+			openGate()
+			return
+		}
+		glog.WarningfCtx(ctx, "rebuild remote deletion tombstones: %v", err)
+		if !time.Now().Before(replayDeadline) {
+			glog.ErrorfCtx(ctx, "rebuild remote deletion tombstones: giving up after %v", remoteDeletionTombstoneTTL)
+			openGate()
+			return
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(backoff):
+		}
+		if backoff < time.Minute {
+			backoff *= 2
+		}
+	}
+}
+
+// remoteDeletionRebuildStartTsNs returns the oldest write-back offset across
+// mounts — the earliest event the daemon may not have applied. Mounts without
+// a recorded offset replay from the TTL floor: events older than it would
+// build tombstones that are already expired.
+func (f *Filer) remoteDeletionRebuildStartTsNs(ctx context.Context, mounts []util.FullPath) int64 {
+	startTsNs := int64(math.MaxInt64)
+	for _, dir := range mounts {
+		offset, err := f.readRemoteSyncOffset(ctx, dir)
+		if err != nil {
+			glog.WarningfCtx(ctx, "read remote sync offset for %s: %v", dir, err)
+			offset = 0
+		}
+		if offset < startTsNs {
+			startTsNs = offset
+		}
+	}
+	ttlFloor := time.Now().Add(-remoteDeletionTombstoneTTL).UnixNano()
+	if startTsNs == int64(math.MaxInt64) || startTsNs < ttlFloor {
+		return ttlFloor
+	}
+	return startTsNs
+}
+
+// onRemoteDeletionEvents folds peer and local metadata events into the
+// tombstone set: a delete or rename source is tombstoned at the event
+// timestamp, a create/update/rename target lifts a file tombstone.
+func (f *Filer) onRemoteDeletionEvents(event *filer_pb.SubscribeMetadataResponse) {
+	message := event.EventNotification
+	if message == nil {
+		return
+	}
+	if message.OldEntry != nil {
+		sourcePath := filer_pb.MetadataEventSourceFullPath(event)
+		if message.NewEntry == nil || sourcePath != filer_pb.MetadataEventTargetFullPath(event) {
+			if f.RemoteStorage != nil && f.remoteTombstones != nil {
+				if _, remoteLoc := f.RemoteStorage.FindMountDirectory(util.FullPath(sourcePath)); remoteLoc != nil {
+					f.remoteTombstones.addFromEvent(sourcePath, message.OldEntry.IsDirectory, event.TsNs)
+				}
+			}
+		}
+	}
+	if message.NewEntry != nil {
+		f.remoteTombstones.clear(filer_pb.MetadataEventTargetFullPath(event), event.TsNs)
+	}
+}

weed/filer/filer_remote_tombstone_test.go

@@ -0,0 +1,437 @@
+package filer
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newMountedTestFiler(t *testing.T, storageType string, stub remote_storage.RemoteStorageClient, listingTtlSeconds int32) (*Filer, *stubFilerStore) {
+	t.Helper()
+	if stub != nil {
+		t.Cleanup(registerStubMaker(t, storageType, stub))
+	}
+	conf := &remote_pb.RemoteConf{Name: "tombstonestore", Type: storageType}
+	rs := NewFilerRemoteStorage()
+	rs.storageNameToConf[conf.Name] = conf
+	rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
+		Name:                   "tombstonestore",
+		Bucket:                 "mybucket",
+		Path:                   "/",
+		ListingCacheTtlSeconds: listingTtlSeconds,
+	})
+	store := newStubFilerStore()
+	return newTestFiler(t, store, rs), store
+}
+
+func putRemoteSyncOffset(t *testing.T, store *stubFilerStore, dir string, offsetTsNs int64) {
+	t.Helper()
+	buf := make([]byte, 8)
+	util.Uint64toBytes(buf, uint64(offsetTsNs))
+	require.NoError(t, store.KvPut(context.Background(), remote_storage.SyncOffsetKey(dir), buf))
+}
+
+func TestRemoteDeletionTombstones_BlocksAndReleases(t *testing.T) {
+	tombs := newRemoteDeletionTombstones()
+
+	tombs.addFromEvent("/m/a.txt", false, 100)
+	ts, _ := tombs.blockedSince("/m/a.txt")
+	assert.Equal(t, int64(100), ts)
+	ts, _ = tombs.blockedSince("/m/b.txt")
+	assert.Zero(t, ts)
+
+	// a delete of a directory blocks its whole subtree
+	tombs.addFromEvent("/m/dir", true, 200)
+	ts, _ = tombs.blockedSince("/m/dir")
+	assert.Equal(t, int64(200), ts)
+	ts, _ = tombs.blockedSince("/m/dir/deep/x.txt")
+	assert.Equal(t, int64(200), ts)
+	ts, _ = tombs.blockedSince("/m/dirx/y.txt")
+	assert.Zero(t, ts)
+
+	// a rewrite at or after the delete lifts only that file's tombstone
+	tombs.clear("/m/a.txt", 100)
+	ts, _ = tombs.blockedSince("/m/a.txt")
+	assert.Zero(t, ts)
+	tombs.addFromEvent("/m/a.txt", false, 300)
+	tombs.clear("/m/a.txt", 250)
+	ts, _ = tombs.blockedSince("/m/a.txt")
+	assert.Equal(t, int64(300), ts, "older create must not lift newer delete")
+
+	tombs.releaseThrough("/m/dir/deep/x.txt", 200)
+	ts, _ = tombs.blockedSince("/m/dir/deep/x.txt")
+	assert.Zero(t, ts)
+	ts, _ = tombs.blockedSince("/m/dir")
+	assert.Zero(t, ts)
+}
+
+func TestRemoteDeletionTombstones_AncestorSubsumesAndCovers(t *testing.T) {
+	tombs := newRemoteDeletionTombstones()
+
+	// a child tombstone recorded before its ancestor is dropped once the
+	// ancestor's newer delete covers the whole subtree
+	tombs.addFromEvent("/m/dir/a.txt", false, 100)
+	tombs.addFromEvent("/m/dir", true, 200)
+	ts, _ := tombs.blockedSince("/m/dir")
+	assert.Equal(t, int64(200), ts)
+	ts, _ = tombs.blockedSince("/m/dir/a.txt")
+	assert.Equal(t, int64(200), ts)
+	_, exists := tombs.files["/m/dir/a.txt"]
+	assert.False(t, exists, "descendant tombstone is subsumed by the ancestor")
+
+	// adds under the covered subtree are skipped while the ancestor stands
+	tombs.addFromEvent("/m/dir/b.txt", false, 150)
+	_, exists = tombs.files["/m/dir/b.txt"]
+	assert.False(t, exists)
+	// ...but a child deleted after the ancestor still records its own tombstone
+	tombs.addFromEvent("/m/dir/c.txt", false, 300)
+	ts, _ = tombs.blockedSince("/m/dir/c.txt")
+	assert.Equal(t, int64(300), ts)
+}
+
+func TestRemoteDeletionTombstones_PendingIgnoresOffset(t *testing.T) {
+	f, store := newMountedTestFiler(t, "stub_tomb_pending_offset", nil, 0)
+
+	// recorded before its event lands: a later event may already have moved
+	// the mount's watermark past the local timestamp, so the offset cannot
+	// vouch for this delete yet
+	filePath := "/buckets/mybucket/dir/a.txt"
+	now := time.Now().UnixNano()
+	f.noteRemoteDeletion(util.FullPath(filePath), false, now)
+	putRemoteSyncOffset(t, store, "/buckets/mybucket", now+100)
+	assert.True(t, f.isRemoteDeletionPending(context.Background(), util.FullPath(filePath), "/buckets/mybucket"))
+
+	// once the delete event stamps the real timestamp, the watermark releases it
+	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/mybucket/dir",
+		TsNs:      now + 50,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: "a.txt"},
+		},
+	})
+	assert.False(t, f.isRemoteDeletionPending(context.Background(), util.FullPath(filePath), "/buckets/mybucket"))
+}
+
+func TestRemoteDeletionTombstones_RaisedLocalTombstoneIsPendingAgain(t *testing.T) {
+	tombs := newRemoteDeletionTombstones()
+
+	tombs.addFromEvent("/m/a.txt", false, 100)
+	_, pending := tombs.blockedSince("/m/a.txt")
+	assert.False(t, pending)
+
+	// a newer local delete restamps the tombstone before its own event
+	// lands, so the write-back offset cannot vouch for it yet
+	tombs.add("/m/a.txt", false, 200)
+	_, pending = tombs.blockedSince("/m/a.txt")
+	assert.True(t, pending)
+
+	// once that delete's event confirms the new timestamp it releases normally
+	tombs.addFromEvent("/m/a.txt", false, 200)
+	_, pending = tombs.blockedSince("/m/a.txt")
+	assert.False(t, pending)
+}
+
+func TestMaybeLazyFetchFromRemote_SkipsTombstonedPath(t *testing.T) {
+	const storageType = "stub_tomb_fetch"
+	stub := &countingRemoteClient{
+		stubRemoteClient: stubRemoteClient{
+			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11},
+		},
+	}
+	f, _ := newMountedTestFiler(t, storageType, stub, 0)
+
+	// a delete under the mount tombstones the path; the remote object is
+	// still there until the write-back daemon consumes the delete event
+	filePath := util.FullPath("/buckets/mybucket/dir/a.txt")
+	f.noteRemoteDeletion(filePath, false, time.Now().UnixNano())
+
+	entry, err := f.maybeLazyFetchFromRemote(context.Background(), filePath)
+	require.NoError(t, err)
+	assert.Nil(t, entry)
+	assert.Equal(t, 0, stub.statCalls, "the remote object must not even be consulted")
+}
+
+func TestMaybeLazyFetchFromRemote_NewerRemoteMtimeStillBlocked(t *testing.T) {
+	const storageType = "stub_tomb_regen"
+	stub := &countingRemoteClient{
+		stubRemoteClient: stubRemoteClient{
+			statResult: &filer_pb.RemoteEntry{RemoteMtime: time.Now().Unix() + 60, RemoteSize: 11},
+		},
+	}
+	f, _ := newMountedTestFiler(t, storageType, stub, 0)
+
+	// a remote object whose mtime postdates the delete is still hidden: the
+	// remote clock cannot distinguish a new generation from the pending
+	// delete's target
+	filePath := util.FullPath("/buckets/mybucket/dir/a.txt")
+	f.noteRemoteDeletion(filePath, false, time.Now().UnixNano())
+
+	entry, err := f.maybeLazyFetchFromRemote(context.Background(), filePath)
+	require.NoError(t, err)
+	assert.Nil(t, entry)
+	assert.Equal(t, 0, stub.statCalls)
+}
+
+func TestMaybeLazyFetchFromRemote_SyncOffsetReleasesTombstone(t *testing.T) {
+	const storageType = "stub_tomb_release"
+	stub := &countingRemoteClient{
+		stubRemoteClient: stubRemoteClient{
+			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11},
+		},
+	}
+	f, store := newMountedTestFiler(t, storageType, stub, 0)
+
+	filePath := util.FullPath("/buckets/mybucket/dir/a.txt")
+	deleteTsNs := time.Now().UnixNano()
+	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/mybucket/dir",
+		TsNs:      deleteTsNs,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: "a.txt"},
+		},
+	})
+
+	// the daemon's persisted watermark is behind the delete: still blocked
+	putRemoteSyncOffset(t, store, "/buckets/mybucket", deleteTsNs-1)
+	entry, err := f.maybeLazyFetchFromRemote(context.Background(), filePath)
+	require.NoError(t, err)
+	assert.Nil(t, entry)
+	assert.Equal(t, 0, stub.statCalls)
+
+	// once the watermark reaches the delete event's own timestamp, the
+	// remote delete has landed and the lookup may consult the remote again
+	putRemoteSyncOffset(t, store, "/buckets/mybucket", deleteTsNs)
+	entry, err = f.maybeLazyFetchFromRemote(context.Background(), filePath)
+	require.NoError(t, err)
+	require.NotNil(t, entry)
+	assert.Equal(t, 1, stub.statCalls)
+}
+
+func TestDeleteEntryMetaAndData_TombstonesPath(t *testing.T) {
+	const storageType = "stub_tomb_delete"
+	stub := &countingRemoteClient{
+		stubRemoteClient: stubRemoteClient{
+			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11},
+		},
+	}
+	f, store := newMountedTestFiler(t, storageType, stub, 0)
+
+	filePath := util.FullPath("/buckets/mybucket/dir/a.txt")
+	store.entries[string(filePath)] = &Entry{
+		FullPath: filePath,
+		Attr: Attr{
+			Mtime:    time.Unix(1700000000, 0),
+			Crtime:   time.Unix(1700000000, 0),
+			Mode:     0644,
+			FileSize: 11,
+		},
+	}
+
+	require.NoError(t, f.DeleteEntryMetaAndData(context.Background(), filePath, false, false, false, false, nil, 0))
+
+	// the deleted path must not resurrect through the lazy fetch even while
+	// the remote object is still present
+	entry, err := f.FindEntry(context.Background(), filePath)
+	assert.ErrorIs(t, err, filer_pb.ErrNotFound)
+	assert.Nil(t, entry)
+	assert.Equal(t, 0, stub.statCalls)
+
+	// a peer-observed create at the path lifts the tombstone
+	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/mybucket/dir",
+		TsNs:      time.Now().UnixNano(),
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{Name: "a.txt"},
+		},
+	})
+	entry, err = f.maybeLazyFetchFromRemote(context.Background(), filePath)
+	require.NoError(t, err)
+	require.NotNil(t, entry)
+	assert.Equal(t, 1, stub.statCalls)
+}
+
+func TestOnMetadataChangeEvent_PeerDeleteTombstones(t *testing.T) {
+	const storageType = "stub_tomb_peer"
+	stub := &countingRemoteClient{
+		stubRemoteClient: stubRemoteClient{
+			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11},
+		},
+	}
+	f, _ := newMountedTestFiler(t, storageType, stub, 0)
+
+	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/mybucket/dir",
+		TsNs:      time.Now().UnixNano(),
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: "a.txt"},
+		},
+	})
+
+	entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/a.txt")
+	require.NoError(t, err)
+	assert.Nil(t, entry)
+	assert.Equal(t, 0, stub.statCalls)
+}
+
+func TestOnMetadataChangeEvent_PeerDirDeleteTombstonesSubtree(t *testing.T) {
+	const storageType = "stub_tomb_peer_dir"
+	stub := &countingRemoteClient{
+		stubRemoteClient: stubRemoteClient{
+			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11},
+		},
+	}
+	f, _ := newMountedTestFiler(t, storageType, stub, 0)
+
+	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/mybucket",
+		TsNs:      time.Now().UnixNano(),
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: "dir", IsDirectory: true},
+		},
+	})
+
+	entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/deep/a.txt")
+	require.NoError(t, err)
+	assert.Nil(t, entry)
+	assert.Equal(t, 0, stub.statCalls)
+}
+
+func TestMaybeLazyListFromRemote_SkipsTombstonedChild(t *testing.T) {
+	const storageType = "stub_tomb_list"
+	stub := &stubRemoteClient{
+		listDirFn: func(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) error {
+			if err := visitFn("/", "deleted.txt", false, &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 10}); err != nil {
+				return err
+			}
+			return visitFn("/", "fresh.txt", false, &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 20})
+		},
+	}
+	f, store := newMountedTestFiler(t, storageType, stub, 300)
+
+	f.noteRemoteDeletion("/buckets/mybucket/deleted.txt", false, time.Now().UnixNano())
+
+	f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/mybucket"))
+	assert.Equal(t, 1, stub.listDirCalls)
+
+	assert.Nil(t, store.getEntry("/buckets/mybucket/deleted.txt"), "deleted child must not resurrect through a listing")
+	require.NotNil(t, store.getEntry("/buckets/mybucket/fresh.txt"), "other remote objects still list")
+}
+
+func TestMaybeLazyListFromRemote_RecreatedDirStillHidesChildren(t *testing.T) {
+	const storageType = "stub_tomb_recreate"
+	deleteTs := time.Now().UnixNano()
+	stub := &stubRemoteClient{
+		listDirFn: func(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) error {
+			if err := visitFn("/", "stale.txt", false, &filer_pb.RemoteEntry{RemoteMtime: deleteTs/int64(time.Second) - 10, RemoteSize: 10}); err != nil {
+				return err
+			}
+			return visitFn("/", "fresh.txt", false, &filer_pb.RemoteEntry{RemoteMtime: deleteTs/int64(time.Second) + 10, RemoteSize: 20})
+		},
+	}
+	f, store := newMountedTestFiler(t, storageType, stub, 300)
+
+	// the directory is deleted then recreated; its remote children stay
+	// hidden — old or new mtime alike — until the remote delete lands
+	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/mybucket",
+		TsNs:      deleteTs,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: "dir", IsDirectory: true},
+		},
+	})
+	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/mybucket",
+		TsNs:      deleteTs + 1,
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{Name: "dir", IsDirectory: true},
+		},
+	})
+
+	f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/mybucket/dir"))
+
+	assert.Nil(t, store.getEntry("/buckets/mybucket/dir/stale.txt"))
+	assert.Nil(t, store.getEntry("/buckets/mybucket/dir/fresh.txt"))
+
+	// after the write-back daemon confirms the remote delete, listing merges again
+	f.remoteTombstones.releaseThrough("/buckets/mybucket/dir/stale.txt", deleteTs)
+	delete(store.getEntry("/buckets/mybucket/dir").Extended, xattrRemoteListingSyncedAt)
+	f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/mybucket/dir"))
+	assert.NotNil(t, store.getEntry("/buckets/mybucket/dir/stale.txt"))
+	assert.NotNil(t, store.getEntry("/buckets/mybucket/dir/fresh.txt"))
+}
+
+func TestRemoteDeletionRebuildStartTsNs_UsesOldestMountOffset(t *testing.T) {
+	f, store := newMountedTestFiler(t, "stub_rebuild", nil, 0)
+	f.RemoteStorage.mapDirectoryToRemoteStorage("/buckets/other", &remote_pb.RemoteStorageLocation{
+		Name: "tombstonestore", Bucket: "other", Path: "/",
+	})
+	mounts := f.RemoteStorage.MountedDirectories()
+	require.Len(t, mounts, 2)
+
+	now := time.Now().UnixNano()
+	putRemoteSyncOffset(t, store, "/buckets/mybucket", now-200)
+	putRemoteSyncOffset(t, store, "/buckets/other", now-300)
+	assert.Equal(t, now-300, f.remoteDeletionRebuildStartTsNs(context.Background(), mounts),
+		"rebuild must replay from the least-synced mount")
+
+	// a mount whose offset was never written replays only within the TTL
+	require.NoError(t, store.KvDelete(context.Background(), remote_storage.SyncOffsetKey("/buckets/other")))
+	floor := f.remoteDeletionRebuildStartTsNs(context.Background(), mounts)
+	assert.GreaterOrEqual(t, floor, time.Now().Add(-remoteDeletionTombstoneTTL-time.Second).UnixNano())
+
+	// an offset older than the TTL floor is raised to it
+	putRemoteSyncOffset(t, store, "/buckets/other", time.Now().Add(-remoteDeletionTombstoneTTL-time.Hour).UnixNano())
+	assert.Greater(t, f.remoteDeletionRebuildStartTsNs(context.Background(), mounts),
+		time.Now().Add(-remoteDeletionTombstoneTTL-time.Hour).UnixNano())
+}
+
+func TestRebuildRemoteDeletionTombstones_EmptyLogReleasesGate(t *testing.T) {
+	f, _ := newMountedTestFiler(t, "stub_rebuild_empty", nil, 0)
+
+	f.RebuildRemoteDeletionTombstones(context.Background())
+	done := f.remoteTombstonesDone.Load()
+	require.NotNil(t, done, "the gate must be set synchronously")
+
+	select {
+	case <-*done:
+	case <-time.After(10 * time.Second):
+		t.Fatal("rebuild never released the gate")
+	}
+	assert.Nil(t, f.remoteTombstonesDone.Load())
+	ts, _ := f.remoteTombstones.blockedSince("/buckets/mybucket/a.txt")
+	assert.Zero(t, ts)
+}
+
+func TestMaybeLazyFetchFromRemote_WaitsForRebuild(t *testing.T) {
+	const storageType = "stub_tomb_pending"
+	stub := &countingRemoteClient{
+		stubRemoteClient: stubRemoteClient{
+			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 10},
+		},
+	}
+	f, _ := newMountedTestFiler(t, storageType, stub, 0)
+
+	// an unfinished rebuild blocks the fetch until the context gives up
+	gate := make(chan struct{})
+	f.remoteTombstonesDone.Store(&gate)
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+	entry, err := f.maybeLazyFetchFromRemote(ctx, "/buckets/mybucket/a.txt")
+	assert.ErrorIs(t, err, context.DeadlineExceeded)
+	assert.Nil(t, entry)
+	assert.Equal(t, 0, stub.statCalls)
+
+	// once the rebuild finishes, the fetch proceeds
+	close(gate)
+	entry, err = f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/a.txt")
+	require.NoError(t, err)
+	require.NotNil(t, entry)
+	assert.Equal(t, 1, stub.statCalls)
+}

weed/filer/reader_at.go

@@ -32,7 +32,9 @@ type ChunkReadAt struct {
 	fileSize      int64
 	readerCache   *ReaderCache
 	readerPattern *ReaderPattern
+	lastChunkMu   sync.Mutex // guards lastChunkFid; mount issues concurrent ReadAt calls
 	lastChunkFid  string
+	stream        chunkStream     // chunk this reader is positioned in, pinned in the shared readerCache
 	prefetchCount int             // Number of chunks to prefetch ahead during sequential reads
 	ctx           context.Context // Context used for cancellation during chunk read operations
 }
@@ -167,10 +169,17 @@ func (c *ChunkReadAt) Size() int64 {
 }
 
 func (c *ChunkReadAt) Close() error {
+	c.ReleaseStream()
 	c.readerCache.destroy()
 	return nil
 }
 
+// ReleaseStream drops this reader's hold on the chunk it is positioned in.
+// Unlike Close it leaves the (possibly shared) ReaderCache intact.
+func (c *ChunkReadAt) ReleaseStream() {
+	c.readerCache.releaseStream(&c.stream)
+}
+
 func (c *ChunkReadAt) ReadAt(p []byte, offset int64) (n int, err error) {
 
 	c.readerPattern.MonitorReadAt(offset, len(p))
@@ -341,6 +350,7 @@ func (c *ChunkReadAt) doReadAt(ctx context.Context, p []byte, offset int64) (n i
 func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunkView *ChunkView, nextChunkViews *Interval[*ChunkView], offset uint64) (n int, err error) {
 
 	if c.readerPattern.IsRandomMode() {
+		c.readerCache.releaseStream(&c.stream)
 		n, err := c.readerCache.chunkCache.ReadChunkAt(buffer, chunkView.FileId, offset)
 		if n > 0 {
 			return n, err
@@ -350,20 +360,20 @@ func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunk
 	}
 
 	shouldCache := (uint64(chunkView.ViewOffset) + chunkView.ChunkSize) <= c.readerCache.chunkCache.GetMaxFilePartSizeInCache()
-	n, err = c.readerCache.ReadChunkAt(ctx, buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), shouldCache)
-	if c.lastChunkFid != chunkView.FileId {
-		if chunkView.OffsetInChunk == 0 { // start of a new chunk
-			if c.lastChunkFid != "" {
-				c.readerCache.UnCache(c.lastChunkFid)
-			}
-			if nextChunkViews != nil && c.prefetchCount > 0 {
-				// Prefetch multiple chunks ahead for better sequential read throughput
-				// This keeps the network pipeline full with parallel chunk fetches
-				c.readerCache.MaybeCache(nextChunkViews, c.prefetchCount)
-			}
+	// The previous chunk is released through the stream pin rather than
+	// UnCache: the buffer is shared, and other streams may still be reading it.
+	n, err = c.readerCache.readChunkAt(ctx, &c.stream, buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), shouldCache)
+	c.lastChunkMu.Lock()
+	enteredChunk := c.lastChunkFid != chunkView.FileId
+	c.lastChunkFid = chunkView.FileId
+	c.lastChunkMu.Unlock()
+	if enteredChunk && chunkView.OffsetInChunk == 0 { // start of a new chunk
+		if nextChunkViews != nil && c.prefetchCount > 0 {
+			// Prefetch multiple chunks ahead for better sequential read throughput
+			// This keeps the network pipeline full with parallel chunk fetches
+			c.readerCache.MaybeCache(nextChunkViews, c.prefetchCount)
 		}
 	}
-	c.lastChunkFid = chunkView.FileId
 	return
 }
 

weed/filer/reader_at_shared_test.go

@@ -0,0 +1,283 @@
+package filer
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+// Two streams (e.g. two S3 GETs) reading the same object through one shared
+// ReaderCache, interleaved in small slices, must share each chunk download:
+// one stream finishing a chunk, or moving on to the next one, must not drop a
+// buffer the other stream is still reading.
+func TestChunkReadAtConcurrentStreamsShareChunks(t *testing.T) {
+	const chunkSize = 64 << 10
+	const chunkCount = 3
+	const sliceSize = 16 << 10
+
+	var mu sync.Mutex
+	fetches := map[string]int{}
+	rc := NewReaderCache(64, (*chunk_cache.TieredChunkCache)(nil), func(context.Context, string) ([]string, error) {
+		return []string{"unused"}, nil
+	}, nil)
+	defer rc.destroy()
+	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, fileId string, _ util_http.RefreshUrlsFunc) (int, error) {
+		mu.Lock()
+		fetches[fileId]++
+		mu.Unlock()
+		for i := range buffer {
+			buffer[i] = fileId[len(fileId)-1]
+		}
+		return len(buffer), nil
+	}
+
+	newStream := func() *ChunkReadAt {
+		views := NewIntervalList[*ChunkView]()
+		for i := 0; i < chunkCount; i++ {
+			views.AppendInterval(&Interval[*ChunkView]{
+				StartOffset: int64(i * chunkSize),
+				StopOffset:  int64((i + 1) * chunkSize),
+				Value: &ChunkView{
+					FileId:     fmt.Sprintf("chunk%d", i),
+					ViewSize:   chunkSize,
+					ViewOffset: int64(i * chunkSize),
+					ChunkSize:  chunkSize,
+				},
+			})
+		}
+		return NewChunkReaderAtFromClient(context.Background(), rc, views, chunkSize*chunkCount, 0)
+	}
+	streams := []*ChunkReadAt{newStream(), newStream()}
+
+	// The streams alternate slice by slice, the second one lagging one slice
+	// behind, so the leader always finishes a chunk while the follower is
+	// still inside it.
+	offsets := make([]int64, len(streams))
+	offsets[1] = -sliceSize
+	for offsets[1] < chunkSize*chunkCount {
+		for i, stream := range streams {
+			if offsets[i] < 0 || offsets[i] >= chunkSize*chunkCount {
+				offsets[i] += sliceSize
+				continue
+			}
+			buf := make([]byte, sliceSize)
+			n, err := stream.ReadAt(buf, offsets[i])
+			if (err != nil && err != io.EOF) || n != sliceSize {
+				t.Fatalf("stream %d at %d: n=%d err=%v", i, offsets[i], n, err)
+			}
+			if want := byte('0' + offsets[i]/chunkSize); buf[0] != want || buf[n-1] != want {
+				t.Fatalf("stream %d at %d: got %q, want %q", i, offsets[i], buf[0], want)
+			}
+			offsets[i] += sliceSize
+		}
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	for i := 0; i < chunkCount; i++ {
+		fileId := fmt.Sprintf("chunk%d", i)
+		if fetches[fileId] != 1 {
+			t.Errorf("%s fetched %d times for two concurrent streams, want 1", fileId, fetches[fileId])
+		}
+	}
+}
+
+func newStreamTestReaderCache(chunkCache chunk_cache.ChunkCache) *ReaderCache {
+	rc := NewReaderCache(64, chunkCache, func(context.Context, string) ([]string, error) {
+		return []string{"unused"}, nil
+	}, nil)
+	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
+		return len(buffer), nil
+	}
+	return rc
+}
+
+func isRetained(rc *ReaderCache, fileId string) bool {
+	rc.Lock()
+	defer rc.Unlock()
+	_, found := rc.downloaders[fileId]
+	return found
+}
+
+// A stream moving on to a chunk served from the chunk cache must still
+// release the chunk it was positioned in before.
+func TestChunkStreamReleasesPreviousChunkOnCacheHit(t *testing.T) {
+	cache := newMockChunkCacheForReaderCache()
+	cache.SetChunk("chunk1", make([]byte, 4<<10))
+	rc := newStreamTestReaderCache(cache)
+	defer rc.destroy()
+
+	stream := &chunkStream{}
+	if _, err := rc.readChunkAt(context.Background(), stream, make([]byte, 1<<10), "chunk0", nil, false, 0, 4<<10, false); err != nil {
+		t.Fatal(err)
+	}
+	if n, err := rc.readChunkAt(context.Background(), stream, make([]byte, 1<<10), "chunk1", nil, false, 0, 4<<10, true); err != nil || n == 0 {
+		t.Fatalf("cache hit read: n=%d err=%v", n, err)
+	}
+	if isRetained(rc, "chunk0") {
+		t.Fatal("chunk0 still retained after the stream moved on to a cached chunk")
+	}
+}
+
+// A chunk the stream leaves while another read is in flight must be dropped
+// once that read ends, even if it did not read the chunk to the end.
+func TestChunkStreamDropsLeftChunkAfterInFlightRead(t *testing.T) {
+	rc := newStreamTestReaderCache(newMockChunkCacheForReaderCache())
+	defer rc.destroy()
+
+	stream := &chunkStream{}
+	if _, err := rc.readChunkAt(context.Background(), stream, make([]byte, 1<<10), "chunk0", nil, false, 0, 4<<10, false); err != nil {
+		t.Fatal(err)
+	}
+	// Another reader is in the middle of a partial read of chunk0.
+	rc.Lock()
+	other := rc.downloaders["chunk0"]
+	other.wg.Add(1)
+	atomic.AddInt32(&other.readers, 1)
+	rc.Unlock()
+
+	if _, err := rc.readChunkAt(context.Background(), stream, make([]byte, 1<<10), "chunk1", nil, false, 0, 4<<10, false); err != nil {
+		t.Fatal(err)
+	}
+	if !isRetained(rc, "chunk0") {
+		t.Fatal("chunk0 dropped while a read was still in flight")
+	}
+
+	// The in-flight read ends without reaching the end of the chunk.
+	other.wg.Done()
+	atomic.AddInt32(&other.readers, -1)
+	rc.removeConsumed(other)
+	if isRetained(rc, "chunk0") {
+		t.Fatal("chunk0 retained after the stream left it and the last read ended")
+	}
+}
+
+// Concurrent ReadAt calls on one ChunkReadAt (as mount does) share its stream
+// pin; they must neither race on it nor leak or double-release pins.
+func TestChunkStreamConcurrentReadsOnOneReader(t *testing.T) {
+	const chunkSize = 16 << 10
+	const chunkCount = 4
+	const sliceSize = 4 << 10
+	rc := newStreamTestReaderCache((*chunk_cache.TieredChunkCache)(nil))
+	defer rc.destroy()
+
+	views := NewIntervalList[*ChunkView]()
+	for i := 0; i < chunkCount; i++ {
+		views.AppendInterval(&Interval[*ChunkView]{
+			StartOffset: int64(i * chunkSize),
+			StopOffset:  int64((i + 1) * chunkSize),
+			Value: &ChunkView{
+				FileId:     fmt.Sprintf("chunk%d", i),
+				ViewSize:   chunkSize,
+				ViewOffset: int64(i * chunkSize),
+				ChunkSize:  chunkSize,
+			},
+		})
+	}
+	reader := NewChunkReaderAtFromClient(context.Background(), rc, views, chunkSize*chunkCount, 0)
+
+	var wg sync.WaitGroup
+	for g := 0; g < 8; g++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for offset := int64(0); offset < chunkSize*chunkCount; offset += sliceSize {
+				if _, err := reader.ReadAt(make([]byte, sliceSize), offset); err != nil && err != io.EOF {
+					t.Error(err)
+					return
+				}
+			}
+		}()
+	}
+	wg.Wait()
+
+	// Finishing the last chunk releases the stream's final pin.
+	if _, err := reader.ReadAt(make([]byte, sliceSize), chunkSize*chunkCount-sliceSize); err != nil && err != io.EOF {
+		t.Fatal(err)
+	}
+	rc.Lock()
+	defer rc.Unlock()
+	for fileId, cacher := range rc.downloaders {
+		t.Errorf("%s retained after all reads finished: pins=%d readers=%d", fileId, atomic.LoadInt32(&cacher.pins), atomic.LoadInt32(&cacher.readers))
+	}
+}
+
+// A chunk a stream is positioned in must outlast downloader-limit eviction:
+// otherwise a busy cache drops the buffer mid-stream and forces a refetch.
+func TestChunkReadAtPinnedChunkSurvivesEviction(t *testing.T) {
+	const chunkSize = 64 << 10
+
+	var fetches int32
+	rc := NewReaderCache(2, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) {
+		return []string{"unused"}, nil
+	}, nil)
+	defer rc.destroy()
+	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, fileId string, _ util_http.RefreshUrlsFunc) (int, error) {
+		if fileId == "chunk0" {
+			atomic.AddInt32(&fetches, 1)
+		}
+		return len(buffer), nil
+	}
+
+	stream := &chunkStream{}
+	buf := make([]byte, 16<<10)
+	// One slice in: the stream is positioned in chunk0 but has not left it.
+	if _, err := rc.readChunkAt(context.Background(), stream, buf, "chunk0", nil, false, 0, chunkSize, false); err != nil {
+		t.Fatal(err)
+	}
+
+	// Fill the downloader map past its limit with unpinned chunks.
+	for _, fileId := range []string{"chunk1", "chunk2", "chunk3"} {
+		if _, err := rc.ReadChunkAt(context.Background(), buf, fileId, nil, false, 0, chunkSize, true); err != nil {
+			t.Fatalf("read %s: %v", fileId, err)
+		}
+	}
+
+	// The stream's next slice must come from the still-pinned buffer.
+	if _, err := rc.readChunkAt(context.Background(), stream, buf, "chunk0", nil, false, 16<<10, chunkSize, false); err != nil {
+		t.Fatal(err)
+	}
+	if got := atomic.LoadInt32(&fetches); got != 1 {
+		t.Errorf("chunk0 fetched %d times, want 1", got)
+	}
+}
+
+// When every downloader is pinned the limit still applies: the oldest pinned
+// buffer is evicted so abandoned streams cannot grow memory past the limit.
+func TestChunkReadAtPinnedEvictionFallsBackWhenAllPinned(t *testing.T) {
+	const chunkSize = 64 << 10
+	rc := NewReaderCache(2, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) {
+		return []string{"unused"}, nil
+	}, nil)
+	defer rc.destroy()
+	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
+		return len(buffer), nil
+	}
+
+	buf := make([]byte, 16<<10)
+	streamA := &chunkStream{}
+	streamB := &chunkStream{}
+	if _, err := rc.readChunkAt(context.Background(), streamA, buf, "chunk0", nil, false, 0, chunkSize, false); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := rc.readChunkAt(context.Background(), streamB, buf, "chunk1", nil, false, 0, chunkSize, false); err != nil {
+		t.Fatal(err)
+	}
+
+	// Every downloader is now pinned; the next chunk must still get in.
+	if _, err := rc.readChunkAt(context.Background(), &chunkStream{}, buf, "chunk2", nil, false, 0, chunkSize, false); err != nil {
+		t.Fatal(err)
+	}
+	if isRetained(rc, "chunk0") {
+		t.Fatal("oldest pinned downloader was not evicted past the limit")
+	}
+	if !isRetained(rc, "chunk2") {
+		t.Fatal("new downloader missing after pinned fallback eviction")
+	}
+}

weed/filer/reader_cache.go

@@ -35,6 +35,8 @@ type SingleChunkCacher struct {
 	completedTimeNew int64
 	readers          int32
 	consumed         int32
+	pins             int32 // streams currently positioned inside this chunk
+	left             int32 // set once the last pinning stream moved on
 	sync.Mutex
 	parent         *ReaderCache
 	chunkFileId    string
@@ -112,7 +114,19 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView], count int) {
 	return
 }
 
+// chunkStream is one sequential reader's position in a shared ReaderCache.
+// The chunk it is reading stays pinned until the stream reads it to the end or
+// moves elsewhere, so another stream finishing or leaving the same chunk does
+// not drop the buffer from under it.
+type chunkStream struct {
+	cacher *SingleChunkCacher
+}
+
 func (rc *ReaderCache) ReadChunkAt(ctx context.Context, buffer []byte, fileId string, cipherKey []byte, isGzipped bool, offset int64, chunkSize int, shouldCache bool) (int, error) {
+	return rc.readChunkAt(ctx, nil, buffer, fileId, cipherKey, isGzipped, offset, chunkSize, shouldCache)
+}
+
+func (rc *ReaderCache) readChunkAt(ctx context.Context, stream *chunkStream, buffer []byte, fileId string, cipherKey []byte, isGzipped bool, offset int64, chunkSize int, shouldCache bool) (int, error) {
 retry:
 	rc.Lock()
 
@@ -130,8 +144,11 @@ retry:
 			// start wg.Wait() on a zero counter while this read is about to register.
 			cacher.wg.Add(1)
 			atomic.AddInt32(&cacher.readers, 1)
+			previous := stream.pin(cacher)
 			rc.Unlock()
+			rc.unpin(previous)
 			n, err := cacher.readChunkAt(ctx, buffer, offset)
+			rc.releaseIfFinished(stream, cacher, offset, n, err, chunkSize)
 			if n > 0 || err != nil {
 				return n, err
 			}
@@ -144,19 +161,34 @@ retry:
 	if shouldCache || rc.lookupFileIdFn == nil {
 		n, err := rc.chunkCache.ReadChunkAt(buffer, fileId, uint64(offset))
 		if n > 0 {
+			// Served from the chunk cache: the stream has left its pinned chunk.
+			previous := stream.unpinLocked()
 			rc.Unlock()
+			rc.unpin(previous)
 			return n, err
 		}
 	}
 
-	// clean up old downloaders
+	// clean up old downloaders; prefer one no stream is positioned in, but
+	// fall back to a pinned one so abandoned pins cannot bypass the limit
 	if len(rc.downloaders) >= rc.limit {
 		oldestFid, oldestTime := "", time.Now().UnixNano()
+		pinnedFid, pinnedTime := "", int64(0)
 		for fid, downloader := range rc.downloaders {
 			completedTime := atomic.LoadInt64(&downloader.completedTimeNew)
-			if completedTime > 0 && completedTime < oldestTime {
-				oldestFid, oldestTime = fid, completedTime
+			if completedTime <= 0 {
+				continue
 			}
+			if atomic.LoadInt32(&downloader.pins) == 0 {
+				if completedTime < oldestTime {
+					oldestFid, oldestTime = fid, completedTime
+				}
+			} else if pinnedFid == "" || completedTime < pinnedTime {
+				pinnedFid, pinnedTime = fid, completedTime
+			}
+		}
+		if oldestFid == "" {
+			oldestFid = pinnedFid
 		}
 		if oldestFid != "" {
 			oldDownloader := rc.downloaders[oldestFid]
@@ -175,9 +207,77 @@ retry:
 	rc.downloaders[fileId] = cacher
 	cacher.wg.Add(1)
 	atomic.AddInt32(&cacher.readers, 1)
+	previous := stream.pin(cacher)
 	rc.Unlock()
+	rc.unpin(previous)
 
-	return cacher.readChunkAt(ctx, buffer, offset)
+	n, err := cacher.readChunkAt(ctx, buffer, offset)
+	rc.releaseIfFinished(stream, cacher, offset, n, err, chunkSize)
+	return n, err
+}
+
+// pin makes cacher the stream's current chunk and returns the chunk it was
+// pinned to before, which the caller unpins once the ReaderCache lock is
+// released. The stream is only touched under the ReaderCache lock, since
+// concurrent ReadAt calls on one ChunkReadAt share it.
+func (stream *chunkStream) pin(cacher *SingleChunkCacher) (previous *SingleChunkCacher) {
+	if stream == nil || stream.cacher == cacher {
+		return nil
+	}
+	previous = stream.cacher
+	stream.cacher = cacher
+	atomic.AddInt32(&cacher.pins, 1)
+	return previous
+}
+
+// unpinLocked detaches the stream from its chunk and returns that chunk for
+// the caller to unpin once the ReaderCache lock is released.
+func (stream *chunkStream) unpinLocked() (previous *SingleChunkCacher) {
+	if stream == nil {
+		return nil
+	}
+	previous = stream.cacher
+	stream.cacher = nil
+	return previous
+}
+
+// releaseIfFinished unpins the stream's chunk once the stream has read it to
+// the end, since the stream will not come back to it.
+func (rc *ReaderCache) releaseIfFinished(stream *chunkStream, cacher *SingleChunkCacher, offset int64, n int, err error, chunkSize int) {
+	if stream == nil || err != nil || offset+int64(n) < int64(chunkSize) {
+		return
+	}
+	var previous *SingleChunkCacher
+	rc.Lock()
+	if stream.cacher == cacher {
+		previous = stream.unpinLocked()
+	}
+	rc.Unlock()
+	rc.unpin(previous)
+}
+
+// releaseStream unpins whatever chunk the stream is positioned in.
+func (rc *ReaderCache) releaseStream(stream *chunkStream) {
+	if stream == nil {
+		return
+	}
+	rc.Lock()
+	previous := stream.unpinLocked()
+	rc.Unlock()
+	rc.unpin(previous)
+}
+
+// unpin drops one stream's pin. Once no stream is positioned in the chunk it
+// is dropped like UnCache would, as soon as no read is in progress either:
+// here if none is, otherwise by the last read's removeConsumed.
+func (rc *ReaderCache) unpin(downloader *SingleChunkCacher) {
+	if downloader == nil {
+		return
+	}
+	if atomic.AddInt32(&downloader.pins, -1) == 0 {
+		atomic.StoreInt32(&downloader.left, 1)
+	}
+	rc.removeConsumed(downloader)
 }
 
 func (rc *ReaderCache) UnCache(fileId string) {
@@ -202,15 +302,34 @@ func (rc *ReaderCache) remove(downloader *SingleChunkCacher) {
 	}
 }
 
-// removeConsumed drops a cacher once its buffer was fully read and no
-// readers remain attached. The checks run under the ReaderCache lock so a
-// reader attaching at the same time either wins (the cacher stays and that
-// reader's detach retries the removal) or misses the map and refetches.
+// removeUnpinned drops a cacher only while no stream is positioned in it.
+// Budget eviction picks its victim under the budget lock, so the pin check
+// and the map removal must happen together under the ReaderCache lock.
+func (rc *ReaderCache) removeUnpinned(downloader *SingleChunkCacher) (removed bool) {
+	rc.Lock()
+	removed = rc.downloaders[downloader.chunkFileId] == downloader &&
+		atomic.LoadInt32(&downloader.pins) == 0
+	if removed {
+		delete(rc.downloaders, downloader.chunkFileId)
+	}
+	rc.Unlock()
+	if removed {
+		downloader.destroy()
+	}
+	return
+}
+
+// removeConsumed drops a cacher once its buffer was fully read, or the
+// streams positioned in it have left, and no readers remain attached or
+// pinned. The checks run under the ReaderCache lock so a reader attaching
+// at the same time either wins (the cacher stays and that reader's detach
+// retries the removal) or misses the map and refetches.
 func (rc *ReaderCache) removeConsumed(downloader *SingleChunkCacher) {
 	rc.Lock()
 	removed := rc.downloaders[downloader.chunkFileId] == downloader &&
 		atomic.LoadInt32(&downloader.readers) == 0 &&
-		atomic.LoadInt32(&downloader.consumed) != 0
+		atomic.LoadInt32(&downloader.pins) == 0 &&
+		(atomic.LoadInt32(&downloader.consumed) != 0 || atomic.LoadInt32(&downloader.left) != 0)
 	if removed {
 		delete(rc.downloaders, downloader.chunkFileId)
 	}

weed/filer/reader_cache_budget.go

@@ -4,6 +4,7 @@ import (
 	"container/list"
 	"fmt"
 	"sync"
+	"sync/atomic"
 
 	"github.com/seaweedfs/seaweedfs/weed/util/mem"
 )
@@ -51,12 +52,39 @@ func (b *ReaderCacheBudget) reserve(s *SingleChunkCacher) error {
 			b.Unlock()
 			return nil
 		}
-		if entry := b.idle.Front(); entry != nil {
-			victim := entry.Value.(*SingleChunkCacher)
+		// Prefer evicting an idle chunk no stream is positioned in; fall back
+		// to the oldest pinned one so abandoned pins cannot block the budget.
+		var victim *SingleChunkCacher
+		var entry *list.Element
+		pinnedVictim := false
+		for e := b.idle.Front(); e != nil; e = e.Next() {
+			c := e.Value.(*SingleChunkCacher)
+			if atomic.LoadInt32(&c.pins) == 0 {
+				victim, entry = c, e
+				pinnedVictim = false
+				break
+			}
+			if victim == nil {
+				victim, entry = c, e
+				pinnedVictim = true
+			}
+		}
+		if entry != nil {
 			b.idle.Remove(entry)
 			delete(b.idleEntries, victim)
 			b.Unlock()
-			victim.parent.remove(victim)
+			if pinnedVictim {
+				victim.parent.remove(victim)
+			} else if !victim.parent.removeUnpinned(victim) {
+				// The victim was pinned between selection and removal: keep it
+				// evictable so a pin abandoned in that gap cannot wedge the
+				// budget, then retry the selection.
+				b.Lock()
+				if _, ok := b.reservations[victim]; ok && b.idleEntries[victim] == nil {
+					b.idleEntries[victim] = b.idle.PushBack(victim)
+				}
+				b.Unlock()
+			}
 			continue
 		}
 		changed := b.changed

weed/filer/remote_storage.go

@@ -36,6 +36,7 @@ type FilerRemoteStorage struct {
 	// whenever /etc/remote changes
 	mu                sync.RWMutex
 	rules             ptrie.Trie[*remote_pb.RemoteStorageLocation]
+	mountDirs         []util.FullPath
 	storageNameToConf map[string]*remote_pb.RemoteConf
 	// confValidator, when set, is applied to every RemoteConf as it is loaded
 	// from /etc/remote. A conf that fails is dropped from storageNameToConf so
@@ -87,11 +88,13 @@ func (rs *FilerRemoteStorage) LoadRemoteStorageConfigurationsAndMapping(filer *F
 	// build into fresh containers so an unmounted directory disappears instead
 	// of lingering in the trie, which has no way to drop a key
 	rules := ptrie.New[*remote_pb.RemoteStorageLocation]()
+	var mountDirs []util.FullPath
 	storageNameToConf := make(map[string]*remote_pb.RemoteConf)
 
 	for _, entry := range entries {
 		if entry.Name() == REMOTE_STORAGE_MOUNT_FILE {
-			if err := loadRemoteStorageMountMapping(rules, entry.Content); err != nil {
+			mountDirs, err = loadRemoteStorageMountMapping(rules, entry.Content)
+			if err != nil {
 				return err
 			}
 			continue
@@ -117,27 +120,46 @@ func (rs *FilerRemoteStorage) LoadRemoteStorageConfigurationsAndMapping(filer *F
 	}
 
 	rs.mu.Lock()
-	rs.rules, rs.storageNameToConf = rules, storageNameToConf
+	rs.rules, rs.mountDirs, rs.storageNameToConf = rules, mountDirs, storageNameToConf
 	rs.mu.Unlock()
 
 	return nil
 }
 
-func loadRemoteStorageMountMapping(rules ptrie.Trie[*remote_pb.RemoteStorageLocation], data []byte) (err error) {
+func loadRemoteStorageMountMapping(rules ptrie.Trie[*remote_pb.RemoteStorageLocation], data []byte) (mountDirs []util.FullPath, err error) {
 	mappings := &remote_pb.RemoteStorageMapping{}
 	if err := proto.Unmarshal(data, mappings); err != nil {
-		return fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, err)
+		return nil, fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, err)
 	}
 	for dir, storageLocation := range mappings.Mappings {
 		putDirectoryToRemoteStorage(rules, util.FullPath(dir), storageLocation)
+		mountDirs = append(mountDirs, util.FullPath(dir))
 	}
-	return nil
+	return mountDirs, nil
 }
 
 func (rs *FilerRemoteStorage) mapDirectoryToRemoteStorage(dir util.FullPath, loc *remote_pb.RemoteStorageLocation) {
 	rs.mu.Lock()
 	defer rs.mu.Unlock()
 	putDirectoryToRemoteStorage(rs.rules, dir, loc)
+	found := false
+	for _, d := range rs.mountDirs {
+		if d == dir {
+			found = true
+			break
+		}
+	}
+	if !found {
+		rs.mountDirs = append(rs.mountDirs, dir)
+	}
+}
+
+// MountedDirectories returns the directories currently mapped to remote
+// storage, for callers that must reason about every mount rather than one path.
+func (rs *FilerRemoteStorage) MountedDirectories() []util.FullPath {
+	rs.mu.RLock()
+	defer rs.mu.RUnlock()
+	return append([]util.FullPath(nil), rs.mountDirs...)
 }
 
 func putDirectoryToRemoteStorage(rules ptrie.Trie[*remote_pb.RemoteStorageLocation], dir util.FullPath, loc *remote_pb.RemoteStorageLocation) {

weed/iam/integration/cached_role_store_generic.go

@@ -116,6 +116,14 @@ func (c *GenericCachedRoleStore) DeleteRole(ctx context.Context, filerAddress st
 	return c.Delete(ctx, filerAddress, roleName)
 }
 
+// UpdateRole implements RoleStore interface. The update reads the filer, not
+// the cache, and the cache is dropped afterwards whatever the outcome: a
+// refused write may mean the cached role is stale.
+func (c *GenericCachedRoleStore) UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error {
+	defer c.ClearCache()
+	return c.adapter.store.UpdateRole(ctx, filerAddress, roleName, update)
+}
+
 // genericCopyRoleDefinition creates a deep copy of a RoleDefinition for the generic cache
 func genericCopyRoleDefinition(role *RoleDefinition) *RoleDefinition {
 	if role == nil {
@@ -123,9 +131,13 @@ func genericCopyRoleDefinition(role *RoleDefinition) *RoleDefinition {
 	}
 
 	result := &RoleDefinition{
-		RoleName:    role.RoleName,
-		RoleArn:     role.RoleArn,
-		Description: role.Description,
+		RoleName:           role.RoleName,
+		RoleArn:            role.RoleArn,
+		Description:        role.Description,
+		MaxSessionDuration: role.MaxSessionDuration,
+		Source:             role.Source,
+		CreatedAt:          role.CreatedAt,
+		RoleId:             role.RoleId,
 	}
 
 	// Deep copy trust policy if it exists

weed/iam/integration/iam_manager.go

@@ -2,7 +2,10 @@ package integration
 
 import (
 	"context"
+	"crypto/rand"
+	"crypto/sha256"
 	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -27,11 +30,30 @@ const maxPoliciesForEvaluation = 1024
 
 // IAMManager orchestrates all IAM components
 type IAMManager struct {
-	stsService           *sts.STSService
-	policyEngine         *policy.PolicyEngine
-	roleStore            RoleStore
-	userStore            UserStore
-	oidcProviderStore    OIDCProviderStore
+	stsService        *sts.STSService
+	policyEngine      *policy.PolicyEngine
+	roleStore         RoleStore
+	userStore         UserStore
+	oidcProviderStore OIDCProviderStore
+	// staticOIDCProviders are the OIDC providers of this server's IAM config
+	// file, by ARN. With an in-memory store they are also written to the
+	// store; a persistent store never holds them (see installOIDCProviderStore).
+	staticOIDCProviders map[string]*OIDCProviderRecord
+	// staticRoles are the roles of this server's IAM config file, by name, once
+	// LoadStaticRoles has run (see installRoleStore).
+	staticRoles map[string]*RoleDefinition
+	// oidcRetryMu guards the background refresh retry and which store is
+	// current: cancelOIDCLoad stops the retry, oidcRetryGen names the one
+	// running (0 when none) so at most one runs, and oidcRetryAgain records a
+	// refresh that failed while it ran, so the retry runs once more.
+	oidcRetryMu    sync.Mutex
+	cancelOIDCLoad context.CancelFunc
+	oidcRetryGen   uint64
+	oidcRetrySeq   uint64
+	oidcRetryAgain bool
+	// oidcRefreshMu serializes refreshes from reading the store to handing
+	// STS the result, so an older snapshot cannot replace a newer one.
+	oidcRefreshMu        sync.Mutex
 	oidcAuditSink        OIDCProviderAuditSink
 	revocationStore      SessionRevocationStore
 	filerAddressProvider func() string // Function to get current filer address
@@ -108,12 +130,13 @@ func (m *IAMManager) PurgeRevokedSessions(ctx context.Context) (int, error) {
 }
 
 // SetOIDCProviderStore configures the IAM-managed OIDC provider store. When
-// nil, OIDC provider IAM actions return ServiceNotReady. The store is the
-// source of truth for AssumeRoleWithWebIdentity provider resolution once
-// Phase 2b lands; in Phase 2a it is read-only and populated from static
-// configuration at boot.
+// nil, OIDC provider IAM actions return ServiceNotReady.
 func (m *IAMManager) SetOIDCProviderStore(store OIDCProviderStore) {
-	m.oidcProviderStore = store
+	var stsConfig *sts.STSConfig
+	if m.stsService != nil {
+		stsConfig = m.stsService.Config
+	}
+	m.installOIDCProviderStore(store, stsConfig)
 }
 
 // GetOIDCProviderStore returns the configured store (may be nil).
@@ -121,13 +144,33 @@ func (m *IAMManager) GetOIDCProviderStore() OIDCProviderStore {
 	return m.oidcProviderStore
 }
 
+// GetRoleStore returns the configured role store.
+func (m *IAMManager) GetRoleStore() RoleStore {
+	return m.roleStore
+}
+
+// SetRoleStore replaces the role store. An S3 server builds the list of
+// directories it watches for peers' changes once, at startup, from the store
+// installed then (RoleStoreDirectory): a filer-backed store installed later
+// with a different basePath is not watched, so peers' changes to it reach this
+// server's cached roles only when the cache expires.
+func (m *IAMManager) SetRoleStore(store RoleStore) {
+	m.installRoleStore(context.Background(), store)
+}
+
 // GetOIDCProvider returns the record for the given ARN, or an error if the
 // store is not configured or the record is missing.
 func (m *IAMManager) GetOIDCProvider(ctx context.Context, arn string) (*OIDCProviderRecord, error) {
 	if m.oidcProviderStore == nil {
 		return nil, fmt.Errorf("OIDC provider store not configured")
 	}
-	return m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	if errors.Is(err, ErrOIDCProviderNotFound) {
+		if static, ok := m.staticOIDCProviders[arn]; ok {
+			return copyOIDCProviderRecord(static), nil
+		}
+	}
+	return rec, err
 }
 
 // ListOIDCProviders enumerates all configured OIDC providers.
@@ -135,7 +178,33 @@ func (m *IAMManager) ListOIDCProviders(ctx context.Context) ([]*OIDCProviderReco
 	if m.oidcProviderStore == nil {
 		return nil, fmt.Errorf("OIDC provider store not configured")
 	}
-	return m.oidcProviderStore.ListProviders(ctx, m.getFilerAddress())
+	records, err := m.oidcProviderStore.ListProviders(ctx, m.getFilerAddress())
+	if err != nil {
+		return nil, err
+	}
+	// A persistent store does not hold the config file's providers.
+	seen := make(map[string]bool, len(records))
+	for _, rec := range records {
+		seen[rec.ARN] = true
+	}
+	for arn, static := range m.staticOIDCProviders {
+		if !seen[arn] {
+			records = append(records, copyOIDCProviderRecord(static))
+		}
+	}
+	return records, nil
+}
+
+// mutableOIDCProvider loads a stored provider for a change. A provider only
+// the IAM config file defines is refused with ErrOIDCProviderStatic.
+func (m *IAMManager) mutableOIDCProvider(ctx context.Context, arn string) (*OIDCProviderRecord, error) {
+	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	if errors.Is(err, ErrOIDCProviderNotFound) {
+		if _, ok := m.staticOIDCProviders[arn]; ok {
+			return nil, fmt.Errorf("%w: %s", ErrOIDCProviderStatic, arn)
+		}
+	}
+	return rec, err
 }
 
 // CreateOIDCProvider persists a new IAM-managed OIDC provider record. Refuses
@@ -150,6 +219,9 @@ func (m *IAMManager) CreateOIDCProvider(ctx context.Context, rec *OIDCProviderRe
 	if err := validateOIDCProviderRecord(rec); err != nil {
 		return err
 	}
+	if _, static := m.staticOIDCProviders[rec.ARN]; static {
+		return fmt.Errorf("%w: %s", ErrOIDCProviderAlreadyExists, rec.ARN)
+	}
 	existing, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), rec.ARN)
 	if err == nil && existing != nil {
 		return fmt.Errorf("%w: %s", ErrOIDCProviderAlreadyExists, rec.ARN)
@@ -173,6 +245,9 @@ func (m *IAMManager) DeleteOIDCProvider(ctx context.Context, arn string) error {
 	if m.oidcProviderStore == nil {
 		return fmt.Errorf("OIDC provider store not configured")
 	}
+	if _, err := m.mutableOIDCProvider(ctx, arn); errors.Is(err, ErrOIDCProviderStatic) {
+		return err
+	}
 	if err := m.oidcProviderStore.DeleteProvider(ctx, m.getFilerAddress(), arn); err != nil {
 		return err
 	}
@@ -190,7 +265,7 @@ func (m *IAMManager) AddClientIDToOIDCProvider(ctx context.Context, arn, clientI
 	if clientID == "" {
 		return fmt.Errorf("ClientID cannot be empty")
 	}
-	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	rec, err := m.mutableOIDCProvider(ctx, arn)
 	if err != nil {
 		return err
 	}
@@ -218,7 +293,7 @@ func (m *IAMManager) RemoveClientIDFromOIDCProvider(ctx context.Context, arn, cl
 	if m.oidcProviderStore == nil {
 		return fmt.Errorf("OIDC provider store not configured")
 	}
-	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	rec, err := m.mutableOIDCProvider(ctx, arn)
 	if err != nil {
 		return err
 	}
@@ -255,7 +330,7 @@ func (m *IAMManager) UpdateOIDCProviderThumbprints(ctx context.Context, arn stri
 			return fmt.Errorf("invalid thumbprint %q: must be 40-character SHA-1 hex", tp)
 		}
 	}
-	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	rec, err := m.mutableOIDCProvider(ctx, arn)
 	if err != nil {
 		return err
 	}
@@ -274,7 +349,7 @@ func (m *IAMManager) TagOIDCProvider(ctx context.Context, arn string, tags map[s
 	if m.oidcProviderStore == nil {
 		return fmt.Errorf("OIDC provider store not configured")
 	}
-	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	rec, err := m.mutableOIDCProvider(ctx, arn)
 	if err != nil {
 		return err
 	}
@@ -297,7 +372,7 @@ func (m *IAMManager) UntagOIDCProvider(ctx context.Context, arn string, keys []s
 	if m.oidcProviderStore == nil {
 		return fmt.Errorf("OIDC provider store not configured")
 	}
-	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
+	rec, err := m.mutableOIDCProvider(ctx, arn)
 	if err != nil {
 		return err
 	}
@@ -415,6 +490,88 @@ type RoleDefinition struct {
 	// set it must satisfy AWS bounds: 3600 ≤ MaxSessionDuration ≤ 43200.
 	// Honoured by AssumeRole, AssumeRoleWithWebIdentity, AssumeRoleWithCredentials.
 	MaxSessionDuration int64 `json:"maxSessionDuration,omitempty"`
+
+	// Source records where the role came from. RoleSourceStaticConfig marks a
+	// role loaded from the IAM config file; empty means it was created at
+	// runtime. Only static-config roles are pruned when they leave the file.
+	Source string `json:"source,omitempty"`
+
+	// CreatedAt is when the role was created through the IAM API. Zero for
+	// roles loaded from the config file.
+	CreatedAt time.Time `json:"createdAt,omitempty"`
+
+	// RoleId uniquely identifies this role, as AWS's RoleId does. A role
+	// deleted and created again under the same name gets a new ID, and a
+	// session is honoured only while the role it was issued for still has
+	// the ID the session carries — so a session outlives neither the role's
+	// deletion nor a later role that reuses its name.
+	RoleId string `json:"roleId,omitempty"`
+}
+
+// NewRoleID returns a fresh, random role ID in AWS's AROA form.
+func NewRoleID() string {
+	const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+	b := make([]byte, 17)
+	if _, err := rand.Read(b); err != nil {
+		panic(fmt.Sprintf("read random role id: %v", err))
+	}
+	for i := range b {
+		b[i] = alphabet[int(b[i])%len(alphabet)]
+	}
+	return "AROA" + string(b)
+}
+
+// StaticRoleID is the ID of a role defined in the IAM config file. Such a
+// role is created again at every start, so its ID is derived rather than
+// random, to keep sessions valid across restarts. It is derived from the name
+// and the trust policy together: a role removed from the file and replaced by
+// a different one under the same name trusts different principals, and must
+// not inherit the old role's sessions. Restoring the same role restores its ID.
+func StaticRoleID(role *RoleDefinition) string {
+	h := sha256.New()
+	h.Write([]byte("static-role:" + role.RoleName + "\x00"))
+	if role.TrustPolicy != nil {
+		trust, err := json.Marshal(role.TrustPolicy)
+		if err != nil {
+			// A trust policy that cannot be encoded cannot be matched on
+			// either; give the role an ID no session can carry.
+			return NewRoleID()
+		}
+		h.Write(trust)
+	}
+	return "AROA" + strings.ToUpper(hex.EncodeToString(h.Sum(nil)))[:17]
+}
+
+// RoleSourceStaticConfig is the Source of a role loaded from the IAM config
+// file.
+const RoleSourceStaticConfig = "static-config"
+
+// checkSessionRoleBinding refuses a session issued for a role that has since
+// been deleted, or replaced by a role reusing its name: the role's current ID
+// must be the one the session carries. It runs for every session carrying a
+// role ID, whatever policies the session embeds — the policies a session
+// embeds are the ones its role had, and outlive the role otherwise. A session
+// issued before role IDs were recorded carries none and is not bound. It
+// returns the role it checked, nil for an unbound session.
+func (m *IAMManager) checkSessionRoleBinding(ctx context.Context, sessionInfo *sts.SessionInfo) (*RoleDefinition, error) {
+	if sessionInfo == nil || sessionInfo.RoleId == "" {
+		return nil, nil
+	}
+	roleName := utils.ExtractRoleNameFromArn(sessionInfo.RoleArn)
+	if roleName == "" {
+		return nil, nil
+	}
+	role, err := m.roleStore.GetRole(ctx, m.getFilerAddress(), roleName)
+	if errors.Is(err, ErrRoleNotFound) {
+		return nil, fmt.Errorf("session was issued for role %s, which no longer exists", roleName)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("resolve role %s for session: %w", roleName, err)
+	}
+	if role.RoleId != sessionInfo.RoleId {
+		return nil, fmt.Errorf("session was issued for an earlier role named %s", roleName)
+	}
+	return role, nil
 }
 
 // ActionRequest represents a request to perform an action
@@ -557,55 +714,174 @@ func (m *IAMManager) initOIDCProviderStore(config *IAMConfig) error {
 	if err != nil {
 		return err
 	}
-	m.oidcProviderStore = store
+	m.installOIDCProviderStore(store, config.STS)
+	return nil
+}
 
-	if config.STS == nil {
-		return nil
+// installOIDCProviderStore makes store the manager's OIDC provider store.
+//
+// The IAM config file's providers are reported by the IAM API alongside the
+// stored ones. An in-memory store holds them as records, as it always has. A
+// persistent store never does: it outlives this process and may be shared by
+// S3 servers with different config files, so a record written from one file
+// would outlive its removal from that file and be trusted by every server.
+// Those providers stay in memory (staticOIDCProviders) and STS keeps serving
+// them from its static configuration; the store holds only providers created
+// through the IAM API, and those are loaded into STS here.
+//
+// A provider stored under the same ARN as a config-file provider takes
+// precedence, in the IAM API as in STS, which already prefers IAM-managed
+// providers so that an API call can shadow a bootstrap entry. Deleting the
+// stored provider brings the config-file one back.
+func (m *IAMManager) installOIDCProviderStore(store OIDCProviderStore, stsConfig *sts.STSConfig) {
+	// Cancel the old store's retry and switch stores in one step, so a failed
+	// refresh of the old store cannot start a retry after the cancel.
+	m.oidcRetryMu.Lock()
+	m.stopOIDCRetryLocked()
+	m.oidcProviderStore = store
+	m.oidcRetryMu.Unlock()
+	m.staticOIDCProviders = staticOIDCProviderRecords(stsConfig)
+	if _, inMemory := store.(*MemoryOIDCProviderStore); inMemory {
+		ctx := context.Background()
+		now := time.Now().UTC()
+		for _, rec := range m.staticOIDCProviders {
+			mirrored := copyOIDCProviderRecord(rec)
+			mirrored.CreatedAt, mirrored.UpdatedAt = now, now
+			if err := store.StoreProvider(ctx, m.getFilerAddress(), mirrored); err != nil {
+				glog.Warningf("mirror static OIDC provider %s into store: %v", rec.ARN, err)
+			}
+		}
+		// The store now holds them and the API may change them, as before;
+		// the overlay is only for stores that must not hold them.
+		m.staticOIDCProviders = nil
+		return
 	}
-	for _, pc := range config.STS.Providers {
+	// The metadata subscription only reports changes made from now on, so
+	// providers already in the store would stay unknown until one changes; a
+	// failed load is retried (RefreshOIDCProvidersFromStore).
+	if err := m.RefreshOIDCProvidersFromStore(context.Background()); err != nil {
+		glog.Warningf("load OIDC providers from the store at startup: %v; retrying in the background", err)
+	}
+}
+
+// startOIDCRetry retries loading store in the background until it succeeds.
+// A store that is no longer current gets no retry: nothing would cancel it,
+// and its eventual success would replace the current store's providers. When
+// a retry is already running, it is asked to run once more instead, because
+// it may already have listed a snapshot older than this failure.
+func (m *IAMManager) startOIDCRetry(store OIDCProviderStore) {
+	m.oidcRetryMu.Lock()
+	defer m.oidcRetryMu.Unlock()
+	if store != m.oidcProviderStore {
+		return
+	}
+	if m.oidcRetryGen != 0 {
+		m.oidcRetryAgain = true
+		return
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	m.oidcRetrySeq++
+	gen := m.oidcRetrySeq
+	m.cancelOIDCLoad, m.oidcRetryGen = cancel, gen
+	bounds := oidcHydrateRetry // read here, not in the goroutine: it outlives its caller
+	go func() {
+		defer cancel()
+		for {
+			m.retryOIDCProviderLoad(ctx, store, bounds)
+			m.oidcRetryMu.Lock()
+			if m.oidcRetryGen != gen {
+				m.oidcRetryMu.Unlock()
+				return // cancelled: another store was installed
+			}
+			if m.oidcRetryAgain && ctx.Err() == nil {
+				m.oidcRetryAgain = false
+				m.oidcRetryMu.Unlock()
+				continue
+			}
+			m.cancelOIDCLoad, m.oidcRetryGen, m.oidcRetryAgain = nil, 0, false
+			m.oidcRetryMu.Unlock()
+			return
+		}
+	}()
+}
+
+// stopOIDCRetryLocked cancels a running retry, as installing another store
+// must. The caller holds oidcRetryMu.
+func (m *IAMManager) stopOIDCRetryLocked() {
+	if m.cancelOIDCLoad != nil {
+		m.cancelOIDCLoad()
+	}
+	m.cancelOIDCLoad, m.oidcRetryGen, m.oidcRetryAgain = nil, 0, false
+}
+
+// currentOIDCProviderStore is the installed store, read under oidcRetryMu.
+func (m *IAMManager) currentOIDCProviderStore() OIDCProviderStore {
+	m.oidcRetryMu.Lock()
+	defer m.oidcRetryMu.Unlock()
+	return m.oidcProviderStore
+}
+
+// staticOIDCProviderRecords describes the enabled OIDC providers of the IAM
+// config file as provider records.
+func staticOIDCProviderRecords(stsConfig *sts.STSConfig) map[string]*OIDCProviderRecord {
+	out := map[string]*OIDCProviderRecord{}
+	if stsConfig == nil {
+		return out
+	}
+	for _, pc := range stsConfig.Providers {
 		if pc == nil || !pc.Enabled || pc.Type != sts.ProviderTypeOIDC {
 			continue
 		}
 		issuer, _ := pc.Config["issuer"].(string)
 		if issuer == "" {
-			glog.Warningf("OIDC provider %s in static config has empty issuer; skipping mirror to store", pc.Name)
+			glog.Warningf("OIDC provider %s in static config has empty issuer; skipping", pc.Name)
 			continue
 		}
-		accountID := ""
-		if config.STS != nil {
-			accountID = config.STS.AccountId
-		}
-		arn, err := DeriveOIDCProviderARN(accountID, issuer)
+		arn, err := DeriveOIDCProviderARN(stsConfig.AccountId, issuer)
 		if err != nil {
 			glog.Warningf("derive ARN for static OIDC provider %s: %v", pc.Name, err)
 			continue
 		}
-		clientIDs := extractClientIDs(pc.Config)
-		ctx := context.Background()
-		// Preserve CreatedAt across reboots when a persistent store already
-		// has this provider — IAM's GetOpenIDConnectProvider response
-		// shouldn't shift its CreateDate every time the server restarts.
-		now := time.Now().UTC()
-		createdAt := now
-		if existing, err := store.GetProviderByARN(ctx, m.getFilerAddress(), arn); err == nil && existing != nil && !existing.CreatedAt.IsZero() {
-			createdAt = existing.CreatedAt
-		}
-		rec := &OIDCProviderRecord{
-			AccountID:               accountID,
+		out[arn] = &OIDCProviderRecord{
+			AccountID:               stsConfig.AccountId,
 			ARN:                     arn,
 			URL:                     issuer,
-			ClientIDs:               clientIDs,
+			ClientIDs:               extractClientIDs(pc.Config),
 			Thumbprints:             extractStringList(pc.Config, "thumbprints"),
 			AllowedPrincipalTagKeys: extractStringList(pc.Config, "allowedPrincipalTagKeys"),
 			PolicyClaim:             extractString(pc.Config, "policyClaim"),
-			CreatedAt:               createdAt,
-			UpdatedAt:               now,
+			// No CreatedAt: a config-file provider has no creation time the
+			// server could report consistently across restarts.
 		}
-		if err := store.StoreProvider(ctx, m.getFilerAddress(), rec); err != nil {
-			glog.Warningf("mirror static OIDC provider %s into store: %v", pc.Name, err)
+	}
+	return out
+}
+
+// oidcHydrateRetry bounds the backoff between startup load attempts.
+var oidcHydrateRetry = struct{ initial, max time.Duration }{initial: time.Second, max: 30 * time.Second}
+
+// retryOIDCProviderLoad retries loading store, the store it was started for,
+// until it succeeds or ctx is cancelled because another store was installed.
+// It calls refreshOIDCProvidersFrom, not RefreshOIDCProvidersFromStore, so it
+// never schedules a retry of its own.
+func (m *IAMManager) retryOIDCProviderLoad(ctx context.Context, store OIDCProviderStore, bounds struct{ initial, max time.Duration }) {
+	delay := bounds.initial
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(delay):
+		}
+		err := m.refreshOIDCProvidersFrom(ctx, store)
+		if err == nil {
+			glog.V(0).Infof("loaded OIDC providers from the store after retrying")
+			return
+		}
+		glog.V(1).Infof("load OIDC providers from the store: %v; retrying in %v", err, delay)
+		if delay *= 2; delay > bounds.max {
+			delay = bounds.max
 		}
 	}
-	return nil
 }
 
 // refreshOIDCProvidersBestEffort calls RefreshOIDCProvidersFromStore and
@@ -626,14 +902,46 @@ func (m *IAMManager) refreshOIDCProvidersBestEffort(ctx context.Context, op, arn
 // the store is empty (clears the IAM-managed map). Records with empty URLs
 // or invalid configuration are logged and skipped so a single bad entry
 // does not stop the rest from refreshing.
+//
+// A refresh that fails keeps retrying in the background until the store
+// answers. Every caller needs that: a metadata-subscription event reports each
+// change ONCE, so a refresh that found the filer unreachable on it would leave
+// a peer's new provider untrusted, or a deleted one trusted, until an unrelated
+// later change; the refresh after a local IAM API mutation and the startup load
+// have the same shape. At most one retry runs.
 func (m *IAMManager) RefreshOIDCProvidersFromStore(ctx context.Context) error {
-	if m.oidcProviderStore == nil || m.stsService == nil {
+	store := m.currentOIDCProviderStore()
+	err := m.refreshOIDCProvidersFrom(ctx, store)
+	if err != nil && store != nil {
+		m.startOIDCRetry(store)
+	}
+	return err
+}
+
+// refreshOIDCProvidersFrom is RefreshOIDCProvidersFromStore for a given store.
+func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCProviderStore) error {
+	if store == nil || m.stsService == nil {
 		return nil
 	}
-	records, err := m.oidcProviderStore.ListProviders(ctx, m.getFilerAddress())
+	// Refreshes run concurrently: after an IAM API change, on a peer's change
+	// and in the startup retry. Unserialized, a refresh that read the store
+	// before a DeleteOIDCProvider could finish after that call's own refresh
+	// and keep the deleted provider trusted.
+	m.oidcRefreshMu.Lock()
+	defer m.oidcRefreshMu.Unlock()
+	records, err := store.ListProviders(ctx, m.getFilerAddress())
 	if err != nil {
 		return fmt.Errorf("list OIDC providers: %w", err)
 	}
+	// A snapshot of a store that has since been replaced must not replace the
+	// current store's providers: a retry is cancelled when another store is
+	// installed, and a refresh may have listed the old store just before.
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if store != m.currentOIDCProviderStore() {
+		return fmt.Errorf("list OIDC providers: the store was replaced during the refresh")
+	}
 	byIssuer := make(map[string][]sts.ScopedOIDCProvider, len(records))
 	for _, rec := range records {
 		if rec == nil || rec.URL == "" {
@@ -664,9 +972,11 @@ func buildOIDCProviderFromRecord(rec *OIDCProviderRecord) (*oidc.OIDCProvider, e
 		return nil, fmt.Errorf("record cannot be nil")
 	}
 	cfg := &oidc.OIDCConfig{
-		Issuer:      rec.URL,
-		ClientIDs:   append([]string(nil), rec.ClientIDs...),
-		Thumbprints: append([]string(nil), rec.Thumbprints...),
+		Issuer:                  rec.URL,
+		ClientIDs:               append([]string(nil), rec.ClientIDs...),
+		Thumbprints:             append([]string(nil), rec.Thumbprints...),
+		AllowedPrincipalTagKeys: append([]string(nil), rec.AllowedPrincipalTagKeys...),
+		PolicyClaim:             rec.PolicyClaim,
 	}
 	provider := oidc.NewOIDCProvider(rec.ARN)
 	if err := provider.Initialize(cfg); err != nil {
@@ -800,14 +1110,51 @@ func (m *IAMManager) CreateRole(ctx context.Context, filerAddress string, roleNa
 	if !m.initialized {
 		return fmt.Errorf("IAM manager not initialized")
 	}
+	if err := PrepareRoleDefinition(roleName, roleDef); err != nil {
+		return err
+	}
+	if roleDef.RoleId == "" {
+		roleDef.RoleId = NewRoleID()
+	}
 
+	// Store role definition
+	return m.roleStore.StoreRole(ctx, "", roleName, roleDef)
+}
+
+// UpdateRole changes a role atomically in the role store (see
+// RoleStore.UpdateRole). update receives the role's current definition, nil
+// when it does not exist, and its result is validated like CreateRole's; a
+// nil result deletes the role.
+// The IAM API's role actions use it, so a change made on one S3 server is
+// neither lost to a concurrent change on another nor written over a delete.
+func (m *IAMManager) UpdateRole(ctx context.Context, roleName string, update RoleUpdate) error {
+	if !m.initialized {
+		return fmt.Errorf("IAM manager not initialized")
+	}
+	return m.roleStore.UpdateRole(ctx, "", roleName, func(current *RoleDefinition) (*RoleDefinition, error) {
+		next, err := update(current)
+		if err != nil || next == nil {
+			return next, err
+		}
+		if err := PrepareRoleDefinition(roleName, next); err != nil {
+			return nil, err
+		}
+		return next, nil
+	})
+}
+
+// PrepareRoleDefinition applies CreateRole's defaults and validation to a role
+// about to be stored or loaded.
+func PrepareRoleDefinition(roleName string, roleDef *RoleDefinition) error {
 	if roleName == "" {
 		return fmt.Errorf("role name cannot be empty")
 	}
-
 	if roleDef == nil {
 		return fmt.Errorf("role definition cannot be nil")
 	}
+	if roleDef.RoleName == "" {
+		roleDef.RoleName = roleName
+	}
 
 	// Set role ARN if not provided
 	if roleDef.RoleArn == "" {
@@ -827,12 +1174,122 @@ func (m *IAMManager) CreateRole(ctx context.Context, filerAddress string, roleNa
 			return fmt.Errorf("MaxSessionDuration must be between 3600 and 43200 seconds, got %d", roleDef.MaxSessionDuration)
 		}
 	}
+	return nil
+}
 
-	// Store role definition
-	return m.roleStore.StoreRole(ctx, "", roleName, roleDef)
+// LoadStaticRoles installs the roles of the IAM config file.
+//
+// An in-memory store holds them as records, as it always has. A persistent
+// store never does: it outlives this process and may be shared by S3 servers
+// with different config files, so a record written from one file would
+// outlive its removal from that file and be honoured by every server. Those
+// roles are served from memory instead, ahead of the store, and cannot be
+// changed or deleted through the store (ErrRoleStatic). A role stored under
+// the same name takes precedence. They report no creation time.
+func (m *IAMManager) LoadStaticRoles(ctx context.Context, roles []*RoleDefinition) {
+	defs := make(map[string]*RoleDefinition, len(roles))
+	for _, role := range roles {
+		if role == nil {
+			continue
+		}
+		role.Source = RoleSourceStaticConfig
+		if role.RoleId == "" {
+			role.RoleId = StaticRoleID(role)
+		}
+		if err := PrepareRoleDefinition(role.RoleName, role); err != nil {
+			glog.Warningf("Failed to load role %s: %v", role.RoleName, err)
+			continue
+		}
+		defs[role.RoleName] = role
+	}
+	m.staticRoles = defs
+	m.installRoleStore(ctx, m.roleStore)
+}
+
+// installRoleStore makes store the role store, with the config-file roles
+// installed in it as LoadStaticRoles describes, so a store set after startup
+// behaves like the one set at startup.
+func (m *IAMManager) installRoleStore(ctx context.Context, store RoleStore) {
+	if overlay, ok := store.(*staticRoleOverlay); ok {
+		store = overlay.inner
+	}
+	if store == nil || m.staticRoles == nil {
+		m.roleStore = store
+		return
+	}
+	if _, inMemory := store.(*MemoryRoleStore); inMemory {
+		for name, role := range m.staticRoles {
+			if err := store.StoreRole(ctx, "", name, role); err != nil {
+				glog.Warningf("Failed to create role %s: %v", name, err)
+			}
+		}
+		m.roleStore = store
+		return
+	}
+	m.roleStore = &staticRoleOverlay{static: m.staticRoles, inner: store}
 }
 
 // GetRole retrieves a role definition by name.
+// ListRoles returns every stored role definition.
+func (m *IAMManager) ListRoles(ctx context.Context) ([]*RoleDefinition, error) {
+	if !m.initialized {
+		return nil, fmt.Errorf("IAM manager not initialized")
+	}
+	names, err := m.roleStore.ListRoles(ctx, m.getFilerAddress())
+	if err != nil {
+		return nil, fmt.Errorf("list roles: %w", err)
+	}
+	roles := make([]*RoleDefinition, 0, len(names))
+	for _, name := range names {
+		role, err := m.roleStore.GetRole(ctx, m.getFilerAddress(), name)
+		if errors.Is(err, ErrRoleNotFound) {
+			continue // deleted between list and read
+		}
+		if err != nil {
+			return nil, fmt.Errorf("get role %s: %w", name, err)
+		}
+		roles = append(roles, role)
+	}
+	return roles, nil
+}
+
+// DeleteRole removes a role definition.
+func (m *IAMManager) DeleteRole(ctx context.Context, roleName string) error {
+	if !m.initialized {
+		return fmt.Errorf("IAM manager not initialized")
+	}
+	if roleName == "" {
+		return fmt.Errorf("role name cannot be empty")
+	}
+	return m.roleStore.DeleteRole(ctx, m.getFilerAddress(), roleName)
+}
+
+// InvalidateRoleCache drops any cached role definitions, so a change written
+// to the store by a peer is seen on the next lookup rather than after the
+// cache TTL.
+func (m *IAMManager) InvalidateRoleCache() {
+	if cached, ok := m.roleStore.(interface{ ClearCache() }); ok {
+		cached.ClearCache()
+	}
+}
+
+// RoleStoreDirectory is the filer directory the role store keeps roles in,
+// its configured basePath; empty when the store is not filer-backed. S3
+// servers watch it to drop cached roles when a peer changes one.
+func (m *IAMManager) RoleStoreDirectory() string {
+	store := m.roleStore
+	if overlay, ok := store.(*staticRoleOverlay); ok {
+		store = overlay.inner
+	}
+	if cached, ok := store.(*GenericCachedRoleStore); ok {
+		store = cached.adapter.store
+	}
+	if filerStore, ok := store.(*FilerRoleStore); ok {
+		return filerStore.basePath
+	}
+	return ""
+}
+
 func (m *IAMManager) GetRole(ctx context.Context, roleName string) (*RoleDefinition, error) {
 	if !m.initialized {
 		return nil, fmt.Errorf("IAM manager not initialized")
@@ -942,7 +1399,8 @@ func (m *IAMManager) AssumeRoleWithWebIdentity(ctx context.Context, request *sts
 	// Apply role-level MaxSessionDuration cap. The STS service still applies
 	// the global MaxSessionLength and the source-token-expiry cap on top of
 	// this; per-role takes precedence whenever it is the tightest bound.
-	request.DurationSeconds = capDurationByRole(request.DurationSeconds, roleDef.MaxSessionDuration)
+	request.DurationSeconds = capDurationByRole(request.DurationSeconds, roleDef.MaxSessionDuration, m.defaultTokenDurationSeconds(), m.maxSessionLengthSeconds())
+	request.RoleId = roleDef.RoleId
 
 	// Use STS service to assume the role
 	return m.stsService.AssumeRoleWithWebIdentity(ctx, request)
@@ -1005,22 +1463,54 @@ func extractIssuerFromJWT(token string) (string, error) {
 	return iss, nil
 }
 
-// capDurationByRole returns the requested duration clamped to the role's
-// MaxSessionDuration. A nil requested duration is left nil so the STS
-// service's calculateSessionDuration applies the global default (typically
-// 1 hour) — substituting the role's max here would silently mint a 12h
-// session for any caller who omitted DurationSeconds, which AWS does not
-// do. The role-max upper bound still applies in the downstream cap chain
-// once the request has a concrete duration.
-func capDurationByRole(requested *int64, roleMax int64) *int64 {
-	if roleMax <= 0 || requested == nil {
-		return requested
+// capDurationByRole returns the session duration clamped to the role's
+// MaxSessionDuration. An omitted DurationSeconds resolves to the configured
+// default first, so the role bound caps defaults and explicit values alike.
+// A nil request is only materialized when something tightened the default
+// and the explicit value still passes the service's own input validation —
+// everything else is left nil so the service resolves the default and its
+// MaxSessionLength cap itself.
+func capDurationByRole(requested *int64, roleMax, defaultSec, serviceMaxSec int64) *int64 {
+	if requested != nil {
+		d := *requested
+		if roleMax > 0 && d > roleMax {
+			d = roleMax
+		}
+		return &d
 	}
-	if *requested > roleMax {
-		v := roleMax
-		return &v
+	d := defaultSec
+	if roleMax > 0 && d > roleMax {
+		d = roleMax
 	}
-	return requested
+	if d > serviceMaxSec {
+		d = serviceMaxSec
+	}
+	if d < 900 && roleMax > 0 {
+		d = 900
+	}
+	if d >= defaultSec || d < 900 {
+		return nil
+	}
+	return &d
+}
+
+func (m *IAMManager) defaultTokenDurationSeconds() int64 {
+	if m.stsService == nil || m.stsService.Config == nil {
+		return sts.DefaultTokenDuration
+	}
+	return int64(m.stsService.Config.TokenDuration.Duration / time.Second)
+}
+
+// maxSessionLengthSeconds mirrors validateSessionDurationSeconds so a
+// materialized default stays inside the bound the service will enforce.
+func (m *IAMManager) maxSessionLengthSeconds() int64 {
+	maxSec := int64(sts.DefaultMaxSessionLength)
+	if m.stsService != nil && m.stsService.Config != nil && m.stsService.Config.MaxSessionLength.Duration > 0 {
+		if configured := int64(m.stsService.Config.MaxSessionLength.Duration / time.Second); configured >= 900 {
+			maxSec = configured
+		}
+	}
+	return maxSec
 }
 
 // AssumeRoleWithCredentials assumes a role using credentials (LDAP)
@@ -1044,9 +1534,10 @@ func (m *IAMManager) AssumeRoleWithCredentials(ctx context.Context, request *sts
 	}
 
 	// Apply role-level MaxSessionDuration cap.
-	request.DurationSeconds = capDurationByRole(request.DurationSeconds, roleDef.MaxSessionDuration)
+	request.DurationSeconds = capDurationByRole(request.DurationSeconds, roleDef.MaxSessionDuration, m.defaultTokenDurationSeconds(), m.maxSessionLengthSeconds())
 
 	// Use STS service to assume the role
+	request.RoleId = roleDef.RoleId
 	return m.stsService.AssumeRoleWithCredentials(ctx, request)
 }
 
@@ -1060,6 +1551,10 @@ func (m *IAMManager) IsActionAllowed(ctx context.Context, request *ActionRequest
 	// We always try to validate with the internal STS service first if it's a SeaweedFS token.
 	// This ensures that session policies embedded in the token are correctly extracted and enforced.
 	var sessionInfo *sts.SessionInfo
+	// boundRole is the role a session carrying a role ID was checked against;
+	// its policies are the ones evaluated, so the check and the evaluation
+	// see one definition even if the role is replaced in between.
+	var boundRole *RoleDefinition
 	if request.SessionToken != "" {
 		// Parse unverified to check issuer
 		parsed, _, err := new(jwt.Parser).ParseUnverified(request.SessionToken, jwt.MapClaims{})
@@ -1092,6 +1587,9 @@ func (m *IAMManager) IsActionAllowed(ctx context.Context, request *ActionRequest
 					return false, fmt.Errorf("session has been revoked")
 				}
 			}
+			if boundRole, err = m.checkSessionRoleBinding(ctx, sessionInfo); err != nil {
+				return false, err
+			}
 		}
 	}
 
@@ -1188,9 +1686,12 @@ func (m *IAMManager) IsActionAllowed(ctx context.Context, request *ActionRequest
 				policies = user.GetPolicyNames()
 			} else {
 				// Get role definition
-				roleDef, err := m.roleStore.GetRole(ctx, m.getFilerAddress(), roleName)
-				if err != nil {
-					return false, fmt.Errorf("role not found: %s", roleName)
+				roleDef := boundRole
+				if roleDef == nil || roleDef.RoleName != roleName {
+					roleDef, err = m.roleStore.GetRole(ctx, m.getFilerAddress(), roleName)
+					if err != nil {
+						return false, fmt.Errorf("role not found: %s", roleName)
+					}
 				}
 
 				hasManagedSubject = true

weed/iam/integration/iam_manager_trust.go

@@ -2,16 +2,30 @@ package integration
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
 	"github.com/seaweedfs/seaweedfs/weed/iam/utils"
 )
 
+// ErrTrustPolicyDenied is wrapped when a role's trust policy does not admit
+// the principal.
+var ErrTrustPolicyDenied = errors.New("trust policy denies access to principal")
+
 // ValidateTrustPolicyForPrincipal validates if a principal is allowed to assume a role
 func (m *IAMManager) ValidateTrustPolicyForPrincipal(ctx context.Context, roleArn, principalArn string) error {
+	_, err := m.ResolveRoleForPrincipal(ctx, roleArn, principalArn)
+	return err
+}
+
+// ResolveRoleForPrincipal returns the role roleArn names if its trust policy
+// admits principalArn. Issuing a session from the definition returned binds
+// the session to the role whose trust was evaluated, not to a role that
+// replaced it under the same name in between.
+func (m *IAMManager) ResolveRoleForPrincipal(ctx context.Context, roleArn, principalArn string) (*RoleDefinition, error) {
 	if !m.initialized {
-		return fmt.Errorf("IAM manager not initialized")
+		return nil, fmt.Errorf("IAM manager not initialized")
 	}
 
 	// Extract role name from ARN
@@ -20,11 +34,11 @@ func (m *IAMManager) ValidateTrustPolicyForPrincipal(ctx context.Context, roleAr
 	// Get role definition
 	roleDef, err := m.roleStore.GetRole(ctx, m.getFilerAddress(), roleName)
 	if err != nil {
-		return fmt.Errorf("failed to get role %s: %w", roleName, err)
+		return nil, fmt.Errorf("failed to get role %s: %w", roleName, err)
 	}
 
 	if roleDef.TrustPolicy == nil {
-		return fmt.Errorf("role has no trust policy")
+		return nil, fmt.Errorf("%w: role has no trust policy", ErrTrustPolicyDenied)
 	}
 
 	// Create evaluation context with RequestContext populated so that
@@ -43,8 +57,8 @@ func (m *IAMManager) ValidateTrustPolicyForPrincipal(ctx context.Context, roleAr
 
 	// Evaluate the trust policy
 	if !m.evaluateTrustPolicy(roleDef.TrustPolicy, evalCtx) {
-		return fmt.Errorf("trust policy denies access to principal: %s", principalArn)
+		return nil, fmt.Errorf("%w: %s", ErrTrustPolicyDenied, principalArn)
 	}
 
-	return nil
+	return roleDef, nil
 }

weed/iam/integration/oidc_provider_persist_test.go

@@ -0,0 +1,561 @@
+package integration
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/golang-jwt/jwt/v5"
+	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
+	"github.com/seaweedfs/seaweedfs/weed/iam/sts"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// persistentTestStore stands in for a store that outlives the process and may
+// be shared (the filer store): anything but *MemoryOIDCProviderStore is
+// treated as persistent.
+type persistentTestStore struct{ *MemoryOIDCProviderStore }
+
+const (
+	persistTestStaticIssuer = "https://static.example"
+	persistTestAPIIssuer    = "https://api.example"
+)
+
+// persistTestConfig is an IAM config whose file defines the given OIDC
+// issuers.
+func persistTestConfig(issuers ...string) *IAMConfig {
+	var providers []*sts.ProviderConfig
+	for i, issuer := range issuers {
+		providers = append(providers, &sts.ProviderConfig{
+			Name:    "static-" + string(rune('a'+i)),
+			Type:    sts.ProviderTypeOIDC,
+			Enabled: true,
+			Config:  map[string]interface{}{"issuer": issuer, "clientId": "aud"},
+		})
+	}
+	return &IAMConfig{
+		STS: &sts.STSConfig{
+			TokenDuration:    sts.FlexibleDuration{Duration: time.Hour},
+			MaxSessionLength: sts.FlexibleDuration{Duration: 12 * time.Hour},
+			Issuer:           "test-sts",
+			SigningKey:       []byte("test-signing-key-32-characters-long"),
+			Providers:        providers,
+		},
+		Policy: &policy.PolicyEngineConfig{DefaultEffect: "Deny", StoreType: "memory"},
+		Roles:  &RoleStoreConfig{StoreType: "memory"},
+	}
+}
+
+// startServer initializes a manager as an S3 server would at boot, with the
+// given config file providers and OIDC provider store.
+func startServer(t *testing.T, store OIDCProviderStore, issuers ...string) *IAMManager {
+	t.Helper()
+	cfg := persistTestConfig(issuers...)
+	mgr := NewIAMManager()
+	require.NoError(t, mgr.Initialize(cfg, func() string { return "localhost:8888" }))
+	mgr.installOIDCProviderStore(store, cfg.STS)
+	return mgr
+}
+
+func arnOf(t *testing.T, issuer string) string {
+	t.Helper()
+	arn, err := DeriveOIDCProviderARN("", issuer)
+	require.NoError(t, err)
+	return arn
+}
+
+func listedARNs(t *testing.T, mgr *IAMManager) map[string]bool {
+	t.Helper()
+	recs, err := mgr.ListOIDCProviders(context.Background())
+	require.NoError(t, err)
+	out := map[string]bool{}
+	for _, r := range recs {
+		out[r.ARN] = true
+	}
+	return out
+}
+
+func storedARNs(t *testing.T, store OIDCProviderStore) map[string]bool {
+	t.Helper()
+	recs, err := store.ListProviders(context.Background(), "")
+	require.NoError(t, err)
+	out := map[string]bool{}
+	for _, r := range recs {
+		out[r.ARN] = true
+	}
+	return out
+}
+
+func createAPIProvider(t *testing.T, mgr *IAMManager, issuer string) {
+	t.Helper()
+	require.NoError(t, mgr.CreateOIDCProvider(context.Background(), &OIDCProviderRecord{
+		ARN: arnOf(t, issuer), URL: issuer, ClientIDs: []string{"aud"},
+	}))
+}
+
+// stsKnowsIssuer reports whether STS resolves a provider for the issuer. The
+// token is not validly signed; the only question is which error comes back.
+func stsKnowsIssuer(t *testing.T, mgr *IAMManager, issuer string) bool {
+	t.Helper()
+	tok, err := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
+		"iss": issuer, "sub": "probe", "aud": "aud", "exp": time.Now().Add(time.Hour).Unix(),
+	}).SignedString([]byte("not-the-providers-key"))
+	require.NoError(t, err)
+	_, _, err = mgr.GetSTSService().ValidateWebIdentityToken(context.Background(), tok)
+	require.Error(t, err, "an unsigned-by-provider token for %s was accepted", issuer)
+	return !strings.Contains(err.Error(), "no identity provider registered")
+}
+
+// A persistent store never receives the config file's providers: it outlives
+// the file and may be shared with servers whose files differ.
+func TestPersistentStoreNeverHoldsConfigFileProviders(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, store, persistTestStaticIssuer)
+
+	assert.Empty(t, storedARNs(t, store), "a config-file provider was written to the persistent store")
+	assert.True(t, listedARNs(t, mgr)[arnOf(t, persistTestStaticIssuer)], "the IAM API no longer lists the config-file provider")
+	rec, err := mgr.GetOIDCProvider(context.Background(), arnOf(t, persistTestStaticIssuer))
+	require.NoError(t, err)
+	assert.Equal(t, persistTestStaticIssuer, rec.URL)
+	assert.True(t, stsKnowsIssuer(t, mgr, persistTestStaticIssuer), "STS stopped trusting the config-file provider")
+}
+
+// Servers sharing a store, with different config files, must not remove or
+// replace each other's providers — including a zero-config server.
+func TestServersSharingAStoreKeepEachOthersProviders(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	configured := startServer(t, store, persistTestStaticIssuer)
+	createAPIProvider(t, configured, persistTestAPIIssuer)
+
+	zeroConfig := startServer(t, store)
+
+	assert.True(t, storedARNs(t, store)[arnOf(t, persistTestAPIIssuer)], "a peer's start removed an API-created provider")
+	assert.True(t, listedARNs(t, configured)[arnOf(t, persistTestStaticIssuer)], "the configured server lost its config-file provider")
+	assert.False(t, listedARNs(t, zeroConfig)[arnOf(t, persistTestStaticIssuer)], "a server lists a provider only its peer's config file defines")
+	assert.True(t, stsKnowsIssuer(t, zeroConfig, persistTestAPIIssuer), "the peer does not trust the shared API-created provider")
+	assert.False(t, stsKnowsIssuer(t, zeroConfig, persistTestStaticIssuer), "the peer trusts a provider only another server's config file defines")
+}
+
+// Removing a provider from the config file revokes it at the next start.
+func TestRemovingAProviderFromTheConfigFileRevokesIt(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	startServer(t, store, persistTestStaticIssuer)
+
+	restarted := startServer(t, store) // the file no longer lists it
+
+	assert.False(t, listedARNs(t, restarted)[arnOf(t, persistTestStaticIssuer)])
+	assert.False(t, stsKnowsIssuer(t, restarted, persistTestStaticIssuer), "a provider removed from the config file is still trusted")
+}
+
+// The API cannot change a provider the config file defines, nor create one
+// with its ARN; the file is where it changes.
+func TestConfigFileProvidersCannotBeChangedThroughTheAPI(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, store, persistTestStaticIssuer)
+	ctx := context.Background()
+	arn := arnOf(t, persistTestStaticIssuer)
+
+	for name, err := range map[string]error{
+		"add client ID":     mgr.AddClientIDToOIDCProvider(ctx, arn, "other"),
+		"remove client ID":  mgr.RemoveClientIDFromOIDCProvider(ctx, arn, "aud"),
+		"update thumbprint": mgr.UpdateOIDCProviderThumbprints(ctx, arn, []string{"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"}),
+		"tag":               mgr.TagOIDCProvider(ctx, arn, map[string]string{"k": "v"}),
+		"untag":             mgr.UntagOIDCProvider(ctx, arn, []string{"k"}),
+		"delete":            mgr.DeleteOIDCProvider(ctx, arn),
+	} {
+		assert.ErrorIs(t, err, ErrOIDCProviderStatic, name)
+	}
+	err := mgr.CreateOIDCProvider(ctx, &OIDCProviderRecord{ARN: arn, URL: persistTestStaticIssuer, ClientIDs: []string{"aud"}})
+	assert.ErrorIs(t, err, ErrOIDCProviderAlreadyExists)
+	assert.Empty(t, storedARNs(t, store), "a refused change still wrote to the store")
+}
+
+// Providers created through the API on an earlier boot, or on a peer, are
+// trusted at startup rather than after the next change.
+func TestStartupLoadsStoredProvidersIntoSTS(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	createAPIProvider(t, startServer(t, store), persistTestAPIIssuer)
+
+	restarted := startServer(t, store)
+	assert.True(t, stsKnowsIssuer(t, restarted, persistTestAPIIssuer))
+}
+
+// An in-memory store keeps its behaviour: the config file's providers are
+// records in it, as before.
+func TestInMemoryStoreStillHoldsConfigFileProviders(t *testing.T) {
+	store := NewMemoryOIDCProviderStore()
+	startServer(t, store, persistTestStaticIssuer)
+	assert.True(t, storedARNs(t, store)[arnOf(t, persistTestStaticIssuer)])
+}
+
+// unreachableThenReadyStore fails its first reads, as a filer that is not up
+// yet when the S3 server starts.
+type unreachableThenReadyStore struct {
+	*MemoryOIDCProviderStore
+	mu        sync.Mutex
+	failsLeft int
+}
+
+func (s *unreachableThenReadyStore) ListProviders(ctx context.Context, addr string) ([]*OIDCProviderRecord, error) {
+	s.mu.Lock()
+	if s.failsLeft > 0 {
+		s.failsLeft--
+		s.mu.Unlock()
+		return nil, errors.New("filer unavailable")
+	}
+	s.mu.Unlock()
+	return s.MemoryOIDCProviderStore.ListProviders(ctx, addr)
+}
+
+// A store that cannot be read at startup is retried: the metadata
+// subscription only reports later changes, so without a retry the providers
+// already stored would stay unknown to STS.
+func TestStartupLoadRetriesUntilTheStoreIsReadable(t *testing.T) {
+	saved := oidcHydrateRetry
+	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, 5*time.Millisecond
+	t.Cleanup(func() { oidcHydrateRetry = saved })
+
+	seeded := NewMemoryOIDCProviderStore()
+	require.NoError(t, seeded.StoreProvider(context.Background(), "", &OIDCProviderRecord{
+		ARN: arnOf(t, persistTestAPIIssuer), URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
+	}))
+	mgr := startServer(t, &unreachableThenReadyStore{MemoryOIDCProviderStore: seeded, failsLeft: 3})
+
+	deadline := time.Now().Add(2 * time.Second)
+	for !stsKnowsIssuer(t, mgr, persistTestAPIIssuer) {
+		if time.Now().After(deadline) {
+			t.Fatal("STS never learned the stored provider after the store became readable")
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+}
+
+// A provider stored under a config-file provider's ARN takes precedence, as it
+// does in STS; the API then changes the stored one, and deleting it brings the
+// config-file provider back.
+func TestStoredProviderTakesPrecedenceOverTheConfigFileOne(t *testing.T) {
+	ctx := context.Background()
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	configured := startServer(t, store, persistTestStaticIssuer)
+	peer := startServer(t, store)
+	arn := arnOf(t, persistTestStaticIssuer)
+	require.NoError(t, peer.CreateOIDCProvider(ctx, &OIDCProviderRecord{ARN: arn, URL: persistTestStaticIssuer, ClientIDs: []string{"stored"}}))
+
+	rec, err := configured.GetOIDCProvider(ctx, arn)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"stored"}, rec.ClientIDs, "the config-file provider hides the stored one")
+	assert.NoError(t, configured.AddClientIDToOIDCProvider(ctx, arn, "more"), "the stored provider cannot be changed")
+
+	require.NoError(t, configured.DeleteOIDCProvider(ctx, arn))
+	rec, err = configured.GetOIDCProvider(ctx, arn)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"aud"}, rec.ClientIDs, "deleting the stored provider did not bring the config-file one back")
+}
+
+// A store installed through SetOIDCProviderStore behaves like one installed
+// at startup: stored providers load into STS and config-file providers stay
+// visible to the IAM API.
+func TestSetOIDCProviderStoreInstallsLikeStartup(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	require.NoError(t, store.StoreProvider(context.Background(), "", &OIDCProviderRecord{
+		ARN: arnOf(t, persistTestAPIIssuer), URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
+	}))
+
+	cfg := persistTestConfig(persistTestStaticIssuer)
+	mgr := NewIAMManager()
+	require.NoError(t, mgr.Initialize(cfg, func() string { return "localhost:8888" }))
+	mgr.SetOIDCProviderStore(store)
+
+	assert.True(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer), "a stored provider was not trusted after install")
+	assert.True(t, listedARNs(t, mgr)[arnOf(t, persistTestStaticIssuer)], "the IAM API no longer lists the config-file provider")
+	assert.ErrorIs(t, mgr.DeleteOIDCProvider(context.Background(), arnOf(t, persistTestStaticIssuer)), ErrOIDCProviderStatic)
+}
+
+// A config-file provider reports no creation time: a time taken at startup
+// would change with every restart.
+func TestConfigFileProvidersReportNoCreationTime(t *testing.T) {
+	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, store, persistTestStaticIssuer)
+	rec, err := mgr.GetOIDCProvider(context.Background(), arnOf(t, persistTestStaticIssuer))
+	require.NoError(t, err)
+	assert.True(t, rec.CreatedAt.IsZero())
+}
+
+// countingUnreadableStore never becomes readable and counts the attempts.
+type countingUnreadableStore struct {
+	*MemoryOIDCProviderStore
+	mu    sync.Mutex
+	reads int
+}
+
+func (s *countingUnreadableStore) ListProviders(context.Context, string) ([]*OIDCProviderRecord, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.reads++
+	return nil, errors.New("filer unavailable")
+}
+
+func (s *countingUnreadableStore) readCount() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.reads
+}
+
+// Installing another store stops the previous one's startup retry.
+func TestInstallingAnotherStoreStopsThePreviousRetry(t *testing.T) {
+	saved := oidcHydrateRetry
+	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, time.Millisecond
+	t.Cleanup(func() { oidcHydrateRetry = saved })
+
+	first := &countingUnreadableStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, first)
+	deadline := time.Now().Add(2 * time.Second)
+	for first.readCount() <= 1 {
+		if time.Now().After(deadline) {
+			t.Fatal("precondition: the first store is never retried")
+		}
+		time.Sleep(time.Millisecond)
+	}
+
+	mgr.installOIDCProviderStore(&persistentTestStore{NewMemoryOIDCProviderStore()}, persistTestConfig().STS)
+	time.Sleep(10 * time.Millisecond) // let an in-flight attempt finish
+	settled := first.readCount()
+	time.Sleep(30 * time.Millisecond)
+	assert.Equal(t, settled, first.readCount(), "the superseded store is still being retried")
+}
+
+// blockingListStore blocks the first ListProviders call after arm, having
+// already read its snapshot, until release is closed.
+type blockingListStore struct {
+	*MemoryOIDCProviderStore
+	mu      sync.Mutex
+	armed   bool
+	entered chan struct{}
+	release chan struct{}
+}
+
+func (s *blockingListStore) arm() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.armed, s.entered, s.release = true, make(chan struct{}), make(chan struct{})
+}
+
+func (s *blockingListStore) ListProviders(ctx context.Context, addr string) ([]*OIDCProviderRecord, error) {
+	records, err := s.MemoryOIDCProviderStore.ListProviders(ctx, addr)
+	s.mu.Lock()
+	block := s.armed
+	s.armed = false
+	s.mu.Unlock()
+	if block {
+		close(s.entered)
+		<-s.release
+	}
+	return records, err
+}
+
+// A refresh that read the store before a provider was deleted cannot leave the
+// deleted provider trusted by finishing after the deletion's own refresh.
+func TestAnOlderRefreshCannotRestoreADeletedProvider(t *testing.T) {
+	store := &blockingListStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, store)
+	createAPIProvider(t, mgr, persistTestAPIIssuer)
+	require.True(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer), "precondition: the provider is trusted")
+
+	store.arm()
+	stale := make(chan struct{})
+	go func() {
+		defer close(stale)
+		_ = mgr.RefreshOIDCProvidersFromStore(context.Background())
+	}()
+	<-store.entered // the stale refresh holds a snapshot with the provider
+
+	deleted := make(chan error, 1)
+	go func() { deleted <- mgr.DeleteOIDCProvider(context.Background(), arnOf(t, persistTestAPIIssuer)) }()
+	// Serialized, the deletion's refresh waits for the stale one; otherwise
+	// let it finish first, which is the ordering that went wrong.
+	var deleteErr error
+	select {
+	case deleteErr = <-deleted:
+		deleted = nil
+	case <-time.After(200 * time.Millisecond):
+	}
+	close(store.release)
+	<-stale
+	if deleted != nil {
+		deleteErr = <-deleted
+	}
+	require.NoError(t, deleteErr)
+	assert.False(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer), "a refresh older than the deletion left the deleted provider trusted")
+}
+
+// A refresh that fails is retried until the store answers. A change event reports each mutation once, so
+// a subscription refresh that found the filer unreachable — mid-restart, say — left the provider set stale
+// until some unrelated later change; a peer's new provider stayed untrusted and a deleted one trusted.
+func TestAFailedRefreshIsRetriedUntilTheStoreAnswers(t *testing.T) {
+	saved := oidcHydrateRetry
+	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, 5*time.Millisecond
+	t.Cleanup(func() { oidcHydrateRetry = saved })
+
+	store := &unreachableThenReadyStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, store)
+	require.NoError(t, store.StoreProvider(context.Background(), "", &OIDCProviderRecord{
+		ARN: arnOf(t, persistTestAPIIssuer), URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
+	}))
+	store.mu.Lock()
+	store.failsLeft = 3
+	store.mu.Unlock()
+	require.Error(t, mgr.RefreshOIDCProvidersFromStore(context.Background()), "precondition: the refresh fails")
+
+	deadline := time.Now().Add(2 * time.Second)
+	for !stsKnowsIssuer(t, mgr, persistTestAPIIssuer) {
+		if time.Now().After(deadline) {
+			t.Fatal("a failed refresh was never retried: the stored provider stays untrusted until an unrelated change")
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+}
+
+// Failures during an outage start ONE retry, not one per event: a filer that is down for a while produces a
+// change event per mutation, and each would otherwise add a goroutine polling the same store. Once the store
+// answers and the retry ends, a later failure starts a new one.
+func TestFailedRefreshesShareOneRetry(t *testing.T) {
+	saved := oidcHydrateRetry
+	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, time.Millisecond
+	t.Cleanup(func() { oidcHydrateRetry = saved })
+
+	store := &unreachableThenReadyStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, store)
+	store.mu.Lock()
+	store.failsLeft = 1 << 30
+	store.mu.Unlock()
+	for range 20 {
+		require.Error(t, mgr.RefreshOIDCProvidersFromStore(context.Background()))
+	}
+	mgr.oidcRetryMu.Lock()
+	started := mgr.oidcRetrySeq
+	mgr.oidcRetryMu.Unlock()
+	assert.Equal(t, uint64(1), started, "twenty failed refreshes started more than one retry")
+
+	store.mu.Lock()
+	store.failsLeft = 0
+	store.mu.Unlock()
+	deadline := time.Now().Add(2 * time.Second)
+	for {
+		mgr.oidcRetryMu.Lock()
+		running := mgr.oidcRetryGen != 0
+		mgr.oidcRetryMu.Unlock()
+		if !running {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("the retry never ended after the store answered")
+		}
+		time.Sleep(time.Millisecond)
+	}
+	store.mu.Lock()
+	store.failsLeft = 1
+	store.mu.Unlock()
+	require.Error(t, mgr.RefreshOIDCProvidersFromStore(context.Background()))
+	mgr.oidcRetryMu.Lock()
+	assert.Equal(t, uint64(2), mgr.oidcRetrySeq, "a failure after the retry ended starts a new one")
+	mgr.oidcRetryMu.Unlock()
+}
+
+// flakyCountingStore fails while fail is set and counts the lists that succeed.
+type flakyCountingStore struct {
+	*MemoryOIDCProviderStore
+	mu   sync.Mutex
+	fail bool
+	ok   int
+}
+
+func (s *flakyCountingStore) ListProviders(ctx context.Context, addr string) ([]*OIDCProviderRecord, error) {
+	s.mu.Lock()
+	if s.fail {
+		s.mu.Unlock()
+		return nil, errors.New("filer unavailable")
+	}
+	s.ok++
+	s.mu.Unlock()
+	return s.MemoryOIDCProviderStore.ListProviders(ctx, addr)
+}
+
+func (s *flakyCountingStore) set(fail bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.fail = fail
+}
+
+func (s *flakyCountingStore) successes() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.ok
+}
+
+// A refresh of store A that fails as store B is installed must not start a retry for A: nothing would cancel
+// it, and when A answered it would replace B's providers (a removed one trusted again, B's own untrusted).
+func TestAFailedRefreshOfASupersededStoreStartsNoRetry(t *testing.T) {
+	a := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	mgr := startServer(t, a)
+	mgr.SetOIDCProviderStore(&persistentTestStore{NewMemoryOIDCProviderStore()})
+	mgr.oidcRetryMu.Lock()
+	before := mgr.oidcRetrySeq
+	mgr.oidcRetryMu.Unlock()
+
+	mgr.startOIDCRetry(a) // the late call of A's failed refresh, after B's install
+
+	mgr.oidcRetryMu.Lock()
+	defer mgr.oidcRetryMu.Unlock()
+	assert.Equal(t, before, mgr.oidcRetrySeq, "a retry was started for the store that was replaced")
+	assert.Zero(t, mgr.oidcRetryGen)
+}
+
+// A refresh that listed store A before B was installed must not hand STS A's providers afterwards.
+func TestASupersededSnapshotIsNotApplied(t *testing.T) {
+	a := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	require.NoError(t, a.StoreProvider(context.Background(), "", &OIDCProviderRecord{
+		ARN: arnOf(t, persistTestAPIIssuer), URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
+	}))
+	b := &persistentTestStore{NewMemoryOIDCProviderStore()}
+	require.NoError(t, b.StoreProvider(context.Background(), "", &OIDCProviderRecord{
+		ARN: arnOf(t, persistTestStaticIssuer), URL: persistTestStaticIssuer, ClientIDs: []string{"aud"},
+	}))
+	mgr := startServer(t, a)
+	mgr.SetOIDCProviderStore(b)
+
+	assert.Error(t, mgr.refreshOIDCProvidersFrom(context.Background(), a), "a snapshot of the replaced store was applied")
+	assert.False(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer), "the replaced store's provider is trusted")
+	assert.True(t, stsKnowsIssuer(t, mgr, persistTestStaticIssuer), "the current store's provider is not")
+}
+
+// A refresh that fails while a retry runs is not dropped: the retry may already have listed an older snapshot,
+// so after its success it runs once more and picks up whatever the failed refresh would have seen.
+func TestAFailureDuringARetryIsNotDropped(t *testing.T) {
+	saved := oidcHydrateRetry
+	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, 2*time.Millisecond
+	t.Cleanup(func() { oidcHydrateRetry = saved })
+
+	store := &flakyCountingStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore(), fail: true}
+	mgr := startServer(t, store) // the startup load fails: a retry is running
+	require.Error(t, mgr.RefreshOIDCProvidersFromStore(context.Background()), "a later refresh fails while it runs")
+	store.set(false)
+
+	deadline := time.Now().Add(2 * time.Second)
+	for {
+		mgr.oidcRetryMu.Lock()
+		running := mgr.oidcRetryGen != 0
+		mgr.oidcRetryMu.Unlock()
+		if !running {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("the retry never ended")
+		}
+		time.Sleep(time.Millisecond)
+	}
+	assert.GreaterOrEqual(t, store.successes(), 2, "the retry ended on its first success, dropping the refresh that failed while it ran")
+}

weed/iam/integration/oidc_provider_store.go

@@ -27,6 +27,9 @@ import (
 var (
 	ErrOIDCProviderNotFound      = errors.New("OIDC provider not found")
 	ErrOIDCProviderAlreadyExists = errors.New("OIDC provider already exists")
+	// ErrOIDCProviderStatic refuses a change to a provider defined in the
+	// server's IAM config file: change it there instead.
+	ErrOIDCProviderStatic = errors.New("OIDC provider is defined in the IAM config file")
 )
 
 // OIDCProviderRecord is the persisted, IAM-managed view of an OIDC identity
@@ -267,15 +270,20 @@ func (f *FilerOIDCProviderStore) GetProviderByARN(ctx context.Context, filerAddr
 
 	var data []byte
 	err := f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
-		resp, err := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
+		resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
 			Directory: f.basePath,
 			Name:      f.fileName(arn),
 		})
+		// Only a confirmed absence is ErrOIDCProviderNotFound: callers create
+		// on it, so an unreachable filer must not read as "no such provider".
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			return fmt.Errorf("%w: %s", ErrOIDCProviderNotFound, arn)
+		}
 		if err != nil {
-			return fmt.Errorf("%w: %v", ErrOIDCProviderNotFound, err)
+			return fmt.Errorf("lookup OIDC provider %s: %w", arn, err)
 		}
 		if resp.Entry == nil {
-			return fmt.Errorf("OIDC provider not found: %s", arn)
+			return fmt.Errorf("%w: %s", ErrOIDCProviderNotFound, arn)
 		}
 		data = resp.Entry.Content
 		return nil

weed/iam/integration/role_id_session_binding_test.go

@@ -0,0 +1,165 @@
+package integration
+
+import (
+	"context"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
+	"github.com/seaweedfs/seaweedfs/weed/iam/sts"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A session is bound to the role it was issued for, not to the role's name:
+// deleting the role ends it, and a role created later under the same name —
+// by whoever may create roles — must not inherit it.
+func TestSessionIsBoundToTheRoleNotItsName(t *testing.T) {
+	ctx := context.Background()
+	m := setupIntegratedIAMSystem(t)
+
+	assume := func() (token, principal string) {
+		resp, err := m.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
+			RoleArn:          "arn:aws:iam::role/S3ReadOnlyRole",
+			WebIdentityToken: createTestJWT(t, "https://test-issuer.com", "test-user-123", "test-signing-key"),
+			RoleSessionName:  "binding-test",
+		})
+		require.NoError(t, err)
+		return resp.Credentials.SessionToken, resp.AssumedRoleUser.Arn
+	}
+	allowed := func(token, principal string) bool {
+		ok, _ := m.IsActionAllowed(ctx, &ActionRequest{
+			Principal:    principal,
+			Action:       "s3:GetObject",
+			Resource:     "arn:aws:s3:::test-bucket/file.txt",
+			SessionToken: token,
+		})
+		return ok
+	}
+
+	original, err := m.GetRole(ctx, "S3ReadOnlyRole")
+	require.NoError(t, err)
+	require.NotEmpty(t, original.RoleId, "a created role has an ID")
+
+	token, principal := assume()
+	require.True(t, allowed(token, principal), "precondition: the session works")
+
+	require.NoError(t, m.DeleteRole(ctx, "S3ReadOnlyRole"))
+	assert.False(t, allowed(token, principal), "the session outlived its role's deletion")
+
+	recreated := *original
+	recreated.RoleId = ""
+	require.NoError(t, m.CreateRole(ctx, "", "S3ReadOnlyRole", &recreated))
+	assert.NotEqual(t, original.RoleId, recreated.RoleId, "a role created again under the same name got the same ID")
+	assert.False(t, allowed(token, principal), "a session of the deleted role works again under a new role of the same name")
+
+	fresh, freshPrincipal := assume()
+	assert.True(t, allowed(fresh, freshPrincipal), "a session issued for the new role is refused")
+}
+
+// A session carries its role's attached policies, and S3 evaluates those
+// rather than looking the role up; the binding must hold on that path too.
+func TestASessionCarryingItsPoliciesIsStillBoundToItsRole(t *testing.T) {
+	ctx := context.Background()
+	m := setupIntegratedIAMSystem(t)
+
+	original, err := m.GetRole(ctx, "S3ReadOnlyRole")
+	require.NoError(t, err)
+	require.NotEmpty(t, original.AttachedPolicies, "precondition: the role attaches a policy")
+	resp, err := m.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
+		RoleArn:          "arn:aws:iam::role/S3ReadOnlyRole",
+		WebIdentityToken: createTestJWT(t, "https://test-issuer.com", "test-user-123", "test-signing-key"),
+		RoleSessionName:  "binding-test",
+	})
+	require.NoError(t, err)
+	allowed := func() bool {
+		ok, _ := m.IsActionAllowed(ctx, &ActionRequest{
+			Principal:    resp.AssumedRoleUser.Arn,
+			Action:       "s3:GetObject",
+			Resource:     "arn:aws:s3:::test-bucket/file.txt",
+			SessionToken: resp.Credentials.SessionToken,
+			PolicyNames:  original.AttachedPolicies,
+		})
+		return ok
+	}
+	require.True(t, allowed(), "precondition: the session works")
+
+	require.NoError(t, m.DeleteRole(ctx, "S3ReadOnlyRole"))
+	assert.False(t, allowed(), "the session's embedded policies outlived its role's deletion")
+
+	recreated := *original
+	recreated.RoleId = ""
+	require.NoError(t, m.CreateRole(ctx, "", "S3ReadOnlyRole", &recreated))
+	assert.False(t, allowed(), "the session's embedded policies work again under a new role of the same name")
+}
+
+func TestStaticRoleIDIsStableAndRuntimeIDsAreUnique(t *testing.T) {
+	trusting := func(principal string) *policy.PolicyDocument {
+		return &policy.PolicyDocument{Version: "2012-10-17", Statement: []policy.Statement{{
+			Effect: "Allow", Action: []string{"sts:AssumeRoleWithWebIdentity"},
+			Principal: map[string]interface{}{"Federated": principal},
+		}}}
+	}
+	app := &RoleDefinition{RoleName: "app", TrustPolicy: trusting("https://a.example")}
+	restarted := &RoleDefinition{RoleName: "app", TrustPolicy: trusting("https://a.example")}
+	replaced := &RoleDefinition{RoleName: "app", TrustPolicy: trusting("https://b.example")}
+
+	assert.Equal(t, StaticRoleID(app), StaticRoleID(restarted), "a config-file role must keep its ID across restarts")
+	assert.NotEqual(t, StaticRoleID(app), StaticRoleID(replaced), "a different role under the same name inherits the old one's ID")
+	assert.NotEqual(t, StaticRoleID(app), StaticRoleID(&RoleDefinition{RoleName: "other", TrustPolicy: trusting("https://a.example")}))
+	assert.Regexp(t, `^AROA[A-Z0-9]{17}$`, StaticRoleID(app))
+	assert.NotEqual(t, NewRoleID(), NewRoleID())
+	assert.Regexp(t, `^AROA[A-Z2-7]{17}$`, NewRoleID())
+}
+
+// replacedAfterFirstReadStore serves the stored role on its first read and a
+// replacement of the same name afterwards: a role replaced while a request
+// is being authorized.
+type replacedAfterFirstReadStore struct {
+	RoleStore
+	reads       int
+	replacement *RoleDefinition
+}
+
+func (s *replacedAfterFirstReadStore) GetRole(ctx context.Context, addr, name string) (*RoleDefinition, error) {
+	s.reads++
+	if s.reads > 1 && name == s.replacement.RoleName {
+		return copyRoleDefinition(s.replacement), nil
+	}
+	return s.RoleStore.GetRole(ctx, addr, name)
+}
+
+// The session's binding is checked against one definition of its role, and
+// that definition's policies are the ones evaluated: a replacement read in
+// between must not lend the session its permissions.
+func TestAuthorizationEvaluatesTheRoleTheBindingCheckSaw(t *testing.T) {
+	ctx := context.Background()
+	m := setupIntegratedIAMSystem(t)
+	require.NoError(t, m.CreatePolicy(ctx, "", "S3WritePolicy", &policy.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy.Statement{{
+			Effect: "Allow", Action: []string{"s3:PutObject"},
+			Resource: []string{"arn:aws:s3:::test-bucket/*"},
+		}},
+	}))
+	resp, err := m.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
+		RoleArn:          "arn:aws:iam::role/S3ReadOnlyRole",
+		WebIdentityToken: createTestJWT(t, "https://test-issuer.com", "test-user-123", "test-signing-key"),
+		RoleSessionName:  "snapshot-test",
+	})
+	require.NoError(t, err)
+
+	original, err := m.GetRole(ctx, "S3ReadOnlyRole")
+	require.NoError(t, err)
+	replacement := *original
+	replacement.RoleId = NewRoleID()
+	replacement.AttachedPolicies = []string{"S3WritePolicy"}
+	m.roleStore = &replacedAfterFirstReadStore{RoleStore: m.roleStore, replacement: &replacement}
+
+	allowed, _ := m.IsActionAllowed(ctx, &ActionRequest{
+		Principal:    resp.AssumedRoleUser.Arn,
+		Action:       "s3:PutObject",
+		Resource:     "arn:aws:s3:::test-bucket/file.txt",
+		SessionToken: resp.Credentials.SessionToken,
+	})
+	assert.False(t, allowed, "the session was authorized by the replacement role's policies")
+}

weed/iam/integration/role_max_session_test.go

@@ -6,21 +6,27 @@ func intPtr(v int64) *int64 { return &v }
 
 func TestCapDurationByRole(t *testing.T) {
 	cases := []struct {
-		name      string
-		requested *int64
-		roleMax   int64
-		want      *int64
+		name          string
+		requested     *int64
+		roleMax       int64
+		defaultSec    int64
+		serviceMaxSec int64
+		want          *int64
 	}{
-		{"no cap, no request", nil, 0, nil},
-		{"no cap, with request", intPtr(7200), 0, intPtr(7200)},
-		{"cap only, no request -> nil so STS default applies", nil, 3600, nil},
-		{"request below cap -> request", intPtr(1800), 3600, intPtr(1800)},
-		{"request equal cap -> request", intPtr(3600), 3600, intPtr(3600)},
-		{"request above cap -> cap", intPtr(43200), 3600, intPtr(3600)},
+		{"no cap, no request -> nil keeps service default", nil, 0, 3600, 43200, nil},
+		{"no cap, with request", intPtr(7200), 0, 3600, 43200, intPtr(7200)},
+		{"cap below default, no request -> cap", nil, 1800, 3600, 43200, intPtr(1800)},
+		{"cap above default, no request -> nil", nil, 43200, 3600, 43200, nil},
+		{"request below cap -> request", intPtr(1800), 3600, 900, 43200, intPtr(1800)},
+		{"request equal cap -> request", intPtr(3600), 3600, 900, 43200, intPtr(3600)},
+		{"request above cap -> cap", intPtr(43200), 3600, 900, 43200, intPtr(3600)},
+		{"default above service max -> service cap materialized", nil, 0, 7200, 3600, intPtr(3600)},
+		{"role bound above service max -> service cap still applies", nil, 40000, 43200, 3600, intPtr(3600)},
+		{"role bound below service floor -> tightest issuable", nil, 500, 3600, 43200, intPtr(900)},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
-			got := capDurationByRole(tc.requested, tc.roleMax)
+			got := capDurationByRole(tc.requested, tc.roleMax, tc.defaultSec, tc.serviceMaxSec)
 			switch {
 			case got == nil && tc.want == nil:
 				return

weed/iam/integration/role_store.go

@@ -3,7 +3,10 @@ package integration
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -13,9 +16,38 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
+// ErrRoleNotFound is wrapped by every RoleStore's GetRole when the role does
+// not exist, so callers can tell a missing role from a store that could not
+// be read.
+var ErrRoleNotFound = errors.New("role not found")
+
+// ErrRoleStatic refuses a change to a role defined in the server's IAM config
+// file: change it there instead.
+var ErrRoleStatic = errors.New("role is defined in the IAM config file")
+
+// ErrRoleExists is returned by an UpdateRole whose update refuses a role that
+// already exists (CreateRole).
+var ErrRoleExists = errors.New("role already exists")
+
+// RoleUpdate computes a role's new definition from its current one, nil when
+// the role does not exist. It returns nil to delete the role, and an error to
+// leave it unchanged. It may run more than once: it is called again with the
+// fresh definition when another writer changed the role in between.
+type RoleUpdate func(current *RoleDefinition) (*RoleDefinition, error)
+
+// maxRoleUpdateAttempts bounds UpdateRole's retries under contention.
+const maxRoleUpdateAttempts = 10
+
+// errRoleUpdateContended is returned when the role kept changing under
+// UpdateRole for maxRoleUpdateAttempts reads.
+var errRoleUpdateContended = errors.New("role changed concurrently; retry")
+
 // RoleStore defines the interface for storing IAM role definitions
 type RoleStore interface {
 	// StoreRole stores a role definition (filerAddress ignored for memory stores)
@@ -29,6 +61,14 @@ type RoleStore interface {
 
 	// DeleteRole deletes a role definition (filerAddress ignored for memory stores)
 	DeleteRole(ctx context.Context, filerAddress string, roleName string) error
+
+	// UpdateRole replaces a role with update's result, atomically: the write
+	// lands only if the role is still as update saw it, absent included, and
+	// update is retried against the current role otherwise. Every change the
+	// IAM API and the filer IAM service make to a role goes through it, so
+	// writers on different servers neither lose each other's changes nor bring
+	// back a role deleted in between.
+	UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error
 }
 
 // MemoryRoleStore implements RoleStore using in-memory storage
@@ -72,7 +112,7 @@ func (m *MemoryRoleStore) GetRole(ctx context.Context, filerAddress string, role
 
 	role, exists := m.roles[roleName]
 	if !exists {
-		return nil, fmt.Errorf("role not found: %s", roleName)
+		return nil, fmt.Errorf("%w: %s", ErrRoleNotFound, roleName)
 	}
 
 	// Return a copy to prevent external modifications
@@ -105,6 +145,26 @@ func (m *MemoryRoleStore) DeleteRole(ctx context.Context, filerAddress string, r
 	return nil
 }
 
+// UpdateRole applies update under the store's lock (filerAddress ignored for
+// memory store).
+func (m *MemoryRoleStore) UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error {
+	if roleName == "" {
+		return fmt.Errorf("role name cannot be empty")
+	}
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	next, err := update(copyRoleDefinition(m.roles[roleName]))
+	if err != nil {
+		return err
+	}
+	if next == nil {
+		delete(m.roles, roleName)
+		return nil
+	}
+	m.roles[roleName] = copyRoleDefinition(next)
+	return nil
+}
+
 // copyRoleDefinition creates a deep copy of a role definition
 func copyRoleDefinition(original *RoleDefinition) *RoleDefinition {
 	if original == nil {
@@ -112,9 +172,13 @@ func copyRoleDefinition(original *RoleDefinition) *RoleDefinition {
 	}
 
 	copied := &RoleDefinition{
-		RoleName:    original.RoleName,
-		RoleArn:     original.RoleArn,
-		Description: original.Description,
+		RoleName:           original.RoleName,
+		RoleArn:            original.RoleArn,
+		Description:        original.Description,
+		MaxSessionDuration: original.MaxSessionDuration,
+		Source:             original.Source,
+		CreatedAt:          original.CreatedAt,
+		RoleId:             original.RoleId,
 	}
 
 	// Deep copy trust policy if it exists
@@ -135,6 +199,10 @@ func copyRoleDefinition(original *RoleDefinition) *RoleDefinition {
 	return copied
 }
 
+// roleListPageSize is the number of entries FilerRoleStore.ListRoles asks the
+// filer for per page.
+const roleListPageSize = 1000
+
 // FilerRoleStore implements RoleStore using SeaweedFS filer
 type FilerRoleStore struct {
 	grpcDialOption       grpc.DialOption
@@ -213,6 +281,139 @@ func (f *FilerRoleStore) StoreRole(ctx context.Context, filerAddress string, rol
 	})
 }
 
+// UpdateRole reads the role's entry, applies update, and writes the result on
+// the condition that the entry is unchanged since the read — absent, when the
+// role did not exist — so the filer refuses a write racing another writer's
+// change or delete, and update is applied again to what that writer left. A
+// delete is made on the same condition, so it removes the role update saw and
+// not one written after it.
+func (f *FilerRoleStore) UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error {
+	if filerAddress == "" && f.filerAddressProvider != nil {
+		filerAddress = f.filerAddressProvider()
+	}
+	if filerAddress == "" {
+		return fmt.Errorf("filer address is required for FilerRoleStore")
+	}
+	if roleName == "" {
+		return fmt.Errorf("role name cannot be empty")
+	}
+	return f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
+		for attempt := 0; attempt < maxRoleUpdateAttempts; attempt++ {
+			var entry *filer_pb.Entry
+			var current *RoleDefinition
+			resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
+				Directory: f.basePath,
+				Name:      f.getRoleFileName(roleName),
+			})
+			switch {
+			case errors.Is(err, filer_pb.ErrNotFound):
+			case err != nil:
+				return fmt.Errorf("lookup role %s: %w", roleName, err)
+			case resp.Entry != nil:
+				entry = resp.Entry
+				current = &RoleDefinition{}
+				if err := json.Unmarshal(entry.Content, current); err != nil {
+					return fmt.Errorf("failed to deserialize role %s: %v", roleName, err)
+				}
+			}
+
+			next, err := update(current)
+			if err != nil {
+				return err
+			}
+			if next == nil {
+				if entry == nil {
+					return nil
+				}
+				deleted, err := f.deleteRoleEntryIfUnchanged(ctx, client, entry)
+				if err != nil {
+					return fmt.Errorf("failed to delete role %s: %w", roleName, err)
+				}
+				if !deleted {
+					glog.V(3).Infof("Role %s changed before its delete; retrying", roleName)
+					continue
+				}
+				return nil
+			}
+			roleData, err := json.MarshalIndent(next, "", "  ")
+			if err != nil {
+				return fmt.Errorf("failed to serialize role: %v", err)
+			}
+
+			clause := &filer_pb.WriteCondition_Clause{Kind: filer_pb.WriteCondition_IF_NOT_EXISTS}
+			if entry != nil {
+				clause = &filer_pb.WriteCondition_Clause{Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, ExpectedEntry: entry}
+			}
+			now := time.Now().Unix()
+			created, err := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
+				Directory: f.basePath,
+				Entry: &filer_pb.Entry{
+					Name: f.getRoleFileName(roleName),
+					Attributes: &filer_pb.FuseAttributes{
+						Mtime:    now,
+						Crtime:   now,
+						FileMode: uint32(0600),
+					},
+					Content: roleData,
+				},
+				Condition: &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{clause}},
+			})
+			if isRoleWriteConflict(created, err) {
+				glog.V(3).Infof("Role %s changed during update; retrying", roleName)
+				continue
+			}
+			if err != nil {
+				return fmt.Errorf("failed to store role %s: %v", roleName, err)
+			}
+			if created.Error != "" {
+				return fmt.Errorf("failed to store role %s: %s", roleName, created.Error)
+			}
+			return nil
+		}
+		return fmt.Errorf("update role %s: %w", roleName, errRoleUpdateContended)
+	})
+}
+
+// deleteRoleEntryIfUnchanged deletes the role's entry if it still equals
+// entry, reporting false when it changed. The delete is routed and locked as
+// the conditional CreateEntry of the same path is, so the two serialize.
+func (f *FilerRoleStore) deleteRoleEntryIfUnchanged(ctx context.Context, client filer_pb.SeaweedFilerClient, entry *filer_pb.Entry) (bool, error) {
+	fullPath := f.basePath + "/" + entry.Name
+	resp, err := client.ObjectTransaction(ctx, &filer_pb.ObjectTransactionRequest{
+		LockKey:  fullPath,
+		RouteKey: s3_constants.ObjectWriteRouteKeyPrefix + fullPath,
+		Condition: &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{
+			Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, ExpectedEntry: entry,
+		}}},
+		Mutations: []*filer_pb.ObjectMutation{{
+			Type: filer_pb.ObjectMutation_DELETE, Directory: f.basePath, Name: entry.Name, IsDeleteData: true,
+		}},
+	})
+	if err != nil {
+		if status.Code(err) == codes.FailedPrecondition {
+			return false, nil
+		}
+		return false, err
+	}
+	if resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED {
+		return false, nil
+	}
+	if resp.Error != "" {
+		return false, errors.New(resp.Error)
+	}
+	return true, nil
+}
+
+// isRoleWriteConflict reports a write the filer refused because its condition
+// no longer held: in the response, or as FailedPrecondition when the write
+// was forwarded to the entry's owner filer.
+func isRoleWriteConflict(resp *filer_pb.CreateEntryResponse, err error) bool {
+	if err != nil {
+		return status.Code(err) == codes.FailedPrecondition
+	}
+	return resp != nil && resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED
+}
+
 // GetRole retrieves a role definition from filer
 func (f *FilerRoleStore) GetRole(ctx context.Context, filerAddress string, roleName string) (*RoleDefinition, error) {
 	// Use provider function if filerAddress is not provided
@@ -234,13 +435,16 @@ func (f *FilerRoleStore) GetRole(ctx context.Context, filerAddress string, roleN
 		}
 
 		glog.V(3).Infof("Looking up role %s", roleName)
-		response, err := client.LookupDirectoryEntry(ctx, request)
+		response, err := filer_pb.LookupEntry(ctx, client, request)
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			return fmt.Errorf("%w: %s", ErrRoleNotFound, roleName)
+		}
 		if err != nil {
-			return fmt.Errorf("role not found: %v", err)
+			return fmt.Errorf("lookup role %s: %w", roleName, err)
 		}
 
 		if response.Entry == nil {
-			return fmt.Errorf("role not found")
+			return fmt.Errorf("%w: %s", ErrRoleNotFound, roleName)
 		}
 
 		roleData = response.Entry.Content
@@ -270,46 +474,50 @@ func (f *FilerRoleStore) ListRoles(ctx context.Context, filerAddress string) ([]
 		return nil, fmt.Errorf("filer address is required for FilerRoleStore")
 	}
 
+	// Page through the directory, and fail on a broken stream rather than
+	// return what arrived: a truncated list would let DeletePolicy miss a role
+	// that still attaches the policy (RolesAttachingPolicy).
 	var roleNames []string
-
 	err := f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
-		request := &filer_pb.ListEntriesRequest{
-			Directory:          f.basePath,
-			Prefix:             "",
-			StartFromFileName:  "",
-			InclusiveStartFrom: false,
-			Limit:              1000, // Process in batches of 1000
-		}
-
 		glog.V(3).Infof("Listing roles in %s", f.basePath)
-		stream, err := client.ListEntries(ctx, request)
-		if err != nil {
-			return fmt.Errorf("failed to list roles: %v", err)
-		}
-
+		startFrom := ""
 		for {
-			resp, err := stream.Recv()
+			stream, err := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
+				Directory:         f.basePath,
+				StartFromFileName: startFrom,
+				Limit:             roleListPageSize,
+			})
 			if err != nil {
-				break // End of stream or error
+				return err
 			}
-
-			if resp.Entry == nil || resp.Entry.IsDirectory {
-				continue
+			received := 0
+			for {
+				resp, err := stream.Recv()
+				if errors.Is(err, io.EOF) {
+					break
+				}
+				if err != nil {
+					return err
+				}
+				if resp.Entry == nil {
+					continue
+				}
+				received++
+				startFrom = resp.Entry.Name
+				if resp.Entry.IsDirectory {
+					continue
+				}
+				if name, ok := strings.CutSuffix(resp.Entry.Name, ".json"); ok {
+					roleNames = append(roleNames, name)
+				}
 			}
-
-			// Extract role name from filename
-			filename := resp.Entry.Name
-			if strings.HasSuffix(filename, ".json") {
-				roleName := strings.TrimSuffix(filename, ".json")
-				roleNames = append(roleNames, roleName)
+			if received < roleListPageSize {
+				return nil
 			}
 		}
-
-		return nil
 	})
-
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to list roles: %w", err)
 	}
 
 	return roleNames, nil
@@ -388,3 +596,146 @@ type CachedFilerRoleStoreConfig struct {
 	ListTTL      string `json:"listTtl,omitempty"`      // e.g., "1m", "30s"
 	MaxCacheSize int    `json:"maxCacheSize,omitempty"` // Maximum number of cached roles
 }
+
+// RolesAttachingPolicy returns the names of the roles that attach the policy.
+// A policy is attached to a role by name, so deleting it while attached would
+// let a policy created later under the same name take effect on the role;
+// callers refuse the delete instead, as AWS does (DeleteConflict).
+//
+// It sees the stored roles, which every server sharing the store sees, and
+// this server's config-file roles. A role defined only in another server's
+// config file is invisible here: servers sharing a role store may have
+// different config files, but a config-file role that attaches a managed
+// (API-created) policy is protected only on the servers whose file defines it.
+// Keep such roles in every server's file, or attach only config-file policies
+// to config-file roles.
+func RolesAttachingPolicy(ctx context.Context, store RoleStore, policyName string) ([]string, error) {
+	names, err := store.ListRoles(ctx, "")
+	if err != nil {
+		return nil, fmt.Errorf("list roles: %w", err)
+	}
+	var attaching []string
+	for _, name := range names {
+		role, err := store.GetRole(ctx, "", name)
+		if errors.Is(err, ErrRoleNotFound) {
+			continue // deleted between list and read
+		}
+		if err != nil {
+			return nil, fmt.Errorf("get role %s: %w", name, err)
+		}
+		for _, attached := range role.AttachedPolicies {
+			if attached == policyName {
+				attaching = append(attaching, name)
+				break
+			}
+		}
+	}
+	return attaching, nil
+}
+
+// MaxManagedPoliciesPerRole caps the managed policies attached to one role,
+// as AWS's default quota does (and as MaxManagedPoliciesPerUser does for users).
+const MaxManagedPoliciesPerRole = 10
+
+var roleNamePattern = regexp.MustCompile(`^[\w+=,.@-]{1,64}$`)
+
+// ValidateRoleName checks a role name against AWS's rules. A role is stored as
+// <name>.json in the filer, so the rules also keep a name from leaving the
+// role store's directory.
+func ValidateRoleName(name string) error {
+	if !roleNamePattern.MatchString(name) {
+		return fmt.Errorf("invalid role name %q: must be 1-64 characters of letters, digits and +=,.@_-", name)
+	}
+	return nil
+}
+
+// staticRoleOverlay serves the IAM config file's roles from memory beside a
+// persistent store, which never holds them (see IAMManager.LoadStaticRoles).
+// A role stored under the same name takes precedence, as a stored OIDC
+// provider does over a config-file one; deleting it brings the config-file
+// role back. A config-file role itself cannot be stored over or deleted.
+type staticRoleOverlay struct {
+	static map[string]*RoleDefinition
+	inner  RoleStore
+}
+
+// storedRoleExists reports whether the store itself holds the role.
+func (o *staticRoleOverlay) storedRoleExists(ctx context.Context, filerAddress, roleName string) (bool, error) {
+	_, err := o.inner.GetRole(ctx, filerAddress, roleName)
+	if errors.Is(err, ErrRoleNotFound) {
+		return false, nil
+	}
+	return err == nil, err
+}
+
+func (o *staticRoleOverlay) StoreRole(ctx context.Context, filerAddress string, roleName string, role *RoleDefinition) error {
+	if _, ok := o.static[roleName]; ok {
+		stored, err := o.storedRoleExists(ctx, filerAddress, roleName)
+		if err != nil {
+			return err
+		}
+		if !stored {
+			return fmt.Errorf("%w: %s", ErrRoleStatic, roleName)
+		}
+	}
+	return o.inner.StoreRole(ctx, filerAddress, roleName, role)
+}
+
+// UpdateRole refuses to create a stored role over a config-file role, as
+// StoreRole does; a stored role of that name may be changed.
+func (o *staticRoleOverlay) UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error {
+	_, static := o.static[roleName]
+	return o.inner.UpdateRole(ctx, filerAddress, roleName, func(current *RoleDefinition) (*RoleDefinition, error) {
+		if current == nil && static {
+			return nil, fmt.Errorf("%w: %s", ErrRoleStatic, roleName)
+		}
+		return update(current)
+	})
+}
+
+func (o *staticRoleOverlay) GetRole(ctx context.Context, filerAddress string, roleName string) (*RoleDefinition, error) {
+	role, err := o.inner.GetRole(ctx, filerAddress, roleName)
+	if errors.Is(err, ErrRoleNotFound) {
+		if static, ok := o.static[roleName]; ok {
+			return copyRoleDefinition(static), nil
+		}
+	}
+	return role, err
+}
+
+func (o *staticRoleOverlay) ListRoles(ctx context.Context, filerAddress string) ([]string, error) {
+	names, err := o.inner.ListRoles(ctx, filerAddress)
+	if err != nil {
+		return nil, err
+	}
+	seen := make(map[string]bool, len(names))
+	for _, name := range names {
+		seen[name] = true
+	}
+	for name := range o.static {
+		if !seen[name] {
+			names = append(names, name)
+		}
+	}
+	return names, nil
+}
+
+func (o *staticRoleOverlay) DeleteRole(ctx context.Context, filerAddress string, roleName string) error {
+	if _, ok := o.static[roleName]; ok {
+		stored, err := o.storedRoleExists(ctx, filerAddress, roleName)
+		if err != nil {
+			return err
+		}
+		if !stored {
+			return fmt.Errorf("%w: %s", ErrRoleStatic, roleName)
+		}
+	}
+	return o.inner.DeleteRole(ctx, filerAddress, roleName)
+}
+
+// ClearCache forwards cache invalidation to the store underneath.
+func (o *staticRoleOverlay) ClearCache() {
+	if cached, ok := o.inner.(interface{ ClearCache() }); ok {
+		cached.ClearCache()
+	}
+}

weed/iam/integration/role_store_filer_test.go

@@ -0,0 +1,286 @@
+package integration
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"slices"
+	"sort"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+)
+
+// roleStoreTestFiler is a filer holding one directory. It evaluates the write
+// conditions FilerRoleStore sends as the filer does, and can run another
+// writer between a role's read and its write (afterLookup) or break a listing
+// stream partway (failListAfter).
+type roleStoreTestFiler struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	mu            sync.Mutex
+	entries       map[string]*filer_pb.Entry
+	afterLookup   func()
+	failListAfter int
+}
+
+func (s *roleStoreTestFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	s.mu.Lock()
+	entry, found := s.entries[req.Name]
+	hook := s.afterLookup
+	s.afterLookup = nil
+	s.mu.Unlock()
+	if hook != nil {
+		defer hook()
+	}
+	if !found {
+		return nil, status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())
+	}
+	return &filer_pb.LookupDirectoryEntryResponse{Entry: proto.Clone(entry).(*filer_pb.Entry)}, nil
+}
+
+func (s *roleStoreTestFiler) CreateEntry(_ context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	current, exists := s.entries[req.Entry.Name]
+	for _, c := range req.GetCondition().GetClauses() {
+		var ok bool
+		switch c.Kind {
+		case filer_pb.WriteCondition_IF_NOT_EXISTS:
+			ok = !exists
+		case filer_pb.WriteCondition_IF_ENTRY_EQUAL:
+			ok = exists && proto.Equal(current, c.ExpectedEntry)
+		default:
+			return nil, fmt.Errorf("unexpected condition %v", c.Kind)
+		}
+		if !ok {
+			return &filer_pb.CreateEntryResponse{Error: "precondition failed", ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
+		}
+	}
+	s.entries[req.Entry.Name] = proto.Clone(req.Entry).(*filer_pb.Entry)
+	return &filer_pb.CreateEntryResponse{}, nil
+}
+
+// ObjectTransaction applies the conditional delete FilerRoleStore sends.
+func (s *roleStoreTestFiler) ObjectTransaction(_ context.Context, req *filer_pb.ObjectTransactionRequest) (*filer_pb.ObjectTransactionResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if len(req.Mutations) != 1 || req.Mutations[0].Type != filer_pb.ObjectMutation_DELETE {
+		return nil, fmt.Errorf("unexpected mutations %v", req.Mutations)
+	}
+	name := req.Mutations[0].Name
+	current, exists := s.entries[name]
+	for _, c := range req.GetCondition().GetClauses() {
+		if c.Kind != filer_pb.WriteCondition_IF_ENTRY_EQUAL {
+			return nil, fmt.Errorf("unexpected condition %v", c.Kind)
+		}
+		if !exists || !proto.Equal(current, c.ExpectedEntry) {
+			return &filer_pb.ObjectTransactionResponse{Error: "precondition failed", ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
+		}
+	}
+	delete(s.entries, name)
+	return &filer_pb.ObjectTransactionResponse{}, nil
+}
+
+func (s *roleStoreTestFiler) ListEntries(req *filer_pb.ListEntriesRequest, stream grpc.ServerStreamingServer[filer_pb.ListEntriesResponse]) error {
+	s.mu.Lock()
+	var names []string
+	for name := range s.entries {
+		if name > req.StartFromFileName {
+			names = append(names, name)
+		}
+	}
+	sort.Strings(names)
+	if req.Limit > 0 && len(names) > int(req.Limit) {
+		names = names[:req.Limit]
+	}
+	page := make([]*filer_pb.Entry, 0, len(names))
+	for _, name := range names {
+		page = append(page, proto.Clone(s.entries[name]).(*filer_pb.Entry))
+	}
+	failAfter := s.failListAfter
+	s.mu.Unlock()
+	for i, entry := range page {
+		if failAfter > 0 && i == failAfter {
+			return status.Error(codes.Unavailable, "filer went away")
+		}
+		if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: entry}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *roleStoreTestFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	delete(s.entries, req.Name)
+	return &filer_pb.DeleteEntryResponse{}, nil
+}
+
+func newTestFilerRoleStore(t *testing.T) (*FilerRoleStore, *roleStoreTestFiler) {
+	t.Helper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	filer := &roleStoreTestFiler{entries: map[string]*filer_pb.Entry{}}
+	server := pb.NewGrpcServer()
+	filer_pb.RegisterSeaweedFilerServer(server, filer)
+	go func() { _ = server.Serve(lis) }()
+	t.Cleanup(func() {
+		server.Stop()
+		_ = lis.Close()
+	})
+	host, port, err := net.SplitHostPort(lis.Addr().String())
+	require.NoError(t, err)
+	grpcPort, err := strconv.Atoi(port)
+	require.NoError(t, err)
+	store, err := NewFilerRoleStore(nil, func() string { return string(pb.NewServerAddress(host, 1, grpcPort)) })
+	require.NoError(t, err)
+	store.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
+	return store, filer
+}
+
+func attachPolicy(policyName string) RoleUpdate {
+	return func(current *RoleDefinition) (*RoleDefinition, error) {
+		if current == nil {
+			return nil, ErrRoleNotFound
+		}
+		current.AttachedPolicies = append(current.AttachedPolicies, policyName)
+		return current, nil
+	}
+}
+
+func createRole(roleID string) RoleUpdate {
+	return func(current *RoleDefinition) (*RoleDefinition, error) {
+		if current != nil {
+			return nil, ErrRoleExists
+		}
+		return &RoleDefinition{RoleName: "app", RoleId: roleID}, nil
+	}
+}
+
+// Two servers changing one role: the change written second is applied to
+// what the first left, not to the role as it read it.
+func TestFilerRoleUpdateIsNotLostToAConcurrentChange(t *testing.T) {
+	ctx := context.Background()
+	store, filer := newTestFilerRoleStore(t)
+	require.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA1")))
+
+	filer.afterLookup = func() { assert.NoError(t, store.UpdateRole(ctx, "", "app", attachPolicy("peer"))) }
+	require.NoError(t, store.UpdateRole(ctx, "", "app", attachPolicy("mine")))
+
+	role, err := store.GetRole(ctx, "", "app")
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"peer", "mine"}, role.AttachedPolicies, "one server's change was lost")
+}
+
+// A change racing a delete must not write the role back.
+func TestFilerRoleUpdateDoesNotReviveADeletedRole(t *testing.T) {
+	ctx := context.Background()
+	store, filer := newTestFilerRoleStore(t)
+	require.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA1")))
+
+	filer.afterLookup = func() { assert.NoError(t, store.DeleteRole(ctx, "", "app")) }
+	err := store.UpdateRole(ctx, "", "app", attachPolicy("mine"))
+	assert.ErrorIs(t, err, ErrRoleNotFound)
+
+	_, err = store.GetRole(ctx, "", "app")
+	assert.ErrorIs(t, err, ErrRoleNotFound, "the deleted role was written back")
+}
+
+// Of two creates of one name, the second sees the first's role.
+func TestFilerRoleCreateRefusesARoleCreatedConcurrently(t *testing.T) {
+	ctx := context.Background()
+	store, filer := newTestFilerRoleStore(t)
+
+	filer.afterLookup = func() { assert.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA-FIRST"))) }
+	err := store.UpdateRole(ctx, "", "app", createRole("AROA-SECOND"))
+	assert.ErrorIs(t, err, ErrRoleExists)
+
+	role, err := store.GetRole(ctx, "", "app")
+	require.NoError(t, err)
+	assert.Equal(t, "AROA-FIRST", role.RoleId, "the second create replaced the first role")
+}
+
+var errRoleHasPolicies = errors.New("role has attached policies")
+
+// deleteUnattached deletes the role only if it attaches no policy, as
+// DeleteRole requires.
+func deleteUnattached(current *RoleDefinition) (*RoleDefinition, error) {
+	if current == nil {
+		return nil, ErrRoleNotFound
+	}
+	if len(current.AttachedPolicies) > 0 {
+		return nil, errRoleHasPolicies
+	}
+	return nil, nil
+}
+
+func TestFilerRoleUpdateCanDeleteTheRole(t *testing.T) {
+	ctx := context.Background()
+	store, _ := newTestFilerRoleStore(t)
+	require.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA1")))
+
+	require.NoError(t, store.UpdateRole(ctx, "", "app", deleteUnattached))
+	_, err := store.GetRole(ctx, "", "app")
+	assert.ErrorIs(t, err, ErrRoleNotFound)
+}
+
+// A delete is decided against the role it removes: a policy attached between
+// the check and the delete makes it refuse, not delete the role anyway.
+func TestFilerRoleDeleteIsDecidedAgainstTheRoleItRemoves(t *testing.T) {
+	ctx := context.Background()
+	store, filer := newTestFilerRoleStore(t)
+	require.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA1")))
+
+	filer.afterLookup = func() { assert.NoError(t, store.UpdateRole(ctx, "", "app", attachPolicy("peer"))) }
+	err := store.UpdateRole(ctx, "", "app", deleteUnattached)
+	assert.ErrorIs(t, err, errRoleHasPolicies)
+
+	role, err := store.GetRole(ctx, "", "app")
+	require.NoError(t, err, "the role was deleted although a policy was attached to it")
+	assert.Equal(t, []string{"peer"}, role.AttachedPolicies)
+}
+
+func TestFilerRoleListingPagesPastTheFirstThousand(t *testing.T) {
+	store, filer := newTestFilerRoleStore(t)
+	for i := range roleListPageSize + 1 {
+		name := fmt.Sprintf("role-%04d.json", i)
+		filer.entries[name] = &filer_pb.Entry{Name: name}
+	}
+	filer.entries["role-0500"] = &filer_pb.Entry{Name: "role-0500", IsDirectory: true}
+
+	names, err := store.ListRoles(context.Background(), "")
+	require.NoError(t, err)
+	assert.Len(t, names, roleListPageSize+1)
+	assert.True(t, slices.Contains(names, fmt.Sprintf("role-%04d", roleListPageSize)), "the role past the first page is missing")
+}
+
+// A listing cut short must fail: DeletePolicy decides from it whether any
+// role still attaches the policy.
+func TestFilerRoleListingFailsOnABrokenStream(t *testing.T) {
+	store, filer := newTestFilerRoleStore(t)
+	for i := range 5 {
+		name := fmt.Sprintf("role-%d.json", i)
+		filer.entries[name] = &filer_pb.Entry{Name: name}
+	}
+	filer.failListAfter = 3
+
+	_, err := store.ListRoles(context.Background(), "")
+	require.Error(t, err, "a partial listing was returned as complete")
+
+	attaching, err := RolesAttachingPolicy(context.Background(), store, "read")
+	assert.Error(t, err)
+	assert.Nil(t, attaching)
+	assert.False(t, errors.Is(err, ErrRoleNotFound))
+}

weed/iam/integration/role_store_notfound_test.go

@@ -0,0 +1,14 @@
+package integration
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+func TestMemoryRoleStoreWrapsErrRoleNotFound(t *testing.T) {
+	_, err := NewMemoryRoleStore().GetRole(context.Background(), "", "missing")
+	if !errors.Is(err, ErrRoleNotFound) {
+		t.Fatalf("missing role error does not wrap ErrRoleNotFound: %v", err)
+	}
+}

weed/iam/integration/role_store_persist_test.go

@@ -0,0 +1,196 @@
+package integration
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// persistentTestRoleStore stands in for a role store that outlives the process
+// and may be shared (the filer store): anything but *MemoryRoleStore is
+// treated as persistent.
+type persistentTestRoleStore struct {
+	*MemoryRoleStore
+	cleared int
+}
+
+func (s *persistentTestRoleStore) ClearCache() { s.cleared++ }
+
+// startRoleServer initializes a manager as an S3 server would at boot, with the
+// given role store and config file roles.
+func startRoleServer(t *testing.T, store RoleStore, staticRoles ...string) *IAMManager {
+	t.Helper()
+	mgr := NewIAMManager()
+	require.NoError(t, mgr.Initialize(persistTestConfig(), func() string { return "localhost:8888" }))
+	mgr.roleStore = store
+	var defs []*RoleDefinition
+	for _, name := range staticRoles {
+		defs = append(defs, &RoleDefinition{RoleName: name})
+	}
+	mgr.LoadStaticRoles(context.Background(), defs)
+	return mgr
+}
+
+func roleNames(t *testing.T, store RoleStore) map[string]bool {
+	t.Helper()
+	names, err := store.ListRoles(context.Background(), "")
+	require.NoError(t, err)
+	out := map[string]bool{}
+	for _, n := range names {
+		out[n] = true
+	}
+	return out
+}
+
+// A persistent store never receives the config file's roles: it outlives the
+// file and may be shared with servers whose files differ.
+func TestPersistentRoleStoreNeverHoldsConfigFileRoles(t *testing.T) {
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	mgr := startRoleServer(t, store, "from-file")
+
+	assert.Empty(t, roleNames(t, store), "a config-file role was written to the persistent store")
+	assert.True(t, roleNames(t, mgr.GetRoleStore())["from-file"], "the config-file role is not listed")
+	role, err := mgr.GetRole(context.Background(), "from-file")
+	require.NoError(t, err)
+	assert.Equal(t, RoleSourceStaticConfig, role.Source)
+	assert.Equal(t, StaticRoleID(&RoleDefinition{RoleName: "from-file"}), role.RoleId)
+}
+
+// Servers sharing a store, with different config files, must not remove or
+// replace each other's roles — including a zero-config server.
+func TestServersSharingARoleStoreKeepEachOthersRoles(t *testing.T) {
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	configured := startRoleServer(t, store, "from-file")
+	require.NoError(t, configured.CreateRole(context.Background(), "", "created-at-runtime", &RoleDefinition{}))
+
+	zeroConfig := startRoleServer(t, store)
+
+	assert.True(t, roleNames(t, store)["created-at-runtime"], "a peer's start removed a role created at runtime")
+	assert.True(t, roleNames(t, configured.GetRoleStore())["from-file"], "the configured server lost its config-file role")
+	_, err := zeroConfig.GetRole(context.Background(), "from-file")
+	assert.ErrorIs(t, err, ErrRoleNotFound, "a server sees a role only its peer's config file defines")
+	_, err = zeroConfig.GetRole(context.Background(), "created-at-runtime")
+	assert.NoError(t, err, "the peer does not see the shared role created at runtime")
+}
+
+// Removing a role from the config file removes it at the next start.
+func TestRemovingARoleFromTheConfigFileRemovesIt(t *testing.T) {
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	startRoleServer(t, store, "from-file")
+
+	restarted := startRoleServer(t, store) // the file no longer lists it
+
+	_, err := restarted.GetRole(context.Background(), "from-file")
+	assert.ErrorIs(t, err, ErrRoleNotFound, "a role removed from the config file is still served")
+}
+
+// The store refuses to replace or delete a role the config file defines; the
+// file is where it changes.
+func TestConfigFileRolesCannotBeReplacedOrDeletedThroughTheStore(t *testing.T) {
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	mgr := startRoleServer(t, store, "from-file")
+
+	err := mgr.CreateRole(context.Background(), "", "from-file", &RoleDefinition{})
+	assert.ErrorIs(t, err, ErrRoleStatic)
+	assert.ErrorIs(t, mgr.DeleteRole(context.Background(), "from-file"), ErrRoleStatic)
+	assert.Empty(t, roleNames(t, store), "a refused change still wrote to the store")
+}
+
+// An in-memory store keeps its behaviour: the config file's roles are records
+// in it.
+func TestInMemoryRoleStoreStillHoldsConfigFileRoles(t *testing.T) {
+	store := NewMemoryRoleStore()
+	startRoleServer(t, store, "from-file")
+	assert.True(t, roleNames(t, store)["from-file"])
+}
+
+// Invalidating the role cache reaches the store beneath the config-file roles.
+func TestRoleCacheInvalidationReachesTheStoreBeneathTheOverlay(t *testing.T) {
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	mgr := startRoleServer(t, store, "from-file")
+	mgr.InvalidateRoleCache()
+	assert.Equal(t, 1, store.cleared)
+}
+
+func TestRoleCopiesKeepSource(t *testing.T) {
+	role := &RoleDefinition{RoleName: "r", Source: RoleSourceStaticConfig}
+	assert.Equal(t, RoleSourceStaticConfig, copyRoleDefinition(role).Source)
+	assert.Equal(t, RoleSourceStaticConfig, genericCopyRoleDefinition(role).Source)
+}
+
+// Config-file roles report no creation time: a time taken at load would
+// change with every restart.
+func TestConfigFileRolesReportNoCreationTime(t *testing.T) {
+	for name, store := range map[string]RoleStore{
+		"persistent": &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()},
+		"in-memory":  NewMemoryRoleStore(),
+	} {
+		t.Run(name, func(t *testing.T) {
+			mgr := startRoleServer(t, store, "from-file")
+			role, err := mgr.GetRole(context.Background(), "from-file")
+			require.NoError(t, err)
+			assert.True(t, role.CreatedAt.IsZero())
+		})
+	}
+}
+
+// A role stored under a config-file role's name takes precedence, as a stored
+// OIDC provider does; it can be changed, and deleting it brings the
+// config-file role back.
+func TestStoredRoleTakesPrecedenceOverTheConfigFileOne(t *testing.T) {
+	ctx := context.Background()
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	configured := startRoleServer(t, store, "shared-name")
+	peer := startRoleServer(t, store)
+	require.NoError(t, peer.CreateRole(ctx, "", "shared-name", &RoleDefinition{Description: "stored"}))
+
+	role, err := configured.GetRole(ctx, "shared-name")
+	require.NoError(t, err)
+	assert.Equal(t, "stored", role.Description, "the config-file role hides the stored one")
+
+	role.Description = "changed"
+	assert.NoError(t, configured.CreateRole(ctx, "", "shared-name", role), "the stored role cannot be changed")
+
+	require.NoError(t, configured.DeleteRole(ctx, "shared-name"))
+	role, err = configured.GetRole(ctx, "shared-name")
+	require.NoError(t, err)
+	assert.Equal(t, RoleSourceStaticConfig, role.Source, "deleting the stored role did not bring the config-file one back")
+}
+
+// A store installed through SetRoleStore behaves like one installed at
+// startup: the config-file roles stay visible and protected, and it serves
+// the roles it holds.
+func TestSetRoleStoreInstallsLikeStartup(t *testing.T) {
+	mgr := startRoleServer(t, &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}, "from-file")
+
+	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
+	require.NoError(t, store.StoreRole(context.Background(), "", "stored", &RoleDefinition{RoleName: "stored"}))
+	mgr.SetRoleStore(store)
+
+	names := roleNames(t, mgr.GetRoleStore())
+	assert.True(t, names["from-file"], "the config-file role is no longer listed")
+	assert.True(t, names["stored"], "the new store's role is not listed")
+	assert.ErrorIs(t, mgr.GetRoleStore().DeleteRole(context.Background(), "", "from-file"), ErrRoleStatic)
+	assert.False(t, roleNames(t, store)["from-file"], "the config-file role was written into the new store")
+}
+
+// S3 servers watch the directory a filer-backed role store keeps roles in, so
+// a store configured with its own basePath must report it, through the
+// cache and the config-file overlay alike.
+func TestRoleStoreDirectoryIsTheConfiguredBasePath(t *testing.T) {
+	provider := func() string { return "localhost:8888" }
+	cached, err := NewGenericCachedRoleStore(map[string]interface{}{"basePath": "/custom/roles"}, provider)
+	require.NoError(t, err)
+	mgr := startRoleServer(t, cached, "from-file")
+	assert.Equal(t, "/custom/roles", mgr.RoleStoreDirectory())
+
+	uncached, err := NewFilerRoleStore(nil, provider)
+	require.NoError(t, err)
+	mgr.SetRoleStore(uncached)
+	assert.Equal(t, "/etc/iam/roles", mgr.RoleStoreDirectory())
+
+	mgr.SetRoleStore(NewMemoryRoleStore())
+	assert.Empty(t, mgr.RoleStoreDirectory(), "a memory store has no directory")
+}

weed/iam/policy/policy_engine.go

@@ -473,8 +473,11 @@ func (e *PolicyEngine) EvaluateTrustPolicy(ctx context.Context, trustPolicy *Pol
 		return nil, fmt.Errorf("trust policy cannot be nil")
 	}
 
+	// A trust policy is deny-by-default, whatever the engine's DefaultEffect:
+	// a role may be assumed only by a principal a trust statement explicitly
+	// allows.
 	result := &EvaluationResult{
-		Effect: Effect(e.config.DefaultEffect),
+		Effect: EffectDeny,
 		EvaluationDetails: &EvaluationDetails{
 			Principal:         evalCtx.Principal,
 			Action:            evalCtx.Action,
@@ -511,7 +514,7 @@ func (e *PolicyEngine) EvaluateTrustPolicy(ctx context.Context, trustPolicy *Pol
 	// AWS IAM evaluation logic:
 	// 1. If there's an explicit Deny, the result is Deny
 	// 2. If there's an Allow and no Deny, the result is Allow
-	// 3. Otherwise, use the default effect
+	// 3. Otherwise, the implicit deny stands
 	if explicitDeny {
 		result.Effect = EffectDeny
 	} else if hasAllow {

weed/iam/policy/trust_policy_implicit_deny_test.go

@@ -0,0 +1,80 @@
+package policy
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+)
+
+const subjectBoundTrustPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+	"Principal":{"Federated":"https://oidc.example"},"Action":["sts:AssumeRoleWithWebIdentity"],
+	"Condition":{"StringEquals":{"oidc:sub":"spiffe://example.org/ns/app/sa/app"}}}]}`
+
+func webIdentityContext(sub string) *EvaluationContext {
+	return &EvaluationContext{
+		Principal: "web-identity-user",
+		Action:    "sts:AssumeRoleWithWebIdentity",
+		Resource:  "arn:aws:iam::role/app",
+		RequestContext: map[string]interface{}{
+			"aws:FederatedProvider": "https://oidc.example",
+			"oidc:iss":              "https://oidc.example",
+			"oidc:sub":              sub,
+		},
+	}
+}
+
+// A trust policy is deny-by-default. The engine's DefaultEffect decides
+// requests no identity policy speaks to; it must not let a principal the trust
+// policy does not allow assume the role.
+func TestEvaluateTrustPolicyIsImplicitDenyWhateverTheDefaultEffect(t *testing.T) {
+	var trust PolicyDocument
+	if err := json.Unmarshal([]byte(subjectBoundTrustPolicy), &trust); err != nil {
+		t.Fatalf("parse test trust policy: %v", err)
+	}
+	for _, defaultEffect := range []string{"Allow", "Deny"} {
+		t.Run("defaultEffect="+defaultEffect, func(t *testing.T) {
+			engine := NewPolicyEngine()
+			if err := engine.Initialize(&PolicyEngineConfig{DefaultEffect: defaultEffect, StoreType: "memory"}); err != nil {
+				t.Fatalf("initialize policy engine: %v", err)
+			}
+
+			res, err := engine.EvaluateTrustPolicy(context.Background(), &trust, webIdentityContext("spiffe://example.org/ns/other/sa/x"))
+			if err != nil {
+				t.Fatalf("evaluate trust policy: %v", err)
+			}
+			if res.Effect != EffectDeny {
+				t.Errorf("a subject the trust policy does not allow got %s", res.Effect)
+			}
+
+			res, err = engine.EvaluateTrustPolicy(context.Background(), &trust, webIdentityContext("spiffe://example.org/ns/app/sa/app"))
+			if err != nil {
+				t.Fatalf("evaluate trust policy: %v", err)
+			}
+			if res.Effect != EffectAllow {
+				t.Errorf("the subject the trust policy allows got %s", res.Effect)
+			}
+		})
+	}
+}
+
+func TestEvaluateTrustPolicyExplicitDenyWins(t *testing.T) {
+	var trust PolicyDocument
+	doc := `{"Version":"2012-10-17","Statement":[
+		{"Effect":"Allow","Principal":{"Federated":"https://oidc.example"},"Action":["sts:AssumeRoleWithWebIdentity"]},
+		{"Effect":"Deny","Principal":{"Federated":"https://oidc.example"},"Action":["sts:AssumeRoleWithWebIdentity"],
+		 "Condition":{"StringEquals":{"oidc:sub":"spiffe://example.org/ns/app/sa/app"}}}]}`
+	if err := json.Unmarshal([]byte(doc), &trust); err != nil {
+		t.Fatalf("parse test trust policy: %v", err)
+	}
+	engine := NewPolicyEngine()
+	if err := engine.Initialize(&PolicyEngineConfig{DefaultEffect: "Allow", StoreType: "memory"}); err != nil {
+		t.Fatalf("initialize policy engine: %v", err)
+	}
+	res, err := engine.EvaluateTrustPolicy(context.Background(), &trust, webIdentityContext("spiffe://example.org/ns/app/sa/app"))
+	if err != nil {
+		t.Fatalf("evaluate trust policy: %v", err)
+	}
+	if res.Effect != EffectDeny {
+		t.Errorf("explicit Deny did not win: %s", res.Effect)
+	}
+}

weed/iam/responses.go

@@ -532,3 +532,83 @@ type ListUserTagsResponse struct {
 	} `xml:"ListUserTagsResult"`
 	CommonResponse
 }
+
+// IAMRole is the Role element returned by the role actions.
+type IAMRole struct {
+	Path                     string `xml:"Path"`
+	RoleName                 string `xml:"RoleName"`
+	RoleId                   string `xml:"RoleId"`
+	Arn                      string `xml:"Arn"`
+	CreateDate               string `xml:"CreateDate,omitempty"`
+	AssumeRolePolicyDocument string `xml:"AssumeRolePolicyDocument,omitempty"`
+	Description              string `xml:"Description,omitempty"`
+	MaxSessionDuration       int64  `xml:"MaxSessionDuration,omitempty"`
+}
+
+// IAMAttachedPolicy is one element of ListAttachedRolePolicies.
+type IAMAttachedPolicy struct {
+	PolicyName string `xml:"PolicyName"`
+	PolicyArn  string `xml:"PolicyArn"`
+}
+
+// CreateRoleResponse is the response for CreateRole.
+type CreateRoleResponse struct {
+	XMLName          xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateRoleResponse"`
+	CreateRoleResult struct {
+		Role IAMRole `xml:"Role"`
+	} `xml:"CreateRoleResult"`
+	CommonResponse
+}
+
+// GetRoleResponse is the response for GetRole.
+type GetRoleResponse struct {
+	XMLName       xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetRoleResponse"`
+	GetRoleResult struct {
+		Role IAMRole `xml:"Role"`
+	} `xml:"GetRoleResult"`
+	CommonResponse
+}
+
+// ListRolesResponse is the response for ListRoles.
+type ListRolesResponse struct {
+	XMLName         xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListRolesResponse"`
+	ListRolesResult struct {
+		Roles       []*IAMRole `xml:"Roles>member"`
+		IsTruncated bool       `xml:"IsTruncated"`
+	} `xml:"ListRolesResult"`
+	CommonResponse
+}
+
+// DeleteRoleResponse is the response for DeleteRole.
+type DeleteRoleResponse struct {
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteRoleResponse"`
+	CommonResponse
+}
+
+// UpdateAssumeRolePolicyResponse is the response for UpdateAssumeRolePolicy.
+type UpdateAssumeRolePolicyResponse struct {
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ UpdateAssumeRolePolicyResponse"`
+	CommonResponse
+}
+
+// AttachRolePolicyResponse is the response for AttachRolePolicy.
+type AttachRolePolicyResponse struct {
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ AttachRolePolicyResponse"`
+	CommonResponse
+}
+
+// DetachRolePolicyResponse is the response for DetachRolePolicy.
+type DetachRolePolicyResponse struct {
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DetachRolePolicyResponse"`
+	CommonResponse
+}
+
+// ListAttachedRolePoliciesResponse is the response for ListAttachedRolePolicies.
+type ListAttachedRolePoliciesResponse struct {
+	XMLName                        xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListAttachedRolePoliciesResponse"`
+	ListAttachedRolePoliciesResult struct {
+		AttachedPolicies []*IAMAttachedPolicy `xml:"AttachedPolicies>member"`
+		IsTruncated      bool                 `xml:"IsTruncated"`
+	} `xml:"ListAttachedRolePoliciesResult"`
+	CommonResponse
+}

weed/iam/sts/session_claims.go

@@ -65,9 +65,10 @@ type STSSessionClaims struct {
 	TokenType   string `json:"typ"`  // token_type
 
 	// Role information
-	RoleArn     string `json:"role"`      // role_arn
-	AssumedRole string `json:"assumed"`   // assumed_role_user
-	Principal   string `json:"principal"` // principal_arn
+	RoleArn     string `json:"role"`          // role_arn
+	RoleId      string `json:"rid,omitempty"` // unique ID of the assumed role
+	AssumedRole string `json:"assumed"`       // assumed_role_user
+	Principal   string `json:"principal"`     // principal_arn
 
 	// Authorization data
 	Policies []string `json:"pol,omitempty"` // policies (abbreviated)
@@ -136,6 +137,7 @@ func (c *STSSessionClaims) ToSessionInfo(credGen *CredentialGenerator) *SessionI
 		SessionId:        c.SessionId,
 		SessionName:      c.SessionName,
 		RoleArn:          c.RoleArn,
+		RoleId:           c.RoleId,
 		AssumedRoleUser:  c.AssumedRole,
 		Principal:        c.Principal,
 		Policies:         c.Policies,
@@ -196,6 +198,13 @@ func (c *STSSessionClaims) WithRoleInfo(roleArn, assumedRole, principal string)
 }
 
 // WithPolicies sets the policies associated with this session
+// WithRoleId binds the session to the assumed role's unique ID. An empty ID
+// leaves the session unbound (roles that predate role IDs).
+func (c *STSSessionClaims) WithRoleId(roleId string) *STSSessionClaims {
+	c.RoleId = roleId
+	return c
+}
+
 func (c *STSSessionClaims) WithPolicies(policies []string) *STSSessionClaims {
 	c.Policies = policies
 	return c

weed/iam/sts/sts_service.go

@@ -187,6 +187,11 @@ type AssumeRoleWithWebIdentityRequest struct {
 
 	// Policy is an optional session policy (optional)
 	Policy *string `json:"Policy,omitempty"`
+
+	// RoleId is the assumed role's unique ID, set by the IAM manager after it
+	// resolves the role. It is embedded in the session so the session is bound
+	// to this role and not to whatever role later holds the same name.
+	RoleId string `json:"-"`
 }
 
 // AssumeRoleWithCredentialsRequest represents a request to assume role with username/password
@@ -211,6 +216,9 @@ type AssumeRoleWithCredentialsRequest struct {
 
 	// Policy is an optional session policy (optional)
 	Policy *string `json:"Policy,omitempty"`
+
+	// RoleId is set by the IAM manager; see AssumeRoleWithWebIdentityRequest.
+	RoleId string `json:"-"`
 }
 
 // AssumeRoleResponse represents the response from assume role operations
@@ -304,6 +312,10 @@ type SessionInfo struct {
 
 	// ParentUser is the stable hashed identity (sub+iss) derived at federation time.
 	ParentUser string `json:"parentUser,omitempty"`
+
+	// RoleId is the unique ID of the role the session was issued for; empty
+	// for sessions issued before role IDs were recorded.
+	RoleId string `json:"roleId,omitempty"`
 }
 
 // NewSTSService creates a new STS service
@@ -604,7 +616,7 @@ func (s *STSService) AssumeRoleWithWebIdentity(ctx context.Context, request *Ass
 	}
 
 	// 4. Calculate session duration
-	sessionDuration := s.calculateSessionDuration(request.DurationSeconds)
+	sessionDuration := s.CalculateSessionDuration(request.DurationSeconds)
 	expiresAt := time.Now().Add(sessionDuration)
 
 	// 5. Generate session ID and credentials
@@ -688,6 +700,7 @@ func (s *STSService) AssumeRoleWithWebIdentity(ctx context.Context, request *Ass
 	sessionClaims := NewSTSSessionClaims(sessionId, s.Config.Issuer, expiresAt).
 		WithSessionName(request.RoleSessionName).
 		WithRoleInfo(effectiveRoleArn, assumedRoleUser.Arn, assumedRoleUser.Arn).
+		WithRoleId(request.RoleId).
 		WithIdentityProvider(provider.Name(), externalIdentity.UserID, externalIdentity.Issuer).
 		WithMaxDuration(sessionDuration).
 		WithRequestContext(requestContext)
@@ -760,16 +773,16 @@ func (s *STSService) AssumeRoleWithCredentials(ctx context.Context, request *Ass
 	}
 
 	// 4-7. Mint the session
-	return s.issueSession(request.RoleArn, request.RoleSessionName, sessionPolicy,
+	return s.issueSession(request.RoleArn, request.RoleId, request.RoleSessionName, sessionPolicy,
 		request.DurationSeconds, provider.Name(), externalIdentity.UserID)
 }
 
 // issueSession mints temporary credentials and the self-contained JWT that
 // carries the whole session, shared by every assume-role entry point.
-func (s *STSService) issueSession(roleArn, roleSessionName, sessionPolicy string,
+func (s *STSService) issueSession(roleArn, roleId, roleSessionName, sessionPolicy string,
 	durationSeconds *int64, providerName, subject string) (*AssumeRoleResponse, error) {
 
-	sessionDuration := s.calculateSessionDuration(durationSeconds)
+	sessionDuration := s.CalculateSessionDuration(durationSeconds)
 	expiresAt := time.Now().Add(sessionDuration)
 
 	sessionId, err := GenerateSessionId()
@@ -791,6 +804,7 @@ func (s *STSService) issueSession(roleArn, roleSessionName, sessionPolicy string
 	sessionClaims := NewSTSSessionClaims(sessionId, s.Config.Issuer, expiresAt).
 		WithSessionName(roleSessionName).
 		WithRoleInfo(roleArn, assumedRoleUser.Arn, assumedRoleUser.Arn).
+		WithRoleId(roleId).
 		WithIdentityProvider(providerName, subject, "").
 		WithMaxDuration(sessionDuration)
 	if sessionPolicy != "" {
@@ -891,7 +905,9 @@ func (s *STSService) AssumeRoleForPrincipal(ctx context.Context, request *Assume
 		return nil, fmt.Errorf("role assumption denied: %w", err)
 	}
 
-	return s.issueSession(request.RoleArn, request.RoleSessionName, sessionPolicy,
+	// Iceberg credential vending resolves no role definition, so its sessions
+	// carry no role ID and stay bound by name.
+	return s.issueSession(request.RoleArn, "", request.RoleSessionName, sessionPolicy,
 		request.DurationSeconds, request.ProviderName, request.Principal)
 }
 
@@ -1120,11 +1136,11 @@ func (s *STSService) validateRoleAssumptionForCredentials(ctx context.Context, r
 	return nil
 }
 
-// calculateSessionDuration returns the requested DurationSeconds, or the
+// CalculateSessionDuration returns the requested DurationSeconds, or the
 // configured TokenDuration default, capped at MaxSessionLength. The source
 // token's exp deliberately plays no part: per AWS semantics the session
 // outlives the (already verified) web identity token.
-func (s *STSService) calculateSessionDuration(durationSeconds *int64) time.Duration {
+func (s *STSService) CalculateSessionDuration(durationSeconds *int64) time.Duration {
 	var duration time.Duration
 	if durationSeconds != nil {
 		duration = time.Duration(*durationSeconds) * time.Second

weed/iam/sts/sts_service_test.go

@@ -30,8 +30,8 @@ func TestCalculateSessionDuration(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			if got := svc.calculateSessionDuration(tc.durationSeconds); got != tc.want {
-				t.Errorf("calculateSessionDuration(%v) = %v, want %v", tc.durationSeconds, got, tc.want)
+			if got := svc.CalculateSessionDuration(tc.durationSeconds); got != tc.want {
+				t.Errorf("CalculateSessionDuration(%v) = %v, want %v", tc.durationSeconds, got, tc.want)
 			}
 		})
 	}

weed/mount/filehandle.go

@@ -211,6 +211,9 @@ func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) {
 }
 
 func (fh *FileHandle) ReleaseHandle() {
+	fhActiveLock := fh.wfs.fhLockTable.AcquireLock("ReleaseHandle", fh.fh, util.ExclusiveLock)
+	defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
+
 	// Release distributed lock before cleaning up, so other mounts can
 	// proceed as soon as this handle is done flushing.
 	if fh.dlmLock != nil {
@@ -219,9 +222,6 @@ func (fh *FileHandle) ReleaseHandle() {
 		glog.V(1).Infof("DLM lock released for inode %d", fh.inode)
 	}
 
-	fhActiveLock := fh.wfs.fhLockTable.AcquireLock("ReleaseHandle", fh.fh, util.ExclusiveLock)
-	defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
-
 	if fh.entryChunkGroup != nil {
 		_ = fh.entryChunkGroup.Close()
 	}

weed/mount/weedfs_file_mkrm.go

@@ -9,6 +9,7 @@ import (
 	"github.com/seaweedfs/go-fuse/v2/fuse"
 	"google.golang.org/protobuf/proto"
 
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -72,10 +73,26 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
 		return code
 	}
 
+	// Acquire the DLM lock before the filer create: a lock failure after an
+	// eager create would return EAGAIN while the file stays persisted.
+	var dlmLock *cluster.LiveLock
+	if wfs.lockClient != nil {
+		owner := fmt.Sprintf("mount-%d", wfs.signature)
+		dlmLock = wfs.lockClient.NewBlockingLongLivedLock(
+			string(entryFullPath), owner, lock_manager.LiveLockTTL,
+		)
+		if dlmLock == nil {
+			return fuse.Status(syscall.EAGAIN)
+		}
+	}
+
 	inode, newEntry, code = wfs.createRegularFile(dirFullPath, name, in.Mode, in.Uid, in.Gid, 0, !wfs.option.EagerFilerCreate, true)
 	if code == fuse.Status(syscall.EEXIST) && in.Flags&syscall.O_EXCL == 0 {
 		// Race: another process created the file between our check and create.
-		// Reopen the winner's entry.
+		// Reopen the winner's entry; AcquireHandle takes its own lock.
+		if dlmLock != nil {
+			dlmLock.Stop()
+		}
 		newEntry, _, code = wfs.maybeLoadEntry(entryFullPath)
 		if code != fuse.OK {
 			return code
@@ -100,6 +117,9 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
 		out.OpenFlags = 0
 		return fuse.OK
 	} else if code != fuse.OK {
+		if dlmLock != nil {
+			dlmLock.Stop()
+		}
 		return code
 	} else {
 		inode = wfs.inodeToPath.Lookup(entryFullPath, newEntry.Attributes.Crtime, false, false, inode, true)
@@ -122,15 +142,15 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
 	// persisted the entry, so its handle starts clean.
 	fileHandle.dirtyMetadata = !wfs.option.EagerFilerCreate
 
-	// Acquire DLM lock for new file creation (Create bypasses AcquireHandle
-	// so we must acquire the lock here). Always lock on Create since file
-	// creation is inherently a write operation.
-	if wfs.lockClient != nil && fileHandle.dlmLock == nil {
-		owner := fmt.Sprintf("mount-%d", wfs.signature)
-		fileHandle.dlmLock = wfs.lockClient.NewBlockingLongLivedLock(
-			string(entryFullPath), owner, lock_manager.LiveLockTTL,
-		)
-		glog.V(1).Infof("DLM lock acquired for new file %s", entryFullPath)
+	// Create bypasses AcquireHandle, so attach the lock acquired above.
+	// A surviving handle may already hold one; ours is redundant then.
+	if dlmLock != nil {
+		if fileHandle.dlmLock == nil {
+			fileHandle.dlmLock = dlmLock
+			glog.V(1).Infof("DLM lock acquired for new file %s", entryFullPath)
+		} else {
+			dlmLock.Stop()
+		}
 	}
 
 	out.Fh = uint64(fileHandle.fh)