remote sync: keep the event entry intact for IF_ENTRY_EQUAL

A FailedPrecondition substring in an unrelated error would have been
swallowed as a skipped stamp; status.FromError already unwraps.

.github/workflows/codeql.yml

@@ -27,7 +27,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v4.38.2
+        uses: github/codeql-action/init@v4.38.1
         # Override language selection by uncommenting this and choosing your languages
         with:
           languages: go
@@ -35,7 +35,7 @@ jobs:
       # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below).
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v4.38.2
+        uses: github/codeql-action/autobuild@v4.38.1
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -49,4 +49,4 @@ jobs:
       #     make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v4.38.2
+        uses: github/codeql-action/analyze@v4.38.1

.github/workflows/container_latest.yml

@@ -156,7 +156,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        platform: [amd64, arm64, arm, 386, ppc64le, s390x]
+        platform: [amd64, arm64, arm, 386]
         variant: ${{ fromJSON(needs.setup.outputs.variants) }}
 
     steps:
@@ -405,7 +405,7 @@ jobs:
           output: trivy-results.sarif
           exit-code: '0'
       - name: Upload Trivy scan results to GitHub Security
-        uses: github/codeql-action/upload-sarif@v4.38.2
+        uses: github/codeql-action/upload-sarif@v4.38.1
         if: always()
         with:
           sarif_file: trivy-results.sarif
@@ -505,9 +505,7 @@ jobs:
             ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
             ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
             ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
-            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 \
-            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-ppc64le \
-            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-s390x
+            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
           # The copy and the signature below use the digest this run pushed, not whatever the tag points at by then.
           DIGEST=$(jq -er '."containerimage.descriptor".digest' /tmp/manifest.json)
           echo "digest=${DIGEST}" >> "$GITHUB_OUTPUT"
@@ -551,15 +549,13 @@ jobs:
             echo "Using skopeo to copy..."
             retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs@${DIGEST} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
           else
-            echo "Using docker buildx imagetools (pulling 6 images from Docker Hub)..."
+            echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
             # Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
             retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
               ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
               ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
               ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
-              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 \
-              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-ppc64le \
-              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-s390x
+              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
           fi
 
       - name: Sign

.github/workflows/container_release_unified.yml

@@ -149,16 +149,12 @@ jobs:
           - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/arm64,  arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
           - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest,    qemu: true }
           - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/386,    arch: i386,  runner: ubuntu-latest,    qemu: false }
-          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/ppc64le, arch: ppc64le, runner: ubuntu-latest,  qemu: true }
-          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/s390x,   arch: s390x, runner: ubuntu-latest,    qemu: true }
 
           # Large disk - multi-arch
           - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64,  arch: amd64, runner: ubuntu-latest,    qemu: false }
           - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64,  arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
           - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest,    qemu: true }
           - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386,    arch: i386,  runner: ubuntu-latest,    qemu: false }
-          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/ppc64le, arch: ppc64le, runner: ubuntu-latest,  qemu: true }
-          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/s390x,   arch: s390x, runner: ubuntu-latest,    qemu: true }
 
           # Full tags - multi-arch
           - { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest,    qemu: false }
@@ -460,7 +456,7 @@ jobs:
 
       - name: Upload Trivy scan results to GitHub Security
         if: always()
-        uses: github/codeql-action/upload-sarif@v4.38.2
+        uses: github/codeql-action/upload-sarif@v4.38.1
         with:
           sarif_file: trivy-results.sarif
           category: trivy-${{ matrix.variant }}

.github/workflows/s3-tables-tests.yml

@@ -439,117 +439,6 @@ jobs:
           path: test/s3tables/catalog_clickhouse/test-output.log
           retention-days: 3
 
-  olake-iceberg-catalog-tests:
-    name: OLake Iceberg Catalog Integration Tests (${{ matrix.tag }})
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Pinned baseline, and latest so new OLake releases are exercised
-          # without a code change. OLake's Iceberg writer is a Java sidecar
-          # whose Iceberg version moves independently of the Go release, so
-          # the latest leg is the one that catches library drift.
-          - olake-image: olakego/source-postgres:v0.10.1
-            tag: "v0.10.1"
-          - olake-image: olakego/source-postgres:latest
-            tag: latest
-
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v7
-
-      - name: Set up Go
-        uses: actions/setup-go@v7
-        with:
-          go-version-file: 'go.mod'
-        id: go
-
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
-
-      - name: Pre-pull images
-        run: |
-          pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
-          pull ${{ matrix.olake-image }}
-          pull postgres:16
-          pull python:3.11-slim
-
-      - name: Run go mod tidy
-        run: go mod tidy
-
-      - name: Install SeaweedFS
-        run: |
-          go install -buildvcs=false ./weed
-
-      - name: Run OLake Iceberg Catalog Integration Tests
-        timeout-minutes: 25
-        working-directory: test/s3tables/catalog_olake
-        env:
-          OLAKE_IMAGE: ${{ matrix.olake-image }}
-        run: |
-          set -x
-          set -o pipefail
-          echo "=== System Information ==="
-          uname -a
-          free -h
-          df -h
-          docker info
-          echo "=== Starting OLake Iceberg Catalog Tests ==="
-
-          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
-            echo "OLake Iceberg catalog integration tests failed"
-            exit 1
-          }
-
-      # The suite skips itself when Docker is unavailable, so a green job is not
-      # by itself evidence that anything ran. Assert execution explicitly.
-      - name: Assert the suite actually ran
-        working-directory: test/s3tables/catalog_olake
-        run: |
-          log=test-output.log
-          if [ ! -f "$log" ]; then
-            echo "::error::no test-output.log; the suite did not run"
-            exit 1
-          fi
-          passes=$(grep -c '^--- PASS' "$log" || true)
-          skips=$(grep -c '^--- SKIP' "$log" || true)
-          echo "top-level PASS=$passes SKIP=$skips"
-          if [ "$skips" -gt 0 ]; then
-            echo "::error::the OLake suite skipped $skips top-level test(s); the environment it needs was not provisioned, so this job proves nothing"
-            grep '^--- SKIP' "$log" | head -20
-            exit 1
-          fi
-          if [ "$passes" -lt 1 ]; then
-            echo "::error::the OLake suite recorded no passing top-level test"
-            exit 1
-          fi
-
-      - name: Show test output on failure
-        if: failure()
-        working-directory: test/s3tables/catalog_olake
-        run: |
-          echo "=== Test Output ==="
-          if [ -f test-output.log ]; then
-            tail -200 test-output.log
-          fi
-
-          echo "=== Process information ==="
-          ps aux | grep -E "(weed|test|docker|olake|postgres)" || true
-          echo "=== Containers ==="
-          docker ps -a | head -30 || true
-
-      - name: Upload test logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v7
-        with:
-          name: olake-iceberg-catalog-test-logs-${{ matrix.tag }}
-          path: test/s3tables/catalog_olake/test-output.log
-          retention-days: 3
-
   polaris-integration-tests:
     name: Polaris Integration Tests
     runs-on: ubuntu-22.04

.github/workflows/s3tests.yml

@@ -289,7 +289,6 @@ jobs:
           s3tests/functional/test_s3.py::test_object_write_check_etag \
           s3tests/functional/test_s3.py::test_object_write_cache_control \
           s3tests/functional/test_s3.py::test_object_write_expires \
-          s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \
           s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
           s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
           s3tests/functional/test_s3.py::test_object_write_file \
@@ -312,7 +311,6 @@ jobs:
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
           s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
-          s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
           s3tests/functional/test_s3.py::test_bucket_head \
           s3tests/functional/test_s3.py::test_bucket_head_notexist \
           s3tests/functional/test_s3.py::test_object_raw_authenticated \
@@ -1151,7 +1149,6 @@ jobs:
           s3tests/functional/test_s3.py::test_object_write_check_etag \
           s3tests/functional/test_s3.py::test_object_write_cache_control \
           s3tests/functional/test_s3.py::test_object_write_expires \
-          s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \
           s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
           s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
           s3tests/functional/test_s3.py::test_object_write_file \
@@ -1174,7 +1171,6 @@ jobs:
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
           s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
-          s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
           s3tests/functional/test_s3.py::test_bucket_head \
           s3tests/functional/test_s3.py::test_bucket_head_notexist \
           s3tests/functional/test_s3.py::test_object_raw_authenticated \

README.md

@@ -21,7 +21,6 @@ One `weed` binary serves an S3 object store, a POSIX file system, and a lakehous
 
 - [Download Binaries for different platforms](https://github.com/seaweedfs/seaweedfs/releases/latest)
 - [Wiki Documentation](https://github.com/seaweedfs/seaweedfs/wiki)
-- [HTTP REST API](REST_API.md) for the filer, master, and volume servers
 - Community: [Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY), [Twitter](https://twitter.com/SeaweedFS), [Telegram](https://t.me/Seaweedfs), [Reddit](https://www.reddit.com/r/SeaweedFS/), [Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [SeaweedFS White Paper](https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS_Architecture.pdf) and introduction slides: [2025.5](https://docs.google.com/presentation/d/1tdkp45J01oRV68dIm4yoTXKJDof-EhainlA0LMXexQE/edit?usp=sharing), [2021.5](https://docs.google.com/presentation/d/1DcxKWlINc-HNCjhYeERkpGXXm6nTCES8mi2W5G0Z4Ts/edit?usp=sharing), [2019.3](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
 

REST_API.md

@@ -1,344 +0,0 @@
-# SeaweedFS HTTP REST API
-
-SeaweedFS exposes three HTTP surfaces:
-
-| Service | Default port | Addressing |
-|---------|--------------|------------|
-| Filer | 8888 | File system paths (`/dir/name`) |
-| Master | 9333 | File id assignment and cluster topology |
-| Volume server | 8080 | File content by file id (`vid,fid`) |
-
-Most clients only need the filer API (paths) or the S3 API. The master and
-volume APIs are the lower-level blob store interface.
-
-Conventions applying to all three:
-
-- Responses are JSON unless noted otherwise. Append `&pretty=y` to pretty-print.
-- A file id (`fid`) has the form `volumeId,fileKeyCookie`, e.g. `3,01637037d6`.
-  An optional suffix selects a reserved id from a `count` assignment
-  (`3,01637037d6_1`, `_2`, ...), and an optional extension
-  (`3,01637037d6.jpg`) sets the content type on reads.
-- `replication` is a 3-digit replica placement `xyz`: `x` copies in other
-  data centers, `y` on other racks in the same data center, `z` on other
-  volume servers on the same rack. `000` = no replication, `001` = one copy
-  on the same rack, `010` = one copy on a different rack, `100` = one copy in
-  another data center, `200` = two copies in two other data centers, `110` =
-  one copy in another data center plus one on another rack.
-- `ttl` units: `m` minute, `h` hour, `d` day, `w` week, `M` month, `y` year.
-
-## Filer API (port 8888)
-
-The filer presents a POSIX-like namespace over the volume servers.
-
-### Upload a file
-
-```bash
-# PUT the raw body to the target path
-curl -T /home/chris/myphoto.jpg "http://localhost:8888/dir/myphoto.jpg"
-
-# or POST as multipart form (the part filename becomes the entry name)
-curl -F file=@/home/chris/myphoto.jpg "http://localhost:8888/dir/"
-```
-
-Response `201 Created`:
-
-```json
-{"name":"myphoto.jpg","size":43234,"eTag":"0x6c656...","mtime":"...","chunks":[...]}
-```
-
-Query parameters:
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `collection` | collection name | empty |
-| `replication` | replica placement code | filer default |
-| `ttl` | file expiration, e.g. `3d` | never |
-| `disk` | disk type to store on | filer default |
-| `fsync` | `true` fsyncs on the volume server | false |
-| `dataCenter` | preferred data center | empty |
-| `rack` | preferred rack | empty |
-| `dataNode` | preferred volume server | empty |
-| `saveInside` | store small content inside the metadata instead of a volume | false |
-| `maxMB` | split the upload into chunks of this many MB | filer `-maxMB` |
-| `mode` | unix permission bits, e.g. `0644` | `0660` |
-| `op` | `append` appends to an existing file | overwrite |
-| `skipCheckParentDir` | `true` skips the parent-directory existence check | false |
-
-### Create a directory
-
-```bash
-curl -X POST "http://localhost:8888/dir/newdir/"
-```
-
-A POST to a path ending in `/` with no content creates the directory,
-including missing parents.
-
-### Read a file
-
-```bash
-curl "http://localhost:8888/dir/myphoto.jpg"
-```
-
-Supports `Range` requests (`Accept-Ranges: bytes`), `ETag`, and the
-`If-None-Match` / `If-Modified-Since` conditional headers. `HEAD` returns
-headers only. Entry headers stored as extended attributes are echoed back,
-minus internal `Seaweed-` and `xattr-` keys.
-
-Entry metadata instead of content:
-
-```bash
-curl "http://localhost:8888/dir/myphoto.jpg?metadata=true"
-```
-
-`metadata=true&resolveManifest=true` additionally resolves chunked-manifest
-entries into their real chunk list.
-
-### List a directory
-
-```bash
-curl -H "Accept: application/json" "http://localhost:8888/dir/?limit=10&lastFileName=a.jpg"
-```
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `limit` | max entries per page | filer `-dirListLimit` |
-| `lastFileName` | resume listing after this entry name | empty |
-| `namePattern` | include only names matching the wildcard | empty |
-| `namePatternExclude` | exclude names matching the wildcard | empty |
-
-The JSON response carries `Path`, `Entries`, `Limit`, `LastFileName`,
-`ShouldDisplayLoadMore`, and `EmptyFolder`. Without the `Accept` header the
-filer renders its HTML browser.
-
-### Move and copy
-
-```bash
-curl -X POST "http://localhost:8888/dir/newname.jpg?mv.from=/dir/myphoto.jpg"
-curl -X POST "http://localhost:8888/dir/copy.jpg?cp.from=/dir/myphoto.jpg"
-```
-
-`mv.from` renames or moves the source to the request path (`204 No Content`).
-`cp.from` copies it.
-
-### Append
-
-```bash
-curl -T chunk2.bin "http://localhost:8888/dir/file.bin?op=append"
-```
-
-### Delete
-
-```bash
-curl -X DELETE "http://localhost:8888/dir/myphoto.jpg"
-curl -X DELETE "http://localhost:8888/dir/?recursive=true"
-```
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `recursive` | delete a non-empty directory tree | false; when the filer runs with `filer.options.recursive_delete=true`, deletes are recursive unless `recursive=false` |
-| `ignoreRecursiveError` | keep deleting remaining entries after an error | false |
-| `skipChunkDeletion` | remove only the metadata, keep volume data | false |
-
-### Tagging
-
-Tags are carried as `Seaweed-`-prefixed request headers, not query
-parameters; `?tagging` selects the tagging handler and `?tagging=K1,K2`
-lists the keys to remove. Header names are canonicalized on write
-(`Seaweed-k1` is stored as `Seaweed-K1`), and the delete list is matched
-case-sensitively against the stored names.
-
-```bash
-curl -X PUT -H "Seaweed-k1: v1" -H "Seaweed-k2: v2" "http://localhost:8888/dir/file.jpg?tagging"
-curl -X DELETE "http://localhost:8888/dir/file.jpg?tagging=K1,K2"
-```
-
-### Read by file id
-
-```bash
-curl "http://localhost:8888/?proxyChunkId=3,01637037d6"
-```
-
-The filer proxies the chunk read to the right volume server, so only the
-filer port needs to be exposed.
-
-### Resumable uploads
-
-The filer serves the [TUS protocol](https://tus.io/) for resumable uploads
-(`POST`, `PATCH`, `HEAD` on upload URLs). It is enabled by default at
-`/.tus`; `-tusBasePath` changes the endpoint base path.
-
-### Health
-
-`GET /healthz` and `GET /readyz` return `200 OK`.
-
-## Master API (port 9333)
-
-Write-affecting endpoints are automatically proxied to the current leader, so
-any master in the quorum can serve them.
-
-### Assign a file id
-
-```bash
-curl "http://localhost:9333/dir/assign?count=1&replication=001&collection=turbo&dataCenter=dc1&ttl=3d&disk=ssd"
-{"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
-```
-
-Upload the file content to `http://<url>/<fid>` afterwards. With `count>1`,
-use `<fid>_1`, `<fid>_2`, ... for the additional ids.
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `count` | file ids to reserve | 1 |
-| `collection` | collection name | empty |
-| `dataCenter` | preferred data center | empty |
-| `rack` | preferred rack | empty |
-| `dataNode` | preferred volume server | empty |
-| `replication` | replica placement | master `-defaultReplication` |
-| `ttl` | file expiration, e.g. `3d` | never |
-| `disk` | disk type | empty |
-| `dataSize` | expected file size in bytes | 0 |
-| `preallocate` | bytes to preallocate for new volumes | master `-volumePreallocate` |
-| `writableVolumeCount` | grow this many volumes when none are writable | master default |
-| `memoryMapMaxSizeMb` | memory-mapped file size (Windows) | 0 |
-
-### Look up a volume or file id
-
-```bash
-curl "http://localhost:9333/dir/lookup?volumeId=3"
-{"locations":[{"url":"localhost:8080","publicUrl":"localhost:8080"}]}
-```
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `volumeId` | volume id; a full `vid,fid` is accepted too | required |
-| `fileId` | like `volumeId`, but also returns a write JWT when security is on | empty |
-| `collection` | speeds up the lookup | empty |
-| `read` | `yes` generates a read JWT instead of a write JWT | empty |
-
-### Store a file in one call
-
-```bash
-curl -F file=@/home/chris/report.pdf "http://localhost:9333/submit?collection=turbo&replication=001"
-{"fileName":"report.pdf","fid":"3,01637037d6","fileUrl":"localhost:8080/3,01637037d6","size":43234,"eTag":"0x6c656..."}
-```
-
-`POST /submit` accepts multipart file data plus the `dir/assign` placement
-parameters (`count`, `collection`, `dataCenter`, `rack`, `replication`,
-`ttl`, `disk`), assigns a file id, uploads to the volume server, and returns
-the result.
-
-### Redirect to a file
-
-```bash
-curl -v "http://localhost:9333/3,01637037d6"
-```
-
-`GET /{fileId}` answers `308 Permanent Redirect` to a volume server holding
-the file, preserving the query string (e.g. image-resize parameters).
-
-### Cluster status
-
-```bash
-curl "http://localhost:9333/dir/status?pretty=y"   # full topology tree
-curl "http://localhost:9333/vol/status?pretty=y"   # every volume on every node
-curl "http://localhost:9333/collection/info?collection=turbo"
-curl "http://localhost:9333/collection/info?collection=turbo&detail=true"
-```
-
-`collection/info` returns aggregated `TotalSize`, `FileCount`, `UsedSize`,
-`VolumeCount`; `detail=true` splits them per volume layout.
-
-### Grow volumes
-
-```bash
-curl "http://localhost:9333/vol/grow?count=4&replication=001&collection=turbo&ttl=5d&disk=ssd&dataCenter=dc1&rack=rack1"
-{"count":4}
-```
-
-`count` is required; the placement parameters match `dir/assign`. One volume
-serves one write at a time, so pre-allocated volumes raise write concurrency.
-
-### Vacuum deleted space
-
-```bash
-curl "http://localhost:9333/vol/vacuum?garbageThreshold=0.4"
-```
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `garbageThreshold` | minimum deleted-bytes ratio before a volume is compacted | master `-garbageThreshold` (0.3) |
-
-Vacuuming makes a volume read-only, copies live needles to a new volume, and
-swaps it in.
-
-### Delete a collection
-
-```bash
-curl "http://localhost:9333/col/delete?collection=benchmark"
-```
-
-Deletes all volumes of the collection, including erasure-coded shards.
-`204 No Content` on success.
-
-### Health
-
-```bash
-curl -I "http://localhost:9333/healthz"     # liveness
-curl -I "http://localhost:9333/readyz"      # readiness
-curl "http://localhost:9333/"               # web UI
-```
-
-## Volume server API (port 8080)
-
-The volume server stores file content by file id. Clients normally get the
-volume URL from `dir/assign` or `dir/lookup`.
-
-### Upload
-
-```bash
-curl -F file=@/home/chris/myphoto.jpg "http://127.0.0.1:8080/3,01637037d6"
-{"name":"myphoto.jpg","size":43234,"eTag":"0x6c656...","mime":"image/jpeg","contentMd5":"..."}
-```
-
-PUT or POST the body (or a multipart `file` part) to `/{vid},{fid}`.
-`204 No Content` is returned when the content is unchanged. `?ts=<unix>`
-sets the stored modification time.
-
-### Read
-
-```bash
-curl "http://127.0.0.1:8080/3,01637037d6"
-curl "http://127.0.0.1:8080/3,01637037d6.jpg"        # sets Content-Type from the extension
-```
-
-Supports `Range` and `HEAD`. Image files can be resized server-side:
-
-| Parameter | Description |
-|-----------|-------------|
-| `width`, `height` | resize bounds in pixels |
-| `mode` | `fit` (contain) or `fill` (cover); omitted resizes to `width`/`height` |
-| `crop_x1`, `crop_y1`, `crop_x2`, `crop_y2` | explicit crop rectangle |
-| `cm` | `false` returns the chunk-manifest blob instead of resolving it |
-| `readDeleted` | `true` reads soft-deleted needles |
-| `collection` | passed through redirects for the right volume |
-
-### Delete
-
-```bash
-curl -X DELETE "http://127.0.0.1:8080/3,01637037d6"
-{"size":43234}
-```
-
-`?ts=<unix>` sets the deletion timestamp. Replicated volumes propagate the
-delete to every replica.
-
-### Status
-
-```bash
-curl "http://localhost:8080/status?pretty=y"  # disk and volume inventory
-curl -I "http://localhost:8080/healthz"       # liveness/readiness
-```
-
-`OPTIONS` preflights answer CORS headers. When `-port.public` differs from
-`-port`, the volume server opens a separate read-only public listener on
-that port; `-publicUrl` sets the address it advertises to clients.

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dustin/go-humanize v1.1.0
+	github.com/dustin/go-humanize v1.0.1
 	github.com/eapache/go-resiliency v1.6.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
@@ -94,7 +94,7 @@ require (
 	golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597
 	golang.org/x/image v0.46.0
 	golang.org/x/net v0.58.0
-	golang.org/x/oauth2 v0.37.0
+	golang.org/x/oauth2 v0.36.0
 	golang.org/x/sys v0.48.0
 	golang.org/x/text v0.42.0 // indirect
 	golang.org/x/tools v0.49.0 // indirect
@@ -111,7 +111,7 @@ require (
 )
 
 require (
-	cloud.google.com/go/kms v1.35.0
+	cloud.google.com/go/kms v1.34.0
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/Jille/raft-grpc-transport v1.6.1
@@ -154,7 +154,7 @@ require (
 	github.com/tarantool/go-tarantool/v3 v3.0.2
 	github.com/testcontainers/testcontainers-go v0.44.0
 	github.com/tikv/client-go/v2 v2.0.7
-	github.com/twmb/avro v1.9.0
+	github.com/twmb/avro v1.8.0
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1
@@ -302,13 +302,13 @@ require (
 	cloud.google.com/go/iam v1.12.0 // indirect
 	cloud.google.com/go/monitoring v1.30.0 // indirect
 	filippo.io/edwards25519 v1.2.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
 	github.com/Azure/go-ntlmssp v0.1.1 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
 	github.com/Files-com/files-sdk-go/v3 v3.3.194 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect

go.sum

@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
 cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
 cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
 cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
-cloud.google.com/go/kms v1.35.0 h1:nJ/ktaqspx1nPM9vIcO0SHbhqCAm8nvAxL1siuVgKm0=
-cloud.google.com/go/kms v1.35.0/go.mod h1:0++71pIHvJL+GmMa8K4jOWFq7gNOX3jm2PRMSJwTKJw=
+cloud.google.com/go/kms v1.34.0 h1:mxWcXEiyjxwFH5gclulLx+B8Y2OEpKJRZ5FOF78c2XE=
+cloud.google.com/go/kms v1.34.0/go.mod h1:FbxZWUiihmyjxlaBha84OK5+fmJHPrS6F5/mBFdJk6A=
 cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
 cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
 cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
@@ -553,10 +553,10 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum
 git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4=
@@ -577,8 +577,8 @@ github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDP
 github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d h1:iG9B49Q218F/XxXNRM7k/vWf7MKmLIS8AcJV9cGN4nA=
@@ -953,8 +953,8 @@ github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.4.0/go.mod h1:gDXhl0OElhzYoDs
 github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
 github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
-github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/eapache/go-resiliency v1.6.0 h1:CqGDTLtpwuWKn6Nj3uNUdflaq+/kIPsg0gfNzHton30=
 github.com/eapache/go-resiliency v1.6.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
 github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws=
@@ -1965,8 +1965,8 @@ github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365/go.mod h1:zj0GJHGvyf
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
-github.com/twmb/avro v1.9.0 h1:JSiqewo3AANj7vlVCQXBsIZTA3LiiBble7Xp8T4lbtA=
-github.com/twmb/avro v1.9.0/go.mod h1:X0fT1dY2xcbV4YuCE4mYro+qljHl4kUF5uA/2z1rgSk=
+github.com/twmb/avro v1.8.0 h1:UMWLg+nH4P3yad5Om7yFSohYLy2RG1s7BcFFiOvmK9Q=
+github.com/twmb/avro v1.8.0/go.mod h1:X0fT1dY2xcbV4YuCE4mYro+qljHl4kUF5uA/2z1rgSk=
 github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=
 github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4=
@@ -2354,8 +2354,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
 golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
 golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
-golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
-golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.48"
+appVersion: "4.47"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.48.0
+version: 4.47.1

k8s/charts/seaweedfs/templates/cert/admin-cert.yaml

@@ -25,10 +25,6 @@ spec:
     organizations:
     - "SeaweedFS CA"
   dnsNames:
-    - '{{ include "seaweedfs.fullname" . }}-admin'
-    - '{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}'
-    - '{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}.svc'
-    - '{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}.svc.cluster.local'
     - '*.{{ include "seaweedfs.fullname" . }}-admin'
     - '*.{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}'
     - '*.{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}.svc'

seaweed-volume/Cargo.lock

@@ -4555,7 +4555,6 @@ dependencies = [
  "aws-config",
  "aws-credential-types",
  "aws-sdk-s3",
- "aws-smithy-runtime-api",
  "aws-types",
  "axum",
  "base64",

seaweed-volume/Cargo.toml

@@ -157,9 +157,6 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] }
 
 [dev-dependencies]
 tempfile = "3"
-# Already a transitive dependency of aws-sdk-s3 at a single locked version;
-# needed directly only for the canned HttpClient in remote_storage::s3 tests.
-aws-smithy-runtime-api = "1"
 
 [build-dependencies]
 tonic-prost-build = "0.14"

seaweed-volume/proto/volume_server.proto

@@ -168,7 +168,6 @@ message VacuumVolumeCheckRequest {
 }
 message VacuumVolumeCheckResponse {
     double garbage_ratio = 1;
-    bool disk_space_low = 4; // the volume is read-only solely because its disk is low on space — a cause compaction itself reclaims
 }
 
 message VacuumVolumeCompactRequest {
@@ -260,10 +259,6 @@ message VolumeDeleteRequest {
     // when true, do not remove the cloud-tier object backing the volume.
     // used for moves where another server is taking over the same .vif.
     bool keep_remote_data = 3;
-    // when true, delete only if every needle is deleted: the volume held
-    // data once but nothing is live anymore. Passing either check,
-    // only_empty or this one, is enough to delete.
-    bool only_garbage = 4;
 }
 message VolumeDeleteResponse {
 }
@@ -476,7 +471,6 @@ message VolumeEcShardsDeleteRequest {
     repeated uint32 shard_ids = 3;
     bool full_teardown = 4; // pre-encode cleanup: wipe every EC artifact + generation for this volume, not just shard_ids
     int64 encode_ts_ns = 5; // full_teardown generation fence: delete only a disk whose .vif generation is strictly OLDER than this; preserve same-or-newer, generation 0, and an unreadable .vif. 0 => wipe-all (shell pre-encode / pre-upgrade)
-    uint32 delete_generations_older_than = 6; // post-commit cleanup: delete only staged <base>.*.v<N> artifacts with N strictly below this; 0 disables
 }
 message VolumeEcShardsDeleteResponse {
     bool full_teardown_done = 1; // set by a new server that performed full_teardown; absent from an old server lets the caller detect the silent no-op

seaweed-volume/src/config.rs

@@ -178,22 +178,12 @@ pub struct Cli {
     pub min_free_space: String,
 
     /// Inflight upload data wait timeout of volume servers.
-    /// Go `time.Duration` syntax: `300ms`, `1.5h`, `2h45m`; a bare number other than `0` is rejected.
-    #[arg(
-        long = "inflightUploadDataTimeout",
-        default_value = "60s",
-        value_parser = parse_go_duration
-    )]
-    pub inflight_upload_data_timeout: std::time::Duration,
+    #[arg(long = "inflightUploadDataTimeout", default_value = "60s")]
+    pub inflight_upload_data_timeout: String,
 
     /// Inflight download data wait timeout of volume servers.
-    /// Go `time.Duration` syntax: `300ms`, `1.5h`, `2h45m`; a bare number other than `0` is rejected.
-    #[arg(
-        long = "inflightDownloadDataTimeout",
-        default_value = "60s",
-        value_parser = parse_go_duration
-    )]
-    pub inflight_download_data_timeout: std::time::Duration,
+    #[arg(long = "inflightDownloadDataTimeout", default_value = "60s")]
+    pub inflight_download_data_timeout: String,
 
     /// <experimental> if true, prevents slow reads from blocking other requests,
     /// but large file read P99 latency will increase.
@@ -481,214 +471,99 @@ fn find_options_arg(args: &[String]) -> String {
     String::new()
 }
 
-/// Parse a duration with Go's `time.ParseDuration` grammar, as Go's
-/// `flag.Duration` flags do: `[0-9]*(\.[0-9]*)?<unit>` repeated, units
-/// `ns`/`us`/`µs`/`μs`/`ms`/`s`/`m`/`h`. A bare `0` is the only unit-less
-/// value; a leading sign parses in Go but a negative has no
-/// `std::time::Duration`, so it is an error, not a clamp.
-fn parse_go_duration(s: &str) -> Result<std::time::Duration, String> {
-    let orig = s;
-    let invalid = || format!("time: invalid duration {:?}", orig);
-
-    let mut s = s;
-    if let Some(rest) = s.strip_prefix('-') {
-        if rest.is_empty() {
-            return Err(invalid());
-        }
-        return Err(format!("negative duration {:?} is not supported", orig));
-    }
-    if let Some(rest) = s.strip_prefix('+') {
-        s = rest;
-    }
-    if s == "0" {
-        return Ok(std::time::Duration::ZERO);
-    }
-    if s.is_empty() {
-        return Err(invalid());
-    }
-
-    // Go bounds the total by 1<<63 (int64 nanoseconds): overflow is an error.
-    const MAX: u64 = 1 << 63;
-    let mut total: u64 = 0;
-    let mut bytes = s.as_bytes();
-    while !bytes.is_empty() {
-        if !(bytes[0] == b'.' || bytes[0].is_ascii_digit()) {
-            return Err(invalid());
-        }
-
-        // Consume [0-9]*
-        let mut i = 0;
-        let mut v: u64 = 0;
-        while i < bytes.len() && bytes[i].is_ascii_digit() {
-            v = v
-                .checked_mul(10)
-                .and_then(|v| v.checked_add(u64::from(bytes[i] - b'0')))
-                .filter(|v| *v <= MAX)
-                .ok_or_else(invalid)?;
-            i += 1;
-        }
-        let pre = i != 0;
-        bytes = &bytes[i..];
-
-        // Consume (\.[0-9]*)?
-        let mut f: u64 = 0;
-        let mut scale: f64 = 1.0;
-        let mut post = false;
-        if let Some((b'.', rest)) = bytes.split_first() {
-            bytes = rest;
-            let mut i = 0;
-            let mut overflow = false;
-            while i < bytes.len() && bytes[i].is_ascii_digit() {
-                if !overflow {
-                    // Like Go's leadingFraction, digits past ~19 are dropped.
-                    match f
-                        .checked_mul(10)
-                        .and_then(|f| f.checked_add(u64::from(bytes[i] - b'0')))
-                        .filter(|f| *f <= MAX)
-                    {
-                        Some(next) => {
-                            f = next;
-                            scale *= 10.0;
-                        }
-                        None => overflow = true,
-                    }
-                }
-                i += 1;
-            }
-            post = i != 0;
-            bytes = &bytes[i..];
-        }
-        if !pre && !post {
-            return Err(invalid());
-        }
-
-        // Consume the unit: everything up to the next digit or '.'.
-        let unit_len = bytes
-            .iter()
-            .position(|c| *c == b'.' || c.is_ascii_digit())
-            .unwrap_or(bytes.len());
-        if unit_len == 0 {
-            return Err(format!("time: missing unit in duration {:?}", orig));
-        }
-        // The boundary is an ASCII byte or the end, so it is a char boundary.
-        let unit = std::str::from_utf8(&bytes[..unit_len]).map_err(|_| invalid())?;
-        bytes = &bytes[unit_len..];
-        let unit_nanos: u64 = match unit {
-            "ns" => 1,
-            "us" | "\u{00b5}s" | "\u{03bc}s" => 1_000,
-            "ms" => 1_000_000,
-            "s" => 1_000_000_000,
-            "m" => 60 * 1_000_000_000,
-            "h" => 3600 * 1_000_000_000,
-            _ => {
-                return Err(format!(
-                    "time: unknown unit {:?} in duration {:?}",
-                    unit, orig
-                ));
-            }
-        };
-
-        if v > MAX / unit_nanos {
-            return Err(invalid());
-        }
-        v *= unit_nanos;
-        if f > 0 {
-            // Go's float math, so `1.5h` rounds the same way on both sides.
-            v += (f as f64 * (unit_nanos as f64 / scale)) as u64;
-            if v > MAX {
-                return Err(invalid());
-            }
-        }
-        total = total
-            .checked_add(v)
-            .filter(|t| *t <= MAX)
-            .ok_or_else(invalid)?;
-    }
-    if total > MAX - 1 {
-        return Err(invalid());
-    }
-    Ok(std::time::Duration::from_nanos(total))
-}
-
-/// Byte-size suffixes of Go's `util.ParseBytes`, lower-cased: SI (`kb`, `k`)
-/// and IEC (`kib`, `ki`) forms, `b`, and no suffix.
-fn byte_unit_multiplier(unit: &str) -> Option<u64> {
-    const K: u64 = 1_000;
-    const KI: u64 = 1 << 10;
-    Some(match unit {
-        "" | "b" => 1,
-        "kb" | "k" => K,
-        "kib" | "ki" => KI,
-        "mb" | "m" => K * K,
-        "mib" | "mi" => KI * KI,
-        "gb" | "g" => K * K * K,
-        "gib" | "gi" => KI * KI * KI,
-        "tb" | "t" => K * K * K * K,
-        "tib" | "ti" => KI * KI * KI * KI,
-        "pb" | "p" => K * K * K * K * K,
-        "pib" | "pi" => KI * KI * KI * KI * KI,
-        "eb" | "e" => K * K * K * K * K * K,
-        "eib" | "ei" => KI * KI * KI * KI * KI * KI,
-        _ => return None,
-    })
-}
-
-/// Parse a human-readable byte count with Go's `util.ParseBytes` grammar:
-/// decimal number, optional whitespace, then a [`byte_unit_multiplier`]
-/// suffix, case-insensitive. `10GiB` → 10737418240, `1.5TB` → 1500000000000.
-///
-/// Thousands commas parse as in Go but never arrive from `-minFreeSpace`,
-/// which is split on commas into per-directory entries first.
-fn parse_bytes(s: &str) -> Result<u64, String> {
-    let digits_end = s
-        .find(|c: char| !(c.is_ascii_digit() || c == '.' || c == ','))
-        .unwrap_or(s.len());
-    let num = s[..digits_end].replace(',', "");
-    let value: f64 = num
-        .parse()
-        .map_err(|e| format!("invalid byte size {:?}: {}", s, e))?;
-    let unit = s[digits_end..].trim().to_lowercase();
-    let multiplier =
-        byte_unit_multiplier(&unit).ok_or_else(|| format!("unhandled size name: {}", unit))?;
-    let bytes = value * multiplier as f64;
-    if bytes >= u64::MAX as f64 {
-        return Err(format!("too large: {}", s));
-    }
-    Ok(bytes as u64)
-}
-
-/// Parse one `-minFreeSpace` entry like Go's `util.ParseMinFreeSpace()`: a
-/// plain number is a percentage in `0..=100`, anything else a byte size
-/// above 100 bytes. `150` errors rather than reading as bytes.
-fn parse_min_free_space(s: &str) -> Result<MinFreeSpace, String> {
+/// Parse a duration string like "60s", "5m", "1h" into a std::time::Duration.
+fn parse_duration(s: &str) -> std::time::Duration {
     let s = s.trim();
-    if let Ok(percent) = s.parse::<f64>() {
-        // Go's range check lets NaN through; rejecting it is the one
-        // intentional difference.
-        if !(0.0..=100.0).contains(&percent) {
-            return Err(format!("minFreeSpace is invalid: {:?}", s));
-        }
-        return Ok(MinFreeSpace::Percent(percent));
+    if s.is_empty() {
+        return std::time::Duration::from_secs(60);
     }
-    match parse_bytes(s) {
-        Ok(bytes) if bytes > 100 => Ok(MinFreeSpace::Bytes(bytes)),
-        _ => Err(format!("minFreeSpace is invalid: {:?}", s)),
+    if let Some(secs) = s.strip_suffix('s')
+        && let Ok(v) = secs.parse::<u64>()
+    {
+        return std::time::Duration::from_secs(v);
     }
+    if let Some(mins) = s.strip_suffix('m')
+        && let Ok(v) = mins.parse::<u64>()
+        && let Some(seconds) = v.checked_mul(60)
+    {
+        return std::time::Duration::from_secs(seconds);
+    }
+    if let Some(hours) = s.strip_suffix('h')
+        && let Ok(v) = hours.parse::<u64>()
+        && let Some(seconds) = v.checked_mul(3600)
+    {
+        return std::time::Duration::from_secs(seconds);
+    }
+    // Fallback: try parsing as raw seconds
+    if let Ok(v) = s.parse::<u64>() {
+        return std::time::Duration::from_secs(v);
+    }
+    std::time::Duration::from_secs(60)
 }
 
-/// Parse minFreeSpace / minFreeSpacePercent like Go's
-/// `util.MustParseMinFreeSpace()`: `--minFreeSpace` wins when set, and any
-/// invalid comma-separated entry fails the whole list.
-fn parse_min_free_spaces(
-    min_free_space: &str,
-    min_free_space_percent: &str,
-) -> Result<Vec<MinFreeSpace>, String> {
+/// Parse minFreeSpace / minFreeSpacePercent into MinFreeSpace values.
+/// Mirrors Go's `util.MustParseMinFreeSpace()`.
+fn parse_min_free_spaces(min_free_space: &str, min_free_space_percent: &str) -> Vec<MinFreeSpace> {
+    // If --minFreeSpace is provided, use it (takes precedence).
     let source = if !min_free_space.is_empty() {
         min_free_space
     } else {
         min_free_space_percent
     };
-    source.split(',').map(parse_min_free_space).collect()
+
+    source
+        .split(',')
+        .map(|s| {
+            let s = s.trim();
+            // Try parsing as a percentage (value <= 100)
+            if let Ok(v) = s.parse::<f64>() {
+                if v <= 100.0 {
+                    return MinFreeSpace::Percent(v);
+                }
+                // Treat as bytes if > 100
+                return MinFreeSpace::Bytes(v as u64);
+            }
+            // Try parsing human-readable bytes: e.g. "10GiB", "500MiB", "1TiB"
+            let s_upper = s.to_uppercase();
+            if let Some(rest) = s_upper.strip_suffix("TIB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1024.0 * 1024.0 * 1024.0 * 1024.0) as u64);
+            }
+            if let Some(rest) = s_upper.strip_suffix("GIB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1024.0 * 1024.0 * 1024.0) as u64);
+            }
+            if let Some(rest) = s_upper.strip_suffix("MIB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1024.0 * 1024.0) as u64);
+            }
+            if let Some(rest) = s_upper.strip_suffix("KIB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1024.0) as u64);
+            }
+            if let Some(rest) = s_upper.strip_suffix("TB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1_000_000_000_000.0) as u64);
+            }
+            if let Some(rest) = s_upper.strip_suffix("GB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1_000_000_000.0) as u64);
+            }
+            if let Some(rest) = s_upper.strip_suffix("MB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1_000_000.0) as u64);
+            }
+            // Default: 1%
+            MinFreeSpace::Percent(1.0)
+        })
+        .collect()
 }
 
 /// Parse comma-separated tag groups like "fast:ssd,archive" into per-folder tag vectors.
@@ -782,9 +657,7 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
 
     // Parse min free spaces
     let mut min_free_spaces =
-        parse_min_free_spaces(&cli.min_free_space, &cli.min_free_space_percent).unwrap_or_else(
-            |e| panic!("The value specified in -minFreeSpace not a valid value: {e}"),
-        );
+        parse_min_free_spaces(&cli.min_free_space, &cli.min_free_space_percent);
     if min_free_spaces.len() == 1 && folder_count > 1 {
         let v = min_free_spaces[0].clone();
         min_free_spaces.resize(folder_count, v);
@@ -899,6 +772,10 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
         .collect();
     white_list.extend(sec.guard_white_list.iter().cloned());
 
+    // Parse durations
+    let inflight_upload_data_timeout = parse_duration(&cli.inflight_upload_data_timeout);
+    let inflight_download_data_timeout = parse_duration(&cli.inflight_download_data_timeout);
+
     VolumeServerConfig {
         port: cli.port,
         grpc_port,
@@ -930,8 +807,8 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
         file_size_limit_bytes: cli.file_size_limit_mb as i64 * 1024 * 1024,
         concurrent_upload_limit: cli.concurrent_upload_limit_mb as i64 * 1024 * 1024,
         concurrent_download_limit: cli.concurrent_download_limit_mb as i64 * 1024 * 1024,
-        inflight_upload_data_timeout: cli.inflight_upload_data_timeout,
-        inflight_download_data_timeout: cli.inflight_download_data_timeout,
+        inflight_upload_data_timeout,
+        inflight_download_data_timeout,
         has_slow_read: cli.has_slow_read,
         read_buffer_size_mb: cli.read_buffer_size_mb,
         ldb_timeout: cli.ldb_timeout,
@@ -1383,224 +1260,25 @@ mod tests {
     }
 
     #[test]
-    fn test_parse_go_duration_accepts_go_grammar() {
-        use std::time::Duration;
-        let cases = [
-            ("60s", Duration::from_secs(60)),
-            ("5m", Duration::from_secs(300)),
-            ("1h", Duration::from_secs(3600)),
-            ("1.5h", Duration::from_secs(5400)),
-            ("2h45m", Duration::from_secs(2 * 3600 + 45 * 60)),
-            ("300ms", Duration::from_millis(300)),
-            ("1.5s", Duration::from_millis(1500)),
-            ("250us", Duration::from_micros(250)),
-            ("250\u{00b5}s", Duration::from_micros(250)),
-            ("250\u{03bc}s", Duration::from_micros(250)),
-            ("7ns", Duration::from_nanos(7)),
-            (".5s", Duration::from_millis(500)),
-            ("5.s", Duration::from_secs(5)),
-            ("+10s", Duration::from_secs(10)),
-            ("0", Duration::ZERO),
-            ("0s", Duration::ZERO),
-            // Go: fraction digits past the 19th are ignored, not an error.
-            ("1.00000000000000000001s", Duration::from_secs(1)),
-        ];
-        for (input, expected) in cases {
-            assert_eq!(parse_go_duration(input), Ok(expected), "input={:?}", input);
-        }
-    }
-
-    #[test]
-    fn test_parse_go_duration_rejects_what_go_rejects() {
-        // Each of these makes Go's flag.Duration fail flag parsing; the old
-        // Rust parser turned them all into the 60 s default.
-        let rejected = [
-            "30",                    // bare number other than 0
-            "30sec",                 // unknown unit
-            "abc",                   // no number at all
-            "",                      // empty
-            "1h ",                   // trailing space is not a unit
-            "s",                     // unit without number
-            ".s",                    // dot without digits
-            "1.5",                   // fraction without unit
-            "1h-30m",                // sign is only allowed at the front
-            "-",                     // sign alone
-            "5124095576030432h",     // overflows int64 nanoseconds
-            "307445734561825861m",   // overflows int64 nanoseconds
-            "9223372036854775808ns", // 1<<63, one past int64
-        ];
-        for input in rejected {
-            let result = parse_go_duration(input);
-            assert!(result.is_err(), "input={:?} parsed as {:?}", input, result);
-            let message = result.unwrap_err();
-            assert!(
-                message.contains(&format!("{:?}", input)) || input.is_empty(),
-                "error for {:?} does not name the value: {}",
-                input,
-                message
-            );
-        }
-        // Largest value Go accepts: 1<<63 - 1 nanoseconds.
+    fn test_parse_duration() {
+        assert_eq!(parse_duration("60s"), std::time::Duration::from_secs(60));
+        assert_eq!(parse_duration("5m"), std::time::Duration::from_secs(300));
+        assert_eq!(parse_duration("1h"), std::time::Duration::from_secs(3600));
+        assert_eq!(parse_duration("30"), std::time::Duration::from_secs(30));
+        assert_eq!(parse_duration(""), std::time::Duration::from_secs(60));
         assert_eq!(
-            parse_go_duration("9223372036854775807ns"),
-            Ok(std::time::Duration::from_nanos(i64::MAX as u64))
-        );
-        // Go accepts negatives; std::time::Duration cannot, so it errors.
-        let negative = parse_go_duration("-1s").unwrap_err();
-        assert!(negative.contains("negative"), "{}", negative);
-    }
-
-    /// Clap refuses the flag so the process exits with a usage error, exactly
-    /// where Go's flag package does.
-    #[test]
-    fn test_cli_rejects_invalid_duration_flags() {
-        for value in ["30sec", "abc", "30"] {
-            let result = Cli::try_parse_from(["bin", "--inflightUploadDataTimeout", value]);
-            assert!(
-                result.is_err(),
-                "--inflightUploadDataTimeout={} was accepted as {:?}",
-                value,
-                result.map(|cli| cli.inflight_upload_data_timeout)
-            );
-            let err = Cli::try_parse_from(["bin", "--inflightDownloadDataTimeout", value])
-                .err()
-                .unwrap_or_else(|| panic!("--inflightDownloadDataTimeout={} was accepted", value));
-            assert_eq!(err.kind(), clap::error::ErrorKind::ValueValidation);
-            assert!(err.to_string().contains(value), "{}", err);
-        }
-        let cli = Cli::parse_from(["bin", "--inflightUploadDataTimeout", "1.5h"]);
-        assert_eq!(
-            cli.inflight_upload_data_timeout,
-            std::time::Duration::from_secs(5400)
-        );
-        // The default still parses through the same grammar.
-        let cli = Cli::parse_from(["bin"]);
-        assert_eq!(
-            cli.inflight_upload_data_timeout,
+            parse_duration("307445734561825861m"),
             std::time::Duration::from_secs(60)
         );
         assert_eq!(
-            cli.inflight_download_data_timeout,
+            parse_duration("5124095576030432h"),
             std::time::Duration::from_secs(60)
         );
     }
 
-    #[test]
-    fn test_parse_bytes_matches_go_parse_bytes() {
-        // Expected values are what Go's util.ParseBytes returns (weed/util/bytes_test.go).
-        let cases: [(&str, u64); 14] = [
-            ("42", 42),
-            ("42MB", 42_000_000),
-            ("42 MB", 42_000_000),
-            ("42 mib", 44_040_192),
-            ("42M", 42_000_000),
-            ("100Ki", 100 * 1024),
-            ("10GiB", 10 * 1024 * 1024 * 1024),
-            ("10gib", 10 * 1024 * 1024 * 1024),
-            ("10GB", 10_000_000_000),
-            ("1.5TB", 1_500_000_000_000),
-            // Go's ParseBytes accepts thousands commas; the -minFreeSpace flag
-            // never delivers one (see test_min_free_space_flag_commas_are_list_separators).
-            ("1,024MB", 1_024_000_000),
-            ("1,024", 1024),
-            ("1PiB", 1 << 50),
-            ("1eb", 1_000_000_000_000_000_000),
-        ];
-        for (input, expected) in cases {
-            assert_eq!(parse_bytes(input), Ok(expected), "input={:?}", input);
-        }
-        for input in [
-            "10GiBx",
-            "GiB",
-            "",
-            "-5GB",
-            "1.2.3GB",
-            "10 Gi B",
-            "99999999999999999999EB",
-        ] {
-            assert!(
-                parse_bytes(input).is_err(),
-                "input={:?} was accepted",
-                input
-            );
-        }
-    }
-
-    fn expect_percent(result: Result<MinFreeSpace, String>, expected: f64, input: &str) {
-        match result {
-            Ok(MinFreeSpace::Percent(v)) => {
-                assert!(
-                    (v - expected).abs() < f64::EPSILON,
-                    "input={:?}: {}",
-                    input,
-                    v
-                )
-            }
-            other => panic!(
-                "input={:?}: expected Percent({}), got {:?}",
-                input, expected, other
-            ),
-        }
-    }
-
-    fn expect_bytes(result: Result<MinFreeSpace, String>, expected: u64, input: &str) {
-        match result {
-            Ok(MinFreeSpace::Bytes(v)) => assert_eq!(v, expected, "input={:?}", input),
-            other => panic!(
-                "input={:?}: expected Bytes({}), got {:?}",
-                input, expected, other
-            ),
-        }
-    }
-
-    #[test]
-    fn test_parse_min_free_space_matches_go() {
-        // Fixtures from weed/util/minfreespace_test.go plus the byte-unit table.
-        expect_percent(parse_min_free_space("42"), 42.0, "42");
-        expect_percent(parse_min_free_space(" 42 "), 42.0, " 42 ");
-        expect_percent(parse_min_free_space("50"), 50.0, "50");
-        expect_percent(parse_min_free_space("100"), 100.0, "100");
-        expect_percent(parse_min_free_space("0"), 0.0, "0");
-        expect_percent(parse_min_free_space("2.5"), 2.5, "2.5");
-        expect_bytes(parse_min_free_space("100Ki"), 100 * 1024, "100Ki");
-        expect_bytes(parse_min_free_space("100GiB"), 100 << 30, "100GiB");
-        expect_bytes(parse_min_free_space("42M"), 42_000_000, "42M");
-        expect_bytes(parse_min_free_space(" 42M "), 42_000_000, " 42M ");
-        expect_bytes(parse_min_free_space("10GiB"), 10 << 30, "10GiB");
-        expect_bytes(parse_min_free_space("10gib"), 10 << 30, "10gib");
-        expect_bytes(parse_min_free_space("1.5TB"), 1_500_000_000_000, "1.5TB");
-        // Comma inputs never arrive whole from the flag; see
-        // test_min_free_space_flag_commas_are_list_separators.
-        expect_bytes(parse_min_free_space("1,024MB"), 1_024_000_000, "1,024MB");
-        expect_bytes(parse_min_free_space("1,000"), 1000, "1,000");
-
-        // Go: percent outside 0..=100, or a byte size of at most 100 bytes.
-        for input in [
-            "150", "101", "-1", "1e3", "inf", "nan", "100B", "50B", "0B", "10GiBx", "", "abc",
-            "10 Gi B",
-        ] {
-            let result = parse_min_free_space(input);
-            assert!(
-                result.is_err(),
-                "input={:?} was accepted as {:?}",
-                input,
-                result
-            );
-            assert!(
-                result
-                    .as_ref()
-                    .unwrap_err()
-                    .contains(&format!("{:?}", input.trim())),
-                "error for {:?} does not name the value: {:?}",
-                input,
-                result
-            );
-        }
-    }
-
     #[test]
     fn test_parse_min_free_spaces_percent() {
-        let result = parse_min_free_spaces("", "1").unwrap();
+        let result = parse_min_free_spaces("", "1");
         assert_eq!(result.len(), 1);
         match &result[0] {
             MinFreeSpace::Percent(v) => assert!((v - 1.0).abs() < f64::EPSILON),
@@ -1610,7 +1288,7 @@ mod tests {
 
     #[test]
     fn test_parse_min_free_spaces_bytes() {
-        let result = parse_min_free_spaces("10GiB", "").unwrap();
+        let result = parse_min_free_spaces("10GiB", "");
         assert_eq!(result.len(), 1);
         match &result[0] {
             MinFreeSpace::Bytes(v) => assert_eq!(*v, 10 * 1024 * 1024 * 1024),
@@ -1618,87 +1296,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_parse_min_free_spaces_list_and_precedence() {
-        // --minFreeSpace wins over --minFreeSpacePercent, as in Go's EmptyTo().
-        let result = parse_min_free_spaces("5, 10GiB", "1").unwrap();
-        assert_eq!(result.len(), 2);
-        assert!(matches!(result[0], MinFreeSpace::Percent(v) if v == 5.0));
-        assert!(matches!(result[1], MinFreeSpace::Bytes(v) if v == 10 << 30));
-        // One bad entry fails the whole list and names that entry.
-        let err = parse_min_free_spaces("5,10GiBx", "1").unwrap_err();
-        assert!(err.contains("\"10GiBx\""), "{}", err);
-        // Both flags empty: Go fatals (ParseMinFreeSpace("") fails) rather than
-        // assuming 1%.
-        assert!(parse_min_free_spaces("", "").is_err());
-    }
-
-    /// A comma in `-minFreeSpace` separates per-directory entries, as in Go's
-    /// `MustParseMinFreeSpace`: `1,024MB` is `1` (percent) and `024MB`, and
-    /// one `-dir` fails the count check.
-    #[test]
-    fn test_min_free_space_flag_commas_are_list_separators() {
-        let result = parse_min_free_spaces("1,024MB", "").unwrap();
-        assert_eq!(result.len(), 2, "{result:?}");
-        assert!(
-            matches!(result[0], MinFreeSpace::Percent(v) if v == 1.0),
-            "{result:?}"
-        );
-        assert!(
-            matches!(result[1], MinFreeSpace::Bytes(v) if v == 24_000_000),
-            "{result:?}"
-        );
-
-        let _guard = process_state_lock();
-        let cli = Cli::parse_from(["bin", "--dir", "/tmp/a", "--minFreeSpace", "1,024MB"]);
-        let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
-            resolve_config_with_env(cli, &|_| None).min_free_spaces
-        }));
-        assert!(
-            outcome.is_err(),
-            "one -dir with two minFreeSpace entries must abort"
-        );
-    }
-
-    /// Negative control at the startup layer: resolving the config with an
-    /// invalid --minFreeSpace must abort, as Go's MustParseMinFreeSpace does.
-    #[test]
-    fn test_resolve_config_rejects_invalid_min_free_space() {
-        // The security-config search reads the working directory.
-        let _guard = process_state_lock();
-        for (flag, value) in [
-            ("--minFreeSpace", "10GiBx"),
-            ("--minFreeSpace", "150"),
-            ("--minFreeSpace", "50B"),
-            ("--minFreeSpacePercent", "150"),
-            ("--minFreeSpacePercent", "abc"),
-        ] {
-            let cli = Cli::parse_from(["bin", flag, value]);
-            let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
-                resolve_config_with_env(cli, &|_| None).min_free_spaces
-            }));
-            let payload = match outcome {
-                Ok(spaces) => panic!("{}={} was accepted as {:?}", flag, value, spaces),
-                Err(payload) => payload,
-            };
-            let message = payload
-                .downcast_ref::<String>()
-                .cloned()
-                .or_else(|| payload.downcast_ref::<&str>().map(|s| s.to_string()))
-                .unwrap_or_default();
-            assert!(
-                message.contains("-minFreeSpace") && message.contains(value),
-                "{}={}: unexpected panic message {:?}",
-                flag,
-                value,
-                message
-            );
-        }
-        let cfg =
-            resolve_config_with_env(Cli::parse_from(["bin", "--minFreeSpace", "100"]), &|_| None);
-        assert!(matches!(cfg.min_free_spaces[0], MinFreeSpace::Percent(v) if v == 100.0));
-    }
-
     #[test]
     fn test_parse_volume_tags_single() {
         let tags = parse_volume_tags("fast:ssd", 3);

seaweed-volume/src/remote_storage/s3.rs

@@ -4,7 +4,6 @@
 
 use aws_sdk_s3::Client;
 use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
-use aws_sdk_s3::error::{DisplayErrorContext, SdkError};
 use aws_sdk_s3::primitives::ByteStream;
 
 use super::{RemoteEntry, RemoteStorageClient, RemoteStorageError};
@@ -26,23 +25,6 @@ impl S3RemoteStorageClient {
         endpoint: &str,
         force_path_style: bool,
     ) -> Self {
-        let client = Client::from_conf(
-            Self::config_builder(access_key, secret_key, region, endpoint, force_path_style)
-                .build(),
-        );
-
-        S3RemoteStorageClient { client, conf }
-    }
-
-    /// Build the SDK config for the given credentials and endpoint. Split out so
-    /// tests can attach a canned HTTP client before building the [`Client`].
-    fn config_builder(
-        access_key: &str,
-        secret_key: &str,
-        region: &str,
-        endpoint: &str,
-        force_path_style: bool,
-    ) -> aws_sdk_s3::config::Builder {
         let region = if region.is_empty() {
             "us-east-1"
         } else {
@@ -67,7 +49,9 @@ impl S3RemoteStorageClient {
             s3_config = s3_config.endpoint_url(endpoint);
         }
 
-        s3_config
+        let client = Client::from_conf(s3_config.build());
+
+        S3RemoteStorageClient { client, conf }
     }
 }
 
@@ -91,14 +75,13 @@ impl RemoteStorageClient for S3RemoteStorageClient {
             req = req.range(format!("bytes={}-", offset));
         }
 
-        let resp = req.send().await.map_err(|e| match e {
-            // Go compares `aerr.Code()` to NoSuchKey on GET
-            // (s3_storage_client.go:436): a bare 404 maps to "NotFound"
-            // and stays a generic error, as it does here.
-            SdkError::ServiceError(ref se) if se.err().is_no_such_key() => {
+        let resp = req.send().await.map_err(|e| {
+            let msg = format!("{}", e);
+            if msg.contains("NoSuchKey") || msg.contains("404") {
                 RemoteStorageError::ObjectNotFound(format!("{}/{}", loc.bucket, key))
+            } else {
+                RemoteStorageError::Other(format!("s3 get object: {}", e))
             }
-            e => RemoteStorageError::Other(format!("s3 get object: {}", DisplayErrorContext(&e))),
         })?;
 
         let data = resp
@@ -125,9 +108,7 @@ impl RemoteStorageClient for S3RemoteStorageClient {
             .body(ByteStream::from(data.to_vec()))
             .send()
             .await
-            .map_err(|e| {
-                RemoteStorageError::Other(format!("s3 put object: {}", DisplayErrorContext(&e)))
-            })?;
+            .map_err(|e| RemoteStorageError::Other(format!("s3 put object: {}", e)))?;
 
         Ok(RemoteEntry {
             size: data.len() as i64,
@@ -153,18 +134,13 @@ impl RemoteStorageClient for S3RemoteStorageClient {
             .key(key)
             .send()
             .await
-            .map_err(|e| match e {
-                // Go checks only the raw HTTP status on HEAD
-                // (s3_storage_client.go:373): a HEAD response carries no
-                // error body, so a 404 is not-found whatever code the SDK
-                // assigns, and a non-404 is not.
-                SdkError::ServiceError(ref se) if se.raw().status().as_u16() == 404 => {
+            .map_err(|e| {
+                let msg = format!("{}", e);
+                if msg.contains("404") || msg.contains("NotFound") {
                     RemoteStorageError::ObjectNotFound(format!("{}/{}", loc.bucket, key))
+                } else {
+                    RemoteStorageError::Other(format!("s3 head object: {}", e))
                 }
-                e => RemoteStorageError::Other(format!(
-                    "s3 head object: {}",
-                    DisplayErrorContext(&e)
-                )),
             })?;
 
         Ok(RemoteEntry {
@@ -184,17 +160,18 @@ impl RemoteStorageClient for S3RemoteStorageClient {
             .key(key)
             .send()
             .await
-            .map_err(|e| {
-                RemoteStorageError::Other(format!("s3 delete object: {}", DisplayErrorContext(&e)))
-            })?;
+            .map_err(|e| RemoteStorageError::Other(format!("s3 delete object: {}", e)))?;
 
         Ok(())
     }
 
     async fn list_buckets(&self) -> Result<Vec<String>, RemoteStorageError> {
-        let resp = self.client.list_buckets().send().await.map_err(|e| {
-            RemoteStorageError::Other(format!("s3 list buckets: {}", DisplayErrorContext(&e)))
-        })?;
+        let resp = self
+            .client
+            .list_buckets()
+            .send()
+            .await
+            .map_err(|e| RemoteStorageError::Other(format!("s3 list buckets: {}", e)))?;
 
         Ok(resp
             .buckets()
@@ -207,178 +184,3 @@ impl RemoteStorageClient for S3RemoteStorageClient {
         &self.conf
     }
 }
-
-#[cfg(test)]
-pub(crate) mod tests {
-    use super::*;
-    use aws_sdk_s3::config::http::{HttpRequest, HttpResponse};
-    use aws_sdk_s3::config::retry::RetryConfig;
-    use aws_sdk_s3::config::{HttpClient, RuntimeComponents};
-    use aws_sdk_s3::primitives::SdkBody;
-    use aws_smithy_runtime_api::client::http::{
-        HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector,
-    };
-    use aws_smithy_runtime_api::http::StatusCode;
-
-    /// An SDK HTTP client that answers every request with one canned response,
-    /// so the error-mapping paths can be exercised without a network or a
-    /// running S3 server.
-    #[derive(Debug, Clone)]
-    pub(crate) struct CannedResponse {
-        pub(crate) status: u16,
-        pub(crate) body: &'static str,
-    }
-
-    impl HttpConnector for CannedResponse {
-        fn call(&self, _request: HttpRequest) -> HttpConnectorFuture {
-            let status = StatusCode::try_from(self.status).expect("valid HTTP status");
-            HttpConnectorFuture::ready(Ok(HttpResponse::new(status, SdkBody::from(self.body))))
-        }
-    }
-
-    impl HttpClient for CannedResponse {
-        fn http_connector(
-            &self,
-            _settings: &HttpConnectorSettings,
-            _components: &RuntimeComponents,
-        ) -> SharedHttpConnector {
-            SharedHttpConnector::new(self.clone())
-        }
-    }
-
-    fn client_with(status: u16, body: &'static str) -> S3RemoteStorageClient {
-        let config = S3RemoteStorageClient::config_builder(
-            "AKIATEST",
-            "secret",
-            "us-east-1",
-            "http://127.0.0.1:1",
-            true,
-        )
-        .http_client(CannedResponse { status, body })
-        .retry_config(RetryConfig::disabled())
-        .build();
-        S3RemoteStorageClient {
-            client: Client::from_conf(config),
-            conf: RemoteConf::default(),
-        }
-    }
-
-    fn location() -> RemoteStorageLocation {
-        RemoteStorageLocation {
-            name: "remote".to_string(),
-            bucket: "bucket".to_string(),
-            path: "/dir/missing".to_string(),
-            ..Default::default()
-        }
-    }
-
-    pub(crate) const NO_SUCH_KEY: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
-<Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><Key>dir/missing</Key></Error>"#;
-
-    const NOT_FOUND_BODY: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
-<Error><Code>NotFound</Code><Message>Not Found</Message></Error>"#;
-
-    const ACCESS_DENIED: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
-<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>"#;
-
-    #[tokio::test]
-    async fn get_no_such_key_is_object_not_found() {
-        let err = client_with(404, NO_SUCH_KEY)
-            .read_file(&location(), 0, 0)
-            .await
-            .unwrap_err();
-        assert!(
-            matches!(&err, RemoteStorageError::ObjectNotFound(path) if path == "bucket/dir/missing"),
-            "expected ObjectNotFound, got {err:?}"
-        );
-    }
-
-    #[tokio::test]
-    async fn get_bare_404_is_not_object_not_found() {
-        // Go compares codes, not statuses, on GET: a body-less 404 stays generic.
-        let err = client_with(404, "")
-            .read_file(&location(), 0, 0)
-            .await
-            .unwrap_err();
-        assert!(
-            matches!(err, RemoteStorageError::Other(_)),
-            "expected Other, got {err:?}"
-        );
-    }
-
-    #[tokio::test]
-    async fn head_404_is_object_not_found() {
-        let err = client_with(404, "")
-            .stat_file(&location())
-            .await
-            .unwrap_err();
-        assert!(
-            matches!(&err, RemoteStorageError::ObjectNotFound(path) if path == "bucket/dir/missing"),
-            "expected ObjectNotFound, got {err:?}"
-        );
-    }
-
-    #[tokio::test]
-    async fn head_404_with_foreign_error_body_is_object_not_found() {
-        // The raw status check makes a 404 not-found whatever body it carries.
-        let err = client_with(404, NO_SUCH_KEY)
-            .stat_file(&location())
-            .await
-            .unwrap_err();
-        assert!(
-            matches!(err, RemoteStorageError::ObjectNotFound(_)),
-            "expected ObjectNotFound, got {err:?}"
-        );
-    }
-
-    #[tokio::test]
-    async fn head_not_found_code_on_a_non_404_status_is_not_object_not_found() {
-        // A NotFound body on a non-404 status stays an error, as in Go.
-        let err = client_with(400, NOT_FOUND_BODY)
-            .stat_file(&location())
-            .await
-            .unwrap_err();
-        assert!(
-            matches!(&err, RemoteStorageError::Other(msg) if msg.contains("NotFound")),
-            "expected Other naming the code, got {err:?}"
-        );
-    }
-
-    #[tokio::test]
-    async fn get_access_denied_keeps_service_error_code() {
-        let err = client_with(403, ACCESS_DENIED)
-            .read_file(&location(), 0, 0)
-            .await
-            .unwrap_err();
-        let msg = err.to_string();
-        assert!(
-            matches!(err, RemoteStorageError::Other(_)),
-            "expected Other, got {err:?}"
-        );
-        assert!(
-            msg.contains("AccessDenied"),
-            "message should carry the S3 error code, got: {msg}"
-        );
-        assert!(
-            !msg.ends_with("service error"),
-            "message should not be the bare SdkError Display, got: {msg}"
-        );
-    }
-
-    #[tokio::test]
-    async fn head_access_denied_keeps_service_error_code() {
-        let err = client_with(403, ACCESS_DENIED)
-            .stat_file(&location())
-            .await
-            .unwrap_err();
-        let msg = err.to_string();
-        assert!(
-            matches!(err, RemoteStorageError::Other(_)),
-            "expected Other, got {err:?}"
-        );
-        assert!(
-            msg.contains("AccessDenied"),
-            "message should carry the S3 error code, got: {msg}"
-        );
-    }
-}

seaweed-volume/src/remote_storage/s3_tier.rs

@@ -8,11 +8,7 @@ use std::future::Future;
 use std::sync::{Arc, OnceLock, RwLock};
 
 use aws_sdk_s3::Client;
-use aws_sdk_s3::config::http::HttpResponse;
 use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
-use aws_sdk_s3::error::{DisplayErrorContext, SdkError};
-use aws_sdk_s3::operation::get_object::GetObjectError;
-use aws_sdk_s3::operation::head_object::HeadObjectError;
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
 use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
 use tokio::sync::Semaphore;
@@ -20,53 +16,6 @@ use tokio::sync::Semaphore;
 /// Concurrency limit for multipart upload/download (matches Go's s3manager).
 const CONCURRENCY: usize = 5;
 
-/// A tier transfer failure. The variant is what callers match on; the
-/// message is the operator-facing text.
-#[derive(Debug, thiserror::Error)]
-pub enum TierError {
-    /// The remote object does not exist.
-    #[error("{0}")]
-    NotFound(String),
-    /// An S3 request or a local file operation failed.
-    #[error("{0}")]
-    Io(String),
-    /// The tier I/O runtime could not be built or dropped the task.
-    #[error("{0}")]
-    RuntimeUnavailable(String),
-    /// The progress callback asked to stop.
-    #[error("{0}")]
-    Aborted(String),
-}
-
-// Not-found rules as in remote_storage/s3.rs: HEAD by the raw 404 status,
-// GET by the NoSuchKey code only.
-fn head_object_error(key: &str, e: SdkError<HeadObjectError, HttpResponse>) -> TierError {
-    let message = format!("failed to head object {}: {}", key, DisplayErrorContext(&e));
-    match e {
-        SdkError::ServiceError(ref se) if se.raw().status().as_u16() == 404 => {
-            TierError::NotFound(message)
-        }
-        _ => TierError::Io(message),
-    }
-}
-
-fn get_object_error(
-    key: &str,
-    range: &str,
-    e: SdkError<GetObjectError, HttpResponse>,
-) -> TierError {
-    let message = format!(
-        "failed to get object {} range {}: {}",
-        key,
-        range,
-        DisplayErrorContext(&e)
-    );
-    match e {
-        SdkError::ServiceError(ref se) if se.err().is_no_such_key() => TierError::NotFound(message),
-        _ => TierError::Io(message),
-    }
-}
-
 /// Configuration for an S3 tier backend.
 #[derive(Debug, Clone)]
 pub struct S3TierConfig {
@@ -140,7 +89,7 @@ impl S3TierBackend {
         &self,
         file_path: &str,
         progress_fn: F,
-    ) -> Result<(String, u64), TierError>
+    ) -> Result<(String, u64), String>
     where
         F: FnMut(i64, f32) -> Result<(), String> + Send + Sync + 'static,
     {
@@ -148,7 +97,7 @@ impl S3TierBackend {
 
         let metadata = tokio::fs::metadata(file_path)
             .await
-            .map_err(|e| TierError::Io(format!("failed to stat file {}: {}", file_path, e)))?;
+            .map_err(|e| format!("failed to stat file {}: {}", file_path, e))?;
         let file_size = metadata.len();
 
         // Calculate part size: start at 64MB, scale up for very large files (matches Go)
@@ -170,16 +119,11 @@ impl S3TierBackend {
             )
             .send()
             .await
-            .map_err(|e| {
-                TierError::Io(format!(
-                    "failed to create multipart upload: {}",
-                    DisplayErrorContext(&e)
-                ))
-            })?;
+            .map_err(|e| format!("failed to create multipart upload: {}", e))?;
 
         let upload_id = create_resp
             .upload_id()
-            .ok_or_else(|| TierError::Io("no upload_id in multipart upload response".to_string()))?
+            .ok_or_else(|| "no upload_id in multipart upload response".to_string())?
             .to_string();
 
         // Build list of (part_number, offset, size) for all parts
@@ -215,21 +159,19 @@ impl S3TierBackend {
                 let _permit = sem
                     .acquire()
                     .await
-                    .map_err(|e| TierError::Io(format!("semaphore error: {}", e)))?;
+                    .map_err(|e| format!("semaphore error: {}", e))?;
 
                 // Read this part's data from the file at the correct offset
                 let mut file = tokio::fs::File::open(&fp)
                     .await
-                    .map_err(|e| TierError::Io(format!("failed to open file {}: {}", fp, e)))?;
+                    .map_err(|e| format!("failed to open file {}: {}", fp, e))?;
                 file.seek(std::io::SeekFrom::Start(off))
                     .await
-                    .map_err(|e| {
-                        TierError::Io(format!("failed to seek to offset {}: {}", off, e))
-                    })?;
+                    .map_err(|e| format!("failed to seek to offset {}: {}", off, e))?;
                 let mut buf = vec![0u8; size];
-                file.read_exact(&mut buf).await.map_err(|e| {
-                    TierError::Io(format!("failed to read file at offset {}: {}", off, e))
-                })?;
+                file.read_exact(&mut buf)
+                    .await
+                    .map_err(|e| format!("failed to read file at offset {}: {}", off, e))?;
 
                 let upload_part_resp = client
                     .upload_part()
@@ -241,12 +183,7 @@ impl S3TierBackend {
                     .send()
                     .await
                     .map_err(|e| {
-                        TierError::Io(format!(
-                            "failed to upload part {} at offset {}: {}",
-                            pn,
-                            off,
-                            DisplayErrorContext(&e)
-                        ))
+                        format!("failed to upload part {} at offset {}: {}", pn, off, e)
                     })?;
 
                 let e_tag = upload_part_resp.e_tag().unwrap_or_default().to_string();
@@ -265,9 +202,9 @@ impl S3TierBackend {
                     };
                     (guard.1)(uploaded as i64, pct)
                 };
-                progress_result.map_err(TierError::Aborted)?;
+                progress_result?;
 
-                Ok::<_, TierError>(
+                Ok::<_, String>(
                     CompletedPart::builder()
                         .e_tag(e_tag)
                         .part_number(pn)
@@ -282,7 +219,7 @@ impl S3TierBackend {
             for handle in handles {
                 let part = handle
                     .await
-                    .map_err(|e| TierError::Io(format!("upload task panicked: {}", e)))??;
+                    .map_err(|e| format!("upload task panicked: {}", e))??;
                 completed_parts.push(part);
             }
 
@@ -299,14 +236,9 @@ impl S3TierBackend {
                 .multipart_upload(completed_upload)
                 .send()
                 .await
-                .map_err(|e| {
-                    TierError::Io(format!(
-                        "failed to complete multipart upload: {}",
-                        DisplayErrorContext(&e)
-                    ))
-                })?;
+                .map_err(|e| format!("failed to complete multipart upload: {}", e))?;
 
-            Ok::<(), TierError>(())
+            Ok::<(), String>(())
         }
         .await;
 
@@ -349,7 +281,7 @@ impl S3TierBackend {
         dest_path: &str,
         key: &str,
         progress_fn: F,
-    ) -> Result<u64, TierError>
+    ) -> Result<u64, String>
     where
         F: FnMut(i64, f32) -> Result<(), String> + Send + Sync + 'static,
     {
@@ -361,7 +293,7 @@ impl S3TierBackend {
             .key(key)
             .send()
             .await
-            .map_err(|e| head_object_error(key, e))?;
+            .map_err(|e| format!("failed to head object {}: {}", key, e))?;
 
         let file_size = head_resp.content_length().unwrap_or(0) as u64;
 
@@ -373,12 +305,10 @@ impl S3TierBackend {
                 .truncate(true)
                 .open(dest_path)
                 .await
-                .map_err(|e| {
-                    TierError::Io(format!("failed to open dest file {}: {}", dest_path, e))
-                })?;
+                .map_err(|e| format!("failed to open dest file {}: {}", dest_path, e))?;
             file.set_len(file_size)
                 .await
-                .map_err(|e| TierError::Io(format!("failed to set file length: {}", e)))?;
+                .map_err(|e| format!("failed to set file length: {}", e))?;
         }
 
         let part_size: u64 = 64 * 1024 * 1024;
@@ -414,7 +344,7 @@ impl S3TierBackend {
                 let _permit = sem
                     .acquire()
                     .await
-                    .map_err(|e| TierError::Io(format!("semaphore error: {}", e)))?;
+                    .map_err(|e| format!("semaphore error: {}", e))?;
 
                 let end = off + size - 1;
                 let range = format!("bytes={}-{}", off, end);
@@ -426,13 +356,13 @@ impl S3TierBackend {
                     .range(&range)
                     .send()
                     .await
-                    .map_err(|e| get_object_error(&key, &range, e))?;
+                    .map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
 
                 let body = get_resp
                     .body
                     .collect()
                     .await
-                    .map_err(|e| TierError::Io(format!("failed to read body: {}", e)))?;
+                    .map_err(|e| format!("failed to read body: {}", e))?;
                 let bytes = body.into_bytes();
 
                 // Write at the correct offset (like Go's WriteAt)
@@ -440,17 +370,13 @@ impl S3TierBackend {
                     .write(true)
                     .open(&dp)
                     .await
-                    .map_err(|e| {
-                        TierError::Io(format!("failed to open dest file {}: {}", dp, e))
-                    })?;
+                    .map_err(|e| format!("failed to open dest file {}: {}", dp, e))?;
                 file.seek(std::io::SeekFrom::Start(off))
                     .await
-                    .map_err(|e| {
-                        TierError::Io(format!("failed to seek to offset {}: {}", off, e))
-                    })?;
+                    .map_err(|e| format!("failed to seek to offset {}: {}", off, e))?;
                 file.write_all(&bytes)
                     .await
-                    .map_err(|e| TierError::Io(format!("failed to write to {}: {}", dp, e)))?;
+                    .map_err(|e| format!("failed to write to {}: {}", dp, e))?;
 
                 // Report progress. The lock is released before the result is
                 // propagated so an aborting callback cannot poison the mutex
@@ -466,9 +392,9 @@ impl S3TierBackend {
                     };
                     (guard.1)(downloaded as i64, pct)
                 };
-                progress_result.map_err(TierError::Aborted)?;
+                progress_result?;
 
-                Ok::<_, TierError>(())
+                Ok::<_, String>(())
             }));
         }
 
@@ -476,7 +402,7 @@ impl S3TierBackend {
         for handle in handles {
             handle
                 .await
-                .map_err(|e| TierError::Io(format!("download task panicked: {}", e)))??;
+                .map_err(|e| format!("download task panicked: {}", e))??;
         }
 
         // fsync the file so its content is durable before the caller trims the .vif
@@ -485,21 +411,16 @@ impl S3TierBackend {
             .write(true)
             .open(dest_path)
             .await
-            .map_err(|e| TierError::Io(format!("failed to open {} for fsync: {}", dest_path, e)))?;
+            .map_err(|e| format!("failed to open {} for fsync: {}", dest_path, e))?;
         synced
             .sync_all()
             .await
-            .map_err(|e| TierError::Io(format!("failed to fsync {}: {}", dest_path, e)))?;
+            .map_err(|e| format!("failed to fsync {}: {}", dest_path, e))?;
 
         Ok(file_size)
     }
 
-    pub async fn read_range(
-        &self,
-        key: &str,
-        offset: u64,
-        size: usize,
-    ) -> Result<Vec<u8>, TierError> {
+    pub async fn read_range(&self, key: &str, offset: u64, size: usize) -> Result<Vec<u8>, String> {
         let end = offset + (size as u64).saturating_sub(1);
         let range = format!("bytes={}-{}", offset, end);
         let resp = self
@@ -510,35 +431,29 @@ impl S3TierBackend {
             .range(&range)
             .send()
             .await
-            .map_err(|e| get_object_error(key, &range, e))?;
+            .map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
 
         let body = resp
             .body
             .collect()
             .await
-            .map_err(|e| TierError::Io(format!("failed to read object {} body: {}", key, e)))?;
+            .map_err(|e| format!("failed to read object {} body: {}", key, e))?;
         Ok(body.into_bytes().to_vec())
     }
 
     /// Delete a file from S3.
-    pub async fn delete_file(&self, key: &str) -> Result<(), TierError> {
+    pub async fn delete_file(&self, key: &str) -> Result<(), String> {
         self.client
             .delete_object()
             .bucket(&self.bucket)
             .key(key)
             .send()
             .await
-            .map_err(|e| {
-                TierError::Io(format!(
-                    "failed to delete object {}: {}",
-                    key,
-                    DisplayErrorContext(&e)
-                ))
-            })?;
+            .map_err(|e| format!("failed to delete object {}: {}", key, e))?;
         Ok(())
     }
 
-    pub fn delete_file_blocking(&self, key: &str) -> Result<(), TierError> {
+    pub fn delete_file_blocking(&self, key: &str) -> Result<(), String> {
         let client = self.client.clone();
         let bucket = self.bucket.clone();
         let key = key.to_string();
@@ -549,13 +464,7 @@ impl S3TierBackend {
                 .key(&key)
                 .send()
                 .await
-                .map_err(|e| {
-                    TierError::Io(format!(
-                        "failed to delete object {}: {}",
-                        key,
-                        DisplayErrorContext(&e)
-                    ))
-                })?;
+                .map_err(|e| format!("failed to delete object {}: {}", key, e))?;
             Ok(())
         })
     }
@@ -565,7 +474,7 @@ impl S3TierBackend {
         key: &str,
         offset: u64,
         size: usize,
-    ) -> Result<Vec<u8>, TierError> {
+    ) -> Result<Vec<u8>, String> {
         let client = self.client.clone();
         let bucket = self.bucket.clone();
         let key = key.to_string();
@@ -579,12 +488,13 @@ impl S3TierBackend {
                 .range(&range)
                 .send()
                 .await
-                .map_err(|e| get_object_error(&key, &range, e))?;
+                .map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
 
-            let body =
-                resp.body.collect().await.map_err(|e| {
-                    TierError::Io(format!("failed to read object {} body: {}", key, e))
-                })?;
+            let body = resp
+                .body
+                .collect()
+                .await
+                .map_err(|e| format!("failed to read object {} body: {}", key, e))?;
             Ok(body.into_bytes().to_vec())
         })
     }
@@ -645,279 +555,18 @@ pub fn global_s3_tier_registry() -> &'static RwLock<S3TierRegistry> {
     GLOBAL_S3_TIER_REGISTRY.get_or_init(|| RwLock::new(S3TierRegistry::new()))
 }
 
-/// The one process-wide runtime for tiered-S3 I/O issued from synchronous
-/// storage code. A per-call runtime tore down the SDK's pooled connections
-/// after every 64 KiB chunk, re-dialing TLS per read; a long-lived runtime
-/// keeps the pool warm.
-///
-/// Built on first use. A build failure is returned, not cached or panicked:
-/// callers sit inside `Volume::destroy` and needle reads, whose own error
-/// paths must run, and a later call may succeed.
-static TIER_RUNTIME: std::sync::Mutex<Option<tokio::runtime::Runtime>> =
-    std::sync::Mutex::new(None);
-
-fn tier_handle() -> Result<tokio::runtime::Handle, TierError> {
-    let mut slot = TIER_RUNTIME
-        .lock()
-        .unwrap_or_else(|poisoned| poisoned.into_inner());
-    if slot.is_none() {
-        let runtime = tokio::runtime::Builder::new_multi_thread()
-            .worker_threads(2)
-            .thread_name("tier-io")
-            .enable_all()
-            .build()
-            .map_err(|e| {
-                TierError::RuntimeUnavailable(format!(
-                    "failed to build the tier I/O tokio runtime: {}",
-                    e
-                ))
-            })?;
-        *slot = Some(runtime);
-    }
-    Ok(slot.as_ref().expect("just initialised").handle().clone())
-}
-
-/// Run `future` on the tier runtime and block the calling thread until it
-/// finishes. The caller may be a worker of *another* tokio runtime, so this
-/// waits on a channel rather than `Handle::block_on`, which panics when
-/// called from inside any runtime context.
-fn block_on_tier_future<F, T>(future: F) -> Result<T, TierError>
+fn block_on_tier_future<F, T>(future: F) -> Result<T, String>
 where
-    F: Future<Output = Result<T, TierError>> + Send + 'static,
+    F: Future<Output = Result<T, String>> + Send + 'static,
     T: Send + 'static,
 {
-    let handle = tier_handle()?;
-    let task = handle.spawn(future);
-    let (tx, rx) = std::sync::mpsc::sync_channel(1);
-    handle.spawn(async move {
-        // The receiver only goes away if the caller was unwound; nothing to
-        // report then.
-        let _ = tx.send(task.await);
-    });
-    match rx.recv() {
-        Ok(Ok(result)) => result,
-        Ok(Err(join_error)) => Err(describe_join_error(join_error)),
-        Err(_) => Err(TierError::RuntimeUnavailable(
-            "tier I/O runtime dropped the task before it finished".to_string(),
-        )),
-    }
-}
-
-/// Turn a `JoinError` into a message that keeps the panic payload, so an
-/// SDK panic surfaces as "boom" rather than a fixed "thread panicked".
-fn describe_join_error(join_error: tokio::task::JoinError) -> TierError {
-    if join_error.is_panic() {
-        let payload = join_error.into_panic();
-        let message = if let Some(s) = payload.downcast_ref::<&str>() {
-            (*s).to_string()
-        } else if let Some(s) = payload.downcast_ref::<String>() {
-            s.clone()
-        } else {
-            "non-string panic payload".to_string()
-        };
-        TierError::Io(format!("tier I/O task panicked: {}", message))
-    } else {
-        TierError::RuntimeUnavailable(format!("tier I/O task failed: {}", join_error))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::remote_storage::s3::tests::{CannedResponse, NO_SUCH_KEY};
-    use std::collections::HashSet;
-    use tokio::runtime::Handle;
-
-    fn probe() -> Result<(tokio::runtime::Id, Option<String>), TierError> {
-        block_on_tier_future(async {
-            Ok((
-                Handle::current().id(),
-                std::thread::current().name().map(str::to_string),
-            ))
-        })
-    }
-
-    #[test]
-    fn block_on_tier_future_reuses_one_runtime() {
-        let (first_runtime, first_thread) = probe().expect("first call");
-        let (second_runtime, second_thread) = probe().expect("second call");
-        assert_eq!(
-            first_runtime, second_runtime,
-            "each call must run on the same long-lived tier runtime"
-        );
-        assert_eq!(first_thread.as_deref(), Some("tier-io"));
-        assert_eq!(second_thread.as_deref(), Some("tier-io"));
-
-        let mut runtimes = HashSet::new();
-        for _ in 0..20 {
-            let (id, _) = probe().expect("probe");
-            runtimes.insert(id);
-        }
-        assert_eq!(runtimes.len(), 1);
-    }
-
-    #[test]
-    fn block_on_tier_future_returns_the_value_and_the_error() {
-        assert_eq!(block_on_tier_future(async { Ok(7u32) }).unwrap(), 7);
-        let err = block_on_tier_future::<_, u32>(async { Err(TierError::NotFound("nope".into())) })
-            .unwrap_err();
-        assert!(
-            matches!(&err, TierError::NotFound(m) if m == "nope"),
-            "{err:?}"
-        );
-    }
-
-    #[test]
-    fn block_on_tier_future_works_from_a_std_thread() {
-        let (id, _) = std::thread::spawn(probe)
-            .join()
-            .expect("probe thread")
-            .expect("probe");
-        assert_eq!(id, tier_handle().expect("tier runtime").id());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn block_on_tier_future_works_from_spawn_blocking() {
-        let (id, _) = tokio::task::spawn_blocking(probe)
-            .await
-            .expect("spawn_blocking")
-            .expect("probe");
-        assert_eq!(id, tier_handle().expect("tier runtime").id());
-        assert_ne!(id, Handle::current().id());
-    }
-
-    // Called straight from another runtime's async context: the case that
-    // would panic with `Handle::block_on` ("Cannot start a runtime from
-    // within a runtime").
-    #[tokio::test]
-    async fn block_on_tier_future_works_from_a_current_thread_runtime() {
-        let (id, _) = probe().expect("probe");
-        assert_eq!(id, tier_handle().expect("tier runtime").id());
-        assert_ne!(id, Handle::current().id());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn block_on_tier_future_works_from_a_multi_thread_runtime_worker() {
-        let (id, _) = probe().expect("probe");
-        assert_eq!(id, tier_handle().expect("tier runtime").id());
-        assert_ne!(id, Handle::current().id());
-    }
-
-    #[test]
-    fn block_on_tier_future_reports_the_panic_payload() {
-        let err = block_on_tier_future::<_, ()>(async {
-            if std::hint::black_box(true) {
-                panic!("boom {}", 42);
-            }
-            Ok(())
-        })
-        .expect_err("a panicking future must be an error");
-        assert!(matches!(err, TierError::Io(_)), "got: {err:?}");
-        assert!(err.to_string().contains("boom 42"), "got: {err}");
-        assert!(err.to_string().contains("panicked"), "got: {err}");
-    }
-
-    #[test]
-    fn block_on_tier_future_reports_a_str_panic_payload() {
-        let err = block_on_tier_future::<_, ()>(async {
-            if std::hint::black_box(true) {
-                panic!("static boom");
-            }
-            Ok(())
-        })
-        .expect_err("a panicking future must be an error");
-        assert!(err.to_string().contains("static boom"), "got: {err}");
-    }
-
-    fn backend_answering(status: u16, body: &'static str) -> S3TierBackend {
-        let config = aws_sdk_s3::Config::builder()
-            .behavior_version(BehaviorVersion::latest())
-            .region(Region::new("us-east-1"))
-            .credentials_provider(Credentials::new("AKIATEST", "secret", None, None, "test"))
-            .endpoint_url("http://127.0.0.1:1")
-            .force_path_style(true)
-            .http_client(CannedResponse { status, body })
-            .retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
-            .build();
-        S3TierBackend {
-            client: Client::from_conf(config),
-            bucket: "bucket".to_string(),
-            storage_class: "STANDARD".to_string(),
-        }
-    }
-
-    #[tokio::test]
-    async fn download_head_404_is_not_found() {
-        let tmp = tempfile::tempdir().unwrap();
-        let dest = tmp.path().join("1.dat");
-        let err = backend_answering(404, "")
-            .download_file(dest.to_str().unwrap(), "missing", |_, _| Ok(()))
-            .await
-            .unwrap_err();
-        assert!(matches!(err, TierError::NotFound(_)), "{err:?}");
-        assert!(
-            err.to_string()
-                .starts_with("failed to head object missing: "),
-            "{err}"
-        );
-    }
-
-    #[tokio::test]
-    async fn download_head_403_is_io() {
-        let tmp = tempfile::tempdir().unwrap();
-        let dest = tmp.path().join("1.dat");
-        let err = backend_answering(403, "")
-            .download_file(dest.to_str().unwrap(), "denied", |_, _| Ok(()))
-            .await
-            .unwrap_err();
-        assert!(matches!(err, TierError::Io(_)), "{err:?}");
-    }
-
-    #[tokio::test]
-    async fn read_range_no_such_key_is_not_found() {
-        let err = backend_answering(404, NO_SUCH_KEY)
-            .read_range("missing", 0, 8)
-            .await
-            .unwrap_err();
-        assert!(matches!(err, TierError::NotFound(_)), "{err:?}");
-        assert!(
-            err.to_string()
-                .starts_with("failed to get object missing range bytes=0-7: "),
-            "{err}"
-        );
-    }
-
-    #[tokio::test]
-    async fn read_range_bare_404_is_io() {
-        // As in Go, GET is not-found by the NoSuchKey code, not the status.
-        let err = backend_answering(404, "")
-            .read_range("missing", 0, 8)
-            .await
-            .unwrap_err();
-        assert!(matches!(err, TierError::Io(_)), "{err:?}");
-    }
-
-    #[test]
-    fn read_range_blocking_no_such_key_is_not_found() {
-        let err = backend_answering(404, NO_SUCH_KEY)
-            .read_range_blocking("missing", 0, 8)
-            .unwrap_err();
-        assert!(matches!(err, TierError::NotFound(_)), "{err:?}");
-    }
-
-    #[test]
-    fn backend_name_to_type_id_splits_on_dot() {
-        assert_eq!(
-            backend_name_to_type_id("s3"),
-            ("s3".to_string(), "default".to_string())
-        );
-        assert_eq!(
-            backend_name_to_type_id("s3.eu"),
-            ("s3".to_string(), "eu".to_string())
-        );
-        assert_eq!(
-            backend_name_to_type_id("s3.a.b"),
-            (String::new(), String::new())
-        );
-    }
+    std::thread::spawn(move || {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .map_err(|e| format!("failed to build tokio runtime: {}", e))?;
+        runtime.block_on(future)
+    })
+    .join()
+    .map_err(|_| "tier runtime thread panicked".to_string())?
 }

seaweed-volume/src/server/handlers.rs

@@ -21,7 +21,6 @@ use super::volume_server::{VolumeServerState, normalize_outgoing_http_url, to_ht
 use crate::config::ReadMode;
 use crate::metrics;
 use crate::pb::volume_server_pb;
-use crate::storage::needle::CRC;
 use crate::storage::needle::needle::Needle;
 use crate::storage::types::*;
 
@@ -143,78 +142,6 @@ const STREAMING_THRESHOLD: u32 = 1024 * 1024; // 1 MB
 /// Default chunk size for streaming reads from the dat file.
 const DEFAULT_STREAMING_CHUNK_SIZE: usize = 64 * 1024; // 64 KB
 
-/// Whether a GET/HEAD can be answered from the needle meta and the data file.
-#[derive(Clone, Copy)]
-struct SourceReadRequest {
-    is_head: bool,
-    has_range: bool,
-    has_image_ops: bool,
-    bypass_cm: bool,
-}
-
-impl SourceReadRequest {
-    /// The payload can be served as stored.
-    fn direct(&self, n: &Needle) -> bool {
-        !n.is_compressed() && !(n.is_chunk_manifest() && !self.bypass_cm) && !self.has_image_ops
-    }
-
-    /// HEAD, a range, or a stream.
-    fn served_from_source(&self, n: &Needle) -> bool {
-        self.is_head || (self.direct(n) && (self.has_range || n.data_size > STREAMING_THRESHOLD))
-    }
-
-    /// Meta first only if the needle could be served from source; its data
-    /// size is below its index size.
-    fn try_meta_first(&self, size: Size) -> bool {
-        self.is_head
-            || (!self.has_image_ops && (self.has_range || size.0 > STREAMING_THRESHOLD as i32))
-    }
-}
-
-/// Blocking part of a GET/HEAD: the store guard is dropped before any needle
-/// I/O, so a slow or tiered read cannot park store writers. `Ok(None)` is a
-/// cookie mismatch.
-fn read_needle_for_get(
-    state: &VolumeServerState,
-    vid: VolumeId,
-    needle_id: NeedleId,
-    cookie: Cookie,
-    read_deleted: bool,
-    request: SourceReadRequest,
-) -> Result<
-    Option<(Needle, Option<crate::storage::volume::NeedleStreamInfo>)>,
-    crate::storage::volume::VolumeError,
-> {
-    let mut plan = state
-        .store
-        .read()
-        .unwrap()
-        .needle_read_plan(vid, needle_id, read_deleted)?;
-    let blank = || Needle {
-        id: needle_id,
-        cookie,
-        ..Needle::default()
-    };
-    let mut n = blank();
-    if request.try_meta_first(plan.size()) {
-        plan.read_meta(&mut n)?;
-        if n.cookie != cookie {
-            return Ok(None);
-        }
-        if request.served_from_source(&n) {
-            let info = plan.into_stream_info(&n);
-            return Ok(Some((n, Some(info))));
-        }
-        // Compressed or a chunk manifest: the payload is needed after all.
-        n = blank();
-    }
-    plan.read_full(&mut n)?;
-    if n.cookie != cookie {
-        return Ok(None);
-    }
-    Ok(Some((n, None)))
-}
-
 /// A body that streams needle data from the dat file in chunks using pread,
 /// avoiding loading the entire payload into memory at once.
 struct StreamingBody {
@@ -240,10 +167,6 @@ struct StreamingBody {
     /// Compaction revision at the time of the initial read; if the volume's revision
     /// changes between chunks, the needle may have moved and we must re-lookup its offset.
     compaction_revision: u16,
-    /// Checksum stored in the needle tail.
-    expected_checksum: u32,
-    /// Running checksum over the emitted data.
-    crc: CRC,
 }
 
 impl http_body::Body for StreamingBody {
@@ -264,32 +187,7 @@ impl http_body::Body for StreamingBody {
                         match result {
                             Ok(Ok(chunk)) => {
                                 let len = chunk.len();
-                                self.crc = self.crc.update(&chunk);
                                 self.pos += len;
-                                // The last frame is held back until the checksum verifies
-                                // (Go parity: readNeedleDataInto). On a mismatch it is
-                                // never emitted, so the response ends short of the declared
-                                // Content-Length and the reader sees a failed transfer.
-                                if self.pos >= self.data_size as usize {
-                                    let ok = self.expected_checksum == self.crc.0
-                                        || self.expected_checksum == self.crc.legacy_value();
-                                    if !ok {
-                                        metrics::HANDLER_COUNTER
-                                            .with_label_values(&[metrics::ERROR_CRC])
-                                            .inc();
-                                        return std::task::Poll::Ready(Some(Err(
-                                            std::io::Error::new(
-                                                std::io::ErrorKind::InvalidData,
-                                                format!(
-                                                    "needle data checksum {} expected {} for needle {}",
-                                                    self.crc.0,
-                                                    self.expected_checksum,
-                                                    self.needle_id
-                                                ),
-                                            ),
-                                        )));
-                                    }
-                                }
                                 return std::task::Poll::Ready(Some(Ok(http_body::Frame::data(
                                     chunk,
                                 ))));
@@ -313,9 +211,6 @@ impl http_body::Body for StreamingBody {
             let relookup_result = {
                 let store = self.server_state.store.read().unwrap();
                 if let Some((_, vol)) = store.find_volume(self.volume_id) {
-                    if let Some(e) = vol.unavailable_error() {
-                        return std::task::Poll::Ready(Some(Err(std::io::Error::other(e))));
-                    }
                     if vol.super_block.compaction_revision != self.compaction_revision {
                         // Compaction occurred — re-lookup the needle's data offset
                         Some(vol.re_lookup_needle_data_offset(self.needle_id))
@@ -950,7 +845,7 @@ async fn proxy_request(
         if name.as_str().eq_ignore_ascii_case("server") {
             continue;
         }
-        response_headers.append(name.clone(), value.clone());
+        response_headers.insert(name.clone(), value.clone());
     }
 
     // Stream the proxy response body instead of buffering it entirely
@@ -1184,7 +1079,11 @@ async fn get_or_head_handler_inner(
 
     // Read needle — branching between regular volume and EC volume paths.
     // EC volumes always do a full read (no streaming/meta-only).
-    let n: Needle;
+    let mut n = Needle {
+        id: needle_id,
+        cookie,
+        ..Needle::default()
+    };
 
     let read_deleted = query.read_deleted.as_deref() == Some("true");
     let has_range = headers.contains_key(header::RANGE);
@@ -1261,49 +1160,24 @@ async fn get_or_head_handler_inner(
         can_handle_range_from_source = false;
     } else {
         // ---- Regular volume read path (with streaming support) ----
-        bypass_cm = query.cm.as_deref() == Some("false");
-        track_download = download_guard.is_some();
-        let request_kind = SourceReadRequest {
-            is_head: method == Method::HEAD,
-            has_range,
-            has_image_ops,
-            bypass_cm,
-        };
 
-        let read_state = state.clone();
-        let read = tokio::task::spawn_blocking(move || {
-            read_needle_for_get(
-                &read_state,
-                vid,
-                needle_id,
-                cookie,
-                read_deleted,
-                request_kind,
-            )
-        })
-        .await;
-        (n, stream_info) = match read {
-            Ok(Ok(Some(found))) => found,
-            // Cookie mismatch
-            Ok(Ok(None)) => return StatusCode::NOT_FOUND.into_response(),
-            Ok(Err(
-                crate::storage::volume::VolumeError::NotFound
-                | crate::storage::volume::VolumeError::Deleted,
-            )) => {
+        // Try meta-only read first for potential streaming
+        let store = state.store.read().unwrap();
+        let si_result = store.read_volume_needle_stream_info(vid, &mut n, read_deleted);
+        stream_info = match si_result {
+            Ok(info) => Some(info),
+            Err(crate::storage::volume::VolumeError::StreamingUnsupported) => None,
+            Err(crate::storage::volume::VolumeError::NotFound) => {
                 metrics::HANDLER_COUNTER
                     .with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
                     .inc();
                 return StatusCode::NOT_FOUND.into_response();
             }
-            Ok(Err(e)) => {
+            Err(crate::storage::volume::VolumeError::Deleted) => {
                 metrics::HANDLER_COUNTER
-                    .with_label_values(&[metrics::ERROR_GET_INTERNAL])
+                    .with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
                     .inc();
-                return (
-                    StatusCode::INTERNAL_SERVER_ERROR,
-                    format!("read error: {}", e),
-                )
-                    .into_response();
+                return StatusCode::NOT_FOUND.into_response();
             }
             Err(e) => {
                 metrics::HANDLER_COUNTER
@@ -1316,9 +1190,19 @@ async fn get_or_head_handler_inner(
                     .into_response();
             }
         };
+        drop(store);
 
-        // Stream info is only returned for a reply served from the data file.
-        let can_direct_source_read = stream_info.is_some() && request_kind.direct(&n);
+        // Validate cookie
+        if n.cookie != cookie {
+            return StatusCode::NOT_FOUND.into_response();
+        }
+
+        bypass_cm = query.cm.as_deref() == Some("false");
+        track_download = download_guard.is_some();
+        let can_direct_source_read = stream_info.is_some()
+            && !n.is_compressed()
+            && !(n.is_chunk_manifest() && !bypass_cm)
+            && !has_image_ops;
 
         // Determine if we can stream (large, direct-source eligible, no range)
         can_stream = can_direct_source_read
@@ -1329,6 +1213,41 @@ async fn get_or_head_handler_inner(
         // Go uses meta-only reads for all HEAD requests, regardless of compression/chunked files.
         can_handle_head_from_meta = stream_info.is_some() && method == Method::HEAD;
         can_handle_range_from_source = can_direct_source_read && has_range;
+
+        // For chunk manifest or any non-streaming path, we need the full data.
+        // If we can't stream, do a full read now.
+        if !can_stream && !can_handle_head_from_meta && !can_handle_range_from_source {
+            // Re-read with full data
+            let mut n_full = Needle {
+                id: needle_id,
+                cookie,
+                ..Needle::default()
+            };
+            let store = state.store.read().unwrap();
+            match store.read_volume_needle_opt(vid, &mut n_full, read_deleted) {
+                Ok(count) => {
+                    if count < 0 {
+                        return StatusCode::NOT_FOUND.into_response();
+                    }
+                }
+                Err(crate::storage::volume::VolumeError::NotFound) => {
+                    return StatusCode::NOT_FOUND.into_response();
+                }
+                Err(crate::storage::volume::VolumeError::Deleted) => {
+                    return StatusCode::NOT_FOUND.into_response();
+                }
+                Err(e) => {
+                    return (
+                        StatusCode::INTERNAL_SERVER_ERROR,
+                        format!("read error: {}", e),
+                    )
+                        .into_response();
+                }
+            }
+            drop(store);
+            // Use the full needle from here (it has the same metadata + data)
+            n = n_full;
+        }
     }
 
     // Build ETag and Last-Modified BEFORE conditional checks and chunk manifest expansion
@@ -1590,8 +1509,6 @@ async fn get_or_head_handler_inner(
             volume_id: info.volume_id,
             needle_id: info.needle_id,
             compaction_revision: info.compaction_revision,
-            expected_checksum: info.checksum,
-            crc: CRC(0),
         };
 
         let body = Body::new(streaming);
@@ -1613,19 +1530,12 @@ async fn get_or_head_handler_inner(
         && let (Some(range_header), Some(info)) = (headers.get(header::RANGE), stream_info)
         && let Ok(range_str) = range_header.to_str()
     {
-        let range_str = range_str.to_string();
-        let tracking = track_download.then(|| state.clone());
-        return tokio::task::spawn_blocking(move || {
-            handle_range_request_from_source(&range_str, info, response_headers, tracking)
-        })
-        .await
-        .unwrap_or_else(|e| {
-            (
-                StatusCode::INTERNAL_SERVER_ERROR,
-                format!("range read error: {}", e),
-            )
-                .into_response()
-        });
+        return handle_range_request_from_source(
+            range_str,
+            info,
+            response_headers,
+            track_download.then(|| state.clone()),
+        );
     }
 
     // ---- Buffered path: small files, compressed, images, range requests ----
@@ -2751,15 +2661,8 @@ pub async fn post_handler(
     // sees it the same way the primary did.
     let fsync = form_value("fsync").as_deref() == Some("true");
 
-    // Go computes the checksum while building the needle (CreateNeedleFromRequest).
-    // The write queue takes the needle away, so the response fields that come
-    // from it are read here, before the write, rather than from a copy.
-    n.checksum = crate::storage::needle::crc::CRC::new(&n.data);
-    let needle_etag = n.etag();
-    let needle_has_name = n.has_name();
-
     let write_result = if let Some(wq) = state.write_queue.get() {
-        wq.submit(vid, n, fsync).await
+        wq.submit(vid, n.clone(), fsync).await
     } else {
         let mut store = state.store.write().unwrap();
         store.write_volume_needle(vid, &mut n, fsync)
@@ -2810,22 +2713,22 @@ pub async fn post_handler(
     let resp = match write_result {
         Ok((_offset, _size, is_unchanged)) => {
             if is_unchanged {
-                let etag = format!("\"{}\"", needle_etag);
+                let etag = format!("\"{}\"", n.etag());
                 (StatusCode::NO_CONTENT, [(header::ETAG, etag)]).into_response()
             } else {
                 // Go only includes contentMd5 when the client provided Content-MD5
                 let result = UploadResult {
-                    name: if needle_has_name {
+                    name: if n.has_name() {
                         filename.clone()
                     } else {
                         String::new()
                     },
                     size: original_data_size, // H3: use original size, not compressed
-                    etag: needle_etag.clone(),
+                    etag: n.etag(),
                     mime: mime_type.clone(),
                     content_md5: original_content_md5.clone(),
                 };
-                let etag = needle_etag;
+                let etag = n.etag();
                 let etag_header = if etag.starts_with('"') {
                     etag.clone()
                 } else {
@@ -4670,463 +4573,4 @@ mod tests {
 
         server.abort();
     }
-
-    /// Repeated target response headers (e.g. Set-Cookie) must all reach the
-    /// client, as Go's `w.Header().Add` does; only `Server` is dropped.
-    #[tokio::test]
-    async fn test_proxy_request_keeps_repeated_response_headers() {
-        use axum::{Router, routing::get};
-
-        let app = Router::new().route(
-            "/3,01637037d6",
-            get(|| async {
-                (
-                    axum::response::AppendHeaders([
-                        (header::SET_COOKIE, "a=1"),
-                        (header::SET_COOKIE, "b=2"),
-                        (header::SERVER, "target"),
-                    ]),
-                    "payload",
-                )
-            }),
-        );
-        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
-        let addr = listener.local_addr().unwrap();
-        let server = tokio::spawn(async move {
-            axum::serve(listener, app).await.unwrap();
-        });
-
-        let info = ProxyRequestInfo {
-            original_headers: HeaderMap::new(),
-            original_query: String::new(),
-            path: "/3,01637037d6".to_string(),
-            vid_str: "3".to_string(),
-            fid_str: "01637037d6".to_string(),
-        };
-        let target = VolumeLocation {
-            url: addr.to_string(),
-            public_url: String::new(),
-            grpc_port: 0,
-            read_only: false,
-            read_only_can_delete: false,
-        };
-
-        let response = proxy_request(&streaming_test_state(), &info, &target).await;
-        assert_eq!(response.status(), StatusCode::OK);
-        let cookies: Vec<_> = response
-            .headers()
-            .get_all(header::SET_COOKIE)
-            .iter()
-            .map(|v| v.to_str().unwrap().to_string())
-            .collect();
-        assert_eq!(cookies, vec!["a=1", "b=2"]);
-        assert!(response.headers().get(header::SERVER).is_none());
-        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
-            .await
-            .unwrap();
-        assert_eq!(&body[..], b"payload");
-
-        server.abort();
-    }
-
-    fn streaming_test_state() -> Arc<VolumeServerState> {
-        use crate::storage::needle_map::NeedleMapKind;
-        use crate::storage::store::Store;
-        test_state_with_store(Store::new(NeedleMapKind::InMemory))
-    }
-
-    fn test_state_with_store(store: crate::storage::store::Store) -> Arc<VolumeServerState> {
-        use crate::security::{Guard, SigningKey};
-        use crate::server::volume_server::RuntimeMetricsConfig;
-        use std::sync::RwLock;
-        use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU32};
-
-        Arc::new(VolumeServerState {
-            store: RwLock::new(store),
-            guard: RwLock::new(Guard::new(
-                &[],
-                SigningKey(vec![]),
-                0,
-                SigningKey(vec![]),
-                0,
-            )),
-            is_stopping: RwLock::new(false),
-            maintenance: AtomicBool::new(false),
-            state_version: AtomicU32::new(0),
-            concurrent_upload_limit: 0,
-            concurrent_download_limit: 0,
-            inflight_upload_data_timeout: std::time::Duration::ZERO,
-            inflight_download_data_timeout: std::time::Duration::ZERO,
-            inflight_upload_bytes: AtomicI64::new(0),
-            inflight_download_bytes: AtomicI64::new(0),
-            upload_notify: tokio::sync::Notify::new(),
-            download_notify: tokio::sync::Notify::new(),
-            data_center: String::new(),
-            rack: String::new(),
-            file_size_limit_bytes: 0,
-            maintenance_byte_per_second: 0,
-            is_heartbeating: AtomicBool::new(false),
-            has_master: false,
-            pre_stop_seconds: 0,
-            volume_state_notify: tokio::sync::Notify::new(),
-            write_queue: std::sync::OnceLock::new(),
-            read_mode: crate::config::ReadMode::Local,
-            allow_untrusted_remote_endpoints: false,
-            master_url: String::new(),
-            master_urls: Vec::new(),
-            seed_master_set: std::collections::HashSet::new(),
-            current_master_url: tokio::sync::RwLock::new(String::new()),
-            self_url: String::new(),
-            http_client: reqwest::Client::new(),
-            outgoing_http_scheme: "http".to_string(),
-            outgoing_grpc_tls: None,
-            metrics_runtime: RwLock::new(RuntimeMetricsConfig::default()),
-            metrics_notify: tokio::sync::Notify::new(),
-            fix_jpg_orientation: false,
-            has_slow_read: false,
-            read_buffer_size_bytes: 4 * 1024 * 1024,
-            security_file: String::new(),
-            cli_white_list: vec![],
-            state_file_path: String::new(),
-        })
-    }
-
-    /// Go's readNeedleDataInto holds back the last chunk of a whole-needle read
-    /// until the checksum verifies; a streamed needle that fails its CRC must
-    /// likewise end short of the declared length instead of looking complete.
-    #[tokio::test]
-    async fn test_streaming_body_checksum_failure_drops_last_chunk() {
-        use futures::StreamExt;
-        use std::io::Write;
-
-        let data: Vec<u8> = (0..10_000u32).map(|i| (i % 251) as u8).collect();
-        let mut tmp = tempfile::NamedTempFile::new().unwrap();
-        tmp.write_all(&data).unwrap();
-        let file = tmp.reopen().unwrap();
-
-        let control = Arc::new(crate::storage::volume::DataFileAccessControl::default());
-        let mk_body = |expected_checksum: u32, file: &std::fs::File| StreamingBody {
-            source: crate::storage::volume::NeedleStreamSource::Local(file.try_clone().unwrap()),
-            data_offset: 0,
-            data_size: data.len() as u32,
-            pos: 0,
-            chunk_size: 4096,
-            data_file_access_control: control.clone(),
-            hold_read_lock_for_stream: true,
-            _held_read_lease: None,
-            pending: None,
-            state: None,
-            tracked_bytes: 0,
-            server_state: streaming_test_state(),
-            volume_id: VolumeId(1),
-            needle_id: NeedleId(7),
-            compaction_revision: 0,
-            expected_checksum,
-            crc: CRC(0),
-        };
-
-        // Healthy needle: every byte is delivered.
-        let body = Body::new(mk_body(CRC::new(&data).0, &file));
-        let mut stream = body.into_data_stream();
-        let mut got = 0usize;
-        while let Some(item) = stream.next().await {
-            got += item.expect("healthy stream errored").len();
-        }
-        assert_eq!(got, data.len());
-
-        // Corrupted needle: the last chunk fails the checksum and is never
-        // emitted — the body ends short of the data size.
-        let body = Body::new(mk_body(CRC::new(&data).0 ^ 0xff, &file));
-        let mut stream = body.into_data_stream();
-        let mut got = 0usize;
-        let mut saw_err = false;
-        while let Some(item) = stream.next().await {
-            match item {
-                Ok(bytes) => got += bytes.len(),
-                Err(e) => {
-                    assert!(
-                        e.to_string().contains("checksum"),
-                        "unexpected stream error: {e}"
-                    );
-                    saw_err = true;
-                }
-            }
-        }
-        assert!(saw_err, "corrupted stream must fail");
-        assert!(got < data.len(), "delivered {got} of {} bytes", data.len());
-    }
-
-    const TEST_COOKIE: u32 = 0x1234_5678;
-
-    /// State whose store holds volume 1 in `tmp`.
-    fn volume_test_state(tmp: &tempfile::TempDir) -> Arc<VolumeServerState> {
-        use crate::storage::needle_map::NeedleMapKind;
-        use crate::storage::store::Store;
-        use crate::storage::volume::VolumeSpec;
-
-        let dir = tmp.path().to_str().unwrap();
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        store
-            .add_location(
-                dir,
-                dir,
-                10,
-                DiskType::HardDrive,
-                crate::config::MinFreeSpace::Percent(1.0),
-                Vec::new(),
-            )
-            .unwrap();
-        store
-            .add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
-            .unwrap();
-        test_state_with_store(store)
-    }
-
-    /// Write a needle to volume 1 and return its URL path.
-    fn put_test_needle(state: &VolumeServerState, id: u64, data: &[u8]) -> String {
-        put_test_needle_with(state, id, data, |_| {})
-    }
-
-    fn put_test_needle_with(
-        state: &VolumeServerState,
-        id: u64,
-        data: &[u8],
-        prepare: impl FnOnce(&mut Needle),
-    ) -> String {
-        let mut n = Needle {
-            id: NeedleId(id),
-            cookie: Cookie(TEST_COOKIE),
-            data: data.to_vec(),
-            data_size: data.len() as u32,
-            ..Needle::default()
-        };
-        prepare(&mut n);
-        state
-            .store
-            .write()
-            .unwrap()
-            .write_volume_needle(VolumeId(1), &mut n, false)
-            .unwrap();
-        format!("/1,{:x}{:08x}", id, TEST_COOKIE)
-    }
-
-    async fn send_read(
-        state: &Arc<VolumeServerState>,
-        method: Method,
-        path: &str,
-        range: Option<&str>,
-    ) -> (StatusCode, HeaderMap, Vec<u8>) {
-        use tower::ServiceExt;
-        let mut req = Request::builder().method(method).uri(path);
-        if let Some(range) = range {
-            req = req.header(header::RANGE, range);
-        }
-        let resp = super::super::volume_server::build_public_router(state.clone())
-            .oneshot(req.body(Body::empty()).unwrap())
-            .await
-            .unwrap();
-        let status = resp.status();
-        let headers = resp.headers().clone();
-        let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
-            .await
-            .unwrap()
-            .to_vec();
-        (status, headers, body)
-    }
-
-    /// A GET whose needle read is parked must not hold the store lock: a
-    /// writer, here a real append to the same volume, must get through.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_get_needle_read_runs_off_the_store_lock() {
-        use crate::storage::volume::needle_read_hook;
-        use std::sync::Mutex;
-        use std::sync::mpsc;
-        use std::time::Duration;
-
-        const ID: u64 = 0x6e7a_0001;
-        let tmp = tempfile::TempDir::new().unwrap();
-        let state = volume_test_state(&tmp);
-        let data = b"parked needle".to_vec();
-        let path = put_test_needle(&state, ID, &data);
-
-        let (entered_tx, entered_rx) = mpsc::channel::<()>();
-        let (release_tx, release_rx) = mpsc::channel::<()>();
-        let release_rx = Mutex::new(release_rx);
-        let _hook = needle_read_hook::register(NeedleId(ID), move |_| {
-            let _ = entered_tx.send(());
-            // Returns once released, or at once when the sender is gone.
-            let _ = release_rx.lock().unwrap().recv();
-        });
-
-        let get = tokio::spawn({
-            let state = state.clone();
-            async move { send_read(&state, Method::GET, &path, None).await }
-        });
-        tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(10)))
-            .await
-            .unwrap()
-            .expect("the GET never reached its needle read");
-
-        let try_write_ok = state.store.try_write().is_ok();
-        let (wrote_tx, wrote_rx) = mpsc::channel();
-        std::thread::spawn({
-            let state = state.clone();
-            move || {
-                put_test_needle(&state, ID + 1, b"written meanwhile");
-                let _ = wrote_tx.send(());
-            }
-        });
-        let write_done = tokio::task::spawn_blocking(move || {
-            wrote_rx.recv_timeout(Duration::from_secs(5)).is_ok()
-        })
-        .await
-        .unwrap();
-        drop(release_tx);
-
-        assert!(
-            try_write_ok,
-            "a parked GET read must not hold the store lock"
-        );
-        assert!(write_done, "an append must not wait for a parked GET read");
-        let (status, _, body) = get.await.unwrap();
-        assert_eq!(status, StatusCode::OK);
-        assert_eq!(body, data);
-    }
-
-    /// A small needle is read from disk once, not once for its meta and
-    /// again in full; a large one never reads its payload before streaming.
-    #[tokio::test]
-    async fn test_get_reads_small_needle_once_and_large_needle_meta_only() {
-        use crate::storage::volume::needle_read_hook;
-        use std::sync::Mutex;
-
-        const SMALL: u64 = 0x6e7a_0101;
-        const LARGE: u64 = 0x6e7a_0102;
-        let tmp = tempfile::TempDir::new().unwrap();
-        let state = volume_test_state(&tmp);
-        let small = b"small needle".to_vec();
-        let large: Vec<u8> = (0..(STREAMING_THRESHOLD as usize + 4096))
-            .map(|i| (i % 251) as u8)
-            .collect();
-        let small_path = put_test_needle(&state, SMALL, &small);
-        let large_path = put_test_needle(&state, LARGE, &large);
-
-        let small_reads = Arc::new(Mutex::new(Vec::new()));
-        let large_reads = Arc::new(Mutex::new(Vec::new()));
-        let _small_hook = needle_read_hook::register(NeedleId(SMALL), {
-            let reads = small_reads.clone();
-            move |len| reads.lock().unwrap().push(len)
-        });
-        let _large_hook = needle_read_hook::register(NeedleId(LARGE), {
-            let reads = large_reads.clone();
-            move |len| reads.lock().unwrap().push(len)
-        });
-
-        let (status, _, body) = send_read(&state, Method::GET, &small_path, None).await;
-        assert_eq!(status, StatusCode::OK);
-        assert_eq!(body, small);
-        let reads = small_reads.lock().unwrap().clone();
-        assert_eq!(reads.len(), 1, "reads of the small needle: {reads:?}");
-
-        let (status, _, body) = send_read(&state, Method::GET, &large_path, None).await;
-        assert_eq!(status, StatusCode::OK);
-        assert_eq!(body, large);
-        let reads = large_reads.lock().unwrap().clone();
-        assert!(
-            !reads.is_empty() && reads.iter().all(|&len| len < 64 * 1024),
-            "the large needle's payload must not be read before streaming: {reads:?}"
-        );
-    }
-
-    /// HEAD and ranged reads keep answering from the needle meta and the
-    /// data file, and a missing or wrong-cookie needle is still a 404.
-    #[tokio::test]
-    async fn test_get_head_range_and_not_found_after_single_read() {
-        const ID: u64 = 0x6e7a_0201;
-        let tmp = tempfile::TempDir::new().unwrap();
-        let state = volume_test_state(&tmp);
-        let data: Vec<u8> = (0..4096u32).map(|i| (i % 251) as u8).collect();
-        let path = put_test_needle(&state, ID, &data);
-
-        let (status, headers, body) = send_read(&state, Method::HEAD, &path, None).await;
-        assert_eq!(status, StatusCode::OK);
-        assert!(body.is_empty());
-        assert_eq!(headers[header::CONTENT_LENGTH], data.len().to_string());
-        assert!(headers.contains_key(header::ETAG));
-
-        let (status, headers, body) =
-            send_read(&state, Method::GET, &path, Some("bytes=10-19")).await;
-        assert_eq!(status, StatusCode::PARTIAL_CONTENT);
-        assert_eq!(body, &data[10..20]);
-        assert_eq!(
-            headers["Content-Range"],
-            format!("bytes 10-19/{}", data.len())
-        );
-
-        let wrong_cookie = format!("/1,{:x}{:08x}", ID, TEST_COOKIE ^ 1);
-        let (status, _, _) = send_read(&state, Method::GET, &wrong_cookie, None).await;
-        assert_eq!(status, StatusCode::NOT_FOUND);
-        let missing = format!("/1,{:x}{:08x}", ID + 1, TEST_COOKIE);
-        let (status, _, _) = send_read(&state, Method::GET, &missing, None).await;
-        assert_eq!(status, StatusCode::NOT_FOUND);
-    }
-
-    /// A large compressed needle cannot be streamed as stored: its meta is
-    /// read first, then the payload exactly once.
-    #[tokio::test]
-    async fn test_get_large_compressed_needle_reads_payload_once() {
-        use crate::storage::volume::needle_read_hook;
-        use std::sync::Mutex;
-
-        const ID: u64 = 0x6e7a_0301;
-        let tmp = tempfile::TempDir::new().unwrap();
-        let state = volume_test_state(&tmp);
-        // Incompressible, so the stored gzip stays above the stream threshold.
-        let mut x = 0x2545_f491_u32;
-        let plain: Vec<u8> = (0..(STREAMING_THRESHOLD as usize + 64 * 1024))
-            .map(|_| {
-                x ^= x << 13;
-                x ^= x >> 17;
-                x ^= x << 5;
-                x as u8
-            })
-            .collect();
-        let gz = try_gzip_data(&plain).unwrap();
-        assert!(gz.len() > STREAMING_THRESHOLD as usize);
-        let path = put_test_needle_with(&state, ID, &gz, |n| n.set_is_compressed());
-
-        let reads = Arc::new(Mutex::new(Vec::new()));
-        let _hook = needle_read_hook::register(NeedleId(ID), {
-            let reads = reads.clone();
-            move |len| reads.lock().unwrap().push(len)
-        });
-        let (status, _, body) = send_read(&state, Method::GET, &path, None).await;
-        assert_eq!(status, StatusCode::OK);
-        assert_eq!(body, plain);
-        let reads = reads.lock().unwrap().clone();
-        let full = reads.iter().filter(|&&len| len >= gz.len()).count();
-        assert_eq!(full, 1, "reads: {reads:?}");
-    }
-
-    /// The single full read still verifies the needle checksum.
-    #[tokio::test]
-    async fn test_get_small_needle_with_bad_checksum_fails() {
-        const ID: u64 = 0x6e7a_0401;
-        let tmp = tempfile::TempDir::new().unwrap();
-        let state = volume_test_state(&tmp);
-        let data = b"checksummed payload, to be corrupted".to_vec();
-        let path = put_test_needle(&state, ID, &data);
-
-        let dat = tmp.path().join("1.dat");
-        let mut bytes = std::fs::read(&dat).unwrap();
-        let at = bytes
-            .windows(data.len())
-            .position(|w| w == data.as_slice())
-            .unwrap();
-        bytes[at] ^= 0xff;
-        std::fs::write(&dat, bytes).unwrap();
-
-        let (status, _, _) = send_read(&state, Method::GET, &path, None).await;
-        assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
-    }
 }

seaweed-volume/src/server/mod.rs

@@ -1,6 +1,5 @@
 use tonic::Status;
 
-use crate::remote_storage::s3_tier::TierError;
 use crate::storage::volume::VolumeError;
 
 #[cfg(unix)]
@@ -24,12 +23,8 @@ impl From<VolumeError> for Status {
     fn from(err: VolumeError) -> Self {
         let message = err.to_string();
         match err {
-            VolumeError::NotFound
-            | VolumeError::VolumeNotFound(_)
-            | VolumeError::Tier(TierError::NotFound(_)) => Status::not_found(message),
-            VolumeError::ReadOnly(_) | VolumeError::NotEmpty => {
-                Status::failed_precondition(message)
-            }
+            VolumeError::NotFound | VolumeError::VolumeNotFound(_) => Status::not_found(message),
+            VolumeError::ReadOnly | VolumeError::NotEmpty => Status::failed_precondition(message),
             VolumeError::InsufficientSpace { .. } => Status::resource_exhausted(message),
             VolumeError::AlreadyExists => Status::already_exists(message),
             _ => Status::internal(message),
@@ -71,14 +66,7 @@ mod tests {
             Code::NotFound
         );
         assert_eq!(code(VolumeError::NotFound), Code::NotFound);
-        assert_eq!(
-            code(VolumeError::ReadOnly(VolumeId(7))),
-            Code::FailedPrecondition
-        );
-        assert_eq!(
-            VolumeError::ReadOnly(VolumeId(7)).to_string(),
-            "volume 7 is read only"
-        );
+        assert_eq!(code(VolumeError::ReadOnly), Code::FailedPrecondition);
         assert_eq!(
             code(VolumeError::InsufficientSpace {
                 vid: VolumeId(7),
@@ -89,27 +77,6 @@ mod tests {
         );
         assert_eq!(code(VolumeError::AlreadyExists), Code::AlreadyExists);
         assert_eq!(code(VolumeError::NotInitialized), Code::Internal);
-        assert_eq!(
-            code(TierError::NotFound("gone".into()).into()),
-            Code::NotFound
-        );
-        for tier in [
-            TierError::Io("io".into()),
-            TierError::RuntimeUnavailable("rt".into()),
-            TierError::Aborted("bye".into()),
-        ] {
-            assert_eq!(code(tier.into()), Code::Internal);
-        }
-
-        let status = status_with_context(
-            "backend s3.default copy file /data/1.dat",
-            TierError::NotFound("failed to head object k: NotFound".into()).into(),
-        );
-        assert_eq!(status.code(), Code::NotFound);
-        assert_eq!(
-            status.message(),
-            "backend s3.default copy file /data/1.dat: failed to head object k: NotFound"
-        );
 
         let status = status_with_context(
             "compact volume 7",

seaweed-volume/src/server/store_ec.rs

@@ -95,41 +95,20 @@ struct Snapshot {
     encode_ts_ns: i64,
 }
 
-/// Why a distributed EC read has no needle to return.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum EcMiss {
-    NotFound,
-    /// Tombstoned in the local `.ecx`/`.ecj`, or reported deleted by a peer.
-    Deleted,
-    VolumeNotFound,
-}
-
-/// Top-level entry point. Returns `Ok(None)` for any miss — absent,
-/// deleted, or volume gone; errors propagate as `io::Error`.
+/// Top-level entry point. Returns `Ok(None)` for "not found" (matches
+/// Go's `ReadEcShardNeedle`); errors propagate as `io::Error`.
 pub async fn read_ec_shard_needle_distributed(
     state: &Arc<VolumeServerState>,
     vid: VolumeId,
     needle_id: NeedleId,
 ) -> io::Result<Option<Needle>> {
-    Ok(read_ec_shard_needle_or_miss(state, vid, needle_id)
-        .await?
-        .ok())
-}
-
-/// Like `read_ec_shard_needle_distributed`, but says why there is no needle,
-/// as Go's `ReadEcShardNeedle` tells `ErrorDeleted` from not-found.
-pub async fn read_ec_shard_needle_or_miss(
-    state: &Arc<VolumeServerState>,
-    vid: VolumeId,
-    needle_id: NeedleId,
-) -> io::Result<Result<Needle, EcMiss>> {
     // Phase A — under the Store read lock, locate the needle, compute
     // intervals, and read any locally-mounted shard intervals. We must
     // not `.await` while holding this guard (std::sync::RwLockReadGuard
     // is !Send).
     let mut snapshot = match snapshot_under_lock(state, vid, needle_id)? {
-        Ok(s) => s,
-        Err(miss) => return Ok(Err(miss)),
+        Some(s) => s,
+        None => return Ok(None),
     };
 
     // Phase B — refresh the shard_locations cache from the master if
@@ -231,11 +210,17 @@ pub async fn read_ec_shard_needle_or_miss(
         .collect()
         .await;
 
-    // A peer reports the needle deleted (a cross-server window where the
-    // local index still shows it live): answer deleted rather than serving zeros.
-    let Some(assembled) = gather_intervals(fetched)? else {
-        return Ok(Err(EcMiss::Deleted));
-    };
+    let mut assembled: Vec<Vec<u8>> = Vec::with_capacity(fetched.len());
+    for res in fetched {
+        let (buf, is_deleted) = res?;
+        // A peer reports the needle deleted (a cross-server window where the
+        // local index still shows it live): treat as not-found rather than
+        // serving zeros, mirroring Go's ErrorDeleted.
+        if is_deleted {
+            return Ok(None);
+        }
+        assembled.push(buf);
+    }
 
     // Phase D — assemble and parse the Needle. Mirrors the tail of
     // `EcVolume::read_ec_shard_needle`.
@@ -267,20 +252,7 @@ pub async fn read_ec_shard_needle_or_miss(
         snapshot.version,
     )
     .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{}", e)))?;
-    Ok(Ok(n))
-}
-
-/// `None` when any holder reported the needle deleted. That outranks another
-/// interval's error: deletes are never invented, so the needle is gone either way.
-fn gather_intervals(fetched: Vec<io::Result<(Vec<u8>, bool)>>) -> io::Result<Option<Vec<Vec<u8>>>> {
-    if fetched.iter().any(|r| matches!(r, Ok((_, true)))) {
-        return Ok(None);
-    }
-    fetched
-        .into_iter()
-        .map(|r| r.map(|(buf, _)| buf))
-        .collect::<io::Result<_>>()
-        .map(Some)
+    Ok(Some(n))
 }
 
 /// What one EC delete RPC carries — `VolumeEcBlobDeleteRequest` minus tonic.
@@ -892,10 +864,11 @@ fn snapshot_under_lock(
     state: &Arc<VolumeServerState>,
     vid: VolumeId,
     needle_id: NeedleId,
-) -> io::Result<Result<Snapshot, EcMiss>> {
+) -> io::Result<Option<Snapshot>> {
     let store = state.store.read().unwrap();
-    let Some(ecv) = store.find_ec_volume(vid) else {
-        return Ok(Err(EcMiss::VolumeNotFound));
+    let ecv = match store.find_ec_volume(vid) {
+        Some(v) => v,
+        None => return Ok(None),
     };
 
     // Reuse EcVolume::locate_needle for offset/size resolution AND
@@ -903,17 +876,11 @@ fn snapshot_under_lock(
     // local-only read path uses, so we stay byte-identical on the
     // shard-size + interval boundaries. locate_needle applies the runtime
     // delete mask, which is correct for serving reads.
-    let Some((offset, size, intervals)) = ecv.locate_needle(needle_id)? else {
-        // locate_needle folds a tombstone into not-found.
-        let deleted =
-            matches!(ecv.find_needle_from_ecx(needle_id)?, Some((_, s)) if s.is_deleted());
-        return Ok(Err(if deleted {
-            EcMiss::Deleted
-        } else {
-            EcMiss::NotFound
-        }));
+    let (offset, size, intervals) = match ecv.locate_needle(needle_id)? {
+        Some(v) => v,
+        None => return Ok(None),
     };
-    build_snapshot(ecv, offset, size, &intervals).map(Ok)
+    build_snapshot(ecv, offset, size, &intervals).map(Some)
 }
 
 /// Like `snapshot_under_lock`, but locates intervals from the RAW .ecx
@@ -1823,21 +1790,6 @@ async fn drain_copy_stream(
 mod tests {
     use super::*;
 
-    #[test]
-    fn gather_intervals_puts_a_reported_deletion_ahead_of_errors() {
-        let failed = || Err(io::Error::other("shard unreachable"));
-        assert!(
-            gather_intervals(vec![failed(), Ok((Vec::new(), true))])
-                .unwrap()
-                .is_none()
-        );
-        assert!(gather_intervals(vec![failed(), Ok((vec![1], false))]).is_err());
-        assert_eq!(
-            gather_intervals(vec![Ok((vec![1], false)), Ok((vec![2], false))]).unwrap(),
-            Some(vec![vec![1], vec![2]])
-        );
-    }
-
     fn locations(count: usize) -> HashMap<ShardId, Vec<String>> {
         (0..count)
             .map(|sid| (sid as ShardId, vec!["127.0.0.1:8080".to_string()]))

seaweed-volume/src/server/write_queue.rs

@@ -3,8 +3,8 @@
 //! Instead of each upload handler directly calling `write_needle`, writes are
 //! submitted to a queue. A background worker drains the queue in batches (up to
 //! 128 entries), groups them by volume ID, and processes them together under a
-//! single store lock. Durable writes to a volume share their .dat and .idx
-//! flushes (see `Volume::write_needles_grouped`).
+//! single store lock. Requests that asked for `fsync` are flushed by
+//! `write_needle` itself, one flush per durable write.
 
 use std::sync::Arc;
 
@@ -159,12 +159,8 @@ fn process_batch(state: Arc<VolumeServerState>, batch: Vec<WriteRequest>) {
     let mut store = state.store.write().unwrap();
 
     for (vid, entries) in groups {
-        let (mut writes, senders): (Vec<_>, Vec<_>) = entries
-            .into_iter()
-            .map(|(needle, fsync, response_tx)| ((needle, fsync), response_tx))
-            .unzip();
-        let results = store.write_volume_needles(vid, &mut writes);
-        for (response_tx, result) in senders.into_iter().zip(results) {
+        for (mut needle, fsync, response_tx) in entries {
+            let result = store.write_volume_needle(vid, &mut needle, fsync);
             // Send result back; ignore error if receiver dropped.
             let _ = response_tx.send(result);
         }
@@ -316,63 +312,6 @@ mod tests {
         }
     }
 
-    /// The queue hands a volume's batch to the grouped path, so ten durable
-    /// writes cost one .dat sync and one .idx sync, not ten of each.
-    #[test]
-    fn test_process_batch_group_commits_fsync_writes() {
-        use crate::config::MinFreeSpace;
-        use crate::storage::types::{DiskType, NeedleId};
-        use crate::storage::volume::VolumeSpec;
-
-        let tmp = tempfile::TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-        let state = make_test_state();
-        {
-            let mut store = state.store.write().unwrap();
-            store
-                .add_location(
-                    dir,
-                    dir,
-                    10,
-                    DiskType::HardDrive,
-                    MinFreeSpace::Percent(1.0),
-                    Vec::new(),
-                )
-                .unwrap();
-            store
-                .add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
-                .unwrap();
-        }
-
-        let mut receivers = Vec::new();
-        let batch = (1..=10u64)
-            .map(|id| {
-                let (response_tx, response_rx) = oneshot::channel();
-                receivers.push(response_rx);
-                WriteRequest {
-                    volume_id: VolumeId(1),
-                    needle: Needle {
-                        id: NeedleId(id),
-                        cookie: 0x1111.into(),
-                        data: vec![id as u8; 8],
-                        data_size: 8,
-                        ..Needle::default()
-                    },
-                    fsync: true,
-                    response_tx,
-                }
-            })
-            .collect();
-        process_batch(state.clone(), batch);
-
-        for mut rx in receivers {
-            assert!(matches!(rx.try_recv().unwrap(), Ok((_, _, false))));
-        }
-        let store = state.store.read().unwrap();
-        let (_, vol) = store.find_volume(VolumeId(1)).unwrap();
-        assert_eq!(vol.sync_counts_for_test(), (1, 1));
-    }
-
     #[tokio::test]
     async fn test_write_queue_dropped_sender() {
         // When the queue is dropped, subsequent submits should fail gracefully.

seaweed-volume/src/storage/disk_location.rs

@@ -18,7 +18,7 @@ use crate::storage::erasure_coding::ec_shard::{
     DATA_SHARDS_COUNT, ERASURE_CODING_LARGE_BLOCK_SIZE, ERASURE_CODING_SMALL_BLOCK_SIZE,
     EcVolumeShard, ShardId,
 };
-use crate::storage::erasure_coding::ec_volume::{EcVolume, is_usable_ecx_file};
+use crate::storage::erasure_coding::ec_volume::EcVolume;
 use crate::storage::needle_map::NeedleMapKind;
 use crate::storage::super_block::SUPER_BLOCK_SIZE;
 use crate::storage::types::*;
@@ -479,13 +479,6 @@ impl DiskLocation {
         if self.idx_directory != self.directory {
             remove_bitrot_sidecars(&idx_base)?;
         }
-
-        // Staged 2PC generations (<base>.ecNN.v<N>, versioned .ecx/.ecj/.vif)
-        // belong to this volume's EC state too; leaving them orphans the files.
-        crate::storage::erasure_coding::ec_shard::remove_ec_generation_files(&base, 0)?;
-        if self.idx_directory != self.directory {
-            crate::storage::erasure_coding::ec_shard::remove_ec_generation_files(&idx_base, 0)?;
-        }
         Ok(())
     }
 
@@ -605,20 +598,13 @@ impl DiskLocation {
         &mut self,
         vid: VolumeId,
         only_empty: bool,
-        only_garbage: bool,
         keep_remote_data: bool,
     ) -> Result<(), VolumeError> {
-        // Refuse before removing: a refused destroy must leave it mounted.
-        if let Some(v) = self.volumes.get(&vid)
-            && v.is_compacting()
-        {
-            return Err(v.compacting_error());
-        }
         if let Some(mut v) = self.volumes.remove(&vid) {
             crate::metrics::VOLUME_GAUGE
                 .with_label_values(&[&v.collection, "volume"])
                 .dec();
-            v.destroy(only_empty, only_garbage, keep_remote_data)?;
+            v.destroy(only_empty, keep_remote_data)?;
             Ok(())
         } else {
             Err(VolumeError::NotFound)
@@ -639,7 +625,7 @@ impl DiskLocation {
                 crate::metrics::VOLUME_GAUGE
                     .with_label_values(&[&v.collection, "volume"])
                     .dec();
-                if let Err(e) = v.destroy(false, false, false) {
+                if let Err(e) = v.destroy(false, false) {
                     warn!(volume_id = vid.0, error = %e, "delete collection: failed to destroy volume");
                 }
             }
@@ -793,17 +779,21 @@ impl DiskLocation {
     /// Mirrors `DiskLocation.HasEcxFileOnDisk` in
     /// `weed/storage/disk_location_ec.go`. Skips entries that are
     /// directories so a stray dir named `<collection>_<vid>.ecx` doesn't
-    /// register as a present index file. A 0-byte `.ecx` is a corrupt stub
-    /// left by a failed EC distribute copy; it must not steer placement
-    /// toward this disk, so it counts as absent (Go requires `Size() > 0`).
+    /// register as a present index file.
     pub fn has_ecx_file_on_disk(&self, collection: &str, vid: VolumeId) -> bool {
         let idx_base = volume_file_name(&self.idx_directory, collection, vid);
-        if is_usable_ecx_file(&format!("{}.ecx", idx_base)) {
+        let idx_path = format!("{}.ecx", idx_base);
+        if let Ok(meta) = fs::metadata(&idx_path)
+            && !meta.is_dir()
+        {
             return true;
         }
         if self.idx_directory != self.directory {
             let data_base = volume_file_name(&self.directory, collection, vid);
-            if is_usable_ecx_file(&format!("{}.ecx", data_base)) {
+            let data_path = format!("{}.ecx", data_base);
+            if let Ok(meta) = fs::metadata(&data_path)
+                && !meta.is_dir()
+            {
                 return true;
             }
         }
@@ -815,20 +805,6 @@ impl DiskLocation {
         self.ec_volumes.remove(&vid)
     }
 
-    /// Drop the in-memory EC volume for vid and close its descriptors without
-    /// deleting files, so a following unlink frees the inodes instead of
-    /// leaving open fds serving the old bytes. Mirrors Go's unloadEcVolume.
-    pub fn unload_ec_volume(&mut self, vid: VolumeId) {
-        if let Some(mut ec_vol) = self.ec_volumes.remove(&vid) {
-            for _ in 0..ec_vol.shard_count() {
-                crate::metrics::VOLUME_GAUGE
-                    .with_label_values(&[&ec_vol.collection, "ec_shards"])
-                    .dec();
-            }
-            ec_vol.close();
-        }
-    }
-
     /// Mount EC shards for a volume on this location.
     ///
     /// `source_disk_type` is the source volume's disk type carried on the
@@ -1761,7 +1737,7 @@ mod tests {
             .unwrap();
         assert_eq!(loc.volumes_len(), 2);
 
-        loc.delete_volume(VolumeId(1), false, false, false).unwrap();
+        loc.delete_volume(VolumeId(1), false, false).unwrap();
         assert_eq!(loc.volumes_len(), 1);
         assert!(loc.find_volume(VolumeId(1)).is_none());
     }
@@ -1814,34 +1790,6 @@ mod tests {
         assert!(loc.find_volume(VolumeId(3)).is_some());
     }
 
-    /// A 0-byte `.ecx` is the stub a failed EC distribute copy leaves behind.
-    /// Go's HasEcxFileOnDisk requires Size() > 0 so the stub cannot pin
-    /// placement to a disk that has no usable index.
-    #[test]
-    fn test_has_ecx_file_on_disk_ignores_zero_byte_stub() {
-        let tmp = TempDir::new().unwrap();
-        let data = tmp.path().join("data");
-        let idx = tmp.path().join("idx");
-        fs::create_dir_all(&data).unwrap();
-        fs::create_dir_all(&idx).unwrap();
-        let loc = DiskLocation::new(
-            data.to_str().unwrap(),
-            idx.to_str().unwrap(),
-            10,
-            DiskType::HardDrive,
-            MinFreeSpace::Percent(1.0),
-            Vec::new(),
-        )
-        .unwrap();
-
-        fs::write(idx.join("pics_7.ecx"), b"").unwrap();
-        assert!(!loc.has_ecx_file_on_disk("pics", VolumeId(7)));
-
-        // A real index in the data dir still counts, stub or no stub.
-        fs::write(data.join("pics_7.ecx"), [0u8; 16]).unwrap();
-        assert!(loc.has_ecx_file_on_disk("pics", VolumeId(7)));
-    }
-
     #[test]
     fn test_disk_location_delete_collection_removes_ec_volumes() {
         let tmp = TempDir::new().unwrap();
@@ -1858,8 +1806,6 @@ mod tests {
 
         let shard_path = format!("{}/pics_7.ec00", dir);
         std::fs::write(&shard_path, b"ec-shard").unwrap();
-        // An EC volume needs its .ecx to mount.
-        std::fs::write(format!("{}/pics_7.ecx", dir), [0u8; 16]).unwrap();
 
         loc.mount_ec_shards(VolumeId(7), "pics", &[0], "").unwrap();
         assert!(loc.has_ec_volume(VolumeId(7)));
@@ -1897,7 +1843,6 @@ mod tests {
         // mount_ec_shards with source_disk_type="ssd" — simulating the
         // VolumeEcShardsMount RPC path.
         std::fs::write(format!("{}/pics_7.ec00", dir), b"ec-shard").unwrap();
-        std::fs::write(format!("{}/pics_7.ecx", dir), [0u8; 16]).unwrap();
         loc.mount_ec_shards(VolumeId(7), "pics", &[0], "ssd")
             .unwrap();
         {
@@ -1997,7 +1942,6 @@ mod tests {
         // A collection name unique to this test: the gauge is process-global
         // and sibling tests running in parallel touch other labels.
         std::fs::write(format!("{}/dupmount_11.ec00", dir), b"shard bytes").unwrap();
-        std::fs::write(format!("{}/dupmount_11.ecx", dir), [0u8; 16]).unwrap();
         let gauge = crate::metrics::VOLUME_GAUGE.with_label_values(&["dupmount", "ec_shards"]);
         let before = gauge.get();
 

seaweed-volume/src/storage/erasure_coding/ec_decoder.rs

@@ -3,12 +3,10 @@
 //! Rebuilds the original .dat + .idx files from data shards (.ec00-.ec09)
 //! and the sorted index (.ecx) + deletion journal (.ecj).
 
-use std::collections::HashSet;
 use std::fs::File;
 use std::io::{self, Read, Write};
 
 use crate::storage::erasure_coding::ec_shard::*;
-use crate::storage::erasure_coding::ec_volume::read_ecj_ids;
 use crate::storage::idx;
 use crate::storage::needle::needle::get_actual_size;
 use crate::storage::super_block::SUPER_BLOCK_SIZE;
@@ -22,21 +20,18 @@ use crate::storage::volume::{fsync_dir, volume_file_name};
 /// `dir` is used both for reading `.ec00` and `.ecx`. For split-disk
 /// reconciled volumes call [`find_dat_file_size_with_dirs`] instead.
 pub fn find_dat_file_size(dir: &str, collection: &str, volume_id: VolumeId) -> io::Result<i64> {
-    let deleted = read_ecj_deletions(&[dir], collection, volume_id)?;
-    find_dat_file_size_with_dirs(dir, dir, collection, volume_id, &deleted)
+    find_dat_file_size_with_dirs(dir, dir, collection, volume_id)
 }
 
 /// Like [`find_dat_file_size`] but lets the caller pass separate dirs
 /// for `.ec00` (the data shard) and `.ecx` (the sealed index). This
 /// is the form needed when shards are split across data dirs and the
-/// `.ecx` lives on a sibling disk's idx dir (#9252). Needles in `deleted`
-/// count as deleted.
+/// `.ecx` lives on a sibling disk's idx dir (#9252).
 pub fn find_dat_file_size_with_dirs(
     ec00_dir: &str,
     ecx_dir: &str,
     collection: &str,
     volume_id: VolumeId,
-    deleted: &HashSet<NeedleId>,
 ) -> io::Result<i64> {
     let ec00_base = volume_file_name(ec00_dir, collection, volume_id);
     let ecx_base = volume_file_name(ecx_dir, collection, volume_id);
@@ -58,9 +53,9 @@ pub fn find_dat_file_size_with_dirs(
 
     for i in 0..entry_count {
         let start = i * NEEDLE_MAP_ENTRY_SIZE;
-        let (key, offset, size) =
+        let (_, offset, size) =
             idx_entry_from_bytes(&ecx_data[start..start + NEEDLE_MAP_ENTRY_SIZE]);
-        if size.is_deleted() || deleted.contains(&key) {
+        if size.is_deleted() {
             continue;
         }
         let entry_stop = offset.to_actual_offset() + get_actual_size(size, version);
@@ -72,48 +67,6 @@ pub fn find_dat_file_size_with_dirs(
     Ok(dat_size)
 }
 
-/// Whether the `.ecx` in `ecx_dir` indexes a needle deleted neither there nor
-/// in `deleted`.
-pub fn has_live_needles(
-    ecx_dir: &str,
-    collection: &str,
-    volume_id: VolumeId,
-    deleted: &HashSet<NeedleId>,
-) -> io::Result<bool> {
-    let ecx_base = volume_file_name(ecx_dir, collection, volume_id);
-    let ecx_data = std::fs::read(format!("{}.ecx", ecx_base))?;
-    let (entries, _) = ecx_data.as_chunks::<NEEDLE_MAP_ENTRY_SIZE>();
-    Ok(entries.iter().any(|entry| {
-        let (key, _, size) = idx_entry_from_bytes(entry);
-        !size.is_deleted() && !deleted.contains(&key)
-    }))
-}
-
-/// Distinct needle ids journaled in the `.ecj` of any of `dirs`. Go folds the
-/// journal into the `.ecx` (RebuildEcxFile) before a decode; reading it leaves
-/// the sealed index untouched. Only NotFound means "no journal".
-pub fn read_ecj_deletions(
-    dirs: &[&str],
-    collection: &str,
-    volume_id: VolumeId,
-) -> io::Result<HashSet<NeedleId>> {
-    let mut ids = HashSet::new();
-    for (i, dir) in dirs.iter().enumerate() {
-        if dirs[..i].contains(dir) {
-            continue;
-        }
-        let path = format!("{}.ecj", volume_file_name(dir, collection, volume_id));
-        let file = match File::open(&path) {
-            Ok(file) => file,
-            Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
-            Err(e) => return Err(e),
-        };
-        let len = file.metadata()?.len();
-        read_ecj_ids(&file, len, &mut ids)?;
-    }
-    Ok(ids)
-}
-
 /// What it takes to rebuild a volume's .dat from its EC data shards.
 ///
 /// Mirrors Go's `WriteDatFile(baseFileName, datFileSize,
@@ -314,61 +267,51 @@ pub fn write_dat_file_from_shards(spec: &DatRebuild<'_>) -> io::Result<()> {
 
 /// Write .idx file from .ecx index + .ecj deletion journal.
 ///
-/// See [`write_idx_file_from_ec_index_with_dirs`]; everything lives in `dir`.
+/// Copies sorted .ecx entries to .idx, then appends tombstones for
+/// deleted needles from .ecj.
 pub fn write_idx_file_from_ec_index(
     dir: &str,
     collection: &str,
     volume_id: VolumeId,
 ) -> io::Result<()> {
-    let deleted = read_ecj_deletions(&[dir], collection, volume_id)?;
-    let dat_file_size = find_dat_file_size_with_dirs(dir, dir, collection, volume_id, &deleted)?;
-    write_idx_file_from_ec_index_with_dirs(dir, dir, collection, volume_id, &deleted, dat_file_size)
-}
-
-/// Write the `.idx` for a `.dat` decoded to `dat_file_size` bytes, from the
-/// `.ecx` in `ecx_dir`, into `idx_dir`.
-///
-/// Copies the `.ecx` rows, then appends one tombstone per row whose needle is
-/// in `deleted`. A deleted needle at or past `dat_file_size` was cut from the
-/// `.dat`, so its row is dropped: a row pointing past the end of the `.dat`
-/// makes the volume load read-only.
-pub fn write_idx_file_from_ec_index_with_dirs(
-    ecx_dir: &str,
-    idx_dir: &str,
-    collection: &str,
-    volume_id: VolumeId,
-    deleted: &HashSet<NeedleId>,
-    dat_file_size: i64,
-) -> io::Result<()> {
-    let ecx_path = format!("{}.ecx", volume_file_name(ecx_dir, collection, volume_id));
-    let idx_path = format!("{}.idx", volume_file_name(idx_dir, collection, volume_id));
+    let base = volume_file_name(dir, collection, volume_id);
+    let ecx_path = format!("{}.ecx", base);
+    let ecj_path = format!("{}.ecj", base);
+    let idx_path = format!("{}.idx", base);
     // Write to a temp file and atomically rename into place, so a crash
     // mid-write never leaves a partial .idx at the final name beside the
     // source shards.
     let tmp_path = format!("{}.tmp", idx_path);
 
     let write_result = (|| -> io::Result<()> {
-        let mut ecx_file = File::open(&ecx_path)?;
-        let mut idx_file = io::BufWriter::new(File::create(&tmp_path)?);
-        let mut tombstoned = Vec::new();
-        idx::walk_index_file(&mut ecx_file, 0, |key, offset, size| {
-            let is_deleted = size.is_deleted() || deleted.contains(&key);
-            if is_deleted && offset.to_actual_offset() >= dat_file_size {
-                return Ok(());
+        // Copy .ecx to the temp .idx
+        std::fs::copy(&ecx_path, &tmp_path)?;
+
+        // Append deletions from .ecj as tombstones. Read the journal directly
+        // and treat only NotFound as "no journal": Path::exists would also
+        // swallow a permission/IO error and silently skip deletions, which
+        // would resurrect deleted needles as live.
+        let mut idx_file = std::fs::OpenOptions::new().append(true).open(&tmp_path)?;
+        match std::fs::read(&ecj_path) {
+            Ok(ecj_data) => {
+                let count = ecj_data.len() / NEEDLE_ID_SIZE;
+                for i in 0..count {
+                    let start = i * NEEDLE_ID_SIZE;
+                    let needle_id = NeedleId::from_bytes(&ecj_data[start..start + NEEDLE_ID_SIZE]);
+                    idx::write_index_entry(
+                        &mut idx_file,
+                        needle_id,
+                        Offset::default(),
+                        TOMBSTONE_FILE_SIZE,
+                    )?;
+                }
             }
-            idx::write_index_entry(&mut idx_file, key, offset, size)?;
-            if !size.is_deleted() && deleted.contains(&key) {
-                tombstoned.push(key);
-            }
-            Ok(())
-        })?;
-        for key in tombstoned {
-            idx::write_index_entry(&mut idx_file, key, Offset::default(), TOMBSTONE_FILE_SIZE)?;
+            Err(e) if e.kind() == io::ErrorKind::NotFound => {}
+            Err(e) => return Err(e),
         }
 
         // fsync, rename, then fsync the dir so the decoded .idx is durable and
         // atomically published before the caller deletes the source shards.
-        let idx_file = idx_file.into_inner().map_err(|e| e.into_error())?;
         idx_file.sync_all()?;
         drop(idx_file);
         // Windows rename does not replace an existing file on every version;
@@ -751,43 +694,4 @@ mod tests {
             .is_err()
         );
     }
-
-    /// A journal many chunks long that repeats a few ids reads back as those
-    /// ids, from each dir once, whatever its length.
-    #[test]
-    fn test_read_ecj_deletions_collects_distinct_ids_across_dirs() {
-        let tmp = TempDir::new().unwrap();
-        let data = tmp.path().join("data");
-        let idx = tmp.path().join("idx");
-        let missing = tmp.path().join("missing");
-        std::fs::create_dir_all(&data).unwrap();
-        std::fs::create_dir_all(&idx).unwrap();
-        let (data, idx, missing) = (
-            data.to_str().unwrap(),
-            idx.to_str().unwrap(),
-            missing.to_str().unwrap(),
-        );
-
-        let entry = |id: u64| {
-            let mut buf = [0u8; NEEDLE_ID_SIZE];
-            NeedleId(id).to_bytes(&mut buf);
-            buf
-        };
-        // Past two load chunks of three repeating ids, then an id only in the
-        // last chunk and a torn trailing record.
-        let mut ecj = Vec::new();
-        while ecj.len() <= 2 * (1 << 20) {
-            for id in [1, 2, 3] {
-                ecj.extend_from_slice(&entry(id));
-            }
-        }
-        ecj.extend_from_slice(&entry(7));
-        ecj.extend_from_slice(&entry(8)[..3]);
-        std::fs::write(format!("{idx}/1.ecj"), &ecj).unwrap();
-        std::fs::write(format!("{data}/1.ecj"), entry(9)).unwrap();
-
-        let ids = read_ecj_deletions(&[data, idx, idx, missing], "", VolumeId(1)).unwrap();
-        let expected: HashSet<NeedleId> = [1, 2, 3, 7, 9].into_iter().map(NeedleId).collect();
-        assert_eq!(ids, expected);
-    }
 }

seaweed-volume/src/storage/erasure_coding/ec_encoder.rs

@@ -813,7 +813,7 @@ impl EncodeRun<'_> {
         // explicitly.
         for (i, buf) in self.buffers[..self.data_shards].iter_mut().enumerate() {
             let read_offset = offset + (i * block_size) as u64;
-            let n = crate::storage::io::read_full_at(self.dat_file, buf, read_offset)?;
+            let n = read_at_most(self.dat_file, buf, read_offset)?;
             buf[n..].fill(0);
         }
 
@@ -834,6 +834,19 @@ impl EncodeRun<'_> {
     }
 }
 
+/// Read into `buf` at `offset` until it is full or EOF; returns bytes read.
+fn read_at_most(dat_file: &File, buf: &mut [u8], offset: u64) -> io::Result<usize> {
+    let mut n = 0;
+    while n < buf.len() {
+        let r = crate::storage::io::read_at(dat_file, &mut buf[n..], offset + n as u64)?;
+        if r == 0 {
+            break;
+        }
+        n += r;
+    }
+    Ok(n)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

seaweed-volume/src/storage/erasure_coding/ec_shard.rs

@@ -87,14 +87,14 @@ impl EcVolumeShard {
         Ok(())
     }
 
-    /// Read data at a specific offset, filling `buf` unless the shard ends first.
+    /// Read data at a specific offset.
     pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize> {
         let file = self
             .ecd_file
             .as_ref()
             .ok_or_else(|| io::Error::other("shard file not open"))?;
 
-        crate::storage::io::read_full_at(file, buf, offset)
+        crate::storage::io::read_at(file, buf, offset)
     }
 
     /// Write data to the shard file (appends).
@@ -195,104 +195,6 @@ impl ShardBits {
     }
 }
 
-/// Parses the generation of a 2PC-staged `<base>.v<N>` file: `None` means the
-/// name is not a generation file of `base`.
-pub fn ec_file_generation(name: &str, base: &str) -> Option<u32> {
-    let suffix = name.strip_prefix(&format!("{}.v", base))?;
-    match suffix.parse::<u32>() {
-        Ok(g) if g > 0 => Some(g),
-        _ => None,
-    }
-}
-
-/// Removes 2PC generation files staged under `base`:
-/// `<base>.ecNN.v<N>`, `<base>.ecx.v<N>`, `<base>.ecj.v<N>`, `<base>.ecsum.v<N>`
-/// and `<base>.vif.v<N>`. `generations_older_than == 0` removes every
-/// generation; otherwise only generations strictly below it. Returns the
-/// first real removal failure. Mirrors Go's `RemoveEcGenerationFiles`.
-pub fn remove_ec_generation_files(base: &str, generations_older_than: u32) -> io::Result<()> {
-    let path = std::path::Path::new(base);
-    let (Some(parent), Some(fname)) = (path.parent(), path.file_name()) else {
-        return Ok(());
-    };
-    let ec_prefix = format!("{}.ec", fname.to_string_lossy());
-    let vif_name = format!("{}.vif", fname.to_string_lossy());
-    let mut first_err: Option<io::Error> = None;
-    let mut record = |res: io::Result<()>| {
-        if let Err(e) = res
-            && first_err.is_none()
-        {
-            first_err = Some(e);
-        }
-    };
-    match fs::read_dir(parent) {
-        Ok(entries) => {
-            for entry in entries {
-                let entry = match entry {
-                    Ok(entry) => entry,
-                    Err(e) => {
-                        // A skipped entry means an incomplete sweep; report it
-                        // instead of pretending the cleanup finished.
-                        record(Err(e));
-                        continue;
-                    }
-                };
-                let name = entry.file_name().to_string_lossy().into_owned();
-                let Some((artifact, _)) = name.rsplit_once(".v") else {
-                    continue;
-                };
-                if artifact != vif_name && !artifact.starts_with(&ec_prefix) {
-                    continue;
-                }
-                let Some(generation) = ec_file_generation(&name, artifact) else {
-                    continue;
-                };
-                if generations_older_than > 0 && generation >= generations_older_than {
-                    continue;
-                }
-                record(match fs::remove_file(entry.path()) {
-                    Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e),
-                    _ => Ok(()),
-                });
-            }
-        }
-        Err(e) if e.kind() != io::ErrorKind::NotFound => record(Err(e)),
-        Err(_) => {}
-    }
-    match first_err {
-        Some(e) => Err(e),
-        None => Ok(()),
-    }
-}
-
-/// Removes every staged generation `<shard_file>.v<N>` of one shard file.
-/// Returns true when at least one generation file was removed.
-pub fn remove_ec_shard_generations(shard_file: &str) -> io::Result<bool> {
-    let path = std::path::Path::new(shard_file);
-    let (Some(parent), Some(fname)) = (path.parent(), path.file_name()) else {
-        return Ok(false);
-    };
-    let fname = fname.to_string_lossy().into_owned();
-    let mut removed = false;
-    match fs::read_dir(parent) {
-        Ok(entries) => {
-            for entry in entries {
-                let entry = entry?;
-                let name = entry.file_name().to_string_lossy().into_owned();
-                if ec_file_generation(&name, &fname).is_some() {
-                    match fs::remove_file(entry.path()) {
-                        Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e),
-                        _ => removed = true,
-                    }
-                }
-            }
-        }
-        Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e),
-        Err(_) => {}
-    }
-    Ok(removed)
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

seaweed-volume/src/storage/erasure_coding/ec_volume.rs

@@ -48,29 +48,6 @@ pub(crate) struct ShardLocationCache {
 /// A multiple of `NEEDLE_ID_SIZE`; 1 MiB is 131072 entries per syscall.
 const ECJ_LOAD_CHUNK_BYTES: usize = 1 << 20;
 
-/// Adds every whole needle id in the first `len` bytes of `ecj_file` to `ids`,
-/// reading `ECJ_LOAD_CHUNK_BYTES` at a time; a trailing partial record is
-/// ignored.
-pub(crate) fn read_ecj_ids(
-    ecj_file: &File,
-    len: u64,
-    ids: &mut HashSet<NeedleId>,
-) -> io::Result<()> {
-    let mut buf = vec![0u8; std::cmp::min(ECJ_LOAD_CHUNK_BYTES as u64, len) as usize];
-    let mut off: u64 = 0;
-    while off + NEEDLE_ID_SIZE as u64 <= len {
-        let mut want = std::cmp::min(ECJ_LOAD_CHUNK_BYTES as u64, len - off) as usize;
-        want -= want % NEEDLE_ID_SIZE;
-        // Positional read: the loader's handle is shared with journal appends.
-        read_exact_at(ecj_file, &mut buf[..want], off)?;
-        for entry in buf[..want].as_chunks::<NEEDLE_ID_SIZE>().0 {
-            ids.insert(NeedleId::from_bytes(entry));
-        }
-        off += want as u64;
-    }
-    Ok(())
-}
-
 /// An erasure-coded volume managing its local shards and index.
 pub struct EcVolume {
     pub volume_id: VolumeId,
@@ -406,26 +383,8 @@ pub fn validate_block_size(block_size: i64) -> io::Result<()> {
     Ok(())
 }
 
-/// Size of the `.ecx` at `path`, or `None` when it is absent or a directory.
-/// Mirrors Go's `statEcxSize`.
-pub(crate) fn ecx_file_size(path: &str) -> Option<u64> {
-    match std::fs::metadata(path) {
-        Ok(meta) if !meta.is_dir() => Some(meta.len()),
-        _ => None,
-    }
-}
-
-/// Whether `path` is an `.ecx` that can steer placement, ownership or a copy:
-/// a regular file with content. A 0-byte `.ecx` is what a failed EC distribute
-/// copy leaves behind, and Go treats it as absent at every such decision
-/// (`HasEcxFileOnDisk`, `findEcxIdxDirForVolume`, `indexEcxOwners`) so the scan
-/// moves on to a sibling disk that may hold a valid index.
-pub(crate) fn is_usable_ecx_file(path: &str) -> bool {
-    ecx_file_size(path).is_some_and(|size| size > 0)
-}
-
 impl EcVolume {
-    /// Create a new EcVolume. Opens the .ecx index (required) and the .ecj journal.
+    /// Create a new EcVolume. Loads .ecx index and .ecj journal if present.
     pub fn new(
         dir: &str,
         dir_idx: &str,
@@ -508,62 +467,27 @@ impl EcVolume {
 
         // Open .ecx file (sorted index) in read/write mode for in-place deletion marking.
         // Matches Go which opens ecx for writing via MarkNeedleDeleted.
-        //
-        // Resolve it the way Go's NewEcVolume does: prefer a non-empty copy,
-        // the one co-located with the shard data first, then the caller's
-        // index directory — either the shared -dir.idx dir or a sibling disk
-        // that owns the .ecx when this disk holds only a 0-byte stub left by an
-        // interrupted copy. A 0-byte .ecx is also a legitimate empty index, so
-        // it yields only to a non-empty copy elsewhere, never to a mere
-        // absence. No .ecx at all fails the mount with NotFound (Go wraps
-        // os.ErrNotExist): an EcVolume without an index would advertise shards
-        // that no read can ever serve.
-        let local_ecx = format!(
-            "{}.ecx",
-            crate::storage::volume::volume_file_name(dir, collection, volume_id)
-        );
-        let shared_ecx = vol.ecx_file_name();
-        let local_size = ecx_file_size(&local_ecx);
-        let shared_size = if dir_idx != dir {
-            ecx_file_size(&shared_ecx)
-        } else {
-            None
-        };
-        let use_local = match (local_size, shared_size) {
-            (Some(n), _) if n > 0 => true,
-            (_, Some(n)) if n > 0 => {
-                tracing::debug!(
+        let ecx_path = vol.ecx_file_name();
+        if std::path::Path::new(&ecx_path).exists() {
+            let file = open_volume_file(OpenOptions::new().read(true).write(true), &ecx_path)?;
+            vol.ecx_file_size = file.metadata()?.len() as i64;
+            vol.ecx_file = Some(file);
+        } else if dir_idx != dir {
+            // Fall back to data directory if .ecx was created before -dir.idx was configured
+            let data_base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
+            let fallback_ecx = format!("{}.ecx", data_base);
+            if std::path::Path::new(&fallback_ecx).exists() {
+                tracing::info!(
                     volume_id = volume_id.0,
-                    "ecx not local at {}, using {}",
-                    local_ecx,
-                    shared_ecx
+                    "ecx file not found in idx dir, falling back to data dir"
                 );
-                false
+                let file =
+                    open_volume_file(OpenOptions::new().read(true).write(true), &fallback_ecx)?;
+                vol.ecx_file_size = file.metadata()?.len() as i64;
+                vol.ecx_file = Some(file);
+                vol.ecx_actual_dir = dir.to_string();
             }
-            // Only 0-byte copies exist: an empty index, local first.
-            (Some(_), _) => true,
-            (None, Some(_)) => false,
-            (None, None) => {
-                let tried = if dir_idx != dir {
-                    format!("{} (or {})", local_ecx, shared_ecx)
-                } else {
-                    local_ecx
-                };
-                return Err(io::Error::new(
-                    io::ErrorKind::NotFound,
-                    format!("cannot open ec volume index {}: not found", tried),
-                ));
-            }
-        };
-        let ecx_path = if use_local {
-            vol.ecx_actual_dir = dir.to_string();
-            local_ecx
-        } else {
-            shared_ecx
-        };
-        let file = open_volume_file(OpenOptions::new().read(true).write(true), &ecx_path)?;
-        vol.ecx_file_size = file.metadata()?.len() as i64;
-        vol.ecx_file = Some(file);
+        }
 
         // Open .ecj file (deletion journal) — use ecx_actual_dir for consistency.
         // Note: Go does NOT replay .ecj into .ecx at volume load (RebuildEcxFile
@@ -833,7 +757,38 @@ impl EcVolume {
         // held the `deleted_needles` write lock for the whole scan, which on a
         // bloated journal is the entire (unbounded) startup.
         let mut loaded: HashSet<NeedleId> = HashSet::new();
-        read_ecj_ids(ecj_file, self.ecj_file_size as u64, &mut loaded)?;
+        let mut buf = vec![0u8; ECJ_LOAD_CHUNK_BYTES];
+        let end = self.ecj_file_size as u64;
+        let mut off: u64 = 0;
+        while off + NEEDLE_ID_SIZE as u64 <= end {
+            // Whole entries only; a trailing partial record is ignored, as the
+            // per-entry loop did by construction.
+            let mut want = std::cmp::min(ECJ_LOAD_CHUNK_BYTES as u64, end - off) as usize;
+            want -= want % NEEDLE_ID_SIZE;
+            if want == 0 {
+                break;
+            }
+            #[cfg(unix)]
+            {
+                use std::os::unix::fs::FileExt;
+                ecj_file.read_exact_at(&mut buf[..want], off)?;
+            }
+            #[cfg(windows)]
+            {
+                // Positional read so concurrent readers of the shared .ecj
+                // handle can't interleave seek/read. Mirrors the
+                // read_exact_at helper at the bottom of this file.
+                read_exact_at(ecj_file, &mut buf[..want], off)?;
+            }
+            #[cfg(not(any(unix, windows)))]
+            {
+                compile_error!("Platform not supported: only unix and windows are supported");
+            }
+            for entry in buf[..want].chunks_exact(NEEDLE_ID_SIZE) {
+                loaded.insert(NeedleId::from_bytes(entry));
+            }
+            off += want as u64;
+        }
 
         let mut set = self
             .deleted_needles
@@ -1909,63 +1864,6 @@ mod tests {
     use super::*;
     use tempfile::TempDir;
 
-    /// Go's NewEcVolume fails with os.ErrNotExist when neither directory has an
-    /// `.ecx`. Mounting anyway advertises shards every read then fails on with
-    /// "ecx file not open", and zeroes the size `add_shard`'s 0-byte guard needs.
-    #[test]
-    fn test_new_without_ecx_is_not_found() {
-        let data = TempDir::new().unwrap();
-        let idx = TempDir::new().unwrap();
-        let (dir, dir_idx) = (data.path().to_str().unwrap(), idx.path().to_str().unwrap());
-        std::fs::write(format!("{}/7.ec00", dir), b"shard").unwrap();
-
-        for idx_dir in [dir, dir_idx] {
-            let err = EcVolume::new(dir, idx_dir, "", VolumeId(7))
-                .err()
-                .expect("an EC volume without an .ecx must not mount");
-            assert_eq!(err.kind(), io::ErrorKind::NotFound, "{}", err);
-        }
-    }
-
-    /// A 0-byte `.ecx` stub left by an interrupted copy yields to a non-empty
-    /// copy in the other directory, whichever side the stub is on.
-    #[test]
-    fn test_new_prefers_non_empty_ecx_over_zero_byte_stub() {
-        for stub_in_idx_dir in [true, false] {
-            let data = TempDir::new().unwrap();
-            let idx = TempDir::new().unwrap();
-            let (dir, dir_idx) = (data.path().to_str().unwrap(), idx.path().to_str().unwrap());
-            let (stub_dir, valid_dir) = if stub_in_idx_dir {
-                (dir_idx, dir)
-            } else {
-                (dir, dir_idx)
-            };
-            std::fs::write(format!("{}/7.ecx", stub_dir), b"").unwrap();
-            std::fs::write(
-                format!("{}/7.ecx", valid_dir),
-                vec![0u8; NEEDLE_MAP_ENTRY_SIZE],
-            )
-            .unwrap();
-
-            let vol = EcVolume::new(dir, dir_idx, "", VolumeId(7)).unwrap();
-            assert_eq!(vol.ecx_actual_dir(), valid_dir);
-            assert_eq!(vol.ecx_file_size, NEEDLE_MAP_ENTRY_SIZE as i64);
-        }
-    }
-
-    /// With no other copy a 0-byte `.ecx` is a legitimate empty index (a volume
-    /// whose needles were all deleted before encoding) and still mounts, as in Go.
-    #[test]
-    fn test_new_accepts_lone_zero_byte_ecx_as_empty_index() {
-        let tmp = TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-        std::fs::write(format!("{}/7.ecx", dir), b"").unwrap();
-
-        let vol = EcVolume::new(dir, dir, "", VolumeId(7)).unwrap();
-        assert_eq!(vol.ecx_file_size, 0);
-        assert!(!is_usable_ecx_file(&vol.ecx_file_name()));
-    }
-
     /// `destroy()` must remove co-located `.ecsum` sidecars (Go Destroy parity).
     /// Without this, `collection.delete` leaves orphaned bitrot files that
     /// inflate EC-health scanners after the shards are gone.
@@ -4660,7 +4558,7 @@ impl EcLocalShard {
             .file
             .as_ref()
             .map_err(|e| io::Error::new(e.kind(), e.to_string()))?;
-        crate::storage::io::read_full_at(file, buf, offset)
+        crate::storage::io::read_at(file, buf, offset)
     }
 }
 

seaweed-volume/src/storage/idx/mod.rs

@@ -21,26 +21,12 @@ where
     let mut buf = vec![0u8; NEEDLE_MAP_ENTRY_SIZE * ROWS_TO_READ];
 
     loop {
-        // Fill the batch before decoding: `read` may return a count that is
-        // not a multiple of the entry size, and a split entry would misalign
-        // every later row. Go is immune: `ReadAt` fills or errors.
-        let mut count = 0;
-        let mut eof = false;
-        while count < buf.len() {
-            match reader.read(&mut buf[count..]) {
-                Ok(0) => {
-                    eof = true;
-                    break;
-                }
-                Ok(n) => count += n,
-                Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
-                Err(ref e) if e.kind() == io::ErrorKind::UnexpectedEof => {
-                    eof = true;
-                    break;
-                }
-                Err(e) => return Err(e),
-            }
-        }
+        let count = match reader.read(&mut buf) {
+            Ok(0) => return Ok(()),
+            Ok(n) => n,
+            Err(ref e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(()),
+            Err(e) => return Err(e),
+        };
 
         let mut i = 0;
         while i + NEEDLE_MAP_ENTRY_SIZE <= count {
@@ -48,11 +34,6 @@ where
             f(key, offset, size)?;
             i += NEEDLE_MAP_ENTRY_SIZE;
         }
-
-        // A trailing partial entry at EOF is ignored, as Go does on `io.EOF`.
-        if eof {
-            return Ok(());
-        }
     }
 }
 
@@ -196,111 +177,6 @@ mod tests {
         data
     }
 
-    /// Reader that hands back at most `chunk` bytes per `read`. 7 is coprime
-    /// with the 17-byte entry size, so nearly every read ends mid-entry. With
-    /// `interrupts`, every other call fails with `ErrorKind::Interrupted`.
-    struct ShortReader {
-        inner: Cursor<Vec<u8>>,
-        chunk: usize,
-        interrupts: bool,
-        interrupt_next: bool,
-    }
-
-    impl ShortReader {
-        fn new(data: Vec<u8>, interrupts: bool) -> Self {
-            ShortReader {
-                inner: Cursor::new(data),
-                chunk: 7,
-                interrupts,
-                interrupt_next: false,
-            }
-        }
-    }
-
-    impl Read for ShortReader {
-        fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-            if self.interrupt_next {
-                self.interrupt_next = false;
-                return Err(io::Error::from(io::ErrorKind::Interrupted));
-            }
-            self.interrupt_next = self.interrupts;
-            let n = buf.len().min(self.chunk);
-            self.inner.read(&mut buf[..n])
-        }
-    }
-
-    impl Seek for ShortReader {
-        fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
-            self.inner.seek(pos)
-        }
-    }
-
-    fn walk_all<R: Read + Seek>(reader: &mut R, start_from: u64) -> Vec<(NeedleId, i64, Size)> {
-        let mut collected = Vec::new();
-        walk_index_file(reader, start_from, |key, offset, size| {
-            collected.push((key, offset.to_actual_offset(), size));
-            Ok(())
-        })
-        .unwrap();
-        collected
-    }
-
-    /// More than one ROWS_TO_READ batch, so the walk crosses a buffer refill.
-    fn many_entries() -> Vec<(NeedleId, Offset, Size)> {
-        (0..(ROWS_TO_READ as u64 * 2 + 37))
-            .map(|i| {
-                (
-                    NeedleId(i * 7 + 1),
-                    Offset::from_actual_offset(i as i64 * 128),
-                    Size(i as i32 + 1),
-                )
-            })
-            .collect()
-    }
-
-    #[test]
-    fn test_walk_index_file_short_reads_keep_alignment() {
-        let data = idx_bytes(&many_entries());
-        let expected = walk_all(&mut Cursor::new(data.clone()), 0);
-        assert_eq!(expected.len(), ROWS_TO_READ * 2 + 37);
-
-        let mut short = ShortReader::new(data, false);
-        assert_eq!(walk_all(&mut short, 0), expected);
-    }
-
-    #[test]
-    fn test_walk_index_file_retries_interrupted_reads() {
-        let data = idx_bytes(&many_entries());
-        let expected = walk_all(&mut Cursor::new(data.clone()), 0);
-
-        let mut short = ShortReader::new(data, true);
-        assert_eq!(walk_all(&mut short, 0), expected);
-    }
-
-    #[test]
-    fn test_walk_index_file_short_reads_start_from() {
-        let data = idx_bytes(&many_entries());
-        let expected = walk_all(&mut Cursor::new(data.clone()), 0);
-
-        let start = ROWS_TO_READ as u64 + 5;
-        let mut short = ShortReader::new(data, false);
-        assert_eq!(walk_all(&mut short, start), expected[start as usize..]);
-    }
-
-    #[test]
-    fn test_walk_index_file_ignores_trailing_partial_entry() {
-        // A torn final entry is dropped without an error, as Go does on io.EOF.
-        let entries = many_entries();
-        let mut data = idx_bytes(&entries);
-        data.extend_from_slice(&[0xAB; NEEDLE_MAP_ENTRY_SIZE - 1]);
-
-        let expected = walk_all(&mut Cursor::new(idx_bytes(&entries)), 0);
-        assert_eq!(walk_all(&mut Cursor::new(data.clone()), 0), expected);
-
-        let mut short = ShortReader::new(data, false);
-        assert_eq!(walk_all(&mut short, 0), expected);
-    }
-
     #[test]
     fn test_check_index_file_clean() {
         let data = idx_bytes(&[

seaweed-volume/src/storage/io.rs

@@ -36,11 +36,23 @@ pub(crate) fn read_exact_at(file: &File, buf: &mut [u8], offset: u64) -> io::Res
     }
     #[cfg(windows)]
     {
-        if read_full_at(file, buf, offset)? < buf.len() {
-            return Err(io::Error::new(
-                io::ErrorKind::UnexpectedEof,
-                "unexpected EOF in seek_read",
-            ));
+        use std::os::windows::fs::FileExt;
+        let mut filled = 0;
+        let mut at = offset;
+        while filled < buf.len() {
+            let n = match file.seek_read(&mut buf[filled..], at) {
+                Ok(n) => n,
+                Err(err) if err.kind() == io::ErrorKind::Interrupted => continue,
+                Err(err) => return Err(err),
+            };
+            if n == 0 {
+                return Err(io::Error::new(
+                    io::ErrorKind::UnexpectedEof,
+                    "unexpected EOF in seek_read",
+                ));
+            }
+            filled += n;
+            at += n as u64;
         }
     }
     #[cfg(not(any(unix, windows)))]
@@ -72,34 +84,9 @@ pub(crate) fn read_at(file: &File, buf: &mut [u8], offset: u64) -> io::Result<us
     }
 }
 
-/// Reads into `buf` at `offset` until it is full or the file ends, retrying
-/// interrupted reads; returns how many bytes were read.
-///
-/// Unlike [`read_at`], a count below `buf.len()` always means end of file.
-pub(crate) fn read_full_at(file: &File, buf: &mut [u8], offset: u64) -> io::Result<usize> {
-    fill_at(|b, at| read_at(file, b, at), buf, offset)
-}
-
-fn fill_at(
-    mut read: impl FnMut(&mut [u8], u64) -> io::Result<usize>,
-    buf: &mut [u8],
-    offset: u64,
-) -> io::Result<usize> {
-    let mut filled = 0;
-    while filled < buf.len() {
-        match read(&mut buf[filled..], offset + filled as u64) {
-            Ok(0) => break,
-            Ok(n) => filled += n,
-            Err(err) if err.kind() == io::ErrorKind::Interrupted => {}
-            Err(err) => return Err(err),
-        }
-    }
-    Ok(filled)
-}
-
 #[cfg(test)]
 mod tests {
-    use super::{fill_at, read_at, read_exact_at, read_full_at};
+    use super::{read_at, read_exact_at};
     use std::io::{ErrorKind, Write};
 
     fn temp_file(bytes: &[u8]) -> tempfile::NamedTempFile {
@@ -152,57 +139,4 @@ mod tests {
         let n = read_at(f.as_file(), &mut buf, 10).expect("read");
         assert_eq!(n, 0);
     }
-
-    /// A source that returns at most `chunk` bytes per call and fails with
-    /// `Interrupted` on its first call, like a network mount under a signal.
-    fn chunked(src: &[u8], chunk: usize) -> impl FnMut(&mut [u8], u64) -> std::io::Result<usize> {
-        let mut interrupted = false;
-        move |buf, at| {
-            if !interrupted {
-                interrupted = true;
-                return Err(ErrorKind::Interrupted.into());
-            }
-            let at = (at as usize).min(src.len());
-            let n = buf.len().min(chunk).min(src.len() - at);
-            buf[..n].copy_from_slice(&src[at..at + n]);
-            Ok(n)
-        }
-    }
-
-    #[test]
-    fn fill_at_fills_across_short_and_interrupted_reads() {
-        let src: Vec<u8> = (0..=255).collect();
-        let mut buf = [0u8; 100];
-        let n = fill_at(chunked(&src, 7), &mut buf, 50).expect("read");
-        assert_eq!(n, buf.len());
-        assert_eq!(&buf[..], &src[50..150]);
-    }
-
-    #[test]
-    fn fill_at_stops_at_end_of_source() {
-        let src: Vec<u8> = (0..=255).collect();
-        let mut buf = [0u8; 100];
-        let n = fill_at(chunked(&src, 7), &mut buf, 200).expect("read");
-        assert_eq!(n, 56);
-        assert_eq!(&buf[..n], &src[200..]);
-    }
-
-    #[test]
-    fn fill_at_propagates_other_errors() {
-        let mut buf = [0u8; 8];
-        let err = fill_at(|_, _| Err(ErrorKind::PermissionDenied.into()), &mut buf, 0)
-            .expect_err("error");
-        assert_eq!(err.kind(), ErrorKind::PermissionDenied);
-    }
-
-    #[test]
-    fn read_full_at_returns_the_short_count_only_at_eof() {
-        let f = temp_file(b"0123456789");
-        let mut buf = [0u8; 8];
-        assert_eq!(read_full_at(f.as_file(), &mut buf, 0).expect("read"), 8);
-        assert_eq!(&buf, b"01234567");
-        assert_eq!(read_full_at(f.as_file(), &mut buf, 6).expect("read"), 4);
-        assert_eq!(&buf[..4], b"6789");
-        assert_eq!(read_full_at(f.as_file(), &mut buf, 10).expect("read"), 0);
-    }
 }

seaweed-volume/src/storage/io_error.rs

@@ -3,7 +3,7 @@
 
 use std::io;
 use std::sync::Mutex;
-use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicI32, Ordering};
 
 /// Consecutive storage-media errors allowed before the volume is quarantined.
 pub(crate) const IO_ERROR_TOLERANCE: i32 = 3;
@@ -34,30 +34,10 @@ pub(crate) fn is_storage_io_error(e: &io::Error) -> bool {
 #[derive(Default)]
 pub(crate) struct IoErrorTracker {
     last: Mutex<Option<String>>,
-    /// The consecutive error count in the low 32 bits and, in the high 32,
-    /// how many times it has been cleared. They share one word so that
-    /// `record_success_at` updates both in one step: reads record their
-    /// outcomes here without the volume's write lock.
-    streak: AtomicU64,
+    count: AtomicI32,
     quarantined: AtomicBool,
 }
 
-const STREAK_COUNT_BITS: u64 = 0xffff_ffff;
-
-fn streak_count(streak: u64) -> i32 {
-    (streak & STREAK_COUNT_BITS) as i32
-}
-
-/// `streak` with its count cleared and one more clear on record.
-fn streak_cleared(streak: u64) -> u64 {
-    (streak >> 32).wrapping_add(1) << 32
-}
-
-/// A point in the error streak, taken where a write landed whose success
-/// is only recorded later. See `IoErrorTracker::record_success_at`.
-#[derive(Clone, Copy)]
-pub(crate) struct StreakMark(u64);
-
 impl IoErrorTracker {
     /// `Some(e)` records a failure, `None` a success. Only storage-media
     /// failures count; every other outcome clears the count and last error.
@@ -65,22 +45,14 @@ impl IoErrorTracker {
         if let Some(e) = err
             && is_storage_io_error(e)
         {
-            self.streak.fetch_add(1, Ordering::Relaxed);
+            self.count.fetch_add(1, Ordering::Relaxed);
             if let Ok(mut guard) = self.last.lock() {
                 *guard = Some(e.to_string());
             }
             crate::metrics::STORAGE_IO_ERROR_COUNTER.inc();
             return;
         }
-        self.clear_count();
-        self.clear_last();
-    }
-
-    fn clear_count(&self) {
-        self.update_streak(|streak| Some(streak_cleared(streak)));
-    }
-
-    fn clear_last(&self) {
+        self.count.store(0, Ordering::Relaxed);
         if let Ok(mut guard) = self.last.lock()
             && guard.is_some()
         {
@@ -88,58 +60,17 @@ impl IoErrorTracker {
         }
     }
 
-    /// Apply `f` to the streak atomically; `None` leaves it as it is.
-    /// Returns the streak `f` produced, if any.
-    fn update_streak(&self, mut f: impl FnMut(u64) -> Option<u64>) -> Option<u64> {
-        let mut updated = None;
-        let _ = self
-            .streak
-            .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |streak| {
-                updated = f(streak);
-                updated
-            });
-        updated
-    }
-
-    pub(crate) fn mark(&self) -> StreakMark {
-        StreakMark(self.streak.load(Ordering::Relaxed))
-    }
-
-    /// Record a success as if it had come at `mark`: the errors counted
-    /// before the mark are cleared and the ones counted since still stand,
-    /// as they would had each outcome been recorded in order. A streak
-    /// cleared since the mark is left as it is.
-    pub(crate) fn record_success_at(&self, mark: StreakMark) {
-        let before = streak_count(mark.0);
-        let updated = self.update_streak(|streak| {
-            if streak >> 32 != mark.0 >> 32 {
-                return None;
-            }
-            if streak_count(streak) <= before {
-                return Some(streak_cleared(streak));
-            }
-            Some(streak - before as u64)
-        });
-        // The last error stays when one counted since the mark is left.
-        if updated.is_some_and(|streak| streak_count(streak) == 0) {
-            self.clear_last();
-        }
-    }
-
     /// The last recorded error, the consecutive count, and the quarantine flag.
     pub(crate) fn get_io_error_state(&self) -> (Option<String>, i32, bool) {
         let err = self.last.lock().ok().and_then(|g| g.clone());
-        let count = self.count();
+        let count = self.count.load(Ordering::Relaxed);
         let quarantined = self.quarantined.load(Ordering::Relaxed);
         (err, count, quarantined)
     }
 
-    fn count(&self) -> i32 {
-        streak_count(self.streak.load(Ordering::Relaxed))
-    }
-
     pub(crate) fn should_quarantine(&self) -> bool {
-        self.quarantined.load(Ordering::Relaxed) || self.count() >= IO_ERROR_TOLERANCE
+        self.quarantined.load(Ordering::Relaxed)
+            || self.count.load(Ordering::Relaxed) >= IO_ERROR_TOLERANCE
     }
 
     pub(crate) fn mark_io_quarantined(&self) {
@@ -147,7 +78,7 @@ impl IoErrorTracker {
     }
 
     pub(crate) fn reset_io_error_state(&self) {
-        self.clear_count();
+        self.count.store(0, Ordering::Relaxed);
         self.quarantined.store(false, Ordering::Relaxed);
         if let Ok(mut guard) = self.last.lock() {
             *guard = None;
@@ -160,11 +91,9 @@ impl IoErrorTracker {
             *guard = err.map(|value| value.to_string());
         }
         if err.is_some() {
-            self.update_streak(|streak| {
-                Some((streak & !STREAK_COUNT_BITS) | IO_ERROR_TOLERANCE as u64)
-            });
+            self.count.store(IO_ERROR_TOLERANCE, Ordering::Relaxed);
         } else {
-            self.clear_count();
+            self.count.store(0, Ordering::Relaxed);
         }
     }
 }
@@ -242,87 +171,6 @@ mod tests {
         assert!(tracker.should_quarantine());
     }
 
-    #[test]
-    fn success_at_a_mark_keeps_only_the_errors_after_it() {
-        let tracker = IoErrorTracker::default();
-        tracker.check_read_write_error(Some(&media_error()));
-        tracker.check_read_write_error(Some(&media_error()));
-        let mark = tracker.mark();
-        tracker.check_read_write_error(Some(&media_error()));
-        tracker.record_success_at(mark);
-
-        assert_eq!(
-            tracker.get_io_error_state(),
-            (Some(media_error().to_string()), 1, false)
-        );
-    }
-
-    #[test]
-    fn success_at_a_mark_with_nothing_after_it_clears_the_streak() {
-        let tracker = IoErrorTracker::default();
-        tracker.check_read_write_error(Some(&media_error()));
-        let mark = tracker.mark();
-        tracker.record_success_at(mark);
-
-        assert_eq!(tracker.get_io_error_state(), (None, 0, false));
-    }
-
-    #[test]
-    fn success_at_a_mark_leaves_a_streak_cleared_since() {
-        let tracker = IoErrorTracker::default();
-        tracker.check_read_write_error(Some(&media_error()));
-        tracker.check_read_write_error(Some(&media_error()));
-        let mark = tracker.mark();
-        tracker.check_read_write_error(None);
-        tracker.check_read_write_error(Some(&media_error()));
-        tracker.record_success_at(mark);
-
-        assert_eq!(
-            tracker.get_io_error_state(),
-            (Some(media_error().to_string()), 1, false)
-        );
-    }
-
-    /// Reads update the tracker without the volume's write lock, so a
-    /// success replayed at a mark must not lose the errors they record
-    /// while it runs.
-    #[test]
-    fn success_at_a_mark_keeps_concurrent_errors() {
-        use std::sync::{Arc, Barrier};
-        const READERS: i32 = 4;
-        const ERRORS: i32 = 200;
-
-        for _ in 0..500 {
-            let tracker = Arc::new(IoErrorTracker::default());
-            tracker.check_read_write_error(Some(&media_error()));
-            tracker.check_read_write_error(Some(&media_error()));
-            let mark = tracker.mark();
-            let start = Arc::new(Barrier::new(READERS as usize + 1));
-            let readers: Vec<_> = (0..READERS)
-                .map(|_| {
-                    let (tracker, start) = (tracker.clone(), start.clone());
-                    std::thread::spawn(move || {
-                        start.wait();
-                        for _ in 0..ERRORS {
-                            tracker.check_read_write_error(Some(&media_error()));
-                        }
-                    })
-                })
-                .collect();
-            start.wait();
-            while tracker.get_io_error_state().1 < 2 + READERS * ERRORS / 2 {
-                std::hint::spin_loop();
-            }
-            tracker.record_success_at(mark);
-            for reader in readers {
-                reader.join().unwrap();
-            }
-            // The two errors before the mark are cleared; every error the
-            // readers recorded after it stands.
-            assert_eq!(tracker.get_io_error_state().1, READERS * ERRORS);
-        }
-    }
-
     #[test]
     fn reset_io_error_state_lifts_the_quarantine() {
         let tracker = IoErrorTracker::default();

seaweed-volume/src/storage/needle_map.rs

@@ -247,8 +247,6 @@ impl CompactNeedleMap {
     pub fn load_from_idx<R: Read + Seek>(reader: &mut R, version: Version) -> io::Result<Self> {
         let mut nm = CompactNeedleMap::new();
         idx::walk_index_file(reader, 0, |key, offset, size| {
-            // A read-only load attaches no writer, so this is its only size.
-            nm.idx_file_offset += NEEDLE_MAP_ENTRY_SIZE as u64;
             nm.metric.maybe_set_max_needle_end(offset, size, version);
             if offset.is_zero() || size.is_deleted() {
                 nm.delete_from_map(key);
@@ -1388,18 +1386,6 @@ impl NeedleMap {
         }
     }
 
-    /// Skew the live file count away from what the `.idx` holds, so tests
-    /// can build a volume whose reported count disagrees with a reload.
-    #[cfg(test)]
-    pub(crate) fn add_file_count_for_test(&self, delta: i64) {
-        let metric = match self {
-            NeedleMap::InMemory(nm) => &nm.metric,
-            NeedleMap::Redb(nm) => &nm.metric,
-            NeedleMap::SortedFile(_) => panic!("sorted-file needle maps are read-only"),
-        };
-        metric.file_count.fetch_add(delta, Ordering::Relaxed);
-    }
-
     /// Largest (offset + actual size) seen during the load walk; 0 if the
     /// map is empty. Used at volume load to detect .idx entries that
     /// reference past the end of .dat (issue #8928) without a second scan.

seaweed-volume/src/storage/store_ec_mirror.rs

@@ -131,12 +131,6 @@ impl Store {
                     let Some(base) = name.strip_suffix(".ecx") else {
                         continue;
                     };
-                    // A 0-byte .ecx is a corrupt stub from a failed copy, not a
-                    // credible owner — skip it so the scan keeps looking for a
-                    // real index on a sibling disk (Go's indexEcxOwners).
-                    if !ent.metadata().is_ok_and(|m| m.len() > 0) {
-                        continue;
-                    }
                     let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
                         continue;
                     };
@@ -423,33 +417,4 @@ mod tests {
         let post = fs::read(dir0.join(format!("{}_{}.ecx", collection, vid))).unwrap();
         assert_eq!(post, ecx_local, "mirror overwrote dir0's existing .ecx");
     }
-
-    /// The mirror shares Go's indexEcxOwners, which skips a 0-byte `.ecx`:
-    /// a stub must not be chosen as the source to mirror from.
-    #[test]
-    fn mirror_owner_index_skips_zero_byte_ecx() {
-        let tmp = TempDir::new().unwrap();
-        let dir0 = tmp.path().join("data0");
-        let dir1 = tmp.path().join("data1");
-        fs::create_dir_all(&dir0).unwrap();
-        fs::create_dir_all(&dir1).unwrap();
-
-        let collection = "video-recordings";
-        let vid = 4123u32;
-        plant_ecx(&dir0, collection, vid, b"");
-        plant_ecx(&dir1, collection, vid, &[0xA1u8; 20]);
-
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        add_loc(&mut store, &dir0);
-        add_loc(&mut store, &dir1);
-
-        let owners = store.index_ecx_owners_for_mirror();
-        let owner = owners
-            .get(&EcKey {
-                collection: collection.to_string(),
-                vid: VolumeId(vid),
-            })
-            .expect("the valid .ecx on disk 1 must be indexed");
-        assert_eq!(owner.location, 1);
-    }
 }

seaweed-volume/src/storage/store_ec_reconcile.rs

@@ -37,7 +37,6 @@ pub(crate) struct EcVolumeMissingIndex {
     pub data_dir: String,
 }
 
-#[cfg(test)]
 pub(crate) fn ec_local_ecx_path(dir: &str, collection: &str, vid: VolumeId) -> String {
     if collection.is_empty() {
         format!("{}/{}.ecx", dir, vid.0)
@@ -118,9 +117,10 @@ impl Store {
                     );
                     continue;
                 };
-                // A 0-byte local stub is not a mirrored index (Go gates this fast
-                // path on HasEcxFileOnDisk); mount against the owner instead.
-                let use_local_idx = loc.has_ecx_file_on_disk(&key.collection, key.vid);
+                let local_ecx = ec_local_ecx_path(&loc.idx_directory, &key.collection, key.vid);
+                let local_ecx_in_data = ec_local_ecx_path(&loc.directory, &key.collection, key.vid);
+                let use_local_idx = std::path::Path::new(&local_ecx).exists()
+                    || std::path::Path::new(&local_ecx_in_data).exists();
 
                 if !use_local_idx && owner.location == loc_idx && owner.idx_dir == loc.idx_directory
                 {
@@ -422,12 +422,6 @@ impl Store {
                     let Some(base) = name.strip_suffix(".ecx") else {
                         continue;
                     };
-                    // A 0-byte .ecx is a corrupt stub from a failed copy, not a
-                    // credible owner — skip it so the scan keeps looking for a
-                    // real index on a sibling disk (Go's indexEcxOwners).
-                    if !ent.metadata().is_ok_and(|m| m.len() > 0) {
-                        continue;
-                    }
                     let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
                         continue;
                     };
@@ -645,34 +639,6 @@ mod tests {
         .unwrap();
     }
 
-    /// A 0-byte `.ecx` is not a credible owner (Go's indexEcxOwners skips
-    /// it): picking the stub would hide the valid index on the sibling disk.
-    #[test]
-    fn test_index_ecx_owners_skips_zero_byte_stub() {
-        let (store, _tmp) = make_test_store(2, None);
-        let d0 = store.locations[0].directory.clone();
-        let d1 = store.locations[1].directory.clone();
-        std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(7)), b"").unwrap();
-        write_index_files(&d1, "pics", 7, 10, 4);
-
-        let owners = store.index_ecx_owners();
-        let owner = owners
-            .get(&EcKey {
-                collection: "pics".to_string(),
-                vid: VolumeId(7),
-            })
-            .expect("the valid .ecx on disk 1 must be indexed");
-        assert_eq!(owner.location, 1);
-        assert_eq!(owner.idx_dir, d1);
-
-        // A stub with no real index anywhere owns nothing.
-        std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(8)), b"").unwrap();
-        assert!(!store.index_ecx_owners().contains_key(&EcKey {
-            collection: "pics".to_string(),
-            vid: VolumeId(8),
-        }));
-    }
-
     /// An empty `.dat` (<= a superblock, i.e. zero needles) for an EC volume
     /// is a leftover stub from the pre-fix loader. It must be swept on startup,
     /// not loaded as a phantom empty volume. With the same vid's stub on two
@@ -1005,56 +971,6 @@ mod tests {
         );
     }
 
-    /// dir0 holds orphan shards next to a 0-byte `.ecx` stub from a failed
-    /// copy; the real index is on dir1. The stub must not count as a
-    /// locally-mirrored index (Go gates that fast path on HasEcxFileOnDisk),
-    /// or the shards get registered against an empty index.
-    #[test]
-    fn test_reconcile_ignores_zero_byte_local_ecx_stub() {
-        let tmp = TempDir::new().unwrap();
-        let dir0 = tmp.path().join("data0");
-        let dir1 = tmp.path().join("data1");
-        std::fs::create_dir_all(&dir0).unwrap();
-        std::fs::create_dir_all(&dir1).unwrap();
-
-        let collection = "grafana-loki";
-        let vid = 1094u32;
-
-        write_shard(dir0.to_str().unwrap(), collection, vid, 0);
-        write_shard(dir1.to_str().unwrap(), collection, vid, 1);
-        write_index_files(dir1.to_str().unwrap(), collection, vid, 10, 4);
-
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        for dir in [&dir0, &dir1] {
-            store
-                .add_location(
-                    dir.to_str().unwrap(),
-                    dir.to_str().unwrap(),
-                    100,
-                    DiskType::HardDrive,
-                    MinFreeSpace::Percent(0.0),
-                    Vec::new(),
-                )
-                .unwrap();
-        }
-        // Plant the stub after the startup scan so only the reconcile decision
-        // is under test, then drop dir0's mount and reconcile again.
-        store.locations[0].remove_ec_volume(VolumeId(vid));
-        std::fs::write(
-            ec_local_ecx_path(dir0.to_str().unwrap(), collection, VolumeId(vid)),
-            b"",
-        )
-        .unwrap();
-
-        store.reconcile_ec_shards_across_disks();
-
-        let ev0 = store.locations[0]
-            .find_ec_volume(VolumeId(vid))
-            .expect("dir0's shard must be mounted against the owner's index");
-        assert!(ev0.has_shard(0));
-        assert_eq!(ev0.ecx_actual_dir(), dir1.to_str().unwrap());
-    }
-
     /// PR 9244 review case: idx_directory is configured but the
     /// owner's .ecx / .ecj / .vif live in the owner's data dir
     /// (the legacy "written before -dir.idx was set" layout). The
@@ -1476,7 +1392,7 @@ mod tests {
         let vid = VolumeId(7004);
         let collection = "grafana-loki";
 
-        store.delete_ec_shards(vid, collection, &[1]).unwrap();
+        store.delete_ec_shards(vid, collection, &[1]);
 
         // Shard 1 file is gone on disk 1.
         let p1 = format!(

seaweed-volume/tests/http_integration.rs

@@ -477,74 +477,6 @@ async fn delete_then_get_returns_404() {
     );
 }
 
-// Go answers both with 500 and an error containing "volume N is read only"
-// (the write behind "failed to write to local disk: ").
-#[tokio::test]
-async fn write_and_delete_on_read_only_volume_say_is_read_only() {
-    let (state, _tmp) = test_state();
-    let uri = "/1,01637037d6";
-
-    let request = |method: &str, body: &[u8]| {
-        Request::builder()
-            .method(method)
-            .uri(uri)
-            .body(Body::from(body.to_vec()))
-            .unwrap()
-    };
-    let error_of = |body: Vec<u8>| -> String {
-        let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
-        json["error"].as_str().unwrap_or_default().to_string()
-    };
-
-    let response = build_admin_router(state.clone())
-        .oneshot(request("POST", b"written before read-only"))
-        .await
-        .unwrap();
-    assert_eq!(response.status(), StatusCode::CREATED);
-
-    state
-        .store
-        .write()
-        .unwrap()
-        .find_volume_mut(VolumeId(1))
-        .unwrap()
-        .1
-        .set_no_write_or_delete(true);
-
-    let response = build_admin_router(state.clone())
-        .oneshot(request("POST", b"refused"))
-        .await
-        .unwrap();
-    assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
-    let error = error_of(body_bytes(response).await);
-    assert!(error.contains("volume 1 is read only"), "{error}");
-
-    let response = build_admin_router(state.clone())
-        .oneshot(request("DELETE", b""))
-        .await
-        .unwrap();
-    assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
-    assert_eq!(
-        error_of(body_bytes(response).await),
-        "Deletion Failed: volume 1 is read only"
-    );
-
-    // The needle is still there to delete once the volume is writable again.
-    state
-        .store
-        .write()
-        .unwrap()
-        .find_volume_mut(VolumeId(1))
-        .unwrap()
-        .1
-        .set_no_write_or_delete(false);
-    let response = build_admin_router(state.clone())
-        .oneshot(request("DELETE", b""))
-        .await
-        .unwrap();
-    assert_eq!(response.status(), StatusCode::ACCEPTED);
-}
-
 // ============================================================================
 // 6. HEAD returns headers without body
 // ============================================================================
@@ -1323,102 +1255,3 @@ async fn non_ascii_fid_and_ttl_are_rejected_not_panicked() {
         "a non-ASCII ttl must behave like any other invalid ttl, not panic"
     );
 }
-
-// ============================================================================
-// The write queue answers an upload with the needle's real ETag
-//
-// The queue worker computes the CRC on the needle it was handed, so the handler
-// has to know the checksum before it submits. Without that every queued upload
-// came back as "00000000". The direct path is the reference: same payload, same
-// ETag, for a plain body and for one the handler gzips before storing.
-// ============================================================================
-
-#[tokio::test]
-async fn write_queue_upload_returns_same_etag_as_direct_write() {
-    use seaweed_volume::server::write_queue::WriteQueue;
-
-    let (direct_state, _direct_tmp) = test_state();
-    let (queued_state, _queued_tmp) = test_state();
-    let wq = WriteQueue::new(queued_state.clone(), 128);
-    let _ = queued_state.write_queue.set(wq);
-
-    let compressible = "seaweedfs ".repeat(200).into_bytes();
-    let uploads: [(&str, &[u8]); 2] = [
-        ("/1,01637037d6", b"hello, seaweedfs!"),
-        ("/1/02637037d6/notes.txt", &compressible),
-    ];
-
-    for (uri, payload) in uploads {
-        let mut etags = Vec::new();
-        for state in [&direct_state, &queued_state] {
-            let app = build_admin_router(state.clone());
-            let response = app
-                .oneshot(
-                    Request::builder()
-                        .method("POST")
-                        .uri(uri)
-                        .body(Body::from(payload.to_vec()))
-                        .unwrap(),
-                )
-                .await
-                .unwrap();
-            assert_eq!(response.status(), StatusCode::CREATED);
-
-            let header = response
-                .headers()
-                .get("ETag")
-                .expect("upload response has no ETag")
-                .to_str()
-                .unwrap()
-                .to_string();
-            let body = body_bytes(response).await;
-            let json: serde_json::Value =
-                serde_json::from_slice(&body).expect("POST response is not valid JSON");
-            let etag = json["eTag"].as_str().unwrap().to_string();
-            assert_eq!(header, format!("\"{}\"", etag));
-            etags.push(etag);
-        }
-        assert_ne!(etags[0], "00000000", "{}: direct ETag is the zero CRC", uri);
-        assert_eq!(
-            etags[1], etags[0],
-            "{}: queued upload must return the direct path's ETag",
-            uri
-        );
-    }
-
-    // The second upload really was stored gzipped, so its ETag is the CRC of
-    // the compressed bytes on both paths.
-    let app = build_admin_router(queued_state.clone());
-    let response = app
-        .oneshot(
-            Request::builder()
-                .uri(uploads[1].0)
-                .header("Accept-Encoding", "gzip")
-                .body(Body::empty())
-                .unwrap(),
-        )
-        .await
-        .unwrap();
-    assert_eq!(response.status(), StatusCode::OK);
-    assert_eq!(response.headers()["Content-Encoding"], "gzip");
-
-    // Re-uploading the same bytes is the unchanged path: 204 with the same ETag.
-    let (uri, payload) = uploads[0];
-    let mut etags = Vec::new();
-    for state in [&direct_state, &queued_state] {
-        let app = build_admin_router(state.clone());
-        let response = app
-            .oneshot(
-                Request::builder()
-                    .method("POST")
-                    .uri(uri)
-                    .body(Body::from(payload.to_vec()))
-                    .unwrap(),
-            )
-            .await
-            .unwrap();
-        assert_eq!(response.status(), StatusCode::NO_CONTENT);
-        etags.push(response.headers()["ETag"].to_str().unwrap().to_string());
-    }
-    assert_eq!(etags[1], etags[0], "unchanged upload ETag differs");
-}

test/java/spark/pom.xml

@@ -21,7 +21,7 @@
         <scala.binary.version>2.12</scala.binary.version>
         <junit.version>4.13.2</junit.version>
         <seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
-        <jackson.version>2.22.3</jackson.version> <!-- Upgraded from 2.15.3 -->
+        <jackson.version>2.22.1</jackson.version> <!-- Upgraded from 2.15.3 -->
         <!-- jackson-annotations dropped the patch digit in 2.20; matches jackson.version's major.minor -->
         <jackson.annotations.version>2.22</jackson.annotations.version>
         <netty.version>4.2.16.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->

test/s3/iam/s3_sts_web_identity_api_test.go

@@ -1,258 +0,0 @@
-package iam
-
-import (
-	"crypto/rand"
-	"crypto/rsa"
-	"encoding/base64"
-	"encoding/json"
-	"encoding/xml"
-	"io"
-	"math/big"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/iam"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/golang-jwt/jwt/v5"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	webIdentityAudience = "web-identity-api-test"
-	webIdentitySubject  = "spiffe://example.org/ns/app/sa/app"
-)
-
-// webIdentityIssuer is an OIDC issuer served by the test: discovery document
-// and JWKS for one RSA key, so the server under test fetches real keys.
-type webIdentityIssuer struct {
-	server *httptest.Server
-	key    *rsa.PrivateKey
-}
-
-func newWebIdentityIssuer(t *testing.T) *webIdentityIssuer {
-	t.Helper()
-	key, err := rsa.GenerateKey(rand.Reader, 2048)
-	require.NoError(t, err)
-	issuer := &webIdentityIssuer{key: key}
-	mux := http.NewServeMux()
-	mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, _ *http.Request) {
-		_ = json.NewEncoder(w).Encode(map[string]any{
-			"issuer":                                issuer.server.URL,
-			"jwks_uri":                              issuer.server.URL + "/keys",
-			"id_token_signing_alg_values_supported": []string{"RS256"},
-		})
-	})
-	mux.HandleFunc("/keys", func(w http.ResponseWriter, _ *http.Request) {
-		b64 := func(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) }
-		_ = json.NewEncoder(w).Encode(map[string]any{"keys": []any{map[string]any{
-			"kty": "RSA", "kid": "k1", "use": "sig", "alg": "RS256",
-			"n": b64(key.PublicKey.N.Bytes()), "e": b64(big.NewInt(int64(key.PublicKey.E)).Bytes()),
-		}}})
-	})
-	issuer.server = httptest.NewServer(mux)
-	t.Cleanup(issuer.server.Close)
-	return issuer
-}
-
-func (i *webIdentityIssuer) claims(sub string) jwt.MapClaims {
-	now := time.Now()
-	return jwt.MapClaims{"iss": i.server.URL, "sub": sub, "aud": webIdentityAudience,
-		"iat": now.Unix(), "exp": now.Add(10 * time.Minute).Unix()}
-}
-
-func (i *webIdentityIssuer) token(t *testing.T, sub string, key *rsa.PrivateKey) string {
-	t.Helper()
-	tok := jwt.NewWithClaims(jwt.SigningMethodRS256, i.claims(sub))
-	tok.Header["kid"] = "k1"
-	signed, err := tok.SignedString(key)
-	require.NoError(t, err)
-	return signed
-}
-
-func (i *webIdentityIssuer) tokenForAudience(t *testing.T, sub, aud string) string {
-	t.Helper()
-	claims := i.claims(sub)
-	claims["aud"] = aud
-	tok := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)
-	tok.Header["kid"] = "k1"
-	signed, err := tok.SignedString(i.key)
-	require.NoError(t, err)
-	return signed
-}
-
-func (i *webIdentityIssuer) unsignedToken(t *testing.T, sub string) string {
-	t.Helper()
-	signed, err := jwt.NewWithClaims(jwt.SigningMethodNone, i.claims(sub)).SignedString(jwt.UnsafeAllowNoneSignatureType)
-	require.NoError(t, err)
-	return signed
-}
-
-func trustPolicyFor(issuerURL, sub string) string {
-	doc, _ := json.Marshal(map[string]any{"Version": "2012-10-17", "Statement": []any{map[string]any{
-		"Effect": "Allow", "Principal": map[string]any{"Federated": issuerURL},
-		"Action":    []string{"sts:AssumeRoleWithWebIdentity"},
-		"Condition": map[string]any{"StringEquals": map[string]any{"oidc:sub": sub}},
-	}}})
-	return string(doc)
-}
-
-// assumeWithWebIdentity returns session credentials, or nil and the error body.
-func assumeWithWebIdentity(t *testing.T, roleArn, token string) (*credentials.Credentials, string) {
-	t.Helper()
-	resp, err := callSTSAPI(t, url.Values{
-		"Action": {"AssumeRoleWithWebIdentity"}, "Version": {"2011-06-15"},
-		"RoleArn": {roleArn}, "RoleSessionName": {"web-identity-api"}, "WebIdentityToken": {token},
-	})
-	require.NoError(t, err)
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
-	require.NoError(t, err)
-	if resp.StatusCode != http.StatusOK {
-		return nil, string(body)
-	}
-	var out AssumeRoleWithWebIdentityTestResponse
-	require.NoError(t, xml.Unmarshal(body, &out), "body: %s", body)
-	c := out.Result.Credentials
-	return credentials.NewStaticCredentials(c.AccessKeyId, c.SecretAccessKey, c.SessionToken), ""
-}
-
-func s3ClientWith(t *testing.T, creds *credentials.Credentials) *s3.S3 {
-	t.Helper()
-	sess, err := session.NewSession(&aws.Config{
-		Region: aws.String(TestRegion), Endpoint: aws.String(TestS3Endpoint),
-		Credentials: creds, S3ForcePathStyle: aws.Bool(true), DisableSSL: aws.Bool(true),
-	})
-	require.NoError(t, err)
-	return s3.New(sess)
-}
-
-// TestWebIdentityWithProviderAndRoleManagedThroughIAMAPI configures STS
-// federation entirely at runtime — an OIDC provider, a managed policy and a role
-// created through the IAM API, with no static configuration — and checks that
-// the role admits exactly the subject its trust policy names, with exactly the
-// permissions of its attached policy.
-func TestWebIdentityWithProviderAndRoleManagedThroughIAMAPI(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-	if !isSTSEndpointRunning(t) {
-		t.Fatal("SeaweedFS STS endpoint is not running at", TestSTSEndpoint, "- please run 'make setup-all-tests' first")
-	}
-
-	framework := NewS3IAMTestFramework(t)
-	defer framework.Cleanup()
-	admin, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
-	require.NoError(t, err)
-	adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
-	require.NoError(t, err)
-
-	issuer := newWebIdentityIssuer(t)
-	bucket := framework.GenerateUniqueBucketName("web-identity")
-	require.NoError(t, framework.CreateBucketWithCleanup(adminS3, bucket))
-
-	provider, err := admin.CreateOpenIDConnectProvider(&iam.CreateOpenIDConnectProviderInput{
-		Url:          aws.String(issuer.server.URL),
-		ClientIDList: []*string{aws.String(webIdentityAudience)},
-		// Required by this SDK version; pinning only applies to a TLS issuer,
-		// and the test issuer is plain HTTP.
-		ThumbprintList: []*string{aws.String("0000000000000000000000000000000000000000")},
-	})
-	require.NoError(t, err)
-	defer admin.DeleteOpenIDConnectProvider(&iam.DeleteOpenIDConnectProviderInput{OpenIDConnectProviderArn: provider.OpenIDConnectProviderArn})
-
-	policyDoc, _ := json.Marshal(map[string]any{"Version": "2012-10-17", "Statement": []any{map[string]any{
-		"Effect": "Allow", "Action": []string{"s3:*"},
-		"Resource": []string{"arn:aws:s3:::" + bucket, "arn:aws:s3:::" + bucket + "/*"},
-	}}})
-	policy, err := admin.CreatePolicy(&iam.CreatePolicyInput{
-		PolicyName: aws.String(bucket + "-rw"), PolicyDocument: aws.String(string(policyDoc)),
-	})
-	require.NoError(t, err)
-	defer admin.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policy.Policy.Arn})
-
-	// Role names are at most 64 characters; the bucket name is longer.
-	roleName := "web-identity-" + strconv.FormatInt(time.Now().UnixNano(), 36)
-	role, err := admin.CreateRole(&iam.CreateRoleInput{
-		RoleName:                 aws.String(roleName),
-		AssumeRolePolicyDocument: aws.String(trustPolicyFor(issuer.server.URL, webIdentitySubject)),
-	})
-	require.NoError(t, err)
-	defer admin.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
-	_, err = admin.AttachRolePolicy(&iam.AttachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
-	require.NoError(t, err)
-	defer admin.DetachRolePolicy(&iam.DetachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
-	roleArn := aws.StringValue(role.Role.Arn)
-
-	t.Run("the trusted subject gets credentials scoped to the attached policy", func(t *testing.T) {
-		creds, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
-		require.NotNil(t, creds, "AssumeRoleWithWebIdentity refused the trusted subject: %s", failure)
-		client := s3ClientWith(t, creds)
-		_, err := client.PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("federated.txt")})
-		assert.NoError(t, err, "the session cannot write the bucket its policy grants")
-		_, err = client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket + "-other")})
-		assert.Error(t, err, "the session created a bucket its policy does not grant")
-	})
-
-	refusals := map[string]string{
-		"another subject": issuer.token(t, "spiffe://example.org/ns/other/sa/other", issuer.key),
-		"a token signed by another key": func() string {
-			other, err := rsa.GenerateKey(rand.Reader, 2048)
-			require.NoError(t, err)
-			return issuer.token(t, webIdentitySubject, other)
-		}(),
-		"an unsigned token": issuer.unsignedToken(t, webIdentitySubject),
-		// The provider accepts only its registered client IDs as audiences.
-		"a token issued for another audience": issuer.tokenForAudience(t, webIdentitySubject, "some-other-service"),
-	}
-	for name, token := range refusals {
-		t.Run("refuses "+name, func(t *testing.T) {
-			creds, _ := assumeWithWebIdentity(t, roleArn, token)
-			assert.Nil(t, creds, "AssumeRoleWithWebIdentity issued credentials for %s", name)
-		})
-	}
-
-	t.Run("an updated trust policy takes effect", func(t *testing.T) {
-		moved := "spiffe://example.org/ns/app/sa/moved"
-		_, err := admin.UpdateAssumeRolePolicy(&iam.UpdateAssumeRolePolicyInput{
-			RoleName: aws.String(roleName), PolicyDocument: aws.String(trustPolicyFor(issuer.server.URL, moved)),
-		})
-		require.NoError(t, err)
-		creds, _ := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
-		assert.Nil(t, creds, "the subject removed from the trust policy still assumes the role")
-		creds, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, moved, issuer.key))
-		assert.NotNil(t, creds, "the subject added to the trust policy was refused: %s", failure)
-	})
-
-	t.Run("a session does not survive its role being deleted and created again", func(t *testing.T) {
-		trust := trustPolicyFor(issuer.server.URL, webIdentitySubject)
-		_, err := admin.UpdateAssumeRolePolicy(&iam.UpdateAssumeRolePolicyInput{RoleName: aws.String(roleName), PolicyDocument: aws.String(trust)})
-		require.NoError(t, err)
-		before, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
-		require.NotNil(t, before, "precondition: %s", failure)
-
-		_, err = admin.DetachRolePolicy(&iam.DetachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
-		require.NoError(t, err)
-		_, err = admin.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
-		require.NoError(t, err)
-		recreated, err := admin.CreateRole(&iam.CreateRoleInput{RoleName: aws.String(roleName), AssumeRolePolicyDocument: aws.String(trust)})
-		require.NoError(t, err)
-		_, err = admin.AttachRolePolicy(&iam.AttachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
-		require.NoError(t, err)
-		assert.NotEqual(t, aws.StringValue(role.Role.RoleId), aws.StringValue(recreated.Role.RoleId), "the recreated role reuses the deleted role's ID")
-
-		_, err = s3ClientWith(t, before).PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("revived.txt")})
-		assert.Error(t, err, "a session of the deleted role works again under the new role of the same name")
-		after, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
-		require.NotNil(t, after, "the new role refuses its trusted subject: %s", failure)
-		_, err = s3ClientWith(t, after).PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("fresh.txt")})
-		assert.NoError(t, err, "a session of the new role cannot use its policy")
-	})
-}

test/s3/remote_cache/remote_cache_evict_test.go

@@ -1,231 +0,0 @@
-package remote_cache
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	evictRemoteS3      = "http://localhost:28334"
-	evictRemoteMaster  = "30334"
-	evictRemoteFiler   = "28889"
-	evictRemoteVolume  = "30341"
-	evictRemoteWebdav  = "27334"
-	evictRemoteMetrics = "30326"
-
-	evictPrimaryS3      = "http://localhost:28333"
-	evictPrimaryMaster  = "30333"
-	evictPrimaryFiler   = "28887"
-	evictPrimaryVolume  = "30340"
-	evictPrimaryWebdav  = "27333"
-	evictPrimaryMetrics = "30327"
-
-	evictBucket = "evictsrc"
-	evictMount  = "evictmnt"
-)
-
-var miniLogPaths []string
-
-func startMini(t *testing.T, dir string, args ...string) {
-	t.Helper()
-	require.NoError(t, os.MkdirAll(dir, 0755))
-	logPath := filepath.Join(dir, "weed.log")
-	logFile, err := os.Create(logPath)
-	require.NoError(t, err)
-	miniLogPaths = append(miniLogPaths, logPath)
-	cmd := exec.Command(weedBinary, append([]string{"mini",
-		"-dir=" + dir,
-		"-s3.config=s3_config.json",
-		"-s3.allowDeleteBucketNotEmpty=true",
-		"-ip=127.0.0.1", "-ip.bind=127.0.0.1",
-	}, args...)...)
-	cmd.Stdout = logFile
-	cmd.Stderr = logFile
-	require.NoError(t, cmd.Start())
-	t.Cleanup(func() {
-		cmd.Process.Kill()
-		cmd.Wait()
-		logFile.Close()
-	})
-}
-
-func waitForHTTP(t *testing.T, url string) {
-	t.Helper()
-	deadline := time.Now().Add(90 * time.Second)
-	for time.Now().Before(deadline) {
-		if resp, err := http.Get(url); err == nil {
-			resp.Body.Close()
-			return
-		}
-		time.Sleep(time.Second)
-	}
-	for _, p := range miniLogPaths {
-		if data, err := os.ReadFile(p); err == nil {
-			lines := strings.Split(string(data), "\n")
-			if len(lines) > 30 {
-				lines = lines[len(lines)-30:]
-			}
-			t.Logf("last lines of %s:\n%s", p, strings.Join(lines, "\n"))
-		}
-	}
-	t.Fatalf("timed out waiting for %s", url)
-}
-
-func shellOn(t *testing.T, masterPort, command string) string {
-	t.Helper()
-	cmd := exec.Command(weedBinary, "shell", "-master=localhost:"+masterPort)
-	cmd.Stdin = strings.NewReader(command + "\nexit\n")
-	out, err := cmd.CombinedOutput()
-	require.NoErrorf(t, err, "shell %q failed: %s", command, out)
-	return stripLogs(string(out))
-}
-
-func chunkCountOn(t *testing.T, masterPort, path string) string {
-	meta := shellOn(t, masterPort, "fs.meta.cat "+path)
-	idx := strings.LastIndex(meta, "chunks ")
-	require.GreaterOrEqualf(t, idx, 0, "no chunk count in %s", meta)
-	return strings.Fields(meta[idx+len("chunks "):])[0]
-}
-
-func volumeStats(t *testing.T, volumePort string) (size, garbage uint64) {
-	t.Helper()
-	resp, err := http.Get("http://localhost:" + volumePort + "/status")
-	require.NoError(t, err)
-	defer resp.Body.Close()
-	var status struct {
-		Volumes []struct {
-			Size             uint64 `json:"Size"`
-			DeletedByteCount uint64 `json:"DeletedByteCount"`
-		} `json:"Volumes"`
-	}
-	require.NoError(t, json.NewDecoder(resp.Body).Decode(&status))
-	for _, v := range status.Volumes {
-		size += v.Size
-		garbage += v.DeletedByteCount
-	}
-	return size, garbage
-}
-
-func readViaFiler(t *testing.T, filerPort, path string) []byte {
-	t.Helper()
-	resp, err := http.Get("http://localhost:" + filerPort + path)
-	require.NoError(t, err)
-	defer resp.Body.Close()
-	require.Equal(t, http.StatusOK, resp.StatusCode, "read %s", path)
-	data, err := io.ReadAll(resp.Body)
-	require.NoError(t, err)
-	return data
-}
-
-// TestRemoteCacheEvictUnderPressure fills a cache constrained to two small
-// volumes with remote-mounted objects until writes fail, then verifies the
-// filer evicts the oldest synced entry, vacuums the garbage, and a later read
-// caches again.
-func TestRemoteCacheEvictUnderPressure(t *testing.T) {
-	if testing.Short() {
-		t.Skip("spawns two weed mini clusters")
-	}
-	if _, err := os.Stat(weedBinary); err != nil {
-		t.Skipf("weed binary not found at %s; run make build-weed", weedBinary)
-	}
-	if isServerRunning(evictRemoteS3) || isServerRunning(evictPrimaryS3) {
-		t.Skip("eviction test ports are already in use")
-	}
-
-	tmp := t.TempDir()
-	startMini(t, filepath.Join(tmp, "remote"),
-		"-s3.port=28334", "-master.port="+evictRemoteMaster,
-		"-filer.port="+evictRemoteFiler, "-volume.port="+evictRemoteVolume,
-		"-webdav.port="+evictRemoteWebdav, "-metricsPort="+evictRemoteMetrics)
-	waitForHTTP(t, evictRemoteS3)
-	startMini(t, filepath.Join(tmp, "primary"),
-		"-s3.port=28333", "-master.port="+evictPrimaryMaster,
-		"-filer.port="+evictPrimaryFiler, "-volume.port="+evictPrimaryVolume,
-		"-webdav.port="+evictPrimaryWebdav, "-metricsPort="+evictPrimaryMetrics,
-		"-volume.allowUntrustedRemoteEndpoints", "-filer.allowUntrustedRemoteEndpoints",
-		"-s3.allowUntrustedRemoteEndpoints",
-		"-master.volumeSizeLimitMB=32", "-volume.max=2",
-		"-filer.remoteCacheEvictThreshold=0.99")
-	waitForHTTP(t, evictPrimaryS3)
-
-	remote := createS3Client(evictRemoteS3)
-	_, err := remote.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(evictBucket)})
-	require.NoError(t, err)
-	for i := 0; i < 5; i++ {
-		data := make([]byte, 16*1024*1024)
-		for j := range data {
-			data[j] = byte(i + j%251)
-		}
-		_, err = remote.PutObject(&s3.PutObjectInput{
-			Bucket: aws.String(evictBucket),
-			Key:    aws.String(fmt.Sprintf("obj%d.bin", i)),
-			Body:   bytes.NewReader(data),
-		})
-		require.NoError(t, err)
-	}
-
-	shellOn(t, evictPrimaryMaster, fmt.Sprintf(
-		"remote.configure -name=evictremote -type=s3 -s3.access_key=%s -s3.secret_key=%s -s3.endpoint=%s -s3.region=us-east-1",
-		accessKey, secretKey, evictRemoteS3))
-	shellOn(t, evictPrimaryMaster, fmt.Sprintf(
-		"remote.mount -dir=/buckets/%s -remote=evictremote/%s -nonempty", evictMount, evictBucket))
-	shellOn(t, evictPrimaryMaster, fmt.Sprintf("remote.meta.sync -dir=/buckets/%s", evictMount))
-	time.Sleep(2 * time.Second)
-
-	mount := "/buckets/" + evictMount
-
-	// obj0 caches alone so it is the oldest evictable entry.
-	first := readViaFiler(t, evictPrimaryFiler, mount+"/obj0.bin")
-	require.Len(t, first, 16*1024*1024)
-	require.NotEqual(t, "0", chunkCountOn(t, evictPrimaryMaster, mount+"/obj0.bin"), "obj0 should be cached")
-
-	// 4 x 16MB against ~64MB of capacity: the fills overrun, hit the
-	// capacity error path, and trigger eviction + vacuum.
-	var wg sync.WaitGroup
-	for i := 1; i <= 4; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			data := readViaFiler(t, evictPrimaryFiler, fmt.Sprintf("%s/obj%d.bin", mount, i))
-			assert.Len(t, data, 16*1024*1024)
-		}(i)
-	}
-	wg.Wait()
-
-	// Eviction must have dropped obj0's local chunks and vacuumed the
-	// tombstoned bytes, leaving only live data within the two volumes.
-	require.True(t, waitForCondition(t, func() bool {
-		size, garbage := volumeStats(t, evictPrimaryVolume)
-		return garbage == 0 && size <= 72*1024*1024
-	}, 2*time.Minute, "volumes to be reclaimed by eviction+vacuum"),
-		"evicted chunks were not reclaimed")
-
-	assert.Equal(t, "0", chunkCountOn(t, evictPrimaryMaster, mount+"/obj0.bin"),
-		"oldest cached entry should be evicted back to remote-only")
-
-	// The cache self-heals: reading the evicted object re-caches it. Fills
-	// still running from the concurrent wave may evict it again, so retry
-	// the read until a commit sticks.
-	var again []byte
-	require.True(t, waitForCondition(t, func() bool {
-		again = readViaFiler(t, evictPrimaryFiler, mount+"/obj0.bin")
-		return chunkCountOn(t, evictPrimaryMaster, mount+"/obj0.bin") != "0"
-	}, 2*time.Minute, "obj0 to re-cache"),
-		"evicted object did not re-cache on read")
-	assert.Equal(t, first, again)
-}

test/s3tables/catalog_olake/Dockerfile.reader

@@ -1,14 +0,0 @@
-# Strict Iceberg reader used by the OLake integration test. PyIceberg is
-# deliberate here: it rejects manifests that omit the spec's field ids and
-# parquet without either field ids or a name mapping, so a passing read proves
-# the catalog served something every engine can consume -- not just something
-# OLake itself can read back.
-FROM python:3.11-slim
-
-WORKDIR /app
-
-RUN pip install --no-cache-dir "pyiceberg[s3fs]==0.11.1" "pyarrow==25.0.0"
-
-COPY inspect_table.py /app/
-
-ENTRYPOINT ["python3", "/app/inspect_table.py"]

test/s3tables/catalog_olake/README.md

@@ -1,104 +0,0 @@
-# OLake Iceberg Catalog Integration Test
-
-An integration test for [OLake](https://github.com/datazip-inc/olake) against
-SeaweedFS's Iceberg REST Catalog, in the same shape as `catalog_clickhouse`.
-
-## Why OLake, given we already test five engines
-
-Two things here are covered by nothing else in this directory.
-
-**It is a strict Java Iceberg client.** OLake does not write Iceberg from Go —
-its Go process spawns a Java sidecar over gRPC and writes through the official
-Apache Iceberg library, because the Go library has no equality deletes and CDC
-needs them. So this test exercises the client class that
-`weed/s3api/iceberg/metadata_compliance.go` exists to serve: the one
-that fails with *"Cannot parse missing long current-snapshot-id"* when the
-catalog omits spec-required keys that `iceberg-go` strips via `omitempty`.
-
-**It produces equality deletes.** OLake is a CDC tool. Its upsert path commits
-`operation=overwrite` with an equality-delete file, and a delete manifest
-alongside the data manifests. ClickHouse, StarRocks, Doris and DuckDB all only
-append.
-
-## What it asserts
-
-`TestOLakeIcebergCatalog` runs six subtests against a `weed mini` cluster with
-a pre-created table bucket and a Postgres source:
-
-| Subtest | What a failure means |
-|---|---|
-| `CheckDestination` | `olake check` did not reach `SUCCEEDED`, or it passed without ever loading `org.apache.iceberg.rest.RESTSessionCatalog` — the second case means the destination was never actually contacted. |
-| `Discover` | OLake could not enumerate the source table, or wrote no `streams.json`. |
-| `FullSyncAppendsRows` | The sync read fewer than the three seeded rows, or committed no Iceberg snapshot. |
-| `StrictReaderSeesRows` | PyIceberg could not read back what the Java writer committed, or the values differ. This is the data path, not just metadata. |
-| `UpsertProducesEqualityDelete` | After an `UPDATE` and a re-sync, no snapshot recorded an overwrite carrying equality deletes, or the current snapshot has no delete manifest. |
-| `CompliantWriterNeedsNoRepair` | The catalog rewrote a manifest the official Iceberg Java writer produced. That is a regression in the repair gate, not a problem with OLake. |
-
-## The one thing this test deliberately does not check
-
-It does **not** assert that a reader sees the updated row and no duplicate.
-
-PyIceberg refuses to scan a table carrying equality deletes
-([apache/iceberg#6568](https://github.com/apache/iceberg/issues/6568)) while
-reading its metadata perfectly well — so a rows-mode read after the upsert would
-raise, not pass. The alternative is an engine that applies equality deletes,
-which for StarRocks means a 3 GB image and 12 GB of RAM in CI.
-
-The line drawn instead: **recording the commit correctly is the catalog's
-contract; applying deletes on read is the query engine's.** The metadata
-assertions cover our half.
-
-This was verified once by hand outside CI, on 2026-09-23, with StarRocks 4.1.4
-attached to the same catalog: after the upsert it read 3 rows / 3 distinct ids
-with `id=1` showing the updated value and `_op_type=u`. If someone later wants
-that inside the gate, add a reader that supports equality deletes — **do not**
-"upgrade" this test to a PyIceberg rows-mode read after the upsert. It would not
-pass; and if PyIceberg ever starts silently skipping deletes instead of raising,
-it would pass by not looking.
-
-## What the config proves
-
-Nothing in the destination config is SeaweedFS-specific:
-
-```json
-{
-  "catalog_type": "rest",
-  "rest_catalog_url": "http://HOST:ICEBERG_PORT",
-  "iceberg_s3_path": "s3://olake-tables",
-  "s3_endpoint": "http://HOST:S3_PORT",
-  "rest_auth_type": "oauth2",
-  "oauth2_uri": "http://HOST:ICEBERG_PORT/v1/oauth/tokens",
-  "credential": "ACCESS_KEY:SECRET_KEY"
-}
-```
-
-`catalog_type` is the generic `rest`, auth is the standard OAuth2
-client-credentials flow, and `s3_path_style` does not even need setting —
-OLake turns it on by itself whenever `s3_endpoint` is non-empty. As of
-OLake v0.10.1 this works with no change on either side.
-
-## Running it
-
-```sh
-go test ./test/s3tables/catalog_olake/ -run TestOLakeIcebergCatalog -v -timeout 25m
-```
-
-Needs Docker and a `weed` binary (at `weed/weed` under the repo root, or on
-`PATH`). Takes about 35 seconds. It skips rather than fails when Docker is
-absent, and `SEAWEEDFS_SKIP_OLAKE_TESTS=1` skips it outright.
-
-Overrides: `OLAKE_IMAGE` (default `olakego/source-postgres:latest`),
-`POSTGRES_IMAGE` (default `postgres:16`).
-
-## In CI
-
-Runs as `olake-iceberg-catalog-tests` in `.github/workflows/s3-tables-tests.yml`,
-on a matrix of a pinned image plus `latest` — the same shape the ClickHouse job
-uses, and for the same reason: OLake's Iceberg writer is a Java sidecar whose
-library version moves independently of the Go release, so the `latest` leg is
-what catches drift in the client rather than in OLake itself.
-
-The job asserts the suite actually ran — at least one top-level `--- PASS` and
-zero `--- SKIP` — rather than trusting a green exit. This suite skips itself
-when Docker is unavailable, and a skipped suite reporting success is how a gate
-quietly stops being one.

test/s3tables/catalog_olake/inspect_table.py

@@ -1,91 +0,0 @@
-#!/usr/bin/env python3
-"""Inspect an Iceberg table written by OLake, through the SeaweedFS REST catalog.
-
-Two modes, because a strict reader cannot do both:
-
-  rows      -- scan the table and print "id,region,amount" per row, ordered by
-               id. Only valid while the table has no equality deletes.
-  snapshots -- print one line per snapshot with its operation and delete-file
-               counters, plus the manifest content kinds of the current
-               snapshot.
-
-The split exists because PyIceberg refuses to scan a table carrying equality
-deletes (apache/iceberg#6568) while reading its metadata perfectly well. OLake
-is a CDC tool, so its upsert path produces exactly those deletes -- asserting
-the commit landed is the catalog's concern, and applying deletes on read is the
-query engine's.
-"""
-
-import argparse
-import sys
-
-from pyiceberg.catalog import load_catalog
-
-
-def main() -> int:
-    p = argparse.ArgumentParser()
-    p.add_argument("mode", choices=["rows", "snapshots"])
-    p.add_argument("--catalog-url", required=True)
-    p.add_argument("--warehouse", required=True)
-    p.add_argument("--prefix", required=True)
-    p.add_argument("--s3-endpoint", required=True)
-    p.add_argument("--access-key", required=True)
-    p.add_argument("--secret-key", required=True)
-    p.add_argument("--region", default="us-east-1")
-    p.add_argument("--namespace", action="append", required=True)
-    p.add_argument("--table", required=True)
-    args = p.parse_args()
-
-    catalog = load_catalog(
-        "rest",
-        **{
-            "type": "rest",
-            "uri": args.catalog_url,
-            "warehouse": args.warehouse,
-            "prefix": args.prefix,
-            "credential": f"{args.access_key}:{args.secret_key}",
-            "s3.access-key-id": args.access_key,
-            "s3.secret-access-key": args.secret_key,
-            "s3.endpoint": args.s3_endpoint,
-            "s3.region": args.region,
-            "s3.path-style-access": "true",
-        },
-    )
-
-    table = catalog.load_table(tuple(args.namespace) + (args.table,))
-
-    if args.mode == "rows":
-        data = table.scan().to_arrow().to_pydict()
-        # amount is decimal(10,2) in Postgres but arrives here as a float, whose
-        # repr drops trailing zeros (120.5, not 120.50). Format it to the source
-        # scale so the expected values in the Go test stay readable.
-        for row_id, region, amount in sorted(
-            zip(data["id"], data["region"], data["amount"])
-        ):
-            print("%s,%s,%.2f" % (row_id, region, float(amount)))
-        return 0
-
-    print("format-version=%d" % table.metadata.format_version)
-    ids = table.metadata.schemas[-1].identifier_field_ids
-    print("identifier-field-ids=%s" % ",".join(str(i) for i in ids))
-    for snap in table.metadata.snapshots:
-        s = snap.summary
-        print(
-            "snapshot operation=%s total-delete-files=%s added-delete-files=%s "
-            "added-equality-deletes=%s total-records=%s"
-            % (
-                s.operation,
-                s.get("total-delete-files", "0"),
-                s.get("added-delete-files", "0"),
-                s.get("added-equality-deletes", "0"),
-                s.get("total-records", "0"),
-            )
-        )
-    current = table.current_snapshot()
-    kinds = [str(m.content).rsplit(".", 1)[-1] for m in current.manifests(table.io)]
-    print("current-manifest-kinds=%s" % ",".join(kinds))
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

test/s3tables/catalog_olake/olake_catalog_test.go

@@ -1,623 +0,0 @@
-// Package catalog_olake provides an integration test for OLake
-// (github.com/datazip-inc/olake) against the SeaweedFS Iceberg REST Catalog.
-//
-// OLake matters here for two reasons that no other engine in this directory
-// covers. First, it does not write Iceberg from Go: it spawns a Java sidecar
-// over gRPC and writes through the official Apache Iceberg library, so this is
-// a strict Java client -- the class that iceberg/metadata_compliance.go exists
-// to serve. Second, it is a CDC tool, so its upsert path emits equality
-// deletes, which neither the ClickHouse nor the Doris test exercises.
-package catalog_olake
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/testutil"
-)
-
-const (
-	olakeDefaultImage    = "olakego/source-postgres:latest"
-	postgresDefaultImage = "postgres:16"
-	readerImage          = "seaweedfs-olake-reader:test"
-
-	tableBucketName = "olake-tables"
-	sourceDatabase  = "olakedb"
-	sourceUser      = "olake"
-	sourcePassword  = "olakepw"
-
-	// OLake derives the destination namespace from the source: it joins the
-	// connector name, database and schema. Asserting the derived name rather
-	// than configuring one keeps the test honest about what OLake actually
-	// does with our catalog.
-	expectedNamespace = "postgres_olakedb_public"
-	expectedTable     = "orders"
-
-	postgresStartTimeout = 90 * time.Second
-	olakeRunTimeout      = 6 * time.Minute
-)
-
-type TestEnvironment struct {
-	seaweedDir string
-	weedBinary string
-	dataDir    string
-	configDir  string
-	bindIP     string
-
-	masterPort     int
-	masterGrpcPort int
-	volumePort     int
-	volumeGrpcPort int
-	filerPort      int
-	filerGrpcPort  int
-	s3Port         int
-	s3GrpcPort     int
-	icebergPort    int
-	postgresPort   int
-
-	accessKey string
-	secretKey string
-
-	weedProcess       *exec.Cmd
-	weedCancel        func()
-	postgresContainer string
-}
-
-func TestOLakeIcebergCatalog(t *testing.T) {
-	requireOLakeRuntime(t)
-
-	env := NewTestEnvironment(t)
-	defer env.Cleanup(t)
-
-	env.StartSeaweedFS(t)
-	env.startPostgres(t)
-	env.seedSource(t)
-
-	buildReaderImage(t)
-	env.writeOLakeConfigs(t)
-
-	t.Run("CheckDestination", func(t *testing.T) {
-		out := env.runOLake(t, "check",
-			"--config", "/mnt/config/source.json",
-			"--destination", "/mnt/config/destination.json")
-		if !strings.Contains(out, `"status":"SUCCEEDED"`) {
-			t.Fatalf("olake check did not report SUCCEEDED.\n%s", tailLines(out, 40))
-		}
-		// The Java sidecar, not the Go process, is what talks to our catalog.
-		// If this line is missing the check passed without exercising the
-		// Iceberg REST path at all.
-		if !strings.Contains(out, "org.apache.iceberg.rest.RESTSessionCatalog") {
-			t.Errorf("check passed but never loaded the Iceberg REST catalog; "+
-				"the destination may not have been contacted.\n%s", tailLines(out, 40))
-		}
-	})
-
-	t.Run("Discover", func(t *testing.T) {
-		out := env.runOLake(t, "discover",
-			"--config", "/mnt/config/source.json",
-			"--destination", "/mnt/config/destination.json")
-		if !strings.Contains(out, `"stream_name":"`+expectedTable+`"`) {
-			t.Fatalf("discover did not report the %s stream.\n%s", expectedTable, tailLines(out, 20))
-		}
-		if _, err := os.Stat(filepath.Join(env.configDir, "streams.json")); err != nil {
-			t.Fatalf("discover did not write streams.json: %v", err)
-		}
-	})
-
-	t.Run("FullSyncAppendsRows", func(t *testing.T) {
-		out := env.runOLake(t, "sync",
-			"--config", "/mnt/config/source.json",
-			"--destination", "/mnt/config/destination.json",
-			"--streams", "/mnt/config/streams.json",
-			"--state", "/mnt/config/state.json")
-		if !strings.Contains(out, "Total records read: 3") {
-			t.Fatalf("sync did not read the three seeded rows.\n%s", tailLines(out, 30))
-		}
-		if !strings.Contains(out, "Committed snapshot") {
-			t.Fatalf("sync never committed an Iceberg snapshot.\n%s", tailLines(out, 30))
-		}
-	})
-
-	t.Run("StrictReaderSeesRows", func(t *testing.T) {
-		rows := env.readTable(t, "rows")
-		want := []string{"1,us-east,120.50", "2,us-west,87.20", "3,eu-west,210.00"}
-		got := nonEmptyLines(rows)
-		if len(got) != len(want) {
-			t.Fatalf("PyIceberg read %d rows, want %d.\n%s", len(got), len(want), rows)
-		}
-		for i := range want {
-			if got[i] != want[i] {
-				t.Errorf("row %d = %q, want %q", i, got[i], want[i])
-			}
-		}
-	})
-
-	t.Run("UpsertProducesEqualityDelete", func(t *testing.T) {
-		// Push the row past the incremental cursor so the next sync re-emits it
-		// as an update rather than skipping it.
-		env.execSQL(t, "UPDATE orders SET amount=999.99, region='ap-south' WHERE id=1")
-
-		out := env.runOLake(t, "sync",
-			"--config", "/mnt/config/source.json",
-			"--destination", "/mnt/config/destination.json",
-			"--streams", "/mnt/config/streams.json",
-			"--state", "/mnt/config/state.json")
-		if !strings.Contains(out, "delete files") {
-			t.Fatalf("re-sync committed no delete files.\n%s", tailLines(out, 30))
-		}
-
-		meta := env.readTable(t, "snapshots")
-		if !strings.Contains(meta, "format-version=2") {
-			t.Errorf("expected Iceberg format-version 2, got:\n%s", meta)
-		}
-		if !strings.Contains(meta, "identifier-field-ids=") ||
-			strings.Contains(meta, "identifier-field-ids=\n") {
-			t.Errorf("table carries no identifier fields, so OLake could not "+
-				"have upserted:\n%s", meta)
-		}
-
-		var sawOverwrite bool
-		for _, line := range nonEmptyLines(meta) {
-			if !strings.HasPrefix(line, "snapshot operation=") {
-				continue
-			}
-			if strings.Contains(line, "operation=Operation.OVERWRITE") &&
-				!strings.Contains(line, "added-equality-deletes=0") {
-				sawOverwrite = true
-			}
-		}
-		if !sawOverwrite {
-			t.Errorf("no snapshot recorded an overwrite carrying equality "+
-				"deletes:\n%s", meta)
-		}
-		if !strings.Contains(meta, "current-manifest-kinds=") ||
-			!strings.Contains(meta, "DELETES") {
-			t.Errorf("current snapshot has no delete manifest:\n%s", meta)
-		}
-		// Deliberately NOT asserted here: that a reader sees the updated value
-		// and no duplicate row. PyIceberg refuses to scan a table carrying
-		// equality deletes (apache/iceberg#6568), so a rows-mode read would
-		// fail rather than pass, and swapping in an engine that can apply them
-		// costs this test a multi-gigabyte image. Applying deletes on read is
-		// the engine's contract; recording the commit correctly is ours, and
-		// that is what the assertions above cover. See the README.
-	})
-
-	t.Run("CompliantWriterNeedsNoRepair", func(t *testing.T) {
-		names := env.listTableMetadata(t)
-		if len(names) == 0 {
-			t.Fatalf("no metadata files found for %s.%s", expectedNamespace, expectedTable)
-		}
-		for _, n := range names {
-			if strings.HasPrefix(n, "repaired-") {
-				t.Errorf("catalog repaired a manifest written by OLake (%s); "+
-					"the official Iceberg Java writer is expected to be "+
-					"spec-compliant, so this is a regression in the repair "+
-					"gate rather than in OLake", n)
-			}
-		}
-	})
-}
-
-func NewTestEnvironment(t *testing.T) *TestEnvironment {
-	t.Helper()
-
-	wd, err := os.Getwd()
-	if err != nil {
-		t.Fatalf("Failed to get working directory: %v", err)
-	}
-
-	seaweedDir := wd
-	for i := 0; i < 6; i++ {
-		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
-			break
-		}
-		seaweedDir = filepath.Dir(seaweedDir)
-	}
-
-	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
-	if info, err := os.Stat(weedBinary); err != nil || info.IsDir() {
-		weedBinary = filepath.Join(seaweedDir, "weed", "weed", "weed")
-		if info, err := os.Stat(weedBinary); err != nil || info.IsDir() {
-			weedBinary = "weed"
-			if _, err := exec.LookPath(weedBinary); err != nil {
-				t.Skip("weed binary not found, skipping integration test")
-			}
-		}
-	}
-
-	dataDir, err := os.MkdirTemp("", "seaweed-olake-test-*")
-	if err != nil {
-		t.Fatalf("Failed to create temp dir: %v", err)
-	}
-	configDir := filepath.Join(dataDir, "olake")
-	if err := os.MkdirAll(configDir, 0755); err != nil {
-		t.Fatalf("Failed to create config dir: %v", err)
-	}
-
-	// 9 for the mini cluster, 1 for the Postgres source mapped on the host.
-	ports := testutil.MustAllocatePorts(t, 10)
-
-	return &TestEnvironment{
-		seaweedDir:     seaweedDir,
-		weedBinary:     weedBinary,
-		dataDir:        dataDir,
-		configDir:      configDir,
-		bindIP:         testutil.FindBindIP(),
-		masterPort:     ports[0],
-		masterGrpcPort: ports[1],
-		volumePort:     ports[2],
-		volumeGrpcPort: ports[3],
-		filerPort:      ports[4],
-		filerGrpcPort:  ports[5],
-		s3Port:         ports[6],
-		s3GrpcPort:     ports[7],
-		icebergPort:    ports[8],
-		postgresPort:   ports[9],
-		accessKey:      "AKIAIOSFODNN7EXAMPLE",
-		secretKey:      "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
-	}
-}
-
-func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
-	t.Helper()
-
-	iamConfigPath, err := testutil.WriteIAMConfig(env.dataDir, env.accessKey, env.secretKey)
-	if err != nil {
-		t.Fatalf("Failed to create IAM config: %v", err)
-	}
-
-	cmd := exec.Command(env.weedBinary, "mini",
-		"-master.port", fmt.Sprintf("%d", env.masterPort),
-		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
-		"-volume.port", fmt.Sprintf("%d", env.volumePort),
-		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
-		"-filer.port", fmt.Sprintf("%d", env.filerPort),
-		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
-		"-s3.port", fmt.Sprintf("%d", env.s3Port),
-		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
-		"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergPort),
-		"-s3.config", iamConfigPath,
-		// Pre-create the table bucket the way an operator would, rather than
-		// reaching for the S3 Tables control plane from the test.
-		"-tableBucket", tableBucketName,
-		"-ip", env.bindIP,
-		"-ip.bind", "0.0.0.0",
-		"-dir", env.dataDir,
-	)
-	cmd.Dir = env.dataDir
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	cmd.Env = append(os.Environ(),
-		"AWS_ACCESS_KEY_ID="+env.accessKey,
-		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
-	)
-
-	if err := cmd.Start(); err != nil {
-		t.Fatalf("Failed to start SeaweedFS: %v", err)
-	}
-	env.weedProcess = cmd
-	env.weedCancel = func() {
-		if cmd.Process != nil {
-			_ = cmd.Process.Kill()
-		}
-	}
-
-	url := fmt.Sprintf("http://%s:%d/v1/config", env.bindIP, env.icebergPort)
-	if !waitForService(url, 45*time.Second) {
-		t.Fatalf("Iceberg REST API did not become ready at %s", url)
-	}
-}
-
-func (env *TestEnvironment) startPostgres(t *testing.T) {
-	t.Helper()
-
-	name := fmt.Sprintf("olake-pg-%d", env.postgresPort)
-	_ = exec.Command("docker", "rm", "-f", name).Run()
-
-	cmd := exec.Command("docker", "run", "-d", "--name", name,
-		"-e", "POSTGRES_USER="+sourceUser,
-		"-e", "POSTGRES_PASSWORD="+sourcePassword,
-		"-e", "POSTGRES_DB="+sourceDatabase,
-		"-p", fmt.Sprintf("%d:5432", env.postgresPort),
-		postgresImage(),
-	)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		t.Fatalf("Failed to start Postgres: %v\n%s", err, out)
-	}
-	env.postgresContainer = name
-
-	deadline := time.Now().Add(postgresStartTimeout)
-	for time.Now().Before(deadline) {
-		probe := exec.Command("docker", "exec", name,
-			"pg_isready", "-U", sourceUser, "-d", sourceDatabase)
-		if err := probe.Run(); err == nil {
-			return
-		}
-		time.Sleep(2 * time.Second)
-	}
-	t.Fatalf("Postgres did not become ready within %s", postgresStartTimeout)
-}
-
-func (env *TestEnvironment) seedSource(t *testing.T) {
-	t.Helper()
-	env.execSQL(t, `
-CREATE TABLE orders (
-    id       int PRIMARY KEY,
-    region   text,
-    amount   numeric(10,2),
-    order_ts timestamp
-);
-INSERT INTO orders VALUES
-    (1,'us-east',120.50,'2026-07-27 09:15'),
-    (2,'us-west', 87.20,'2026-07-27 09:20'),
-    (3,'eu-west',210.00,'2026-07-27 09:31');
-ALTER TABLE orders REPLICA IDENTITY FULL;`)
-}
-
-func (env *TestEnvironment) execSQL(t *testing.T, sqlText string) {
-	t.Helper()
-	cmd := exec.Command("docker", "exec", env.postgresContainer,
-		"psql", "-v", "ON_ERROR_STOP=1", "-U", sourceUser, "-d", sourceDatabase,
-		"-c", sqlText)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		t.Fatalf("psql failed: %v\n%s", err, out)
-	}
-}
-
-// writeOLakeConfigs writes the source and destination configs. The destination
-// is the point of this test: nothing in it is SeaweedFS-specific. catalog_type
-// is the generic "rest", auth is the standard OAuth2 client-credentials flow,
-// and path-style access is not even set here because OLake turns it on by
-// itself whenever s3_endpoint is non-empty.
-func (env *TestEnvironment) writeOLakeConfigs(t *testing.T) {
-	t.Helper()
-
-	source := map[string]any{
-		"host":            env.bindIP,
-		"port":            env.postgresPort,
-		"database":        sourceDatabase,
-		"username":        sourceUser,
-		"password":        sourcePassword,
-		"jdbc_url_params": map[string]any{},
-		"ssl":             map[string]any{"mode": "disable"},
-		"update_method":   map[string]any{"type": "Standalone"},
-		"max_threads":     2,
-		"retry_count":     0,
-	}
-
-	catalogURL := fmt.Sprintf("http://%s:%d", env.bindIP, env.icebergPort)
-	destination := map[string]any{
-		"type": "ICEBERG",
-		"writer": map[string]any{
-			"catalog_type":     "rest",
-			"rest_catalog_url": catalogURL,
-			"catalog_name":     "olake",
-			"iceberg_s3_path":  "s3://" + tableBucketName,
-			"s3_endpoint":      fmt.Sprintf("http://%s:%d", env.bindIP, env.s3Port),
-			"s3_use_ssl":       false,
-			"s3_path_style":    true,
-			"aws_region":       "us-east-1",
-			"aws_access_key":   env.accessKey,
-			"aws_secret_key":   env.secretKey,
-			"rest_auth_type":   "oauth2",
-			"oauth2_uri":       catalogURL + "/v1/oauth/tokens",
-			"credential":       env.accessKey + ":" + env.secretKey,
-		},
-	}
-
-	writeJSON(t, filepath.Join(env.configDir, "source.json"), source)
-	writeJSON(t, filepath.Join(env.configDir, "destination.json"), destination)
-	if err := os.WriteFile(filepath.Join(env.configDir, "state.json"), []byte("{}\n"), 0644); err != nil {
-		t.Fatalf("Failed to write state.json: %v", err)
-	}
-}
-
-func (env *TestEnvironment) runOLake(t *testing.T, args ...string) string {
-	t.Helper()
-
-	full := append([]string{
-		"run", "--rm",
-		"-v", dockerMount(env.configDir) + ":/mnt/config",
-		olakeImage(),
-	}, args...)
-
-	cmd := exec.Command("docker", full...)
-	done := make(chan struct{})
-	var out []byte
-	var err error
-	go func() {
-		out, err = cmd.CombinedOutput()
-		close(done)
-	}()
-	select {
-	case <-done:
-	case <-time.After(olakeRunTimeout):
-		if cmd.Process != nil {
-			_ = cmd.Process.Kill()
-		}
-		t.Fatalf("olake %s did not finish within %s", args[0], olakeRunTimeout)
-	}
-	if err != nil {
-		t.Fatalf("olake %s failed: %v\n%s", args[0], err, tailLines(string(out), 40))
-	}
-	return string(out)
-}
-
-func (env *TestEnvironment) readTable(t *testing.T, mode string) string {
-	t.Helper()
-
-	cmd := exec.Command("docker", "run", "--rm", readerImage, mode,
-		"--catalog-url", fmt.Sprintf("http://%s:%d", env.bindIP, env.icebergPort),
-		"--warehouse", "s3://"+tableBucketName,
-		"--prefix", tableBucketName,
-		"--s3-endpoint", fmt.Sprintf("http://%s:%d", env.bindIP, env.s3Port),
-		"--access-key", env.accessKey,
-		"--secret-key", env.secretKey,
-		"--namespace", expectedNamespace,
-		"--table", expectedTable,
-	)
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("reader (%s) failed: %v\n%s", mode, err, tailLines(string(out), 30))
-	}
-	return string(out)
-}
-
-// listTableMetadata lists the table's metadata directory through the filer so
-// the repair assertion looks at real objects rather than at what the catalog
-// reports about itself.
-func (env *TestEnvironment) listTableMetadata(t *testing.T) []string {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s:%d/buckets/%s/%s/%s/metadata/?limit=200",
-		env.bindIP, env.filerPort, tableBucketName, expectedNamespace, expectedTable)
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		t.Fatalf("Failed to build filer request: %v", err)
-	}
-	req.Header.Set("Accept", "application/json")
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		t.Fatalf("filer listing failed: %v", err)
-	}
-	defer resp.Body.Close()
-	body, _ := io.ReadAll(resp.Body)
-	if resp.StatusCode != http.StatusOK {
-		t.Fatalf("filer listing returned %d: %s", resp.StatusCode, body)
-	}
-
-	var parsed struct {
-		Entries []struct {
-			FullPath string `json:"FullPath"`
-		} `json:"Entries"`
-	}
-	if err := json.Unmarshal(body, &parsed); err != nil {
-		t.Fatalf("Failed to parse filer listing: %v\n%s", err, body)
-	}
-
-	names := make([]string, 0, len(parsed.Entries))
-	for _, e := range parsed.Entries {
-		names = append(names, e.FullPath[strings.LastIndex(e.FullPath, "/")+1:])
-	}
-	return names
-}
-
-func (env *TestEnvironment) Cleanup(t *testing.T) {
-	t.Helper()
-
-	if env.postgresContainer != "" {
-		_ = exec.Command("docker", "rm", "-f", env.postgresContainer).Run()
-	}
-	if env.weedCancel != nil {
-		env.weedCancel()
-	}
-	if env.weedProcess != nil {
-		_ = env.weedProcess.Wait()
-	}
-	if env.dataDir != "" {
-		_ = os.RemoveAll(env.dataDir)
-	}
-}
-
-func buildReaderImage(t *testing.T) {
-	t.Helper()
-
-	wd, err := os.Getwd()
-	if err != nil {
-		t.Fatalf("Failed to get working directory: %v", err)
-	}
-	cmd := exec.Command("docker", "build",
-		"-f", filepath.Join(wd, "Dockerfile.reader"),
-		"-t", readerImage, wd)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		t.Fatalf("Failed to build reader image: %v\n%s", err, tailLines(string(out), 30))
-	}
-}
-
-func olakeImage() string {
-	if v := os.Getenv("OLAKE_IMAGE"); v != "" {
-		return v
-	}
-	return olakeDefaultImage
-}
-
-func postgresImage() string {
-	if v := os.Getenv("POSTGRES_IMAGE"); v != "" {
-		return v
-	}
-	return postgresDefaultImage
-}
-
-func requireOLakeRuntime(t *testing.T) {
-	t.Helper()
-	if os.Getenv("SEAWEEDFS_SKIP_OLAKE_TESTS") != "" {
-		t.Skip("SEAWEEDFS_SKIP_OLAKE_TESTS set")
-	}
-	if _, err := exec.LookPath("docker"); err != nil {
-		t.Skip("docker not available, skipping OLake integration test")
-	}
-	if err := exec.Command("docker", "info").Run(); err != nil {
-		t.Skip("docker daemon not reachable, skipping OLake integration test")
-	}
-}
-
-func waitForService(url string, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	client := &http.Client{Timeout: 3 * time.Second}
-	for time.Now().Before(deadline) {
-		resp, err := client.Get(url)
-		if err == nil {
-			resp.Body.Close()
-			if resp.StatusCode < 500 {
-				return true
-			}
-		}
-		time.Sleep(time.Second)
-	}
-	return false
-}
-
-func writeJSON(t *testing.T, path string, v any) {
-	t.Helper()
-	body, err := json.MarshalIndent(v, "", "  ")
-	if err != nil {
-		t.Fatalf("Failed to marshal %s: %v", path, err)
-	}
-	if err := os.WriteFile(path, append(body, '\n'), 0644); err != nil {
-		t.Fatalf("Failed to write %s: %v", path, err)
-	}
-}
-
-// dockerMount normalises a host path for a -v bind mount. Docker Desktop
-// accepts forward slashes on Windows; the native separator it does not.
-func dockerMount(path string) string {
-	return strings.ReplaceAll(path, `\`, "/")
-}
-
-func nonEmptyLines(s string) []string {
-	var out []string
-	for _, line := range strings.Split(s, "\n") {
-		if trimmed := strings.TrimSpace(line); trimmed != "" {
-			out = append(out, trimmed)
-		}
-	}
-	return out
-}
-
-func tailLines(s string, n int) string {
-	lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
-	if len(lines) > n {
-		lines = lines[len(lines)-n:]
-	}
-	return strings.Join(lines, "\n")
-}

weed/cluster/lock_client.go

@@ -3,7 +3,6 @@ package cluster
 import (
 	"context"
 	"fmt"
-	"slices"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -28,7 +27,6 @@ type LockClient struct {
 	// correct: the filer forwards to the real primary as a fallback.
 	ringMu      sync.RWMutex
 	ring        *lock_manager.HashRing
-	ringServers []pb.ServerAddress
 	ringVersion int64
 
 	// priorRing is the ring before the most recent change, kept for priorWindow so a
@@ -37,24 +35,15 @@ type LockClient struct {
 	priorRing     *lock_manager.HashRing
 	ringChangedAt time.Time
 	priorWindow   time.Duration
-
-	// noLockServerRetryPeriod bounds retries when every filer reports "no
-	// lock server found": an empty lock ring is a systemic fault that waiting
-	// on a lock holder cannot resolve, unlike ordinary contention. The bound
-	// is long enough to ride out a master leader change (ring reset + fresh
-	// snapshot) but short enough that a write fails instead of hanging
-	// forever.
-	noLockServerRetryPeriod time.Duration
 }
 
 func NewLockClient(grpcDialOption grpc.DialOption, seedFiler pb.ServerAddress) *LockClient {
 	return &LockClient{
-		grpcDialOption:          grpcDialOption,
-		maxLockDuration:         5 * time.Second,
-		sleepDuration:           2473 * time.Millisecond,
-		seedFiler:               seedFiler,
-		priorWindow:             5 * time.Second,
-		noLockServerRetryPeriod: 15 * time.Second,
+		grpcDialOption:  grpcDialOption,
+		maxLockDuration: 5 * time.Second,
+		sleepDuration:   2473 * time.Millisecond,
+		seedFiler:       seedFiler,
+		priorWindow:     5 * time.Second,
 	}
 }
 
@@ -69,16 +58,10 @@ func (lc *LockClient) SetRing(servers []pb.ServerAddress, version int64) {
 		return
 	}
 	lc.ringVersion = version
-	sorted := slices.Clone(servers)
-	slices.Sort(sorted)
-	if slices.Equal(sorted, lc.ringServers) {
-		return
-	}
-	lc.ringServers = sorted
 	// Build a fresh ring (not an in-place mutation) so the outgoing ring survives as
 	// priorRing with its own servers for the cooling-off window.
 	newRing := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
-	newRing.SetServers(sorted)
+	newRing.SetServers(servers)
 	if lc.ring != nil {
 		lc.priorRing = lc.ring
 		lc.ringChangedAt = time.Now()
@@ -86,18 +69,6 @@ func (lc *LockClient) SetRing(servers []pb.ServerAddress, version int64) {
 	lc.ring = newRing
 }
 
-// ResetRing clears only the version gate so the first update from a
-// different master applies unconditionally: ring versions are per-master
-// monotonic and a high version accepted from a former leader must not
-// reject the new leader's snapshot. The last ring keeps routing during the
-// gap rather than falling back to the seed filer, and the arriving ring
-// becomes the prior ring for the cooling-off window.
-func (lc *LockClient) ResetRing() {
-	lc.ringMu.Lock()
-	defer lc.ringMu.Unlock()
-	lc.ringVersion = 0
-}
-
 // hostForKey returns the filer that should own key per the current ring view,
 // falling back to the seed filer when no view has been received yet.
 func (lc *LockClient) hostForKey(key string) pb.ServerAddress {
@@ -162,9 +133,7 @@ type LiveLock struct {
 	consecutiveFailures int // Track connection failures to trigger fallback
 }
 
-// NewShortLivedLock creates a lock with a 5-second duration.
-// It returns nil when the lock cannot be acquired because no lock server
-// exists; ordinary contention is still waited out.
+// NewShortLivedLock creates a lock with a 5-second duration
 func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLock) {
 	lock = &LiveLock{
 		key:            key,
@@ -175,10 +144,7 @@ func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLoc
 		self:           owner,
 		lc:             lc,
 	}
-	if err := lock.retryUntilLocked(5 * time.Second); err != nil {
-		glog.Warningf("create lock %s: %v", key, err)
-		return nil
-	}
+	lock.retryUntilLocked(5 * time.Second)
 	return
 }
 
@@ -201,10 +167,7 @@ func (lc *LockClient) NewBlockingLongLivedLock(key, owner string, lockTTL time.D
 		lockTTL:        lockTTL,
 	}
 	// Block until acquired
-	if err := lock.retryUntilLocked(lockTTL); err != nil {
-		glog.Warningf("create lock %s: %v", key, err)
-		return nil
-	}
+	lock.retryUntilLocked(lockTTL)
 	// Start renewal goroutine using a ticker for interruptible sleep
 	lock.renewalDone = make(chan struct{})
 	go func() {
@@ -303,31 +266,12 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
 // several seconds): when a holder on another mount releases the lock, the
 // waiter must pick it up promptly, otherwise cross-mount write handoff stalls
 // long enough to time out clients.
-func (lock *LiveLock) retryUntilLocked(lockDuration time.Duration) error {
-	var unavailableSince time.Time
+func (lock *LiveLock) retryUntilLocked(lockDuration time.Duration) {
 	for lock.renewToken == "" {
-		err := lock.AttemptToLock(lockDuration)
-		if err == nil {
-			unavailableSince = time.Time{}
-			continue
-		}
-		glog.V(1).Infof("create lock %s: %v", lock.key, err)
-		if strings.Contains(err.Error(), "lock already owned") {
-			// Ordinary contention: a reachable server holds the lock, so
-			// waiting is the point and has no bound.
-			unavailableSince = time.Time{}
-			continue
-		}
-		// Anything else — "no lock server found", a dead ring member refusing
-		// connections — is a systemic fault waiting cannot fix; give up once
-		// it persists past the retry period.
-		if unavailableSince.IsZero() {
-			unavailableSince = time.Now()
-		} else if time.Since(unavailableSince) > lock.lc.noLockServerRetryPeriod {
-			return err
+		if err := lock.AttemptToLock(lockDuration); err != nil {
+			glog.V(1).Infof("create lock %s: %v", lock.key, err)
 		}
 	}
-	return nil
 }
 
 func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {

weed/cluster/lock_client_test.go

@@ -1,18 +1,13 @@
 package cluster
 
 import (
-	"context"
 	"fmt"
-	"net"
 	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
 )
 
 // The gateway must resolve a lock key to the same primary the filers do,
@@ -157,135 +152,6 @@ func TestLockClientPriorOwnerForKeyExpires(t *testing.T) {
 	}
 }
 
-// A master change clears the version gate so the new leader's (lower-versioned)
-// snapshot applies — versions are only comparable within one master's stream.
-func TestLockClientResetRing(t *testing.T) {
-	lc := NewLockClient(nil, "seed:8888")
-
-	lc.SetRing([]pb.ServerAddress{"filer-a:8888", "filer-b:8888"}, 100)
-	lc.ResetRing()
-
-	// The last ring keeps routing during the gap; only version acceptance
-	// is reset so the new leader's (lower-versioned) snapshot applies.
-	if got := lc.hostForKey("k"); got == "seed:8888" {
-		t.Fatal("expected the previous ring to keep routing after reset")
-	}
-	lc.SetRing([]pb.ServerAddress{"filer-z:8888"}, 50)
-	if got := lc.hostForKey("k"); got != "filer-z:8888" {
-		t.Fatalf("lower version from new master not applied, got %q", got)
-	}
-}
-
-type noLockServerFiler struct {
-	filer_pb.UnimplementedSeaweedFilerServer
-}
-
-func (s *noLockServerFiler) DistributedLock(ctx context.Context, req *filer_pb.LockRequest) (*filer_pb.LockResponse, error) {
-	return &filer_pb.LockResponse{Error: lock_manager.NoLockServerError.Error()}, nil
-}
-
-// When every filer reports an empty lock ring, lock acquisition must fail
-// after a bounded period instead of hanging the write forever.
-func TestNewShortLivedLockFailsFastOnNoLockServer(t *testing.T) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		t.Fatalf("listen: %v", err)
-	}
-	grpcServer := grpc.NewServer()
-	filer_pb.RegisterSeaweedFilerServer(grpcServer, &noLockServerFiler{})
-	go grpcServer.Serve(listener)
-	defer grpcServer.Stop()
-
-	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
-	host, port, err := net.SplitHostPort(listener.Addr().String())
-	if err != nil {
-		t.Fatalf("split host port: %v", err)
-	}
-	// "host:httpPort.grpcPort" dials the fake filer's port directly.
-	lc := NewLockClient(dialOption, pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
-	lc.noLockServerRetryPeriod = 200 * time.Millisecond
-
-	start := time.Now()
-	lock := lc.NewShortLivedLock("test-key", "test-owner")
-	elapsed := time.Since(start)
-
-	if lock != nil {
-		t.Fatal("expected nil lock when no lock server exists")
-	}
-	if elapsed > 10*time.Second {
-		t.Fatalf("lock acquisition took %v, expected fail-fast", elapsed)
-	}
-}
-
-type contendedLockFiler struct {
-	filer_pb.UnimplementedSeaweedFilerServer
-}
-
-func (s *contendedLockFiler) DistributedLock(ctx context.Context, req *filer_pb.LockRequest) (*filer_pb.LockResponse, error) {
-	return &filer_pb.LockResponse{Error: "lock already owned by someone"}, nil
-}
-
-// A lock held by another owner is ordinary contention: acquisition waits it
-// out rather than failing on the unavailability bound.
-func TestNewShortLivedLockWaitsOutContention(t *testing.T) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		t.Fatalf("listen: %v", err)
-	}
-	grpcServer := grpc.NewServer()
-	filer_pb.RegisterSeaweedFilerServer(grpcServer, &contendedLockFiler{})
-	go grpcServer.Serve(listener)
-	defer grpcServer.Stop()
-
-	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
-	host, port, err := net.SplitHostPort(listener.Addr().String())
-	if err != nil {
-		t.Fatalf("split host port: %v", err)
-	}
-	lc := NewLockClient(dialOption, pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
-	lc.noLockServerRetryPeriod = 200 * time.Millisecond
-
-	done := make(chan *LiveLock, 1)
-	go func() {
-		done <- lc.NewShortLivedLock("test-key", "test-owner")
-	}()
-	select {
-	case <-done:
-		t.Fatal("ordinary lock contention must not hit the unavailability bound")
-	case <-time.After(3 * lc.noLockServerRetryPeriod):
-	}
-}
-
-// A ring member that refuses connections is unavailability, not contention:
-// acquisition fails on the same bound as "no lock server found".
-func TestNewShortLivedLockFailsFastOnUnreachableFiler(t *testing.T) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		t.Fatalf("listen: %v", err)
-	}
-	addr := listener.Addr().String()
-	listener.Close()
-
-	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
-	host, port, err := net.SplitHostPort(addr)
-	if err != nil {
-		t.Fatalf("split host port: %v", err)
-	}
-	lc := NewLockClient(dialOption, pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
-	lc.noLockServerRetryPeriod = 200 * time.Millisecond
-
-	start := time.Now()
-	lock := lc.NewShortLivedLock("test-key", "test-owner")
-	elapsed := time.Since(start)
-
-	if lock != nil {
-		t.Fatal("expected nil lock when the ring member is unreachable")
-	}
-	if elapsed > 10*time.Second {
-		t.Fatalf("lock acquisition took %v, expected fail-fast", elapsed)
-	}
-}
-
 // LiveLock.generation is a fencing token written and read with 64-bit atomic
 // operations. On 32-bit platforms (GOARCH=386 and GOARCH=arm) a 64-bit atomic
 // op requires an 8-byte-aligned address, which Go only guarantees for the

weed/cluster/lock_manager/lock_ring.go

@@ -1,7 +1,6 @@
 package lock_manager
 
 import (
-	"slices"
 	"sort"
 	"sync"
 	"time"
@@ -55,14 +54,6 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress, version int64) bool {
 		r.Unlock()
 		return false
 	}
-	// An unchanged member list is only a version refresh: installing it as a
-	// new snapshot would run the topology-change callback and restart the
-	// prior-owner window on every periodic rebroadcast.
-	if len(r.snapshots) > 0 && slices.Equal(servers, r.snapshots[0].servers) {
-		r.version = version
-		r.Unlock()
-		return true
-	}
 	r.version = version
 	// Update the ring while holding the lock so version and ring state
 	// are always consistent — prevents a concurrent SetSnapshot from
@@ -83,19 +74,6 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress, version int64) bool {
 	return true
 }
 
-// Reset clears only the version gate so the first update from a different
-// master always applies: ring versions are per-master monotonic, and a high
-// version accepted from a former leader must not reject the new leader's
-// view. The ring itself stays installed — writes keep routing to the last
-// known owner during the gap instead of every filer treating itself as the
-// owner, and the arriving snapshot transitions off it with the usual
-// prior-owner window.
-func (r *LockRing) Reset() {
-	r.Lock()
-	defer r.Unlock()
-	r.version = 0
-}
-
 // Version returns the current ring version.
 func (r *LockRing) Version() int64 {
 	r.RLock()

weed/cluster/lock_manager/lock_ring_test.go

@@ -96,42 +96,3 @@ func TestLockRing_VersionRejectsStale(t *testing.T) {
 	assert.True(t, ok)
 	assert.Equal(t, 1, len(r.GetSnapshot()))
 }
-
-func TestLockRing_SetSnapshotUnchangedOnlyBumpsVersion(t *testing.T) {
-	r := NewLockRing(100 * time.Millisecond)
-	callbacks := 0
-	r.SetTakeSnapshotCallback(func(snapshot []pb.ServerAddress) { callbacks++ })
-
-	assert.True(t, r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2"}, 100))
-	assert.Equal(t, 1, callbacks)
-
-	// A periodic rebroadcast with the same members refreshes the version
-	// without a new snapshot or another topology-change callback.
-	assert.True(t, r.SetSnapshot([]pb.ServerAddress{"b:2", "a:1"}, 200))
-	assert.Equal(t, int64(200), r.Version())
-	assert.Equal(t, 1, r.GetSnapshotCount())
-	assert.Equal(t, 1, callbacks, "unchanged ring must not fire the topology callback")
-
-	assert.True(t, r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2", "c:3"}, 300))
-	assert.Equal(t, 2, callbacks)
-}
-
-func TestLockRing_Reset(t *testing.T) {
-	r := NewLockRing(100 * time.Millisecond)
-
-	// A high version accepted from a former leader must not reject the new
-	// leader's view once the client has moved masters.
-	ok := r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2"}, 100)
-	assert.True(t, ok)
-
-	r.Reset()
-	assert.Equal(t, int64(0), r.Version())
-	// The operational ring survives the reset: writes keep routing to the
-	// last known owner until the new leader's snapshot arrives.
-	assert.Equal(t, 2, len(r.GetSnapshot()))
-	assert.NotEqual(t, "", string(r.GetPrimary("key")))
-
-	ok = r.SetSnapshot([]pb.ServerAddress{"c:1"}, 50)
-	assert.True(t, ok, "lower version from a different master must apply after reset")
-	assert.Equal(t, 1, len(r.GetSnapshot()))
-}

weed/cluster/lock_ring_manager.go

@@ -11,38 +11,29 @@ import (
 
 const LockRingStabilizationInterval = 1 * time.Second
 
-// LockRingRebroadcastInterval is how often the ring is re-sent even when
-// membership has not changed. Broadcasts are otherwise purely event-driven,
-// so a single lost or poisoned update would be permanent without this.
-const LockRingRebroadcastInterval = 30 * time.Second
-
 // LockRingManager tracks filer membership for the distributed lock ring.
 // It batches rapid topology changes (e.g., node drop + join) with a
 // stabilization timer, then broadcasts the complete member list atomically
 // so filers receive a single consistent ring update instead of multiple
 // intermediate states.
 type LockRingManager struct {
-	mu                  sync.Mutex
-	members             map[FilerGroupName]map[pb.ServerAddress]struct{}
-	version             map[FilerGroupName]int64
-	lastBroadcast       map[FilerGroupName]*master_pb.LockRingUpdate
-	pendingTimer        map[FilerGroupName]*time.Timer
-	rebroadcastTimer    map[FilerGroupName]*time.Timer
-	broadcastFn         func(resp *master_pb.KeepConnectedResponse)
-	stabilizeDelay      time.Duration
-	rebroadcastInterval time.Duration
+	mu             sync.Mutex
+	members        map[FilerGroupName]map[pb.ServerAddress]struct{}
+	version        map[FilerGroupName]int64
+	lastBroadcast  map[FilerGroupName]*master_pb.LockRingUpdate
+	pendingTimer   map[FilerGroupName]*time.Timer
+	broadcastFn    func(resp *master_pb.KeepConnectedResponse)
+	stabilizeDelay time.Duration
 }
 
 func NewLockRingManager(broadcastFn func(resp *master_pb.KeepConnectedResponse)) *LockRingManager {
 	return &LockRingManager{
-		members:             make(map[FilerGroupName]map[pb.ServerAddress]struct{}),
-		version:             make(map[FilerGroupName]int64),
-		lastBroadcast:       make(map[FilerGroupName]*master_pb.LockRingUpdate),
-		pendingTimer:        make(map[FilerGroupName]*time.Timer),
-		rebroadcastTimer:    make(map[FilerGroupName]*time.Timer),
-		broadcastFn:         broadcastFn,
-		stabilizeDelay:      LockRingStabilizationInterval,
-		rebroadcastInterval: LockRingRebroadcastInterval,
+		members:        make(map[FilerGroupName]map[pb.ServerAddress]struct{}),
+		version:        make(map[FilerGroupName]int64),
+		lastBroadcast:  make(map[FilerGroupName]*master_pb.LockRingUpdate),
+		pendingTimer:   make(map[FilerGroupName]*time.Timer),
+		broadcastFn:    broadcastFn,
+		stabilizeDelay: LockRingStabilizationInterval,
 	}
 }
 
@@ -125,64 +116,16 @@ func (lrm *LockRingManager) scheduleBroadcast(filerGroup FilerGroupName) {
 
 func (lrm *LockRingManager) doBroadcast(filerGroup FilerGroupName) {
 	lrm.mu.Lock()
-	delete(lrm.pendingTimer, filerGroup)
-	lrm.mu.Unlock()
-	lrm.emit(filerGroup)
-}
-
-// rebroadcast re-sends the current ring unless a membership broadcast is
-// still stabilizing — emitting mid-window would publish an intermediate
-// topology that the pending timer immediately replaces. The check and the
-// update must sit in one critical section or a membership change can slip
-// a pending timer in between.
-func (lrm *LockRingManager) rebroadcast(filerGroup FilerGroupName) {
-	lrm.mu.Lock()
-	var update *master_pb.LockRingUpdate
-	if _, pending := lrm.pendingTimer[filerGroup]; !pending {
-		update = lrm.nextBroadcastUpdate(filerGroup)
-	}
-	lrm.mu.Unlock()
-	lrm.sendUpdate(filerGroup, update)
-}
-
-func (lrm *LockRingManager) emit(filerGroup FilerGroupName) {
-	lrm.mu.Lock()
-	update := lrm.nextBroadcastUpdate(filerGroup)
-	lrm.mu.Unlock()
-	lrm.sendUpdate(filerGroup, update)
-}
-
-func (lrm *LockRingManager) sendUpdate(filerGroup FilerGroupName, update *master_pb.LockRingUpdate) {
-	if update == nil {
-		return
-	}
-	glog.V(0).Infof("LockRing: broadcasting ring update for group %q version %d: %v", filerGroup, update.Version, update.Servers)
-	if lrm.broadcastFn != nil {
-		lrm.broadcastFn(&master_pb.KeepConnectedResponse{
-			LockRingUpdate: update,
-		})
-	}
-}
-
-// nextBroadcastUpdate stamps the current members into an update and re-arms
-// the periodic rebroadcast. It returns nil for an empty member list: an empty
-// lock ring is never usable, so a late "last member removed" event from a
-// former leader must not propagate and wedge every lock client. The last
-// non-empty broadcast stays in lastBroadcast for reconnecting clients.
-// Caller must hold lrm.mu.
-func (lrm *LockRingManager) nextBroadcastUpdate(filerGroup FilerGroupName) *master_pb.LockRingUpdate {
-	members := lrm.members[filerGroup]
-	if len(members) == 0 {
-		return nil
-	}
 	// Use wall-clock nanoseconds so the version survives master restarts
 	// without persistence — a restarted master produces a version greater
 	// than any pre-restart value (assuming clocks don't jump backward).
 	version := time.Now().UnixNano()
 	lrm.version[filerGroup] = version
-	servers := make([]string, 0, len(members))
-	for addr := range members {
-		servers = append(servers, string(addr))
+	servers := make([]string, 0)
+	if members, ok := lrm.members[filerGroup]; ok {
+		for addr := range members {
+			servers = append(servers, string(addr))
+		}
 	}
 	update := &master_pb.LockRingUpdate{
 		FilerGroup: string(filerGroup),
@@ -190,13 +133,16 @@ func (lrm *LockRingManager) nextBroadcastUpdate(filerGroup FilerGroupName) *mast
 		Version:    version,
 	}
 	lrm.lastBroadcast[filerGroup] = update
-	if timer, ok := lrm.rebroadcastTimer[filerGroup]; ok {
-		timer.Stop()
+	delete(lrm.pendingTimer, filerGroup)
+	lrm.mu.Unlock()
+
+	glog.V(0).Infof("LockRing: broadcasting ring update for group %q version %d: %v", filerGroup, version, servers)
+
+	if lrm.broadcastFn != nil {
+		lrm.broadcastFn(&master_pb.KeepConnectedResponse{
+			LockRingUpdate: update,
+		})
 	}
-	lrm.rebroadcastTimer[filerGroup] = time.AfterFunc(lrm.rebroadcastInterval, func() {
-		lrm.rebroadcast(filerGroup)
-	})
-	return update
 }
 
 // FlushPending fires any pending timer immediately (for testing or shutdown).

weed/cluster/lock_ring_manager_test.go

@@ -213,111 +213,6 @@ func TestLockRingManager_NoBroadcastWithoutFn(t *testing.T) {
 	time.Sleep(50 * time.Millisecond) // should not panic
 }
 
-func TestLockRingManager_EmptyRingNotBroadcast(t *testing.T) {
-	var mu sync.Mutex
-	var broadcasts []*master_pb.LockRingUpdate
-
-	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
-		mu.Lock()
-		if resp.LockRingUpdate != nil {
-			broadcasts = append(broadcasts, resp.LockRingUpdate)
-		}
-		mu.Unlock()
-	})
-	lrm.stabilizeDelay = 50 * time.Millisecond
-
-	group := FilerGroupName("default")
-
-	lrm.AddServer(group, "filer1:8888")
-	lrm.FlushPending(group)
-
-	mu.Lock()
-	broadcasts = nil
-	mu.Unlock()
-
-	// Removing the last member must not propagate an empty ring.
-	lrm.RemoveServer(group, "filer1:8888")
-	time.Sleep(100 * time.Millisecond)
-
-	mu.Lock()
-	assert.Equal(t, 0, len(broadcasts), "empty ring must not be broadcast")
-	mu.Unlock()
-
-	// The last non-empty snapshot is still served to reconnecting clients.
-	update := lrm.GetLastUpdate(group)
-	require.NotNil(t, update)
-	assert.Equal(t, []string{"filer1:8888"}, update.Servers)
-}
-
-func TestLockRingManager_PeriodicRebroadcast(t *testing.T) {
-	var mu sync.Mutex
-	var broadcasts []*master_pb.LockRingUpdate
-
-	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
-		mu.Lock()
-		if resp.LockRingUpdate != nil {
-			broadcasts = append(broadcasts, resp.LockRingUpdate)
-		}
-		mu.Unlock()
-	})
-	lrm.stabilizeDelay = 20 * time.Millisecond
-	lrm.rebroadcastInterval = 60 * time.Millisecond
-
-	group := FilerGroupName("default")
-	lrm.AddServer(group, "filer1:8888")
-
-	// Without any further membership change, the ring keeps being re-sent so
-	// a lost or poisoned update cannot be permanent.
-	time.Sleep(200 * time.Millisecond)
-
-	mu.Lock()
-	require.GreaterOrEqual(t, len(broadcasts), 2, "ring should rebroadcast periodically")
-	for i := 1; i < len(broadcasts); i++ {
-		assert.Greater(t, broadcasts[i].Version, broadcasts[i-1].Version)
-	}
-	mu.Unlock()
-}
-
-func TestLockRingManager_RebroadcastDefersToPendingStabilization(t *testing.T) {
-	var mu sync.Mutex
-	var broadcasts []*master_pb.LockRingUpdate
-
-	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
-		mu.Lock()
-		if resp.LockRingUpdate != nil {
-			broadcasts = append(broadcasts, resp.LockRingUpdate)
-		}
-		mu.Unlock()
-	})
-	lrm.stabilizeDelay = 100 * time.Millisecond
-	lrm.rebroadcastInterval = 30 * time.Millisecond
-
-	group := FilerGroupName("default")
-	lrm.AddServer(group, "filer1:8888")
-	lrm.FlushPending(group)
-
-	mu.Lock()
-	require.Len(t, broadcasts, 1)
-	mu.Unlock()
-
-	// A membership change just before the periodic tick: the rebroadcast must
-	// not publish the unsettled ring ahead of the stabilization timer.
-	lrm.RemoveServer(group, "filer1:8888")
-	lrm.AddServer(group, "filer2:8888")
-	time.Sleep(2 * lrm.rebroadcastInterval)
-
-	mu.Lock()
-	assert.Len(t, broadcasts, 1, "rebroadcast during stabilization should be deferred")
-	mu.Unlock()
-
-	time.Sleep(2 * lrm.stabilizeDelay)
-
-	mu.Lock()
-	require.GreaterOrEqual(t, len(broadcasts), 2)
-	assert.Equal(t, []string{"filer2:8888"}, broadcasts[1].Servers)
-	mu.Unlock()
-}
-
 func TestLockRingManager_GetLastUpdateReturnsBroadcastState(t *testing.T) {
 	lrm := NewLockRingManager(nil)
 

weed/command/backup.go

@@ -156,7 +156,7 @@ func backupFromLocation(volumeServer pb.ServerAddress, grpcDialOption grpc.DialO
 
 	// If local volume is larger than remote, recreate it
 	if datSize > stats.TailOffset {
-		if err := v.Destroy(false, false, false); err != nil {
+		if err := v.Destroy(false, false); err != nil {
 			v.Close()
 			return fmt.Errorf("destroying volume: %w", err), false
 		}

weed/command/filer.go

@@ -89,7 +89,6 @@ type FilerOptions struct {
 	s3ConfigFile              *string // optional path to static S3 identity config
 
 	allowUntrustedRemoteEndpoints *bool
-	remoteCacheEvictThreshold     *float64
 	// shutdownCtx, when non-nil, tells startFiler to gracefully shut down its
 	// HTTP/gRPC servers once the ctx is cancelled. Used by integration tests
 	// and by weed mini; nil for standalone weed filer.
@@ -135,7 +134,6 @@ func init() {
 	f.tusMaxSizeMB = cmdFiler.Flag.Int("tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
 	f.tusSessionExpiry = cmdFiler.Flag.Duration("tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"")
 	f.allowUntrustedRemoteEndpoints = cmdFiler.Flag.Bool("allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
-	f.remoteCacheEvictThreshold = cmdFiler.Flag.Float64("remoteCacheEvictThreshold", 0.9, "evict remote-cached objects (oldest first) when any volume disk exceeds this usage fraction; 0 disables")
 
 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -409,7 +407,6 @@ func (fo *FilerOptions) startFiler() {
 		CredentialManager:         credentialManager,
 
 		AllowUntrustedRemoteEndpoints: *fo.allowUntrustedRemoteEndpoints,
-		RemoteCacheEvictThreshold:     *fo.remoteCacheEvictThreshold,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)

weed/command/master.go

@@ -46,19 +46,17 @@ const (
 )
 
 type MasterOptions struct {
-	port                          *int
-	portGrpc                      *int
-	ip                            *string
-	ipBind                        *string
-	metaFolder                    *string
-	peers                         *string
-	mastersDeprecated             *string // deprecated, for backward compatibility in master.follower
-	volumeSizeLimitMB             *uint
-	fileSizeLimitMB               *int
-	volumePreallocate             *bool
-	maxParallelVacuumPerServer    *int
-	vacuumIntervalSeconds         *int
-	vacuumDeleteEmptyAfterSeconds *int
+	port                       *int
+	portGrpc                   *int
+	ip                         *string
+	ipBind                     *string
+	metaFolder                 *string
+	peers                      *string
+	mastersDeprecated          *string // deprecated, for backward compatibility in master.follower
+	volumeSizeLimitMB          *uint
+	fileSizeLimitMB            *int
+	volumePreallocate          *bool
+	maxParallelVacuumPerServer *int
 	// pulseSeconds       *int
 	defaultReplication *string
 	garbageThreshold   *float64
@@ -95,8 +93,6 @@ func init() {
 	m.fileSizeLimitMB = cmdMaster.Flag.Int("fileSizeLimitMB", 256, "limit the file size accepted by /submit, should match the volume servers' -fileSizeLimitMB (-volume.fileSizeLimitMB under weed server or weed mini, which set this for you)")
 	m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
 	m.maxParallelVacuumPerServer = cmdMaster.Flag.Int("maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel per volume server")
-	m.vacuumIntervalSeconds = cmdMaster.Flag.Int("vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps")
-	m.vacuumDeleteEmptyAfterSeconds = cmdMaster.Flag.Int("vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables")
 	// m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
 	m.defaultReplication = cmdMaster.Flag.String("defaultReplication", "", "Default replication type if not specified.")
 	m.garbageThreshold = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
@@ -471,14 +467,12 @@ func peerIndex(self pb.ServerAddress, peers []pb.ServerAddress) int {
 func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOption {
 	masterAddress := pb.NewServerAddress(*m.ip, *m.port, *m.portGrpc)
 	return &weed_server.MasterOption{
-		Master:                        masterAddress,
-		MetaFolder:                    *m.metaFolder,
-		VolumeSizeLimitMB:             uint32(*m.volumeSizeLimitMB),
-		FileSizeLimitMB:               *m.fileSizeLimitMB,
-		VolumePreallocate:             *m.volumePreallocate,
-		MaxParallelVacuumPerServer:    *m.maxParallelVacuumPerServer,
-		VacuumIntervalSeconds:         *m.vacuumIntervalSeconds,
-		VacuumDeleteEmptyAfterSeconds: *m.vacuumDeleteEmptyAfterSeconds,
+		Master:                     masterAddress,
+		MetaFolder:                 *m.metaFolder,
+		VolumeSizeLimitMB:          uint32(*m.volumeSizeLimitMB),
+		FileSizeLimitMB:            *m.fileSizeLimitMB,
+		VolumePreallocate:          *m.volumePreallocate,
+		MaxParallelVacuumPerServer: *m.maxParallelVacuumPerServer,
 		// PulseSeconds:            *m.pulseSeconds,
 		DefaultReplicaPlacement: *m.defaultReplication,
 		GarbageThreshold:        *m.garbageThreshold,

weed/command/master_follower.go

@@ -47,8 +47,6 @@ func init() {
 	mf.metricsIntervalSec = aws.Int(0)
 	mf.raftResumeState = aws.Bool(false)
 	mf.maxParallelVacuumPerServer = aws.Int(1)
-	mf.vacuumIntervalSeconds = aws.Int(840)
-	mf.vacuumDeleteEmptyAfterSeconds = aws.Int(0)
 	mf.telemetryUrl = aws.String("https://telemetry.seaweedfs.com/api/collect")
 	mf.telemetryEnabled = aws.Bool(false)
 }

weed/command/mini.go

@@ -427,8 +427,6 @@ func initMiniMasterFlags() {
 	miniMasterOptions.volumeSizeLimitMB = cmdMini.Flag.Uint("master.volumeSizeLimitMB", defaultMiniVolumeSizeMB, "Master stops directing writes to oversized volumes (default: 128MB for mini)")
 	miniMasterOptions.volumePreallocate = cmdMini.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
 	miniMasterOptions.maxParallelVacuumPerServer = cmdMini.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server")
-	miniMasterOptions.vacuumIntervalSeconds = cmdMini.Flag.Int("master.vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps")
-	miniMasterOptions.vacuumDeleteEmptyAfterSeconds = cmdMini.Flag.Int("master.vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables")
 	miniMasterOptions.defaultReplication = cmdMini.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
 	miniMasterOptions.garbageThreshold = cmdMini.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	miniMasterOptions.metricsAddress = cmdMini.Flag.String("master.metrics.address", "", "Prometheus gateway address")
@@ -467,7 +465,6 @@ func initMiniFilerFlags() {
 	miniFilerOptions.tusMaxSizeMB = cmdMini.Flag.Int("filer.tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
 	miniFilerOptions.tusSessionExpiry = cmdMini.Flag.Duration("filer.tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration")
 	miniFilerOptions.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("filer.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
-	miniFilerOptions.remoteCacheEvictThreshold = cmdMini.Flag.Float64("filer.remoteCacheEvictThreshold", 0.9, "evict remote-cached objects (oldest first) when any volume disk exceeds this usage fraction; 0 disables")
 }
 
 // initMiniVolumeFlags initializes Volume server flag options

weed/command/s3.go

@@ -618,7 +618,6 @@ func (s3opt *S3Options) startLanceServer(s3ApiServer *s3api.S3ApiServer) {
 	lanceRouter.Use(util_http.EscapeSemicolonsInQuery)
 
 	lanceServer := lance.NewServer(s3ApiServer, s3ApiServer)
-	lanceServer.SetCredentialValidator(s3ApiServer)
 	if s3opt.icebergCredentialRole != nil && *s3opt.icebergCredentialRole != "" {
 		lanceServer.SetCredentialVendor(lanceCredentialVendor{s3ApiServer})
 	}

weed/command/scaffold/master.toml

@@ -41,7 +41,6 @@ copy_2 = 6                # create 2 x 6 = 12 actual volumes
 copy_3 = 3                # create 3 x 3 = 9 actual volumes
 copy_other = 1            # create n x 1 = n actual volumes
 threshold = 0.9           # create threshold
-reservation_timeout = "5m"# capacity reservation timeout before unreleased reservations expire
 disable = false           # disables volume growth if true
 
 # configuration flags for replication

weed/command/server.go

@@ -98,8 +98,6 @@ func init() {
 	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", util.DefaultVolumeSizeLimitMB, "Master stops directing writes to oversized volumes.")
 	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
 	masterOptions.maxParallelVacuumPerServer = cmdServer.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server")
-	masterOptions.vacuumIntervalSeconds = cmdServer.Flag.Int("master.vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps")
-	masterOptions.vacuumDeleteEmptyAfterSeconds = cmdServer.Flag.Int("master.vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables")
 	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
 	masterOptions.garbageThreshold = cmdServer.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterOptions.metricsAddress = cmdServer.Flag.String("master.metrics.address", "", "Prometheus gateway address")
@@ -136,7 +134,6 @@ func init() {
 	filerOptions.tusMaxSizeMB = cmdServer.Flag.Int("filer.tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
 	filerOptions.tusSessionExpiry = cmdServer.Flag.Duration("filer.tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"")
 	filerOptions.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("filer.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
-	filerOptions.remoteCacheEvictThreshold = cmdServer.Flag.Float64("filer.remoteCacheEvictThreshold", 0.9, "evict remote-cached objects (oldest first) when any volume disk exceeds this usage fraction; 0 disables")
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")

weed/ec/ec_encode.go

@@ -43,7 +43,7 @@ func markVolumeReplicaWritable(ctx context.Context, grpcDialOption grpc.DialOpti
 // deleteVolume removes the volume from sourceVolumeServer via the canonical
 // volume_move helper.
 func deleteVolume(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer pb.ServerAddress, onlyEmpty bool, keepRemoteData bool) (err error) {
-	return volume_move.NewMover(grpcDialOption).DeleteVolume(ctx, volumeId, sourceVolumeServer, onlyEmpty, false, keepRemoteData)
+	return volume_move.NewMover(grpcDialOption).DeleteVolume(ctx, volumeId, sourceVolumeServer, onlyEmpty, keepRemoteData)
 }
 
 func ChunkVolumeIds(volumeIds []needle.VolumeId, batchSize int) [][]needle.VolumeId {

weed/filer/filer.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"sort"
 	"strings"
-	"sync/atomic"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
@@ -23,7 +22,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
 
 	"google.golang.org/grpc"
-	"google.golang.org/protobuf/proto"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -75,11 +73,6 @@ type Filer struct {
 	EmptyFolderCleanupDelay       time.Duration
 	persistedLogCache             *persistedLogCache
 	metaLogInflight               metaLogInflight
-	remoteTombstones              *remoteDeletionTombstones
-	// remoteTombstonesDone, when non-nil, is closed once the startup tombstone
-	// rebuild finishes; lazy remote reads wait on it so a pending delete
-	// cannot resurrect in the gap.
-	remoteTombstonesDone atomic.Pointer[chan struct{}]
 }
 
 func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer {
@@ -95,7 +88,6 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH
 		deletionQuit:        make(chan struct{}),
 		DeletionRetryQueue:  NewDeletionRetryQueue(),
 		persistedLogCache:   newPersistedLogCache(persistedLogCacheMaxBytes),
-		remoteTombstones:    newRemoteDeletionTombstones(),
 	}
 	if f.UniqueFilerId < 0 {
 		f.UniqueFilerId = -f.UniqueFilerId
@@ -182,10 +174,6 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
 		glog.V(0).Infof("LockRing: applying master ring update v%d: %v", update.Version, servers)
 		f.Dlm.LockRing.SetSnapshot(servers, update.Version)
 	})
-	f.MasterClient.SetOnMasterChangeFn(func(previous, current pb.ServerAddress) {
-		glog.V(0).Infof("LockRing: master changed %s -> %s, resetting ring", previous, current)
-		f.Dlm.LockRing.Reset()
-	})
 
 	// Subscribe to the local filer first: its events reach the aggregated
 	// buffer only through this subscription, and the peer watermarks must
@@ -277,13 +265,7 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
 
 	oldEntry := existing
 	if oldEntry == nil {
-		var findErr error
-		oldEntry, findErr = f.FindEntry(ctx, entry.FullPath)
-		if o_excl && findErr != nil && !errors.Is(findErr, filer_pb.ErrNotFound) {
-			// An exclusive create cannot decide whether the path exists when
-			// the lookup itself failed; proceeding would upsert over it.
-			return fmt.Errorf("find entry %s: %w", entry.FullPath, findErr)
-		}
+		oldEntry, _ = f.FindEntry(ctx, entry.FullPath)
 	}
 
 	/*
@@ -328,7 +310,7 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
 			return fmt.Errorf("%s: %w", entry.FullPath, filer_pb.ErrEntryAlreadyExists)
 		}
 		glog.V(4).InfofCtx(ctx, "UpdateEntry %s: old entry: %v", entry.FullPath, oldEntry.Name())
-		if err := f.UpdateEntry(ctx, oldEntry, entry, isFromOtherCluster); err != nil {
+		if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
 			if errors.Is(err, filer_pb.ErrExistingIsDirectory) || errors.Is(err, filer_pb.ErrExistingIsFile) {
 				glog.V(2).InfofCtx(ctx, "update entry %s: %v", entry.FullPath, err)
 			} else {
@@ -483,7 +465,7 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
 		narrowed := existing.ShallowClone()
 		narrowed.Mode = existing.Mode&^restorableModeBits | kept
 		glog.V(1).InfofCtx(ctx, "restore directory %s: narrowing %v to %v", dirPath, existing.Mode, narrowed.Mode)
-		if err := f.UpdateEntry(ctx, existing, narrowed, false); err != nil {
+		if err := f.UpdateEntry(ctx, existing, narrowed); err != nil {
 			return err
 		}
 		f.NotifyUpdateEvent(ctx, existing, narrowed, false, false, nil)
@@ -516,7 +498,7 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
 	return nil
 }
 
-func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromOtherCluster bool) (err error) {
+func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
 	if oldEntry != nil {
 		entry.Attr.Crtime = oldEntry.Attr.Crtime
 		if oldEntry.Attr.Inode != 0 {
@@ -536,17 +518,6 @@ func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromO
 			glog.V(2).InfofCtx(ctx, "existing %s is a file", oldEntry.FullPath)
 			return fmt.Errorf("%s: %w", oldEntry.FullPath, filer_pb.ErrExistingIsFile)
 		}
-		// A local write to a remote-backed entry leaves the copy on remote
-		// stale until a sync re-uploads it. Content changes that arrive
-		// without a fresh sync stamp are unsynced; clear the stamp so nothing
-		// mistakes the still-local chunks for a re-fetchable cache copy.
-		// Replicated updates carry the writer's authoritative stamp.
-		if !isFromOtherCluster && oldEntry.Remote != nil && entry.Remote != nil &&
-			oldEntry.Remote.LastLocalSyncTsNs == entry.Remote.LastLocalSyncTsNs &&
-			!chunksEqual(oldEntry.Chunks, entry.Chunks) {
-			entry.Remote = proto.Clone(entry.Remote).(*filer_pb.RemoteEntry)
-			entry.Remote.LastLocalSyncTsNs = 0
-		}
 	}
 	if entry.Attr.Atime.IsZero() {
 		entry.Attr.Atime = entryInitialAtime(entry.Attr)
@@ -618,7 +589,7 @@ func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, sta
 	lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) (bool, error) {
 		select {
 		case <-ctx.Done():
-			glog.V(1).InfofCtx(ctx, "listing %q canceled: %v", p, ctx.Err())
+			glog.Errorf("Context is done.")
 			return false, fmt.Errorf("context canceled: %w", ctx.Err())
 		default:
 			if entry.TtlSec > 0 && !entry.IsDirectory() {

weed/filer/filer_delete_collection_test.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
@@ -97,7 +96,7 @@ func newFilerWithFakeMaster(t *testing.T) (*Filer, *hookedStore, *collectionDele
 
 	mc := wdclient.NewMasterClient(
 		grpc.WithTransportCredentials(insecure.NewCredentials()),
-		"", cluster.FilerType, pb.ServerAddress("localhost:0"), "", "",
+		"test", cluster.FilerType, pb.ServerAddress("localhost:0"), "", "",
 		*pb.NewServiceDiscoveryFromMap(map[string]pb.ServerAddress{"m": masterAddress}),
 	)
 
@@ -182,206 +181,3 @@ func TestDeleteEntryMetaAndDataDeletesCollectionWhenTheRequestIsCancelledMidDele
 		t.Error("the bucket entry survived the delete")
 	}
 }
-
-func seedBucket(t *testing.T, store *hookedStore, path util.FullPath) {
-	t.Helper()
-	if err := store.InsertEntry(context.Background(), &Entry{
-		FullPath: path,
-		Attr:     Attr{Mode: os.ModeDir | 0755},
-	}); err != nil {
-		t.Fatalf("seed bucket %s: %v", path, err)
-	}
-}
-
-// Two buckets resolving to one collection: deleting either must leave the
-// collection for the other. Previously the delete dropped the collection
-// named after the bucket regardless of where its data actually lived.
-func TestDeleteBucketKeepsSharedCollection(t *testing.T) {
-	f, store, master := newFilerWithFakeMaster(t)
-	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
-		LocationPrefix: "/buckets",
-		Collection:     "shared",
-	})
-	seedBucket(t, store, util.FullPath("/buckets/a"))
-	seedBucket(t, store, util.FullPath("/buckets/b"))
-
-	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/a", true, false, true, false, nil, 0); err != nil {
-		t.Fatalf("DeleteEntryMetaAndData: %v", err)
-	}
-
-	select {
-	case call := <-master.calls:
-		t.Fatalf("shared collection was deleted: %q", call.name)
-	default:
-	}
-	if store.getEntry("/buckets/b") == nil {
-		t.Error("the surviving bucket's entry is gone")
-	}
-}
-
-// A bucket named after a collection other buckets resolve to is still just a
-// bucket: deleting it must not take the shared collection down with it.
-func TestDeleteBucketNamedAfterSharedCollection(t *testing.T) {
-	f, store, master := newFilerWithFakeMaster(t)
-	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
-		LocationPrefix: "/buckets",
-		Collection:     "shared",
-	})
-	seedBucket(t, store, util.FullPath("/buckets/shared"))
-	seedBucket(t, store, util.FullPath("/buckets/other"))
-
-	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/shared", true, false, true, false, nil, 0); err != nil {
-		t.Fatalf("DeleteEntryMetaAndData: %v", err)
-	}
-
-	select {
-	case call := <-master.calls:
-		t.Fatalf("collection backing other buckets was deleted: %q", call.name)
-	default:
-	}
-}
-
-// A rule pointing a non-bucket path at the same collection keeps it: the
-// collection serves files the bucket delete must not orphan.
-func TestDeleteBucketKeepsCollectionUsedByNonBucketPath(t *testing.T) {
-	f, store, master := newFilerWithFakeMaster(t)
-	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
-		LocationPrefix: "/buckets/a",
-		Collection:     "cold",
-	})
-	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
-		LocationPrefix: "/archives",
-		Collection:     "cold",
-	})
-	seedBucket(t, store, util.FullPath("/buckets/a"))
-
-	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/a", true, false, true, false, nil, 0); err != nil {
-		t.Fatalf("DeleteEntryMetaAndData: %v", err)
-	}
-
-	select {
-	case call := <-master.calls:
-		t.Fatalf("collection used by /archives was deleted: %q", call.name)
-	default:
-	}
-}
-
-// A broad prefix rule covering the whole tree keeps the collection even for a
-// lone bucket: the same collection backs non-bucket paths too.
-func TestDeleteBucketKeepsCollectionFromBroadRule(t *testing.T) {
-	f, store, master := newFilerWithFakeMaster(t)
-	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
-		LocationPrefix: "/",
-		Collection:     "everything",
-	})
-	seedBucket(t, store, util.FullPath("/buckets/a"))
-
-	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/a", true, false, true, false, nil, 0); err != nil {
-		t.Fatalf("DeleteEntryMetaAndData: %v", err)
-	}
-
-	select {
-	case call := <-master.calls:
-		t.Fatalf("collection from a / rule was deleted: %q", call.name)
-	default:
-	}
-}
-
-// A bucket resolving to the filer's default collection keeps it: rule-less
-// writes outside buckets land there too, so it is never one bucket's alone.
-func TestDeleteBucketKeepsDefaultCollection(t *testing.T) {
-	f, store, master := newFilerWithFakeMaster(t)
-	f.metaLogCollection = "everything"
-	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
-		LocationPrefix: "/buckets/a",
-		Collection:     "everything",
-	})
-	seedBucket(t, store, util.FullPath("/buckets/a"))
-
-	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/a", true, false, true, false, nil, 0); err != nil {
-		t.Fatalf("DeleteEntryMetaAndData: %v", err)
-	}
-
-	select {
-	case call := <-master.calls:
-		t.Fatalf("the filer's default collection was deleted: %q", call.name)
-	default:
-	}
-}
-
-// A rule nested under a surviving bucket keeps the collection: the other
-// bucket resolves elsewhere at its root, but objects deeper inside it still
-// land in the shared collection.
-func TestDeleteBucketKeepsCollectionFromNestedSiblingRule(t *testing.T) {
-	f, store, master := newFilerWithFakeMaster(t)
-	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
-		LocationPrefix: "/buckets/a",
-		Collection:     "shared",
-	})
-	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
-		LocationPrefix: "/buckets/b/deep",
-		Collection:     "shared",
-	})
-	seedBucket(t, store, util.FullPath("/buckets/a"))
-	seedBucket(t, store, util.FullPath("/buckets/b"))
-
-	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/a", true, false, true, false, nil, 0); err != nil {
-		t.Fatalf("DeleteEntryMetaAndData: %v", err)
-	}
-
-	select {
-	case call := <-master.calls:
-		t.Fatalf("collection used under /buckets/b/deep was deleted: %q", call.name)
-	default:
-	}
-}
-
-// A grouped gateway writes to <group>_<bucket> regardless of the storage
-// rules, so that is the collection the delete must drop -- and a rule-named
-// collection the bucket never used must survive.
-func TestDeleteBucketUnderFilerGroup(t *testing.T) {
-	f, store, master := newFilerWithFakeMaster(t)
-	f.MasterClient.FilerGroup = "tenant1"
-	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
-		LocationPrefix: "/buckets/photos",
-		Collection:     "archive",
-	})
-	seedBucket(t, store, util.FullPath("/buckets/photos"))
-
-	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/photos", true, false, true, false, nil, 0); err != nil {
-		t.Fatalf("DeleteEntryMetaAndData: %v", err)
-	}
-
-	select {
-	case call := <-master.calls:
-		if call.name != "tenant1_photos" {
-			t.Fatalf("CollectionDelete = %q, want %q", call.name, "tenant1_photos")
-		}
-	case <-time.After(20 * time.Second):
-		t.Fatal("CollectionDelete never reached the master")
-	}
-}
-
-// A collection only the deleted bucket resolves to is dropped under its real
-// name, so a dedicated custom collection does not leak its volumes.
-func TestDeleteBucketDeletesResolvedCollection(t *testing.T) {
-	f, store, master := newFilerWithFakeMaster(t)
-	f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
-		LocationPrefix: "/buckets/only",
-		Collection:     "custom",
-	})
-	seedBucket(t, store, util.FullPath("/buckets/only"))
-
-	if err := f.DeleteEntryMetaAndData(context.Background(), "/buckets/only", true, false, true, false, nil, 0); err != nil {
-		t.Fatalf("DeleteEntryMetaAndData: %v", err)
-	}
-
-	select {
-	case call := <-master.calls:
-		if call.name != "custom" {
-			t.Fatalf("CollectionDelete = %q, want %q", call.name, "custom")
-		}
-	case <-time.After(20 * time.Second):
-		t.Fatal("CollectionDelete never reached the master")
-	}
-}

weed/filer/filer_delete_entry.go

@@ -68,10 +68,6 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 		return nil
 	}
 	isDeleteCollection := f.IsBucket(entry)
-	collectionName := ""
-	if isDeleteCollection {
-		collectionName = f.bucketCollection(ctx, entry.Name())
-	}
 	if entry.IsDirectory() {
 		// delete the folder children, not including the folder itself
 		err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures, func(hardLinkIds []HardLinkId) error {
@@ -106,18 +102,17 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 	}
 
 	if isDeleteCollection {
-		if collectionName != "" {
-			// the entry is already gone: a caller that hung up must not leave the
-			// collection behind, so this cleanup outlives the request -- bounded all
-			// the same, or a master that is down parks this handler indefinitely and
-			// every client retry behind it parks another
-			collectionCtx, cancelCollection := context.WithTimeout(context.WithoutCancel(ctx), collectionDeleteTimeout)
-			f.DoDeleteCollection(collectionCtx, collectionName)
-			cancelCollection()
-		}
+		collectionName := entry.Name()
+		// the entry is already gone: a caller that hung up must not leave the
+		// collection behind, so this cleanup outlives the request -- bounded all
+		// the same, or a master that is down parks this handler indefinitely and
+		// every client retry behind it parks another
+		collectionCtx, cancelCollection := context.WithTimeout(context.WithoutCancel(ctx), collectionDeleteTimeout)
+		f.DoDeleteCollection(collectionCtx, collectionName)
+		cancelCollection()
 		// drop bucket-labeled series held by this process; the S3 gateway
 		// only cleans its own registry
-		stats.DeleteBucketMetrics(entry.Name())
+		stats.DeleteBucketMetrics(collectionName)
 	}
 
 	return nil
@@ -125,15 +120,6 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 
 func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32, onHardLinkIdsFn OnHardLinkIdsFunc) (err error) {
 
-	var dirTombstoneTs int64
-	if isRecursive {
-		// Tombstone the directory before its children: when the store drops
-		// the subtree without listing it, or a child error aborts the walk,
-		// the ancestor tombstone still covers every descendant.
-		dirTombstoneTs = time.Now().UnixNano()
-		f.noteRemoteDeletion(entry.FullPath, true, dirTombstoneTs)
-	}
-
 	//collect all the chunks of this layer and delete them together at the end
 	var chunksToDelete []*filer_pb.FileChunk
 	lastFileName := ""
@@ -143,9 +129,6 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 		for {
 			entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "", "")
 			if err != nil {
-				// nothing was deleted; a leftover tombstone would hide the
-				// still-existing remote children
-				f.unnoteRemoteDeletion(entry.FullPath, dirTombstoneTs)
 				glog.ErrorfCtx(ctx, "list folder %s: %v", entry.FullPath, err)
 				return fmt.Errorf("list folder %s: %v", entry.FullPath, err)
 			}
@@ -157,7 +140,6 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 
 			for _, sub := range entries {
 				lastFileName = sub.Name()
-				f.noteRemoteDeletion(sub.FullPath, sub.IsDirectory(), time.Now().UnixNano())
 				if sub.IsDirectory() {
 					subIsDeletingBucket := f.IsBucket(sub)
 					err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, isFromOtherCluster, nil, onHardLinkIdsFn)
@@ -220,8 +202,6 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
 		}
 	}
 
-	f.noteRemoteDeletion(entry.FullPath, entry.IsDirectory(), time.Now().UnixNano())
-
 	if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil {
 		return fmt.Errorf("filer store delete: %w", storeDeletionErr)
 	}
@@ -241,75 +221,6 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
 // DeleteCollection, still has retry budget left.
 const collectionDeleteTimeout = 15 * time.Second
 
-// bucketCollection resolves the collection a bucket's objects land in
-// through the same chain the write path uses -- a grouped gateway's explicit
-// collection, then the storage rules, then the bucket name -- and reports it
-// only when nothing outside the bucket can still route into it. A shared
-// collection must survive the bucket delete: dropping it removes volumes
-// other paths still write to. A listing failure keeps the collection, the
-// safe side of an unknown.
-func (f *Filer) bucketCollection(ctx context.Context, bucket string) (collection string) {
-	bucketDir := f.DirBucketsPath + "/" + bucket + "/"
-	resolve := func(dir, name string) string {
-		if f.MasterClient != nil {
-			if group := f.MasterClient.FilerGroup; group != "" {
-				return group + "_" + name
-			}
-		}
-		return util.Nvl(f.FilerConf.MatchStorageRule(dir).Collection, name)
-	}
-	collection = resolve(bucketDir, bucket)
-
-	// Rule-less writes outside buckets fall back to the filer's default
-	// collection, so a bucket resolving there shares it with them.
-	if collection == f.metaLogCollection {
-		return ""
-	}
-
-	// A rule whose prefix escapes the bucket can route other paths into the
-	// same collection, including prefixes nested under surviving buckets.
-	for _, rule := range f.FilerConf.ToProto().Locations {
-		prefix := strings.TrimSuffix(rule.LocationPrefix, "/") + "/"
-		if strings.HasPrefix(prefix, bucketDir) {
-			continue
-		}
-		if f.FilerConf.MatchStorageRule(prefix).Collection == collection {
-			return ""
-		}
-	}
-
-	siblings, err := f.listBuckets(ctx)
-	if err != nil {
-		glog.ErrorfCtx(ctx, "list buckets for collection check: %v", err)
-		return ""
-	}
-	for _, sibling := range siblings {
-		if sibling != bucket && resolve(f.DirBucketsPath+"/"+sibling+"/", sibling) == collection {
-			return ""
-		}
-	}
-	return collection
-}
-
-func (f *Filer) listBuckets(ctx context.Context) (buckets []string, err error) {
-	lastFileName := ""
-	for {
-		entries, _, listErr := f.ListDirectoryEntries(ctx, util.FullPath(f.DirBucketsPath), lastFileName, false, PaginationSize, "", "", "")
-		if listErr != nil {
-			return nil, listErr
-		}
-		for _, entry := range entries {
-			lastFileName = entry.Name()
-			if f.IsBucket(entry) {
-				buckets = append(buckets, entry.Name())
-			}
-		}
-		if len(entries) < PaginationSize {
-			return buckets, nil
-		}
-	}
-}
-
 func (f *Filer) DoDeleteCollection(ctx context.Context, collectionName string) (err error) {
 
 	return f.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {

weed/filer/filer_deletion.go

@@ -306,29 +306,24 @@ func (f *Filer) loopProcessingDeletion() {
 			glog.V(0).Infof("deletion processor shutting down")
 			return
 		case <-ticker.C:
-			f.FlushFileIdDeletionQueue(context.Background(), lookupFunc)
+			f.FileIdDeletionQueue.Consume(func(fileIds []string) {
+				for i := 0; i < len(fileIds); i += DeletionBatchSize {
+					end := i + DeletionBatchSize
+					if end > len(fileIds) {
+						end = len(fileIds)
+					}
+					toDeleteFileIds := fileIds[i:end]
+					f.processDeletionBatch(toDeleteFileIds, lookupFunc)
+				}
+			})
 		}
 	}
 }
 
-func (f *Filer) FlushFileIdDeletionQueue(ctx context.Context, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) (consumed []string) {
-	f.FileIdDeletionQueue.Consume(func(fileIds []string) {
-		consumed = fileIds
-		for i := 0; i < len(fileIds); i += DeletionBatchSize {
-			end := i + DeletionBatchSize
-			if end > len(fileIds) {
-				end = len(fileIds)
-			}
-			f.processDeletionBatch(ctx, fileIds[i:end], lookupFunc)
-		}
-	})
-	return consumed
-}
-
 // processDeletionBatch handles deletion of a batch of file IDs and processes results.
 // It classifies errors into retryable and permanent categories, adds retryable failures
 // to the retry queue, and logs appropriate messages.
-func (f *Filer) processDeletionBatch(ctx context.Context, toDeleteFileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) {
+func (f *Filer) processDeletionBatch(toDeleteFileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) {
 	// Deduplicate file IDs to prevent incorrect retry count increments for the same file ID within a single batch.
 	uniqueFileIdsSlice := make([]string, 0, len(toDeleteFileIds))
 	processed := make(map[string]struct{}, len(toDeleteFileIds))
@@ -344,7 +339,7 @@ func (f *Filer) processDeletionBatch(ctx context.Context, toDeleteFileIds []stri
 	}
 
 	// Delete files and classify outcomes
-	outcomes := deleteFilesAndClassify(ctx, f.GrpcDialOption, uniqueFileIdsSlice, lookupFunc)
+	outcomes := deleteFilesAndClassify(f.GrpcDialOption, uniqueFileIdsSlice, lookupFunc)
 
 	// Process outcomes
 	var successCount, notFoundCount, retryableErrorCount, permanentErrorCount int
@@ -410,9 +405,9 @@ type deletionOutcome struct {
 }
 
 // deleteFilesAndClassify performs deletion and classifies outcomes for a list of file IDs
-func deleteFilesAndClassify(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) map[string]deletionOutcome {
+func deleteFilesAndClassify(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) map[string]deletionOutcome {
 	// Perform deletion
-	results := operation.DeleteFileIdsWithLookupVolumeId(ctx, grpcDialOption, fileIds, lookupFunc)
+	results := operation.DeleteFileIdsWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
 
 	// Group results by file ID to handle multiple results for replicated volumes
 	resultsByFileId := make(map[string][]*volume_server_pb.DeleteResult)
@@ -550,7 +545,7 @@ func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc fu
 	}
 
 	// Delete files and classify outcomes
-	outcomes := deleteFilesAndClassify(context.Background(), f.GrpcDialOption, fileIds, lookupFunc)
+	outcomes := deleteFilesAndClassify(f.GrpcDialOption, fileIds, lookupFunc)
 
 	// Process outcomes - iterate over readyItems to ensure all items are accounted for
 	var successCount, notFoundCount, retryCount, permanentErrorCount int

weed/filer/filer_inode_test.go

@@ -2,13 +2,11 @@ package filer
 
 import (
 	"context"
-	"errors"
 	"os"
 	"testing"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -103,57 +101,6 @@ func TestCreateEntryAssignsInodesToAutoCreatedParents(t *testing.T) {
 	}
 }
 
-func TestCreateEntryOExclPreservesExistingEntry(t *testing.T) {
-	f, store := newTestFilerWithStubStore()
-
-	original := &Entry{
-		FullPath: util.FullPath("/buckets/my-bucket"),
-		Attr:     Attr{Mode: os.ModeDir | 0o777},
-		Extended: map[string][]byte{
-			"lifecycle": []byte(`<LifecycleConfiguration/>`),
-			"owner":     []byte("alice"),
-		},
-	}
-	require.NoError(t, store.InsertEntry(context.Background(), original))
-
-	replacement := &Entry{
-		FullPath: util.FullPath("/buckets/my-bucket"),
-		Attr:     Attr{Mode: os.ModeDir | 0o777},
-	}
-	err := f.CreateEntry(context.Background(), replacement, original, true, false, nil, false, f.MaxFilenameLength)
-	require.ErrorIs(t, err, filer_pb.ErrEntryAlreadyExists)
-
-	stored, findErr := store.FindEntry(context.Background(), original.FullPath)
-	require.NoError(t, findErr)
-	assert.Equal(t, original.Extended, stored.Extended)
-}
-
-func TestCreateEntryOExclFailsOnLookupError(t *testing.T) {
-	f, store := newTestFilerWithStubStore()
-
-	original := &Entry{
-		FullPath: util.FullPath("/buckets/my-bucket"),
-		Attr:     Attr{Mode: os.ModeDir | 0o777},
-		Extended: map[string][]byte{"owner": []byte("alice")},
-	}
-	require.NoError(t, store.InsertEntry(context.Background(), original))
-
-	// a failed lookup must not masquerade as "not found": without the check
-	// the insert path would upsert over the stored bucket entry
-	store.findErr = errors.New("transient store failure")
-	err := f.CreateEntry(context.Background(), &Entry{
-		FullPath: util.FullPath("/buckets/my-bucket"),
-		Attr:     Attr{Mode: os.ModeDir | 0o777},
-	}, nil, true, false, nil, false, f.MaxFilenameLength)
-	require.Error(t, err)
-	assert.NotErrorIs(t, err, filer_pb.ErrEntryAlreadyExists)
-
-	store.findErr = nil
-	stored, findErr := store.FindEntry(context.Background(), original.FullPath)
-	require.NoError(t, findErr)
-	assert.Equal(t, original.Extended, stored.Extended)
-}
-
 func TestUpdateEntryPreservesExistingInode(t *testing.T) {
 	f, store := newTestFilerWithStubStore()
 
@@ -173,7 +120,7 @@ func TestUpdateEntryPreservesExistingInode(t *testing.T) {
 		},
 	}
 
-	err := f.UpdateEntry(context.Background(), original, updated, false)
+	err := f.UpdateEntry(context.Background(), original, updated)
 	require.Error(t, err)
 
 	updated = &Entry{
@@ -182,7 +129,7 @@ func TestUpdateEntryPreservesExistingInode(t *testing.T) {
 			Mode: 0o600,
 		},
 	}
-	err = f.UpdateEntry(context.Background(), original, updated, false)
+	err = f.UpdateEntry(context.Background(), original, updated)
 	require.NoError(t, err)
 
 	stored, findErr := store.FindEntry(context.Background(), original.FullPath)
@@ -208,7 +155,7 @@ func TestUpdateEntryBackfillsMissingLegacyInode(t *testing.T) {
 			Mode: 0o640,
 		},
 	}
-	err := f.UpdateEntry(context.Background(), original, updated, false)
+	err := f.UpdateEntry(context.Background(), original, updated)
 	require.NoError(t, err)
 
 	stored, findErr := store.FindEntry(context.Background(), original.FullPath)

weed/filer/filer_lazy_remote.go

@@ -7,10 +7,7 @@ import (
 	"strings"
 	"time"
 
-	"google.golang.org/protobuf/proto"
-
 	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
 	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -46,21 +43,6 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (
 		return nil, nil
 	}
 
-	// A startup tombstone rebuild may still be replaying the meta log; wait
-	// for it so a pending delete cannot resurrect here.
-	if done := f.remoteTombstonesDone.Load(); done != nil {
-		select {
-		case <-*done:
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		}
-	}
-
-	if f.isRemoteDeletionPending(ctx, p, mountDir) {
-		glog.V(2).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s deleted locally, remote delete pending", p)
-		return nil, nil
-	}
-
 	remoteConf, found := f.RemoteStorage.FindRemoteStorageConf(p)
 	if !found {
 		return nil, nil
@@ -121,27 +103,10 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (
 		persistBaseCtx, cancelPersist := context.WithTimeout(context.Background(), 30*time.Second)
 		defer cancelPersist()
 		persistCtx := context.WithValue(persistBaseCtx, lazyFetchContextKey{}, true)
-		// A delete may have landed while StatFile was in flight; re-check so
-		// the fetched object cannot resurrect a path whose delete is pending.
-		if f.isRemoteDeletionPending(persistCtx, p, mountDir) {
-			glog.V(2).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s deleted during remote stat", p)
-			return lazyFetchResult{nil}, nil
-		}
 		saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, true, f.MaxFilenameLength)
 		if saveErr != nil {
 			glog.Warningf("maybeLazyFetchFromRemote: failed to persist filer entry for %s: %v", p, saveErr)
 			f.lazyFetchGroup.Forget(key)
-			return lazyFetchResult{entry}, nil
-		}
-
-		// A delete records its tombstone before removing the entry, so a
-		// tombstone visible now means the insert raced a delete that already
-		// ran: retract the persisted entry so the path stays deleted.
-		if f.isRemoteDeletionPending(persistCtx, p, mountDir) {
-			glog.V(2).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s deleted while persisting", p)
-			f.lazyFetchGroup.Forget(key)
-			f.retractLazyRemoteEntry(persistCtx, entry)
-			return lazyFetchResult{nil}, nil
 		}
 
 		return lazyFetchResult{entry}, nil
@@ -157,25 +122,6 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (
 	return result.entry, nil
 }
 
-// retractLazyRemoteEntry deletes the entry at entry.FullPath only when it is
-// still the entry a lazy remote read just materialized — a concurrent write
-// may have replaced it, and deleting by path alone would take that write down.
-func (f *Filer) retractLazyRemoteEntry(ctx context.Context, entry *Entry) {
-	existing, findErr := f.FindEntry(ctx, entry.FullPath)
-	if findErr != nil || existing == nil {
-		return
-	}
-	// The stored entry must still be exactly what this read materialized —
-	// an intervening write (appended chunks, touched attributes) means a
-	// real update owns the path now.
-	if !proto.Equal(existing.ToProtoEntry(), entry.ToProtoEntry()) {
-		return
-	}
-	if err := f.doDeleteEntryMetaAndData(ctx, existing, false, false, nil); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
-		glog.Warningf("retractLazyRemoteEntry %s: %v", entry.FullPath, err)
-	}
-}
-
 func (f *Filer) maybeDeleteFromRemote(ctx context.Context, entry *Entry) (bool, error) {
 	if entry == nil || f.RemoteStorage == nil {
 		return false, nil

weed/filer/filer_lazy_remote_listing.go

@@ -56,14 +56,6 @@ func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) {
 		}
 	}
 
-	if done := f.remoteTombstonesDone.Load(); done != nil {
-		select {
-		case <-*done:
-		case <-ctx.Done():
-			return
-		}
-	}
-
 	// Lazy listing is opt-in: disabled when TTL is 0
 	if remoteLoc.ListingCacheTtlSeconds <= 0 {
 		return
@@ -117,10 +109,6 @@ func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) {
 				return nil
 			}
 
-			if existingEntry == nil && f.isRemoteDeletionPending(persistCtx, childPath, mountDir) {
-				return nil
-			}
-
 			if existingEntry != nil {
 				// Merge: update remote metadata while preserving local state
 				// (Chunks, Extended, Uid/Gid/Mode, etc.)
@@ -173,9 +161,6 @@ func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) {
 				}
 				if saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, true, f.MaxFilenameLength); saveErr != nil {
 					glog.Warningf("maybeLazyListFromRemote: persist %s: %v", childPath, saveErr)
-				} else if f.isRemoteDeletionPending(persistCtx, childPath, mountDir) {
-					// a delete landed between the check above and the insert
-					f.retractLazyRemoteEntry(persistCtx, entry)
 				}
 			}
 			return nil

weed/filer/filer_lazy_remote_test.go

@@ -34,7 +34,6 @@ type stubFilerStore struct {
 	entries         map[string]*Entry
 	kv              map[string][]byte
 	insertErr       error
-	findErr         error
 	deleteErrByPath map[string]error
 }
 
@@ -175,9 +174,6 @@ func (s *stubFilerStore) UpdateEntry(_ context.Context, entry *Entry) error {
 func (s *stubFilerStore) FindEntry(_ context.Context, p util.FullPath) (*Entry, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	if s.findErr != nil {
-		return nil, s.findErr
-	}
 	if e, ok := s.entries[string(p)]; ok {
 		return e, nil
 	}
@@ -283,7 +279,6 @@ func newTestFiler(t *testing.T, store *stubFilerStore, rs *FilerRemoteStorage) *
 		MasterClient:        mc,
 		FileIdDeletionQueue: util.NewUnboundedQueue(),
 		deletionQuit:        make(chan struct{}),
-		remoteTombstones:    newRemoteDeletionTombstones(),
 		LocalMetaLogBuffer: log_buffer.NewLogBuffer("test", time.Minute,
 			func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, func() {}),
 	}

weed/filer/filer_on_meta_event.go

@@ -15,7 +15,6 @@ func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse)
 	f.maybeReloadRemoteStorageConfigurationAndMapping(event)
 	f.onBucketEvents(event)
 	f.onEmptyFolderCleanupEvents(event)
-	f.onRemoteDeletionEvents(event)
 }
 
 func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {

weed/filer/filer_prefix_object_test.go

@@ -104,23 +104,3 @@ func TestExpiredFileIsDeletedOnRead(t *testing.T) {
 	require.ErrorIs(t, err, filer_pb.ErrNotFound,
 		"native TTL lookup should remove the expired metadata row")
 }
-
-func TestListDirectoryEntriesContextCanceled(t *testing.T) {
-	f, store := newTestFilerWithStubStore()
-	ctx := context.Background()
-
-	require.NoError(t, store.InsertEntry(ctx, &Entry{
-		FullPath: util.FullPath("/dir/file1.txt"),
-		Attr:     Attr{Mode: 0o644},
-	}))
-
-	canceledCtx, cancel := context.WithCancel(ctx)
-	cancel()
-
-	_, _, err := f.doListDirectoryEntries(canceledCtx, util.FullPath("/dir"), "", false, 10, "", func(entry *Entry) (bool, error) {
-		return true, nil
-	})
-	require.Error(t, err)
-	assert.ErrorContains(t, err, "context canceled")
-}
-

weed/filer/filer_remote_evict.go

@@ -1,89 +0,0 @@
-package filer
-
-import (
-	"context"
-	"sort"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-)
-
-func chunksEqual(a, b []*filer_pb.FileChunk) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for i := range a {
-		if a[i].GetFileIdString() != b[i].GetFileIdString() ||
-			a[i].Size != b[i].Size ||
-			a[i].Offset != b[i].Offset ||
-			a[i].ModifiedTsNs != b[i].ModifiedTsNs {
-			return false
-		}
-	}
-	return true
-}
-
-// IsEvictableRemoteEntry reports whether an entry's local chunks are backed by
-// a synchronized remote copy, mirroring the checks remote.uncache applies:
-// remote-backed, chunks present, and not ahead of the remote version.
-func IsEvictableRemoteEntry(entry *Entry) bool {
-	if entry.IsDirectory() || entry.Remote == nil {
-		return false
-	}
-	if entry.Remote.LastLocalSyncTsNs <= 0 || len(entry.GetChunks()) == 0 {
-		return false
-	}
-	if entry.Remote.LastLocalSyncTsNs < entry.Mtime.UnixNano() {
-		return false
-	}
-	return true
-}
-
-// ListEvictableRemoteEntries walks every mounted directory and returns
-// synchronized remote entries holding local chunks, oldest cached first.
-func (f *Filer) ListEvictableRemoteEntries(ctx context.Context, mounts []util.FullPath, minCacheAge time.Duration) (out []*Entry) {
-	cutoffNs := time.Now().UnixNano() - minCacheAge.Nanoseconds()
-	for _, dir := range mounts {
-		if err := f.collectEvictableRemoteEntries(ctx, dir, cutoffNs, &out); err != nil {
-			glog.WarningfCtx(ctx, "list evictable remote entries under %s: %v", dir, err)
-		}
-	}
-	sort.Slice(out, func(i, j int) bool {
-		return out[i].Remote.LastLocalSyncTsNs < out[j].Remote.LastLocalSyncTsNs
-	})
-	return out
-}
-
-func (f *Filer) collectEvictableRemoteEntries(ctx context.Context, dir util.FullPath, cutoffNs int64, out *[]*Entry) error {
-	startFileName := ""
-	for {
-		var subDirs []util.FullPath
-		lastFileName, err := f.Store.ListDirectoryEntries(ctx, dir, startFileName, false, 1024, func(entry *Entry) (bool, error) {
-			if err := ctx.Err(); err != nil {
-				return false, err
-			}
-			if entry.IsDirectory() {
-				subDirs = append(subDirs, entry.FullPath)
-				return true, nil
-			}
-			if entry.Remote != nil && entry.Remote.LastLocalSyncTsNs > 0 && entry.Remote.LastLocalSyncTsNs <= cutoffNs && IsEvictableRemoteEntry(entry) {
-				*out = append(*out, entry)
-			}
-			return true, nil
-		})
-		if err != nil {
-			return err
-		}
-		for _, subDir := range subDirs {
-			if err := f.collectEvictableRemoteEntries(ctx, subDir, cutoffNs, out); err != nil {
-				return err
-			}
-		}
-		if lastFileName == "" {
-			return nil
-		}
-		startFileName = lastFileName
-	}
-}

weed/filer/filer_remote_evict_test.go

@@ -1,164 +0,0 @@
-package filer
-
-import (
-	"context"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func remoteCachedEntry(path string, syncTsNs int64, mtime time.Time, chunks int) *Entry {
-	entry := &Entry{
-		FullPath: util.FullPath(path),
-		Attr: Attr{
-			Mtime:    mtime,
-			Crtime:   mtime,
-			Mode:     0644,
-			Uid:      1,
-			Gid:      1,
-			Mime:     "application/octet-stream",
-			Md5:      nil,
-			FileSize: 0,
-		},
-		Remote: &filer_pb.RemoteEntry{
-			RemoteMtime:       mtime.Unix(),
-			LastLocalSyncTsNs: syncTsNs,
-			RemoteETag:        "etag",
-			RemoteSize:        100,
-		},
-	}
-	for i := 0; i < chunks; i++ {
-		entry.Chunks = append(entry.Chunks, &filer_pb.FileChunk{FileId: "1,01637037d6", Size: 100})
-	}
-	return entry
-}
-
-func TestIsEvictableRemoteEntry(t *testing.T) {
-	now := time.Now()
-	synced := now.Add(-time.Hour).UnixNano()
-	mtime := now.Add(-time.Hour)
-
-	tests := []struct {
-		name     string
-		entry    *Entry
-		expected bool
-	}{
-		{"directory", &Entry{FullPath: "/d", Attr: Attr{Mode: os.ModeDir | 0755}}, false},
-		{"local only, no remote", &Entry{FullPath: "/f", Attr: Attr{Mtime: mtime, Mode: 0644}}, false},
-		{"remote only, never cached", remoteCachedEntry("/f", 0, mtime, 0), false},
-		{"remote, chunks cleared", remoteCachedEntry("/f", synced, mtime, 0), false},
-		{"dirty, newer than remote", remoteCachedEntry("/f", synced, now, 1), false},
-		{"same-second write after sync", remoteCachedEntry("/f",
-			now.Truncate(time.Second).Add(100*time.Millisecond).UnixNano(),
-			now.Truncate(time.Second).Add(900*time.Millisecond), 1), false},
-		{"same-second sync after write", remoteCachedEntry("/f",
-			now.Truncate(time.Second).Add(900*time.Millisecond).UnixNano(),
-			now.Truncate(time.Second).Add(100*time.Millisecond), 1), true},
-		{"synced and cached", remoteCachedEntry("/f", synced, mtime, 1), true},
-	}
-	for _, tt := range tests {
-		assert.Equal(t, tt.expected, IsEvictableRemoteEntry(tt.entry), tt.name)
-	}
-}
-
-func TestListEvictableRemoteEntries(t *testing.T) {
-	now := time.Now()
-	mtime := now.Add(-2 * time.Hour)
-	oldSync := now.Add(-time.Hour).UnixNano()
-	newSync := now.Add(-time.Second).UnixNano()
-
-	store := newStubFilerStore()
-	f := newTestFiler(t, store, NewFilerRemoteStorage())
-	mounts := []util.FullPath{"/buckets/mybucket"}
-
-	seed := func(e *Entry) {
-		require.NoError(t, f.CreateEntry(context.Background(), e, nil, false, false, nil, false, 255))
-	}
-	seed(remoteCachedEntry("/buckets/mybucket/old.bin", oldSync, mtime, 1))
-	seed(remoteCachedEntry("/buckets/mybucket/fresh.bin", newSync, mtime, 1))
-	seed(remoteCachedEntry("/buckets/mybucket/remoteonly.bin", 0, mtime, 0))
-	seed(remoteCachedEntry("/buckets/mybucket/dirty.bin", oldSync, now, 1))
-	seed(&Entry{FullPath: "/buckets/mybucket/plain.bin", Attr: Attr{Mtime: mtime, Mode: 0644}})
-
-	store.entries["/buckets/mybucket/sub"] = &Entry{
-		FullPath: "/buckets/mybucket/sub",
-		Attr:     Attr{Mode: os.ModeDir | 0755, Mtime: mtime},
-	}
-	seed(remoteCachedEntry("/buckets/mybucket/sub/nested.bin", oldSync-1000, mtime, 1))
-
-	got := f.ListEvictableRemoteEntries(context.Background(), mounts, 0)
-	var paths []string
-	for _, e := range got {
-		paths = append(paths, string(e.FullPath))
-	}
-	assert.Equal(t, []string{
-		"/buckets/mybucket/sub/nested.bin",
-		"/buckets/mybucket/old.bin",
-		"/buckets/mybucket/fresh.bin",
-	}, paths, "oldest LastLocalSyncTsNs first; remote-only, dirty, local entries skipped")
-
-	got = f.ListEvictableRemoteEntries(context.Background(), mounts, 30*time.Second)
-	paths = paths[:0]
-	for _, e := range got {
-		paths = append(paths, string(e.FullPath))
-	}
-	assert.Equal(t, []string{
-		"/buckets/mybucket/sub/nested.bin",
-		"/buckets/mybucket/old.bin",
-	}, paths, "minCacheAge excludes freshly cached entries")
-}
-
-func TestUpdateEntryInvalidatesStaleSyncStamp(t *testing.T) {
-	now := time.Now()
-	synced := now.Add(-time.Hour).UnixNano()
-	mtime := now.Add(-time.Hour)
-	path := "/buckets/mybucket/a.bin"
-
-	newFiler := func() *Filer {
-		f := newTestFiler(t, newStubFilerStore(), NewFilerRemoteStorage())
-		require.NoError(t, f.CreateEntry(context.Background(), remoteCachedEntry(path, synced, mtime, 1), nil, false, false, nil, false, 255))
-		return f
-	}
-	syncStamp := func(f *Filer) int64 {
-		entry, err := f.FindEntry(context.Background(), util.FullPath(path))
-		require.NoError(t, err)
-		return entry.Remote.LastLocalSyncTsNs
-	}
-
-	t.Run("local chunk change clears stamp", func(t *testing.T) {
-		f := newFiler()
-		update := remoteCachedEntry(path, synced, mtime, 1)
-		update.Chunks[0].FileId = "2,01637037d6"
-		require.NoError(t, f.CreateEntry(context.Background(), update, nil, false, false, nil, false, 255))
-		assert.Zero(t, syncStamp(f))
-	})
-
-	t.Run("metadata-only update keeps stamp", func(t *testing.T) {
-		f := newFiler()
-		update := remoteCachedEntry(path, synced, mtime, 1)
-		update.Attr.Mime = "text/plain"
-		require.NoError(t, f.CreateEntry(context.Background(), update, nil, false, false, nil, false, 255))
-		assert.Equal(t, synced, syncStamp(f))
-	})
-
-	t.Run("fresh sync stamp survives chunk change", func(t *testing.T) {
-		f := newFiler()
-		update := remoteCachedEntry(path, now.UnixNano(), mtime, 1)
-		update.Chunks[0].FileId = "2,01637037d6"
-		require.NoError(t, f.CreateEntry(context.Background(), update, nil, false, false, nil, false, 255))
-		assert.Equal(t, now.UnixNano(), syncStamp(f))
-	})
-
-	t.Run("replicated update keeps authoritative stamp", func(t *testing.T) {
-		f := newFiler()
-		update := remoteCachedEntry(path, synced, mtime, 1)
-		update.Chunks[0].FileId = "2,01637037d6"
-		require.NoError(t, f.CreateEntry(context.Background(), update, nil, false, true, nil, false, 255))
-		assert.Equal(t, synced, syncStamp(f))
-	})
-}

weed/filer/filer_remote_tombstone.go

@@ -1,423 +0,0 @@
-package filer
-
-import (
-	"context"
-	"math"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
-	"google.golang.org/protobuf/proto"
-)
-
-const (
-	// remoteDeletionTombstoneTTL bounds a tombstone when no write-back sync
-	// offset ever confirms the remote delete. A remote object re-created
-	// outside the filer at a deleted path stays hidden for this long.
-	remoteDeletionTombstoneTTL = 24 * time.Hour
-	// remoteDeletionTombstoneLimit bounds tracked paths; past it new
-	// tombstones are dropped after an expired sweep still leaves no room.
-	remoteDeletionTombstoneLimit = 1 << 16
-)
-
-// remoteDeletionTombstones tracks paths deleted under a remote mount whose
-// remote object may still exist because the write-back daemon has not
-// consumed the delete event yet. A lazy remote fetch or listing must not
-// resurrect them. A tombstone lifts when the path is written again, when the
-// mount's persisted sync offset passes the delete event, or on TTL.
-//
-// Tombstones recorded before their metadata event lands (the synchronous
-// delete path) are marked pending: the sync offset orders against event
-// timestamps, which only the event itself knows, so a pending tombstone can
-// only be lifted by the event confirming it or by TTL. Once the event stamps
-// the real timestamp the tombstone is releasable by the offset.
-type remoteDeletionTombstones struct {
-	mu      sync.Mutex
-	files   map[string]int64 // file path -> delete event TsNs
-	dirs    map[string]int64 // deleted directory path -> event TsNs; covers its subtree
-	pending map[string]bool  // tombstone path recorded ahead of its event
-}
-
-func newRemoteDeletionTombstones() *remoteDeletionTombstones {
-	return &remoteDeletionTombstones{
-		files:   make(map[string]int64),
-		dirs:    make(map[string]int64),
-		pending: make(map[string]bool),
-	}
-}
-
-// add records a tombstone ahead of its metadata event — the timestamp is the
-// local delete time, a lower bound the sync offset cannot order against.
-func (t *remoteDeletionTombstones) add(path string, isDir bool, tsNs int64) {
-	t.upsert(path, isDir, tsNs, false)
-}
-
-// addFromEvent records a tombstone stamped by the delete event itself, so the
-// mount's sync offset can release it once the daemon passes that event.
-func (t *remoteDeletionTombstones) addFromEvent(path string, isDir bool, tsNs int64) {
-	t.upsert(path, isDir, tsNs, true)
-}
-
-func (t *remoteDeletionTombstones) upsert(path string, isDir bool, tsNs int64, fromEvent bool) {
-	if t == nil {
-		return
-	}
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	// An ancestor directory tombstone at least as new already covers the
-	// path; recording it again only spends capacity.
-	for p := path; ; {
-		i := strings.LastIndexByte(p, '/')
-		if i <= 0 {
-			break
-		}
-		p = p[:i]
-		if ancestorTs, ok := t.dirs[p]; ok && ancestorTs >= tsNs {
-			return
-		}
-	}
-	m := t.files
-	if isDir {
-		m = t.dirs
-	}
-	if cur, ok := m[path]; ok {
-		if tsNs > cur {
-			m[path] = tsNs
-			if !fromEvent {
-				// a newer local delete restamps the tombstone ahead of its
-				// event — the offset cannot vouch for it until the event lands
-				t.pending[path] = true
-			}
-		}
-		if fromEvent && m[path] == tsNs {
-			delete(t.pending, path)
-		}
-		return
-	}
-	if len(t.files)+len(t.dirs) >= remoteDeletionTombstoneLimit {
-		t.evictExpiredLocked(time.Now().UnixNano())
-		if len(t.files)+len(t.dirs) >= remoteDeletionTombstoneLimit {
-			glog.V(0).Infof("remote deletion tombstones full (%d), skipping %s", remoteDeletionTombstoneLimit, path)
-			return
-		}
-	}
-	m[path] = tsNs
-	if fromEvent {
-		delete(t.pending, path)
-	} else {
-		t.pending[path] = true
-	}
-	if isDir {
-		t.dropCoveredLocked(path, tsNs)
-	}
-}
-
-// dropCoveredLocked removes descendant tombstones a new directory tombstone
-// subsumes: their deletes predate it, so the ancestor already hides those
-// remote objects. Descendants deleted later keep their own tombstone.
-// Caller must hold t.mu.
-func (t *remoteDeletionTombstones) dropCoveredLocked(dirPath string, tsNs int64) {
-	prefix := dirPath + "/"
-	for p, ts := range t.files {
-		if ts <= tsNs && strings.HasPrefix(p, prefix) {
-			delete(t.files, p)
-			delete(t.pending, p)
-		}
-	}
-	for p, ts := range t.dirs {
-		if ts <= tsNs && strings.HasPrefix(p, prefix) {
-			delete(t.dirs, p)
-			delete(t.pending, p)
-		}
-	}
-}
-
-// drop removes the exact tombstone recorded for path, e.g. when the delete
-// that recorded it fails before touching anything.
-func (t *remoteDeletionTombstones) drop(path string, tsNs int64) {
-	if t == nil {
-		return
-	}
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if cur, ok := t.dirs[path]; ok && cur <= tsNs {
-		delete(t.dirs, path)
-		delete(t.pending, path)
-	}
-	if cur, ok := t.files[path]; ok && cur <= tsNs {
-		delete(t.files, path)
-		delete(t.pending, path)
-	}
-}
-
-// clear drops a file tombstone when a write at the path is at least as new as
-// the delete; a replayed older create must not lift a newer delete.
-func (t *remoteDeletionTombstones) clear(path string, tsNs int64) {
-	if t == nil {
-		return
-	}
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if cur, ok := t.files[path]; ok && tsNs >= cur {
-		delete(t.files, path)
-		delete(t.pending, path)
-	}
-}
-
-// blockedSince returns the newest delete timestamp governing path — its own
-// file tombstone or one from a deleted ancestor directory — and whether that
-// tombstone is still waiting for its event. 0 means clear.
-func (t *remoteDeletionTombstones) blockedSince(path string) (tsNs int64, pending bool) {
-	if t == nil {
-		return 0, false
-	}
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if ts, ok := t.files[path]; ok {
-		tsNs = ts
-		pending = t.pending[path]
-	}
-	for p := path; ; {
-		if ts, ok := t.dirs[p]; ok && ts > tsNs {
-			tsNs = ts
-			pending = t.pending[p]
-		}
-		i := strings.LastIndexByte(p, '/')
-		if i <= 0 {
-			break
-		}
-		p = p[:i]
-	}
-	return tsNs, pending
-}
-
-// releaseThrough drops the tombstones governing path that are no newer than
-// tsNs, once their remote deletes are confirmed consumed.
-func (t *remoteDeletionTombstones) releaseThrough(path string, tsNs int64) {
-	if t == nil {
-		return
-	}
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if cur, ok := t.files[path]; ok && cur <= tsNs {
-		delete(t.files, path)
-		delete(t.pending, path)
-	}
-	for p := path; ; {
-		if cur, ok := t.dirs[p]; ok && cur <= tsNs {
-			delete(t.dirs, p)
-			delete(t.pending, p)
-		}
-		i := strings.LastIndexByte(p, '/')
-		if i <= 0 {
-			break
-		}
-		p = p[:i]
-	}
-}
-
-func (t *remoteDeletionTombstones) evictExpiredLocked(nowNs int64) {
-	for p, ts := range t.files {
-		if nowNs-ts >= int64(remoteDeletionTombstoneTTL) {
-			delete(t.files, p)
-			delete(t.pending, p)
-		}
-	}
-	for p, ts := range t.dirs {
-		if nowNs-ts >= int64(remoteDeletionTombstoneTTL) {
-			delete(t.dirs, p)
-			delete(t.pending, p)
-		}
-	}
-}
-
-// noteRemoteDeletion records a delete of path under a remote mount so lazy
-// remote reads skip it until the remote delete is confirmed.
-func (f *Filer) noteRemoteDeletion(p util.FullPath, isDir bool, tsNs int64) {
-	if f.RemoteStorage == nil || f.remoteTombstones == nil {
-		return
-	}
-	if _, remoteLoc := f.RemoteStorage.FindMountDirectory(p); remoteLoc == nil {
-		return
-	}
-	f.remoteTombstones.add(string(p), isDir, tsNs)
-}
-
-// unnoteRemoteDeletion retracts a tombstone when the delete that recorded it
-// fails before touching anything under path.
-func (f *Filer) unnoteRemoteDeletion(p util.FullPath, tsNs int64) {
-	if f.remoteTombstones == nil {
-		return
-	}
-	f.remoteTombstones.drop(string(p), tsNs)
-}
-
-// isRemoteDeletionPending reports whether a remote write-back delete for p is
-// still owed: p was deleted under mountDir and neither a rewrite, the mount's
-// sync offset, nor the TTL has lifted the tombstone.
-func (f *Filer) isRemoteDeletionPending(ctx context.Context, p util.FullPath, mountDir util.FullPath) bool {
-	if f.remoteTombstones == nil {
-		return false
-	}
-	tsNs, pending := f.remoteTombstones.blockedSince(string(p))
-	if tsNs == 0 {
-		return false
-	}
-	if pending {
-		// Recorded ahead of its delete event — the write-back offset cannot
-		// vouch for it yet; only the TTL lifts it.
-		if f.remoteDeletionExpired(tsNs) {
-			f.remoteTombstones.releaseThrough(string(p), tsNs)
-			return false
-		}
-		return true
-	}
-	if f.remoteDeletionConsumed(ctx, mountDir, tsNs) {
-		f.remoteTombstones.releaseThrough(string(p), tsNs)
-		return false
-	}
-	return true
-}
-
-func (f *Filer) remoteDeletionExpired(tsNs int64) bool {
-	return time.Now().UnixNano()-tsNs >= int64(remoteDeletionTombstoneTTL)
-}
-
-func (f *Filer) remoteDeletionConsumed(ctx context.Context, mountDir util.FullPath, tsNs int64) bool {
-	if f.remoteDeletionExpired(tsNs) {
-		return true
-	}
-	offset, err := f.readRemoteSyncOffset(ctx, mountDir)
-	return err == nil && offset >= tsNs
-}
-
-// readRemoteSyncOffset reads the write-back daemon's persisted watermark for
-// mountDir straight from the local store: every event at or below it has been
-// applied to the remote.
-func (f *Filer) readRemoteSyncOffset(ctx context.Context, mountDir util.FullPath) (int64, error) {
-	value, err := f.Store.KvGet(ctx, remote_storage.SyncOffsetKey(string(mountDir)))
-	if err != nil {
-		return 0, err
-	}
-	if len(value) < 8 {
-		return 0, nil
-	}
-	return int64(util.BytesToUint64(value)), nil
-}
-
-// RebuildRemoteDeletionTombstones gates lazy remote reads and replays the
-// persisted metadata log from the oldest write-back offset across mounts,
-// restoring tombstones for deletes committed before a restart but not yet
-// applied to the remote. Every filer writes its log files under the same
-// directory, so pending peer deletes replay too; only events still inside
-// the unflushed buffer tail are missed. The gate is set synchronously so no
-// lazy read can slip in before replay starts, and it stays closed until a
-// replay succeeds.
-func (f *Filer) RebuildRemoteDeletionTombstones(ctx context.Context) {
-	if f.RemoteStorage == nil || f.remoteTombstones == nil {
-		return
-	}
-	mounts := f.RemoteStorage.MountedDirectories()
-	if len(mounts) == 0 {
-		return
-	}
-	done := make(chan struct{})
-	f.remoteTombstonesDone.Store(&done)
-	go f.rebuildRemoteDeletionTombstones(ctx, mounts, done)
-}
-
-func (f *Filer) rebuildRemoteDeletionTombstones(ctx context.Context, mounts []util.FullPath, done chan struct{}) {
-	// the replay itself lists directories; do not let it wait on its own gate
-	ctx = context.WithValue(ctx, lazyFetchContextKey{}, true)
-	startTsNs := f.remoteDeletionRebuildStartTsNs(ctx, mounts)
-	openGate := func() {
-		close(done)
-		f.remoteTombstonesDone.Store(nil)
-	}
-	// Deletes made after startup are recorded through the live delete and
-	// event paths, so replay can only be missing deletes committed before
-	// the restart — and those have all crossed the tombstone TTL once this
-	// deadline passes. Holding the gate longer protects nothing.
-	replayDeadline := time.Now().Add(remoteDeletionTombstoneTTL)
-	backoff := 2 * time.Second
-	for {
-		_, _, err := f.ReadPersistedLogBuffer(ctx, log_buffer.NewMessagePosition(startTsNs, 0), 0,
-			func(logEntry *filer_pb.LogEntry) (bool, error) {
-				event := &filer_pb.SubscribeMetadataResponse{}
-				if err := proto.Unmarshal(logEntry.Data, event); err != nil {
-					return false, nil
-				}
-				f.onRemoteDeletionEvents(event)
-				return false, nil
-			})
-		if err == nil {
-			openGate()
-			return
-		}
-		glog.WarningfCtx(ctx, "rebuild remote deletion tombstones: %v", err)
-		if !time.Now().Before(replayDeadline) {
-			glog.ErrorfCtx(ctx, "rebuild remote deletion tombstones: giving up after %v", remoteDeletionTombstoneTTL)
-			openGate()
-			return
-		}
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(backoff):
-		}
-		if backoff < time.Minute {
-			backoff *= 2
-		}
-	}
-}
-
-// remoteDeletionRebuildStartTsNs returns the oldest write-back offset across
-// mounts — the earliest event the daemon may not have applied. Mounts without
-// a recorded offset replay from the TTL floor: events older than it would
-// build tombstones that are already expired.
-func (f *Filer) remoteDeletionRebuildStartTsNs(ctx context.Context, mounts []util.FullPath) int64 {
-	startTsNs := int64(math.MaxInt64)
-	for _, dir := range mounts {
-		offset, err := f.readRemoteSyncOffset(ctx, dir)
-		if err != nil {
-			glog.WarningfCtx(ctx, "read remote sync offset for %s: %v", dir, err)
-			offset = 0
-		}
-		if offset < startTsNs {
-			startTsNs = offset
-		}
-	}
-	ttlFloor := time.Now().Add(-remoteDeletionTombstoneTTL).UnixNano()
-	if startTsNs == int64(math.MaxInt64) || startTsNs < ttlFloor {
-		return ttlFloor
-	}
-	return startTsNs
-}
-
-// onRemoteDeletionEvents folds peer and local metadata events into the
-// tombstone set: a delete or rename source is tombstoned at the event
-// timestamp, a create/update/rename target lifts a file tombstone.
-func (f *Filer) onRemoteDeletionEvents(event *filer_pb.SubscribeMetadataResponse) {
-	message := event.EventNotification
-	if message == nil {
-		return
-	}
-	if message.OldEntry != nil {
-		sourcePath := filer_pb.MetadataEventSourceFullPath(event)
-		if message.NewEntry == nil || sourcePath != filer_pb.MetadataEventTargetFullPath(event) {
-			if f.RemoteStorage != nil && f.remoteTombstones != nil {
-				if _, remoteLoc := f.RemoteStorage.FindMountDirectory(util.FullPath(sourcePath)); remoteLoc != nil {
-					f.remoteTombstones.addFromEvent(sourcePath, message.OldEntry.IsDirectory, event.TsNs)
-				}
-			}
-		}
-	}
-	if message.NewEntry != nil {
-		f.remoteTombstones.clear(filer_pb.MetadataEventTargetFullPath(event), event.TsNs)
-	}
-}

weed/filer/filer_remote_tombstone_test.go

@@ -1,437 +0,0 @@
-package filer
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
-	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func newMountedTestFiler(t *testing.T, storageType string, stub remote_storage.RemoteStorageClient, listingTtlSeconds int32) (*Filer, *stubFilerStore) {
-	t.Helper()
-	if stub != nil {
-		t.Cleanup(registerStubMaker(t, storageType, stub))
-	}
-	conf := &remote_pb.RemoteConf{Name: "tombstonestore", Type: storageType}
-	rs := NewFilerRemoteStorage()
-	rs.storageNameToConf[conf.Name] = conf
-	rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
-		Name:                   "tombstonestore",
-		Bucket:                 "mybucket",
-		Path:                   "/",
-		ListingCacheTtlSeconds: listingTtlSeconds,
-	})
-	store := newStubFilerStore()
-	return newTestFiler(t, store, rs), store
-}
-
-func putRemoteSyncOffset(t *testing.T, store *stubFilerStore, dir string, offsetTsNs int64) {
-	t.Helper()
-	buf := make([]byte, 8)
-	util.Uint64toBytes(buf, uint64(offsetTsNs))
-	require.NoError(t, store.KvPut(context.Background(), remote_storage.SyncOffsetKey(dir), buf))
-}
-
-func TestRemoteDeletionTombstones_BlocksAndReleases(t *testing.T) {
-	tombs := newRemoteDeletionTombstones()
-
-	tombs.addFromEvent("/m/a.txt", false, 100)
-	ts, _ := tombs.blockedSince("/m/a.txt")
-	assert.Equal(t, int64(100), ts)
-	ts, _ = tombs.blockedSince("/m/b.txt")
-	assert.Zero(t, ts)
-
-	// a delete of a directory blocks its whole subtree
-	tombs.addFromEvent("/m/dir", true, 200)
-	ts, _ = tombs.blockedSince("/m/dir")
-	assert.Equal(t, int64(200), ts)
-	ts, _ = tombs.blockedSince("/m/dir/deep/x.txt")
-	assert.Equal(t, int64(200), ts)
-	ts, _ = tombs.blockedSince("/m/dirx/y.txt")
-	assert.Zero(t, ts)
-
-	// a rewrite at or after the delete lifts only that file's tombstone
-	tombs.clear("/m/a.txt", 100)
-	ts, _ = tombs.blockedSince("/m/a.txt")
-	assert.Zero(t, ts)
-	tombs.addFromEvent("/m/a.txt", false, 300)
-	tombs.clear("/m/a.txt", 250)
-	ts, _ = tombs.blockedSince("/m/a.txt")
-	assert.Equal(t, int64(300), ts, "older create must not lift newer delete")
-
-	tombs.releaseThrough("/m/dir/deep/x.txt", 200)
-	ts, _ = tombs.blockedSince("/m/dir/deep/x.txt")
-	assert.Zero(t, ts)
-	ts, _ = tombs.blockedSince("/m/dir")
-	assert.Zero(t, ts)
-}
-
-func TestRemoteDeletionTombstones_AncestorSubsumesAndCovers(t *testing.T) {
-	tombs := newRemoteDeletionTombstones()
-
-	// a child tombstone recorded before its ancestor is dropped once the
-	// ancestor's newer delete covers the whole subtree
-	tombs.addFromEvent("/m/dir/a.txt", false, 100)
-	tombs.addFromEvent("/m/dir", true, 200)
-	ts, _ := tombs.blockedSince("/m/dir")
-	assert.Equal(t, int64(200), ts)
-	ts, _ = tombs.blockedSince("/m/dir/a.txt")
-	assert.Equal(t, int64(200), ts)
-	_, exists := tombs.files["/m/dir/a.txt"]
-	assert.False(t, exists, "descendant tombstone is subsumed by the ancestor")
-
-	// adds under the covered subtree are skipped while the ancestor stands
-	tombs.addFromEvent("/m/dir/b.txt", false, 150)
-	_, exists = tombs.files["/m/dir/b.txt"]
-	assert.False(t, exists)
-	// ...but a child deleted after the ancestor still records its own tombstone
-	tombs.addFromEvent("/m/dir/c.txt", false, 300)
-	ts, _ = tombs.blockedSince("/m/dir/c.txt")
-	assert.Equal(t, int64(300), ts)
-}
-
-func TestRemoteDeletionTombstones_PendingIgnoresOffset(t *testing.T) {
-	f, store := newMountedTestFiler(t, "stub_tomb_pending_offset", nil, 0)
-
-	// recorded before its event lands: a later event may already have moved
-	// the mount's watermark past the local timestamp, so the offset cannot
-	// vouch for this delete yet
-	filePath := "/buckets/mybucket/dir/a.txt"
-	now := time.Now().UnixNano()
-	f.noteRemoteDeletion(util.FullPath(filePath), false, now)
-	putRemoteSyncOffset(t, store, "/buckets/mybucket", now+100)
-	assert.True(t, f.isRemoteDeletionPending(context.Background(), util.FullPath(filePath), "/buckets/mybucket"))
-
-	// once the delete event stamps the real timestamp, the watermark releases it
-	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
-		Directory: "/buckets/mybucket/dir",
-		TsNs:      now + 50,
-		EventNotification: &filer_pb.EventNotification{
-			OldEntry: &filer_pb.Entry{Name: "a.txt"},
-		},
-	})
-	assert.False(t, f.isRemoteDeletionPending(context.Background(), util.FullPath(filePath), "/buckets/mybucket"))
-}
-
-func TestRemoteDeletionTombstones_RaisedLocalTombstoneIsPendingAgain(t *testing.T) {
-	tombs := newRemoteDeletionTombstones()
-
-	tombs.addFromEvent("/m/a.txt", false, 100)
-	_, pending := tombs.blockedSince("/m/a.txt")
-	assert.False(t, pending)
-
-	// a newer local delete restamps the tombstone before its own event
-	// lands, so the write-back offset cannot vouch for it yet
-	tombs.add("/m/a.txt", false, 200)
-	_, pending = tombs.blockedSince("/m/a.txt")
-	assert.True(t, pending)
-
-	// once that delete's event confirms the new timestamp it releases normally
-	tombs.addFromEvent("/m/a.txt", false, 200)
-	_, pending = tombs.blockedSince("/m/a.txt")
-	assert.False(t, pending)
-}
-
-func TestMaybeLazyFetchFromRemote_SkipsTombstonedPath(t *testing.T) {
-	const storageType = "stub_tomb_fetch"
-	stub := &countingRemoteClient{
-		stubRemoteClient: stubRemoteClient{
-			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11},
-		},
-	}
-	f, _ := newMountedTestFiler(t, storageType, stub, 0)
-
-	// a delete under the mount tombstones the path; the remote object is
-	// still there until the write-back daemon consumes the delete event
-	filePath := util.FullPath("/buckets/mybucket/dir/a.txt")
-	f.noteRemoteDeletion(filePath, false, time.Now().UnixNano())
-
-	entry, err := f.maybeLazyFetchFromRemote(context.Background(), filePath)
-	require.NoError(t, err)
-	assert.Nil(t, entry)
-	assert.Equal(t, 0, stub.statCalls, "the remote object must not even be consulted")
-}
-
-func TestMaybeLazyFetchFromRemote_NewerRemoteMtimeStillBlocked(t *testing.T) {
-	const storageType = "stub_tomb_regen"
-	stub := &countingRemoteClient{
-		stubRemoteClient: stubRemoteClient{
-			statResult: &filer_pb.RemoteEntry{RemoteMtime: time.Now().Unix() + 60, RemoteSize: 11},
-		},
-	}
-	f, _ := newMountedTestFiler(t, storageType, stub, 0)
-
-	// a remote object whose mtime postdates the delete is still hidden: the
-	// remote clock cannot distinguish a new generation from the pending
-	// delete's target
-	filePath := util.FullPath("/buckets/mybucket/dir/a.txt")
-	f.noteRemoteDeletion(filePath, false, time.Now().UnixNano())
-
-	entry, err := f.maybeLazyFetchFromRemote(context.Background(), filePath)
-	require.NoError(t, err)
-	assert.Nil(t, entry)
-	assert.Equal(t, 0, stub.statCalls)
-}
-
-func TestMaybeLazyFetchFromRemote_SyncOffsetReleasesTombstone(t *testing.T) {
-	const storageType = "stub_tomb_release"
-	stub := &countingRemoteClient{
-		stubRemoteClient: stubRemoteClient{
-			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11},
-		},
-	}
-	f, store := newMountedTestFiler(t, storageType, stub, 0)
-
-	filePath := util.FullPath("/buckets/mybucket/dir/a.txt")
-	deleteTsNs := time.Now().UnixNano()
-	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
-		Directory: "/buckets/mybucket/dir",
-		TsNs:      deleteTsNs,
-		EventNotification: &filer_pb.EventNotification{
-			OldEntry: &filer_pb.Entry{Name: "a.txt"},
-		},
-	})
-
-	// the daemon's persisted watermark is behind the delete: still blocked
-	putRemoteSyncOffset(t, store, "/buckets/mybucket", deleteTsNs-1)
-	entry, err := f.maybeLazyFetchFromRemote(context.Background(), filePath)
-	require.NoError(t, err)
-	assert.Nil(t, entry)
-	assert.Equal(t, 0, stub.statCalls)
-
-	// once the watermark reaches the delete event's own timestamp, the
-	// remote delete has landed and the lookup may consult the remote again
-	putRemoteSyncOffset(t, store, "/buckets/mybucket", deleteTsNs)
-	entry, err = f.maybeLazyFetchFromRemote(context.Background(), filePath)
-	require.NoError(t, err)
-	require.NotNil(t, entry)
-	assert.Equal(t, 1, stub.statCalls)
-}
-
-func TestDeleteEntryMetaAndData_TombstonesPath(t *testing.T) {
-	const storageType = "stub_tomb_delete"
-	stub := &countingRemoteClient{
-		stubRemoteClient: stubRemoteClient{
-			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11},
-		},
-	}
-	f, store := newMountedTestFiler(t, storageType, stub, 0)
-
-	filePath := util.FullPath("/buckets/mybucket/dir/a.txt")
-	store.entries[string(filePath)] = &Entry{
-		FullPath: filePath,
-		Attr: Attr{
-			Mtime:    time.Unix(1700000000, 0),
-			Crtime:   time.Unix(1700000000, 0),
-			Mode:     0644,
-			FileSize: 11,
-		},
-	}
-
-	require.NoError(t, f.DeleteEntryMetaAndData(context.Background(), filePath, false, false, false, false, nil, 0))
-
-	// the deleted path must not resurrect through the lazy fetch even while
-	// the remote object is still present
-	entry, err := f.FindEntry(context.Background(), filePath)
-	assert.ErrorIs(t, err, filer_pb.ErrNotFound)
-	assert.Nil(t, entry)
-	assert.Equal(t, 0, stub.statCalls)
-
-	// a peer-observed create at the path lifts the tombstone
-	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
-		Directory: "/buckets/mybucket/dir",
-		TsNs:      time.Now().UnixNano(),
-		EventNotification: &filer_pb.EventNotification{
-			NewEntry: &filer_pb.Entry{Name: "a.txt"},
-		},
-	})
-	entry, err = f.maybeLazyFetchFromRemote(context.Background(), filePath)
-	require.NoError(t, err)
-	require.NotNil(t, entry)
-	assert.Equal(t, 1, stub.statCalls)
-}
-
-func TestOnMetadataChangeEvent_PeerDeleteTombstones(t *testing.T) {
-	const storageType = "stub_tomb_peer"
-	stub := &countingRemoteClient{
-		stubRemoteClient: stubRemoteClient{
-			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11},
-		},
-	}
-	f, _ := newMountedTestFiler(t, storageType, stub, 0)
-
-	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
-		Directory: "/buckets/mybucket/dir",
-		TsNs:      time.Now().UnixNano(),
-		EventNotification: &filer_pb.EventNotification{
-			OldEntry: &filer_pb.Entry{Name: "a.txt"},
-		},
-	})
-
-	entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/a.txt")
-	require.NoError(t, err)
-	assert.Nil(t, entry)
-	assert.Equal(t, 0, stub.statCalls)
-}
-
-func TestOnMetadataChangeEvent_PeerDirDeleteTombstonesSubtree(t *testing.T) {
-	const storageType = "stub_tomb_peer_dir"
-	stub := &countingRemoteClient{
-		stubRemoteClient: stubRemoteClient{
-			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11},
-		},
-	}
-	f, _ := newMountedTestFiler(t, storageType, stub, 0)
-
-	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
-		Directory: "/buckets/mybucket",
-		TsNs:      time.Now().UnixNano(),
-		EventNotification: &filer_pb.EventNotification{
-			OldEntry: &filer_pb.Entry{Name: "dir", IsDirectory: true},
-		},
-	})
-
-	entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/deep/a.txt")
-	require.NoError(t, err)
-	assert.Nil(t, entry)
-	assert.Equal(t, 0, stub.statCalls)
-}
-
-func TestMaybeLazyListFromRemote_SkipsTombstonedChild(t *testing.T) {
-	const storageType = "stub_tomb_list"
-	stub := &stubRemoteClient{
-		listDirFn: func(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) error {
-			if err := visitFn("/", "deleted.txt", false, &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 10}); err != nil {
-				return err
-			}
-			return visitFn("/", "fresh.txt", false, &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 20})
-		},
-	}
-	f, store := newMountedTestFiler(t, storageType, stub, 300)
-
-	f.noteRemoteDeletion("/buckets/mybucket/deleted.txt", false, time.Now().UnixNano())
-
-	f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/mybucket"))
-	assert.Equal(t, 1, stub.listDirCalls)
-
-	assert.Nil(t, store.getEntry("/buckets/mybucket/deleted.txt"), "deleted child must not resurrect through a listing")
-	require.NotNil(t, store.getEntry("/buckets/mybucket/fresh.txt"), "other remote objects still list")
-}
-
-func TestMaybeLazyListFromRemote_RecreatedDirStillHidesChildren(t *testing.T) {
-	const storageType = "stub_tomb_recreate"
-	deleteTs := time.Now().UnixNano()
-	stub := &stubRemoteClient{
-		listDirFn: func(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) error {
-			if err := visitFn("/", "stale.txt", false, &filer_pb.RemoteEntry{RemoteMtime: deleteTs/int64(time.Second) - 10, RemoteSize: 10}); err != nil {
-				return err
-			}
-			return visitFn("/", "fresh.txt", false, &filer_pb.RemoteEntry{RemoteMtime: deleteTs/int64(time.Second) + 10, RemoteSize: 20})
-		},
-	}
-	f, store := newMountedTestFiler(t, storageType, stub, 300)
-
-	// the directory is deleted then recreated; its remote children stay
-	// hidden — old or new mtime alike — until the remote delete lands
-	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
-		Directory: "/buckets/mybucket",
-		TsNs:      deleteTs,
-		EventNotification: &filer_pb.EventNotification{
-			OldEntry: &filer_pb.Entry{Name: "dir", IsDirectory: true},
-		},
-	})
-	f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
-		Directory: "/buckets/mybucket",
-		TsNs:      deleteTs + 1,
-		EventNotification: &filer_pb.EventNotification{
-			NewEntry: &filer_pb.Entry{Name: "dir", IsDirectory: true},
-		},
-	})
-
-	f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/mybucket/dir"))
-
-	assert.Nil(t, store.getEntry("/buckets/mybucket/dir/stale.txt"))
-	assert.Nil(t, store.getEntry("/buckets/mybucket/dir/fresh.txt"))
-
-	// after the write-back daemon confirms the remote delete, listing merges again
-	f.remoteTombstones.releaseThrough("/buckets/mybucket/dir/stale.txt", deleteTs)
-	delete(store.getEntry("/buckets/mybucket/dir").Extended, xattrRemoteListingSyncedAt)
-	f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/mybucket/dir"))
-	assert.NotNil(t, store.getEntry("/buckets/mybucket/dir/stale.txt"))
-	assert.NotNil(t, store.getEntry("/buckets/mybucket/dir/fresh.txt"))
-}
-
-func TestRemoteDeletionRebuildStartTsNs_UsesOldestMountOffset(t *testing.T) {
-	f, store := newMountedTestFiler(t, "stub_rebuild", nil, 0)
-	f.RemoteStorage.mapDirectoryToRemoteStorage("/buckets/other", &remote_pb.RemoteStorageLocation{
-		Name: "tombstonestore", Bucket: "other", Path: "/",
-	})
-	mounts := f.RemoteStorage.MountedDirectories()
-	require.Len(t, mounts, 2)
-
-	now := time.Now().UnixNano()
-	putRemoteSyncOffset(t, store, "/buckets/mybucket", now-200)
-	putRemoteSyncOffset(t, store, "/buckets/other", now-300)
-	assert.Equal(t, now-300, f.remoteDeletionRebuildStartTsNs(context.Background(), mounts),
-		"rebuild must replay from the least-synced mount")
-
-	// a mount whose offset was never written replays only within the TTL
-	require.NoError(t, store.KvDelete(context.Background(), remote_storage.SyncOffsetKey("/buckets/other")))
-	floor := f.remoteDeletionRebuildStartTsNs(context.Background(), mounts)
-	assert.GreaterOrEqual(t, floor, time.Now().Add(-remoteDeletionTombstoneTTL-time.Second).UnixNano())
-
-	// an offset older than the TTL floor is raised to it
-	putRemoteSyncOffset(t, store, "/buckets/other", time.Now().Add(-remoteDeletionTombstoneTTL-time.Hour).UnixNano())
-	assert.Greater(t, f.remoteDeletionRebuildStartTsNs(context.Background(), mounts),
-		time.Now().Add(-remoteDeletionTombstoneTTL-time.Hour).UnixNano())
-}
-
-func TestRebuildRemoteDeletionTombstones_EmptyLogReleasesGate(t *testing.T) {
-	f, _ := newMountedTestFiler(t, "stub_rebuild_empty", nil, 0)
-
-	f.RebuildRemoteDeletionTombstones(context.Background())
-	done := f.remoteTombstonesDone.Load()
-	require.NotNil(t, done, "the gate must be set synchronously")
-
-	select {
-	case <-*done:
-	case <-time.After(10 * time.Second):
-		t.Fatal("rebuild never released the gate")
-	}
-	assert.Nil(t, f.remoteTombstonesDone.Load())
-	ts, _ := f.remoteTombstones.blockedSince("/buckets/mybucket/a.txt")
-	assert.Zero(t, ts)
-}
-
-func TestMaybeLazyFetchFromRemote_WaitsForRebuild(t *testing.T) {
-	const storageType = "stub_tomb_pending"
-	stub := &countingRemoteClient{
-		stubRemoteClient: stubRemoteClient{
-			statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 10},
-		},
-	}
-	f, _ := newMountedTestFiler(t, storageType, stub, 0)
-
-	// an unfinished rebuild blocks the fetch until the context gives up
-	gate := make(chan struct{})
-	f.remoteTombstonesDone.Store(&gate)
-	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
-	defer cancel()
-	entry, err := f.maybeLazyFetchFromRemote(ctx, "/buckets/mybucket/a.txt")
-	assert.ErrorIs(t, err, context.DeadlineExceeded)
-	assert.Nil(t, entry)
-	assert.Equal(t, 0, stub.statCalls)
-
-	// once the rebuild finishes, the fetch proceeds
-	close(gate)
-	entry, err = f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/a.txt")
-	require.NoError(t, err)
-	require.NotNil(t, entry)
-	assert.Equal(t, 1, stub.statCalls)
-}

weed/filer/reader_at.go

@@ -32,9 +32,7 @@ type ChunkReadAt struct {
 	fileSize      int64
 	readerCache   *ReaderCache
 	readerPattern *ReaderPattern
-	lastChunkMu   sync.Mutex // guards lastChunkFid; mount issues concurrent ReadAt calls
 	lastChunkFid  string
-	stream        chunkStream     // chunk this reader is positioned in, pinned in the shared readerCache
 	prefetchCount int             // Number of chunks to prefetch ahead during sequential reads
 	ctx           context.Context // Context used for cancellation during chunk read operations
 }
@@ -169,17 +167,10 @@ func (c *ChunkReadAt) Size() int64 {
 }
 
 func (c *ChunkReadAt) Close() error {
-	c.ReleaseStream()
 	c.readerCache.destroy()
 	return nil
 }
 
-// ReleaseStream drops this reader's hold on the chunk it is positioned in.
-// Unlike Close it leaves the (possibly shared) ReaderCache intact.
-func (c *ChunkReadAt) ReleaseStream() {
-	c.readerCache.releaseStream(&c.stream)
-}
-
 func (c *ChunkReadAt) ReadAt(p []byte, offset int64) (n int, err error) {
 
 	c.readerPattern.MonitorReadAt(offset, len(p))
@@ -350,7 +341,6 @@ func (c *ChunkReadAt) doReadAt(ctx context.Context, p []byte, offset int64) (n i
 func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunkView *ChunkView, nextChunkViews *Interval[*ChunkView], offset uint64) (n int, err error) {
 
 	if c.readerPattern.IsRandomMode() {
-		c.readerCache.releaseStream(&c.stream)
 		n, err := c.readerCache.chunkCache.ReadChunkAt(buffer, chunkView.FileId, offset)
 		if n > 0 {
 			return n, err
@@ -360,20 +350,20 @@ func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunk
 	}
 
 	shouldCache := (uint64(chunkView.ViewOffset) + chunkView.ChunkSize) <= c.readerCache.chunkCache.GetMaxFilePartSizeInCache()
-	// The previous chunk is released through the stream pin rather than
-	// UnCache: the buffer is shared, and other streams may still be reading it.
-	n, err = c.readerCache.readChunkAt(ctx, &c.stream, buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), shouldCache)
-	c.lastChunkMu.Lock()
-	enteredChunk := c.lastChunkFid != chunkView.FileId
-	c.lastChunkFid = chunkView.FileId
-	c.lastChunkMu.Unlock()
-	if enteredChunk && chunkView.OffsetInChunk == 0 { // start of a new chunk
-		if nextChunkViews != nil && c.prefetchCount > 0 {
-			// Prefetch multiple chunks ahead for better sequential read throughput
-			// This keeps the network pipeline full with parallel chunk fetches
-			c.readerCache.MaybeCache(nextChunkViews, c.prefetchCount)
+	n, err = c.readerCache.ReadChunkAt(ctx, buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), shouldCache)
+	if c.lastChunkFid != chunkView.FileId {
+		if chunkView.OffsetInChunk == 0 { // start of a new chunk
+			if c.lastChunkFid != "" {
+				c.readerCache.UnCache(c.lastChunkFid)
+			}
+			if nextChunkViews != nil && c.prefetchCount > 0 {
+				// Prefetch multiple chunks ahead for better sequential read throughput
+				// This keeps the network pipeline full with parallel chunk fetches
+				c.readerCache.MaybeCache(nextChunkViews, c.prefetchCount)
+			}
 		}
 	}
+	c.lastChunkFid = chunkView.FileId
 	return
 }
 

weed/filer/reader_at_shared_test.go

@@ -1,283 +0,0 @@
-package filer
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"sync"
-	"sync/atomic"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
-	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
-)
-
-// Two streams (e.g. two S3 GETs) reading the same object through one shared
-// ReaderCache, interleaved in small slices, must share each chunk download:
-// one stream finishing a chunk, or moving on to the next one, must not drop a
-// buffer the other stream is still reading.
-func TestChunkReadAtConcurrentStreamsShareChunks(t *testing.T) {
-	const chunkSize = 64 << 10
-	const chunkCount = 3
-	const sliceSize = 16 << 10
-
-	var mu sync.Mutex
-	fetches := map[string]int{}
-	rc := NewReaderCache(64, (*chunk_cache.TieredChunkCache)(nil), func(context.Context, string) ([]string, error) {
-		return []string{"unused"}, nil
-	}, nil)
-	defer rc.destroy()
-	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, fileId string, _ util_http.RefreshUrlsFunc) (int, error) {
-		mu.Lock()
-		fetches[fileId]++
-		mu.Unlock()
-		for i := range buffer {
-			buffer[i] = fileId[len(fileId)-1]
-		}
-		return len(buffer), nil
-	}
-
-	newStream := func() *ChunkReadAt {
-		views := NewIntervalList[*ChunkView]()
-		for i := 0; i < chunkCount; i++ {
-			views.AppendInterval(&Interval[*ChunkView]{
-				StartOffset: int64(i * chunkSize),
-				StopOffset:  int64((i + 1) * chunkSize),
-				Value: &ChunkView{
-					FileId:     fmt.Sprintf("chunk%d", i),
-					ViewSize:   chunkSize,
-					ViewOffset: int64(i * chunkSize),
-					ChunkSize:  chunkSize,
-				},
-			})
-		}
-		return NewChunkReaderAtFromClient(context.Background(), rc, views, chunkSize*chunkCount, 0)
-	}
-	streams := []*ChunkReadAt{newStream(), newStream()}
-
-	// The streams alternate slice by slice, the second one lagging one slice
-	// behind, so the leader always finishes a chunk while the follower is
-	// still inside it.
-	offsets := make([]int64, len(streams))
-	offsets[1] = -sliceSize
-	for offsets[1] < chunkSize*chunkCount {
-		for i, stream := range streams {
-			if offsets[i] < 0 || offsets[i] >= chunkSize*chunkCount {
-				offsets[i] += sliceSize
-				continue
-			}
-			buf := make([]byte, sliceSize)
-			n, err := stream.ReadAt(buf, offsets[i])
-			if (err != nil && err != io.EOF) || n != sliceSize {
-				t.Fatalf("stream %d at %d: n=%d err=%v", i, offsets[i], n, err)
-			}
-			if want := byte('0' + offsets[i]/chunkSize); buf[0] != want || buf[n-1] != want {
-				t.Fatalf("stream %d at %d: got %q, want %q", i, offsets[i], buf[0], want)
-			}
-			offsets[i] += sliceSize
-		}
-	}
-
-	mu.Lock()
-	defer mu.Unlock()
-	for i := 0; i < chunkCount; i++ {
-		fileId := fmt.Sprintf("chunk%d", i)
-		if fetches[fileId] != 1 {
-			t.Errorf("%s fetched %d times for two concurrent streams, want 1", fileId, fetches[fileId])
-		}
-	}
-}
-
-func newStreamTestReaderCache(chunkCache chunk_cache.ChunkCache) *ReaderCache {
-	rc := NewReaderCache(64, chunkCache, func(context.Context, string) ([]string, error) {
-		return []string{"unused"}, nil
-	}, nil)
-	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
-		return len(buffer), nil
-	}
-	return rc
-}
-
-func isRetained(rc *ReaderCache, fileId string) bool {
-	rc.Lock()
-	defer rc.Unlock()
-	_, found := rc.downloaders[fileId]
-	return found
-}
-
-// A stream moving on to a chunk served from the chunk cache must still
-// release the chunk it was positioned in before.
-func TestChunkStreamReleasesPreviousChunkOnCacheHit(t *testing.T) {
-	cache := newMockChunkCacheForReaderCache()
-	cache.SetChunk("chunk1", make([]byte, 4<<10))
-	rc := newStreamTestReaderCache(cache)
-	defer rc.destroy()
-
-	stream := &chunkStream{}
-	if _, err := rc.readChunkAt(context.Background(), stream, make([]byte, 1<<10), "chunk0", nil, false, 0, 4<<10, false); err != nil {
-		t.Fatal(err)
-	}
-	if n, err := rc.readChunkAt(context.Background(), stream, make([]byte, 1<<10), "chunk1", nil, false, 0, 4<<10, true); err != nil || n == 0 {
-		t.Fatalf("cache hit read: n=%d err=%v", n, err)
-	}
-	if isRetained(rc, "chunk0") {
-		t.Fatal("chunk0 still retained after the stream moved on to a cached chunk")
-	}
-}
-
-// A chunk the stream leaves while another read is in flight must be dropped
-// once that read ends, even if it did not read the chunk to the end.
-func TestChunkStreamDropsLeftChunkAfterInFlightRead(t *testing.T) {
-	rc := newStreamTestReaderCache(newMockChunkCacheForReaderCache())
-	defer rc.destroy()
-
-	stream := &chunkStream{}
-	if _, err := rc.readChunkAt(context.Background(), stream, make([]byte, 1<<10), "chunk0", nil, false, 0, 4<<10, false); err != nil {
-		t.Fatal(err)
-	}
-	// Another reader is in the middle of a partial read of chunk0.
-	rc.Lock()
-	other := rc.downloaders["chunk0"]
-	other.wg.Add(1)
-	atomic.AddInt32(&other.readers, 1)
-	rc.Unlock()
-
-	if _, err := rc.readChunkAt(context.Background(), stream, make([]byte, 1<<10), "chunk1", nil, false, 0, 4<<10, false); err != nil {
-		t.Fatal(err)
-	}
-	if !isRetained(rc, "chunk0") {
-		t.Fatal("chunk0 dropped while a read was still in flight")
-	}
-
-	// The in-flight read ends without reaching the end of the chunk.
-	other.wg.Done()
-	atomic.AddInt32(&other.readers, -1)
-	rc.removeConsumed(other)
-	if isRetained(rc, "chunk0") {
-		t.Fatal("chunk0 retained after the stream left it and the last read ended")
-	}
-}
-
-// Concurrent ReadAt calls on one ChunkReadAt (as mount does) share its stream
-// pin; they must neither race on it nor leak or double-release pins.
-func TestChunkStreamConcurrentReadsOnOneReader(t *testing.T) {
-	const chunkSize = 16 << 10
-	const chunkCount = 4
-	const sliceSize = 4 << 10
-	rc := newStreamTestReaderCache((*chunk_cache.TieredChunkCache)(nil))
-	defer rc.destroy()
-
-	views := NewIntervalList[*ChunkView]()
-	for i := 0; i < chunkCount; i++ {
-		views.AppendInterval(&Interval[*ChunkView]{
-			StartOffset: int64(i * chunkSize),
-			StopOffset:  int64((i + 1) * chunkSize),
-			Value: &ChunkView{
-				FileId:     fmt.Sprintf("chunk%d", i),
-				ViewSize:   chunkSize,
-				ViewOffset: int64(i * chunkSize),
-				ChunkSize:  chunkSize,
-			},
-		})
-	}
-	reader := NewChunkReaderAtFromClient(context.Background(), rc, views, chunkSize*chunkCount, 0)
-
-	var wg sync.WaitGroup
-	for g := 0; g < 8; g++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			for offset := int64(0); offset < chunkSize*chunkCount; offset += sliceSize {
-				if _, err := reader.ReadAt(make([]byte, sliceSize), offset); err != nil && err != io.EOF {
-					t.Error(err)
-					return
-				}
-			}
-		}()
-	}
-	wg.Wait()
-
-	// Finishing the last chunk releases the stream's final pin.
-	if _, err := reader.ReadAt(make([]byte, sliceSize), chunkSize*chunkCount-sliceSize); err != nil && err != io.EOF {
-		t.Fatal(err)
-	}
-	rc.Lock()
-	defer rc.Unlock()
-	for fileId, cacher := range rc.downloaders {
-		t.Errorf("%s retained after all reads finished: pins=%d readers=%d", fileId, atomic.LoadInt32(&cacher.pins), atomic.LoadInt32(&cacher.readers))
-	}
-}
-
-// A chunk a stream is positioned in must outlast downloader-limit eviction:
-// otherwise a busy cache drops the buffer mid-stream and forces a refetch.
-func TestChunkReadAtPinnedChunkSurvivesEviction(t *testing.T) {
-	const chunkSize = 64 << 10
-
-	var fetches int32
-	rc := NewReaderCache(2, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) {
-		return []string{"unused"}, nil
-	}, nil)
-	defer rc.destroy()
-	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, fileId string, _ util_http.RefreshUrlsFunc) (int, error) {
-		if fileId == "chunk0" {
-			atomic.AddInt32(&fetches, 1)
-		}
-		return len(buffer), nil
-	}
-
-	stream := &chunkStream{}
-	buf := make([]byte, 16<<10)
-	// One slice in: the stream is positioned in chunk0 but has not left it.
-	if _, err := rc.readChunkAt(context.Background(), stream, buf, "chunk0", nil, false, 0, chunkSize, false); err != nil {
-		t.Fatal(err)
-	}
-
-	// Fill the downloader map past its limit with unpinned chunks.
-	for _, fileId := range []string{"chunk1", "chunk2", "chunk3"} {
-		if _, err := rc.ReadChunkAt(context.Background(), buf, fileId, nil, false, 0, chunkSize, true); err != nil {
-			t.Fatalf("read %s: %v", fileId, err)
-		}
-	}
-
-	// The stream's next slice must come from the still-pinned buffer.
-	if _, err := rc.readChunkAt(context.Background(), stream, buf, "chunk0", nil, false, 16<<10, chunkSize, false); err != nil {
-		t.Fatal(err)
-	}
-	if got := atomic.LoadInt32(&fetches); got != 1 {
-		t.Errorf("chunk0 fetched %d times, want 1", got)
-	}
-}
-
-// When every downloader is pinned the limit still applies: the oldest pinned
-// buffer is evicted so abandoned streams cannot grow memory past the limit.
-func TestChunkReadAtPinnedEvictionFallsBackWhenAllPinned(t *testing.T) {
-	const chunkSize = 64 << 10
-	rc := NewReaderCache(2, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) {
-		return []string{"unused"}, nil
-	}, nil)
-	defer rc.destroy()
-	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
-		return len(buffer), nil
-	}
-
-	buf := make([]byte, 16<<10)
-	streamA := &chunkStream{}
-	streamB := &chunkStream{}
-	if _, err := rc.readChunkAt(context.Background(), streamA, buf, "chunk0", nil, false, 0, chunkSize, false); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := rc.readChunkAt(context.Background(), streamB, buf, "chunk1", nil, false, 0, chunkSize, false); err != nil {
-		t.Fatal(err)
-	}
-
-	// Every downloader is now pinned; the next chunk must still get in.
-	if _, err := rc.readChunkAt(context.Background(), &chunkStream{}, buf, "chunk2", nil, false, 0, chunkSize, false); err != nil {
-		t.Fatal(err)
-	}
-	if isRetained(rc, "chunk0") {
-		t.Fatal("oldest pinned downloader was not evicted past the limit")
-	}
-	if !isRetained(rc, "chunk2") {
-		t.Fatal("new downloader missing after pinned fallback eviction")
-	}
-}

weed/filer/reader_cache.go

@@ -35,8 +35,6 @@ type SingleChunkCacher struct {
 	completedTimeNew int64
 	readers          int32
 	consumed         int32
-	pins             int32 // streams currently positioned inside this chunk
-	left             int32 // set once the last pinning stream moved on
 	sync.Mutex
 	parent         *ReaderCache
 	chunkFileId    string
@@ -114,19 +112,7 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView], count int) {
 	return
 }
 
-// chunkStream is one sequential reader's position in a shared ReaderCache.
-// The chunk it is reading stays pinned until the stream reads it to the end or
-// moves elsewhere, so another stream finishing or leaving the same chunk does
-// not drop the buffer from under it.
-type chunkStream struct {
-	cacher *SingleChunkCacher
-}
-
 func (rc *ReaderCache) ReadChunkAt(ctx context.Context, buffer []byte, fileId string, cipherKey []byte, isGzipped bool, offset int64, chunkSize int, shouldCache bool) (int, error) {
-	return rc.readChunkAt(ctx, nil, buffer, fileId, cipherKey, isGzipped, offset, chunkSize, shouldCache)
-}
-
-func (rc *ReaderCache) readChunkAt(ctx context.Context, stream *chunkStream, buffer []byte, fileId string, cipherKey []byte, isGzipped bool, offset int64, chunkSize int, shouldCache bool) (int, error) {
 retry:
 	rc.Lock()
 
@@ -144,11 +130,8 @@ retry:
 			// start wg.Wait() on a zero counter while this read is about to register.
 			cacher.wg.Add(1)
 			atomic.AddInt32(&cacher.readers, 1)
-			previous := stream.pin(cacher)
 			rc.Unlock()
-			rc.unpin(previous)
 			n, err := cacher.readChunkAt(ctx, buffer, offset)
-			rc.releaseIfFinished(stream, cacher, offset, n, err, chunkSize)
 			if n > 0 || err != nil {
 				return n, err
 			}
@@ -161,34 +144,19 @@ retry:
 	if shouldCache || rc.lookupFileIdFn == nil {
 		n, err := rc.chunkCache.ReadChunkAt(buffer, fileId, uint64(offset))
 		if n > 0 {
-			// Served from the chunk cache: the stream has left its pinned chunk.
-			previous := stream.unpinLocked()
 			rc.Unlock()
-			rc.unpin(previous)
 			return n, err
 		}
 	}
 
-	// clean up old downloaders; prefer one no stream is positioned in, but
-	// fall back to a pinned one so abandoned pins cannot bypass the limit
+	// clean up old downloaders
 	if len(rc.downloaders) >= rc.limit {
 		oldestFid, oldestTime := "", time.Now().UnixNano()
-		pinnedFid, pinnedTime := "", int64(0)
 		for fid, downloader := range rc.downloaders {
 			completedTime := atomic.LoadInt64(&downloader.completedTimeNew)
-			if completedTime <= 0 {
-				continue
+			if completedTime > 0 && completedTime < oldestTime {
+				oldestFid, oldestTime = fid, completedTime
 			}
-			if atomic.LoadInt32(&downloader.pins) == 0 {
-				if completedTime < oldestTime {
-					oldestFid, oldestTime = fid, completedTime
-				}
-			} else if pinnedFid == "" || completedTime < pinnedTime {
-				pinnedFid, pinnedTime = fid, completedTime
-			}
-		}
-		if oldestFid == "" {
-			oldestFid = pinnedFid
 		}
 		if oldestFid != "" {
 			oldDownloader := rc.downloaders[oldestFid]
@@ -207,77 +175,9 @@ retry:
 	rc.downloaders[fileId] = cacher
 	cacher.wg.Add(1)
 	atomic.AddInt32(&cacher.readers, 1)
-	previous := stream.pin(cacher)
 	rc.Unlock()
-	rc.unpin(previous)
 
-	n, err := cacher.readChunkAt(ctx, buffer, offset)
-	rc.releaseIfFinished(stream, cacher, offset, n, err, chunkSize)
-	return n, err
-}
-
-// pin makes cacher the stream's current chunk and returns the chunk it was
-// pinned to before, which the caller unpins once the ReaderCache lock is
-// released. The stream is only touched under the ReaderCache lock, since
-// concurrent ReadAt calls on one ChunkReadAt share it.
-func (stream *chunkStream) pin(cacher *SingleChunkCacher) (previous *SingleChunkCacher) {
-	if stream == nil || stream.cacher == cacher {
-		return nil
-	}
-	previous = stream.cacher
-	stream.cacher = cacher
-	atomic.AddInt32(&cacher.pins, 1)
-	return previous
-}
-
-// unpinLocked detaches the stream from its chunk and returns that chunk for
-// the caller to unpin once the ReaderCache lock is released.
-func (stream *chunkStream) unpinLocked() (previous *SingleChunkCacher) {
-	if stream == nil {
-		return nil
-	}
-	previous = stream.cacher
-	stream.cacher = nil
-	return previous
-}
-
-// releaseIfFinished unpins the stream's chunk once the stream has read it to
-// the end, since the stream will not come back to it.
-func (rc *ReaderCache) releaseIfFinished(stream *chunkStream, cacher *SingleChunkCacher, offset int64, n int, err error, chunkSize int) {
-	if stream == nil || err != nil || offset+int64(n) < int64(chunkSize) {
-		return
-	}
-	var previous *SingleChunkCacher
-	rc.Lock()
-	if stream.cacher == cacher {
-		previous = stream.unpinLocked()
-	}
-	rc.Unlock()
-	rc.unpin(previous)
-}
-
-// releaseStream unpins whatever chunk the stream is positioned in.
-func (rc *ReaderCache) releaseStream(stream *chunkStream) {
-	if stream == nil {
-		return
-	}
-	rc.Lock()
-	previous := stream.unpinLocked()
-	rc.Unlock()
-	rc.unpin(previous)
-}
-
-// unpin drops one stream's pin. Once no stream is positioned in the chunk it
-// is dropped like UnCache would, as soon as no read is in progress either:
-// here if none is, otherwise by the last read's removeConsumed.
-func (rc *ReaderCache) unpin(downloader *SingleChunkCacher) {
-	if downloader == nil {
-		return
-	}
-	if atomic.AddInt32(&downloader.pins, -1) == 0 {
-		atomic.StoreInt32(&downloader.left, 1)
-	}
-	rc.removeConsumed(downloader)
+	return cacher.readChunkAt(ctx, buffer, offset)
 }
 
 func (rc *ReaderCache) UnCache(fileId string) {
@@ -302,34 +202,15 @@ func (rc *ReaderCache) remove(downloader *SingleChunkCacher) {
 	}
 }
 
-// removeUnpinned drops a cacher only while no stream is positioned in it.
-// Budget eviction picks its victim under the budget lock, so the pin check
-// and the map removal must happen together under the ReaderCache lock.
-func (rc *ReaderCache) removeUnpinned(downloader *SingleChunkCacher) (removed bool) {
-	rc.Lock()
-	removed = rc.downloaders[downloader.chunkFileId] == downloader &&
-		atomic.LoadInt32(&downloader.pins) == 0
-	if removed {
-		delete(rc.downloaders, downloader.chunkFileId)
-	}
-	rc.Unlock()
-	if removed {
-		downloader.destroy()
-	}
-	return
-}
-
-// removeConsumed drops a cacher once its buffer was fully read, or the
-// streams positioned in it have left, and no readers remain attached or
-// pinned. The checks run under the ReaderCache lock so a reader attaching
-// at the same time either wins (the cacher stays and that reader's detach
-// retries the removal) or misses the map and refetches.
+// removeConsumed drops a cacher once its buffer was fully read and no
+// readers remain attached. The checks run under the ReaderCache lock so a
+// reader attaching at the same time either wins (the cacher stays and that
+// reader's detach retries the removal) or misses the map and refetches.
 func (rc *ReaderCache) removeConsumed(downloader *SingleChunkCacher) {
 	rc.Lock()
 	removed := rc.downloaders[downloader.chunkFileId] == downloader &&
 		atomic.LoadInt32(&downloader.readers) == 0 &&
-		atomic.LoadInt32(&downloader.pins) == 0 &&
-		(atomic.LoadInt32(&downloader.consumed) != 0 || atomic.LoadInt32(&downloader.left) != 0)
+		atomic.LoadInt32(&downloader.consumed) != 0
 	if removed {
 		delete(rc.downloaders, downloader.chunkFileId)
 	}

weed/filer/reader_cache_budget.go

@@ -4,7 +4,6 @@ import (
 	"container/list"
 	"fmt"
 	"sync"
-	"sync/atomic"
 
 	"github.com/seaweedfs/seaweedfs/weed/util/mem"
 )
@@ -52,39 +51,12 @@ func (b *ReaderCacheBudget) reserve(s *SingleChunkCacher) error {
 			b.Unlock()
 			return nil
 		}
-		// Prefer evicting an idle chunk no stream is positioned in; fall back
-		// to the oldest pinned one so abandoned pins cannot block the budget.
-		var victim *SingleChunkCacher
-		var entry *list.Element
-		pinnedVictim := false
-		for e := b.idle.Front(); e != nil; e = e.Next() {
-			c := e.Value.(*SingleChunkCacher)
-			if atomic.LoadInt32(&c.pins) == 0 {
-				victim, entry = c, e
-				pinnedVictim = false
-				break
-			}
-			if victim == nil {
-				victim, entry = c, e
-				pinnedVictim = true
-			}
-		}
-		if entry != nil {
+		if entry := b.idle.Front(); entry != nil {
+			victim := entry.Value.(*SingleChunkCacher)
 			b.idle.Remove(entry)
 			delete(b.idleEntries, victim)
 			b.Unlock()
-			if pinnedVictim {
-				victim.parent.remove(victim)
-			} else if !victim.parent.removeUnpinned(victim) {
-				// The victim was pinned between selection and removal: keep it
-				// evictable so a pin abandoned in that gap cannot wedge the
-				// budget, then retry the selection.
-				b.Lock()
-				if _, ok := b.reservations[victim]; ok && b.idleEntries[victim] == nil {
-					b.idleEntries[victim] = b.idle.PushBack(victim)
-				}
-				b.Unlock()
-			}
+			victim.parent.remove(victim)
 			continue
 		}
 		changed := b.changed

weed/filer/remote_storage.go

@@ -36,7 +36,6 @@ type FilerRemoteStorage struct {
 	// whenever /etc/remote changes
 	mu                sync.RWMutex
 	rules             ptrie.Trie[*remote_pb.RemoteStorageLocation]
-	mountDirs         []util.FullPath
 	storageNameToConf map[string]*remote_pb.RemoteConf
 	// confValidator, when set, is applied to every RemoteConf as it is loaded
 	// from /etc/remote. A conf that fails is dropped from storageNameToConf so
@@ -88,13 +87,11 @@ func (rs *FilerRemoteStorage) LoadRemoteStorageConfigurationsAndMapping(filer *F
 	// build into fresh containers so an unmounted directory disappears instead
 	// of lingering in the trie, which has no way to drop a key
 	rules := ptrie.New[*remote_pb.RemoteStorageLocation]()
-	var mountDirs []util.FullPath
 	storageNameToConf := make(map[string]*remote_pb.RemoteConf)
 
 	for _, entry := range entries {
 		if entry.Name() == REMOTE_STORAGE_MOUNT_FILE {
-			mountDirs, err = loadRemoteStorageMountMapping(rules, entry.Content)
-			if err != nil {
+			if err := loadRemoteStorageMountMapping(rules, entry.Content); err != nil {
 				return err
 			}
 			continue
@@ -120,46 +117,27 @@ func (rs *FilerRemoteStorage) LoadRemoteStorageConfigurationsAndMapping(filer *F
 	}
 
 	rs.mu.Lock()
-	rs.rules, rs.mountDirs, rs.storageNameToConf = rules, mountDirs, storageNameToConf
+	rs.rules, rs.storageNameToConf = rules, storageNameToConf
 	rs.mu.Unlock()
 
 	return nil
 }
 
-func loadRemoteStorageMountMapping(rules ptrie.Trie[*remote_pb.RemoteStorageLocation], data []byte) (mountDirs []util.FullPath, err error) {
+func loadRemoteStorageMountMapping(rules ptrie.Trie[*remote_pb.RemoteStorageLocation], data []byte) (err error) {
 	mappings := &remote_pb.RemoteStorageMapping{}
 	if err := proto.Unmarshal(data, mappings); err != nil {
-		return nil, fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, err)
+		return fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, err)
 	}
 	for dir, storageLocation := range mappings.Mappings {
 		putDirectoryToRemoteStorage(rules, util.FullPath(dir), storageLocation)
-		mountDirs = append(mountDirs, util.FullPath(dir))
 	}
-	return mountDirs, nil
+	return nil
 }
 
 func (rs *FilerRemoteStorage) mapDirectoryToRemoteStorage(dir util.FullPath, loc *remote_pb.RemoteStorageLocation) {
 	rs.mu.Lock()
 	defer rs.mu.Unlock()
 	putDirectoryToRemoteStorage(rs.rules, dir, loc)
-	found := false
-	for _, d := range rs.mountDirs {
-		if d == dir {
-			found = true
-			break
-		}
-	}
-	if !found {
-		rs.mountDirs = append(rs.mountDirs, dir)
-	}
-}
-
-// MountedDirectories returns the directories currently mapped to remote
-// storage, for callers that must reason about every mount rather than one path.
-func (rs *FilerRemoteStorage) MountedDirectories() []util.FullPath {
-	rs.mu.RLock()
-	defer rs.mu.RUnlock()
-	return append([]util.FullPath(nil), rs.mountDirs...)
 }
 
 func putDirectoryToRemoteStorage(rules ptrie.Trie[*remote_pb.RemoteStorageLocation], dir util.FullPath, loc *remote_pb.RemoteStorageLocation) {

weed/iam/integration/cached_role_store_generic.go

@@ -116,14 +116,6 @@ func (c *GenericCachedRoleStore) DeleteRole(ctx context.Context, filerAddress st
 	return c.Delete(ctx, filerAddress, roleName)
 }
 
-// UpdateRole implements RoleStore interface. The update reads the filer, not
-// the cache, and the cache is dropped afterwards whatever the outcome: a
-// refused write may mean the cached role is stale.
-func (c *GenericCachedRoleStore) UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error {
-	defer c.ClearCache()
-	return c.adapter.store.UpdateRole(ctx, filerAddress, roleName, update)
-}
-
 // genericCopyRoleDefinition creates a deep copy of a RoleDefinition for the generic cache
 func genericCopyRoleDefinition(role *RoleDefinition) *RoleDefinition {
 	if role == nil {
@@ -131,13 +123,9 @@ func genericCopyRoleDefinition(role *RoleDefinition) *RoleDefinition {
 	}
 
 	result := &RoleDefinition{
-		RoleName:           role.RoleName,
-		RoleArn:            role.RoleArn,
-		Description:        role.Description,
-		MaxSessionDuration: role.MaxSessionDuration,
-		Source:             role.Source,
-		CreatedAt:          role.CreatedAt,
-		RoleId:             role.RoleId,
+		RoleName:    role.RoleName,
+		RoleArn:     role.RoleArn,
+		Description: role.Description,
 	}
 
 	// Deep copy trust policy if it exists

weed/iam/integration/iam_manager.go

@@ -2,10 +2,7 @@ package integration
 
 import (
 	"context"
-	"crypto/rand"
-	"crypto/sha256"
 	"encoding/base64"
-	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -30,30 +27,11 @@ const maxPoliciesForEvaluation = 1024
 
 // IAMManager orchestrates all IAM components
 type IAMManager struct {
-	stsService        *sts.STSService
-	policyEngine      *policy.PolicyEngine
-	roleStore         RoleStore
-	userStore         UserStore
-	oidcProviderStore OIDCProviderStore
-	// staticOIDCProviders are the OIDC providers of this server's IAM config
-	// file, by ARN. With an in-memory store they are also written to the
-	// store; a persistent store never holds them (see installOIDCProviderStore).
-	staticOIDCProviders map[string]*OIDCProviderRecord
-	// staticRoles are the roles of this server's IAM config file, by name, once
-	// LoadStaticRoles has run (see installRoleStore).
-	staticRoles map[string]*RoleDefinition
-	// oidcRetryMu guards the background refresh retry and which store is
-	// current: cancelOIDCLoad stops the retry, oidcRetryGen names the one
-	// running (0 when none) so at most one runs, and oidcRetryAgain records a
-	// refresh that failed while it ran, so the retry runs once more.
-	oidcRetryMu    sync.Mutex
-	cancelOIDCLoad context.CancelFunc
-	oidcRetryGen   uint64
-	oidcRetrySeq   uint64
-	oidcRetryAgain bool
-	// oidcRefreshMu serializes refreshes from reading the store to handing
-	// STS the result, so an older snapshot cannot replace a newer one.
-	oidcRefreshMu        sync.Mutex
+	stsService           *sts.STSService
+	policyEngine         *policy.PolicyEngine
+	roleStore            RoleStore
+	userStore            UserStore
+	oidcProviderStore    OIDCProviderStore
 	oidcAuditSink        OIDCProviderAuditSink
 	revocationStore      SessionRevocationStore
 	filerAddressProvider func() string // Function to get current filer address
@@ -130,13 +108,12 @@ func (m *IAMManager) PurgeRevokedSessions(ctx context.Context) (int, error) {
 }
 
 // SetOIDCProviderStore configures the IAM-managed OIDC provider store. When
-// nil, OIDC provider IAM actions return ServiceNotReady.
+// nil, OIDC provider IAM actions return ServiceNotReady. The store is the
+// source of truth for AssumeRoleWithWebIdentity provider resolution once
+// Phase 2b lands; in Phase 2a it is read-only and populated from static
+// configuration at boot.
 func (m *IAMManager) SetOIDCProviderStore(store OIDCProviderStore) {
-	var stsConfig *sts.STSConfig
-	if m.stsService != nil {
-		stsConfig = m.stsService.Config
-	}
-	m.installOIDCProviderStore(store, stsConfig)
+	m.oidcProviderStore = store
 }
 
 // GetOIDCProviderStore returns the configured store (may be nil).
@@ -144,33 +121,13 @@ func (m *IAMManager) GetOIDCProviderStore() OIDCProviderStore {
 	return m.oidcProviderStore
 }
 
-// GetRoleStore returns the configured role store.
-func (m *IAMManager) GetRoleStore() RoleStore {
-	return m.roleStore
-}
-
-// SetRoleStore replaces the role store. An S3 server builds the list of
-// directories it watches for peers' changes once, at startup, from the store
-// installed then (RoleStoreDirectory): a filer-backed store installed later
-// with a different basePath is not watched, so peers' changes to it reach this
-// server's cached roles only when the cache expires.
-func (m *IAMManager) SetRoleStore(store RoleStore) {
-	m.installRoleStore(context.Background(), store)
-}
-
 // GetOIDCProvider returns the record for the given ARN, or an error if the
 // store is not configured or the record is missing.
 func (m *IAMManager) GetOIDCProvider(ctx context.Context, arn string) (*OIDCProviderRecord, error) {
 	if m.oidcProviderStore == nil {
 		return nil, fmt.Errorf("OIDC provider store not configured")
 	}
-	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
-	if errors.Is(err, ErrOIDCProviderNotFound) {
-		if static, ok := m.staticOIDCProviders[arn]; ok {
-			return copyOIDCProviderRecord(static), nil
-		}
-	}
-	return rec, err
+	return m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
 }
 
 // ListOIDCProviders enumerates all configured OIDC providers.
@@ -178,33 +135,7 @@ func (m *IAMManager) ListOIDCProviders(ctx context.Context) ([]*OIDCProviderReco
 	if m.oidcProviderStore == nil {
 		return nil, fmt.Errorf("OIDC provider store not configured")
 	}
-	records, err := m.oidcProviderStore.ListProviders(ctx, m.getFilerAddress())
-	if err != nil {
-		return nil, err
-	}
-	// A persistent store does not hold the config file's providers.
-	seen := make(map[string]bool, len(records))
-	for _, rec := range records {
-		seen[rec.ARN] = true
-	}
-	for arn, static := range m.staticOIDCProviders {
-		if !seen[arn] {
-			records = append(records, copyOIDCProviderRecord(static))
-		}
-	}
-	return records, nil
-}
-
-// mutableOIDCProvider loads a stored provider for a change. A provider only
-// the IAM config file defines is refused with ErrOIDCProviderStatic.
-func (m *IAMManager) mutableOIDCProvider(ctx context.Context, arn string) (*OIDCProviderRecord, error) {
-	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
-	if errors.Is(err, ErrOIDCProviderNotFound) {
-		if _, ok := m.staticOIDCProviders[arn]; ok {
-			return nil, fmt.Errorf("%w: %s", ErrOIDCProviderStatic, arn)
-		}
-	}
-	return rec, err
+	return m.oidcProviderStore.ListProviders(ctx, m.getFilerAddress())
 }
 
 // CreateOIDCProvider persists a new IAM-managed OIDC provider record. Refuses
@@ -219,9 +150,6 @@ func (m *IAMManager) CreateOIDCProvider(ctx context.Context, rec *OIDCProviderRe
 	if err := validateOIDCProviderRecord(rec); err != nil {
 		return err
 	}
-	if _, static := m.staticOIDCProviders[rec.ARN]; static {
-		return fmt.Errorf("%w: %s", ErrOIDCProviderAlreadyExists, rec.ARN)
-	}
 	existing, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), rec.ARN)
 	if err == nil && existing != nil {
 		return fmt.Errorf("%w: %s", ErrOIDCProviderAlreadyExists, rec.ARN)
@@ -245,9 +173,6 @@ func (m *IAMManager) DeleteOIDCProvider(ctx context.Context, arn string) error {
 	if m.oidcProviderStore == nil {
 		return fmt.Errorf("OIDC provider store not configured")
 	}
-	if _, err := m.mutableOIDCProvider(ctx, arn); errors.Is(err, ErrOIDCProviderStatic) {
-		return err
-	}
 	if err := m.oidcProviderStore.DeleteProvider(ctx, m.getFilerAddress(), arn); err != nil {
 		return err
 	}
@@ -265,7 +190,7 @@ func (m *IAMManager) AddClientIDToOIDCProvider(ctx context.Context, arn, clientI
 	if clientID == "" {
 		return fmt.Errorf("ClientID cannot be empty")
 	}
-	rec, err := m.mutableOIDCProvider(ctx, arn)
+	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
 	if err != nil {
 		return err
 	}
@@ -293,7 +218,7 @@ func (m *IAMManager) RemoveClientIDFromOIDCProvider(ctx context.Context, arn, cl
 	if m.oidcProviderStore == nil {
 		return fmt.Errorf("OIDC provider store not configured")
 	}
-	rec, err := m.mutableOIDCProvider(ctx, arn)
+	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
 	if err != nil {
 		return err
 	}
@@ -330,7 +255,7 @@ func (m *IAMManager) UpdateOIDCProviderThumbprints(ctx context.Context, arn stri
 			return fmt.Errorf("invalid thumbprint %q: must be 40-character SHA-1 hex", tp)
 		}
 	}
-	rec, err := m.mutableOIDCProvider(ctx, arn)
+	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
 	if err != nil {
 		return err
 	}
@@ -349,7 +274,7 @@ func (m *IAMManager) TagOIDCProvider(ctx context.Context, arn string, tags map[s
 	if m.oidcProviderStore == nil {
 		return fmt.Errorf("OIDC provider store not configured")
 	}
-	rec, err := m.mutableOIDCProvider(ctx, arn)
+	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
 	if err != nil {
 		return err
 	}
@@ -372,7 +297,7 @@ func (m *IAMManager) UntagOIDCProvider(ctx context.Context, arn string, keys []s
 	if m.oidcProviderStore == nil {
 		return fmt.Errorf("OIDC provider store not configured")
 	}
-	rec, err := m.mutableOIDCProvider(ctx, arn)
+	rec, err := m.oidcProviderStore.GetProviderByARN(ctx, m.getFilerAddress(), arn)
 	if err != nil {
 		return err
 	}
@@ -490,88 +415,6 @@ type RoleDefinition struct {
 	// set it must satisfy AWS bounds: 3600 ≤ MaxSessionDuration ≤ 43200.
 	// Honoured by AssumeRole, AssumeRoleWithWebIdentity, AssumeRoleWithCredentials.
 	MaxSessionDuration int64 `json:"maxSessionDuration,omitempty"`
-
-	// Source records where the role came from. RoleSourceStaticConfig marks a
-	// role loaded from the IAM config file; empty means it was created at
-	// runtime. Only static-config roles are pruned when they leave the file.
-	Source string `json:"source,omitempty"`
-
-	// CreatedAt is when the role was created through the IAM API. Zero for
-	// roles loaded from the config file.
-	CreatedAt time.Time `json:"createdAt,omitempty"`
-
-	// RoleId uniquely identifies this role, as AWS's RoleId does. A role
-	// deleted and created again under the same name gets a new ID, and a
-	// session is honoured only while the role it was issued for still has
-	// the ID the session carries — so a session outlives neither the role's
-	// deletion nor a later role that reuses its name.
-	RoleId string `json:"roleId,omitempty"`
-}
-
-// NewRoleID returns a fresh, random role ID in AWS's AROA form.
-func NewRoleID() string {
-	const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
-	b := make([]byte, 17)
-	if _, err := rand.Read(b); err != nil {
-		panic(fmt.Sprintf("read random role id: %v", err))
-	}
-	for i := range b {
-		b[i] = alphabet[int(b[i])%len(alphabet)]
-	}
-	return "AROA" + string(b)
-}
-
-// StaticRoleID is the ID of a role defined in the IAM config file. Such a
-// role is created again at every start, so its ID is derived rather than
-// random, to keep sessions valid across restarts. It is derived from the name
-// and the trust policy together: a role removed from the file and replaced by
-// a different one under the same name trusts different principals, and must
-// not inherit the old role's sessions. Restoring the same role restores its ID.
-func StaticRoleID(role *RoleDefinition) string {
-	h := sha256.New()
-	h.Write([]byte("static-role:" + role.RoleName + "\x00"))
-	if role.TrustPolicy != nil {
-		trust, err := json.Marshal(role.TrustPolicy)
-		if err != nil {
-			// A trust policy that cannot be encoded cannot be matched on
-			// either; give the role an ID no session can carry.
-			return NewRoleID()
-		}
-		h.Write(trust)
-	}
-	return "AROA" + strings.ToUpper(hex.EncodeToString(h.Sum(nil)))[:17]
-}
-
-// RoleSourceStaticConfig is the Source of a role loaded from the IAM config
-// file.
-const RoleSourceStaticConfig = "static-config"
-
-// checkSessionRoleBinding refuses a session issued for a role that has since
-// been deleted, or replaced by a role reusing its name: the role's current ID
-// must be the one the session carries. It runs for every session carrying a
-// role ID, whatever policies the session embeds — the policies a session
-// embeds are the ones its role had, and outlive the role otherwise. A session
-// issued before role IDs were recorded carries none and is not bound. It
-// returns the role it checked, nil for an unbound session.
-func (m *IAMManager) checkSessionRoleBinding(ctx context.Context, sessionInfo *sts.SessionInfo) (*RoleDefinition, error) {
-	if sessionInfo == nil || sessionInfo.RoleId == "" {
-		return nil, nil
-	}
-	roleName := utils.ExtractRoleNameFromArn(sessionInfo.RoleArn)
-	if roleName == "" {
-		return nil, nil
-	}
-	role, err := m.roleStore.GetRole(ctx, m.getFilerAddress(), roleName)
-	if errors.Is(err, ErrRoleNotFound) {
-		return nil, fmt.Errorf("session was issued for role %s, which no longer exists", roleName)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("resolve role %s for session: %w", roleName, err)
-	}
-	if role.RoleId != sessionInfo.RoleId {
-		return nil, fmt.Errorf("session was issued for an earlier role named %s", roleName)
-	}
-	return role, nil
 }
 
 // ActionRequest represents a request to perform an action
@@ -714,174 +557,55 @@ func (m *IAMManager) initOIDCProviderStore(config *IAMConfig) error {
 	if err != nil {
 		return err
 	}
-	m.installOIDCProviderStore(store, config.STS)
-	return nil
-}
-
-// installOIDCProviderStore makes store the manager's OIDC provider store.
-//
-// The IAM config file's providers are reported by the IAM API alongside the
-// stored ones. An in-memory store holds them as records, as it always has. A
-// persistent store never does: it outlives this process and may be shared by
-// S3 servers with different config files, so a record written from one file
-// would outlive its removal from that file and be trusted by every server.
-// Those providers stay in memory (staticOIDCProviders) and STS keeps serving
-// them from its static configuration; the store holds only providers created
-// through the IAM API, and those are loaded into STS here.
-//
-// A provider stored under the same ARN as a config-file provider takes
-// precedence, in the IAM API as in STS, which already prefers IAM-managed
-// providers so that an API call can shadow a bootstrap entry. Deleting the
-// stored provider brings the config-file one back.
-func (m *IAMManager) installOIDCProviderStore(store OIDCProviderStore, stsConfig *sts.STSConfig) {
-	// Cancel the old store's retry and switch stores in one step, so a failed
-	// refresh of the old store cannot start a retry after the cancel.
-	m.oidcRetryMu.Lock()
-	m.stopOIDCRetryLocked()
 	m.oidcProviderStore = store
-	m.oidcRetryMu.Unlock()
-	m.staticOIDCProviders = staticOIDCProviderRecords(stsConfig)
-	if _, inMemory := store.(*MemoryOIDCProviderStore); inMemory {
-		ctx := context.Background()
-		now := time.Now().UTC()
-		for _, rec := range m.staticOIDCProviders {
-			mirrored := copyOIDCProviderRecord(rec)
-			mirrored.CreatedAt, mirrored.UpdatedAt = now, now
-			if err := store.StoreProvider(ctx, m.getFilerAddress(), mirrored); err != nil {
-				glog.Warningf("mirror static OIDC provider %s into store: %v", rec.ARN, err)
-			}
-		}
-		// The store now holds them and the API may change them, as before;
-		// the overlay is only for stores that must not hold them.
-		m.staticOIDCProviders = nil
-		return
-	}
-	// The metadata subscription only reports changes made from now on, so
-	// providers already in the store would stay unknown until one changes; a
-	// failed load is retried (RefreshOIDCProvidersFromStore).
-	if err := m.RefreshOIDCProvidersFromStore(context.Background()); err != nil {
-		glog.Warningf("load OIDC providers from the store at startup: %v; retrying in the background", err)
-	}
-}
 
-// startOIDCRetry retries loading store in the background until it succeeds.
-// A store that is no longer current gets no retry: nothing would cancel it,
-// and its eventual success would replace the current store's providers. When
-// a retry is already running, it is asked to run once more instead, because
-// it may already have listed a snapshot older than this failure.
-func (m *IAMManager) startOIDCRetry(store OIDCProviderStore) {
-	m.oidcRetryMu.Lock()
-	defer m.oidcRetryMu.Unlock()
-	if store != m.oidcProviderStore {
-		return
+	if config.STS == nil {
+		return nil
 	}
-	if m.oidcRetryGen != 0 {
-		m.oidcRetryAgain = true
-		return
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	m.oidcRetrySeq++
-	gen := m.oidcRetrySeq
-	m.cancelOIDCLoad, m.oidcRetryGen = cancel, gen
-	bounds := oidcHydrateRetry // read here, not in the goroutine: it outlives its caller
-	go func() {
-		defer cancel()
-		for {
-			m.retryOIDCProviderLoad(ctx, store, bounds)
-			m.oidcRetryMu.Lock()
-			if m.oidcRetryGen != gen {
-				m.oidcRetryMu.Unlock()
-				return // cancelled: another store was installed
-			}
-			if m.oidcRetryAgain && ctx.Err() == nil {
-				m.oidcRetryAgain = false
-				m.oidcRetryMu.Unlock()
-				continue
-			}
-			m.cancelOIDCLoad, m.oidcRetryGen, m.oidcRetryAgain = nil, 0, false
-			m.oidcRetryMu.Unlock()
-			return
-		}
-	}()
-}
-
-// stopOIDCRetryLocked cancels a running retry, as installing another store
-// must. The caller holds oidcRetryMu.
-func (m *IAMManager) stopOIDCRetryLocked() {
-	if m.cancelOIDCLoad != nil {
-		m.cancelOIDCLoad()
-	}
-	m.cancelOIDCLoad, m.oidcRetryGen, m.oidcRetryAgain = nil, 0, false
-}
-
-// currentOIDCProviderStore is the installed store, read under oidcRetryMu.
-func (m *IAMManager) currentOIDCProviderStore() OIDCProviderStore {
-	m.oidcRetryMu.Lock()
-	defer m.oidcRetryMu.Unlock()
-	return m.oidcProviderStore
-}
-
-// staticOIDCProviderRecords describes the enabled OIDC providers of the IAM
-// config file as provider records.
-func staticOIDCProviderRecords(stsConfig *sts.STSConfig) map[string]*OIDCProviderRecord {
-	out := map[string]*OIDCProviderRecord{}
-	if stsConfig == nil {
-		return out
-	}
-	for _, pc := range stsConfig.Providers {
+	for _, pc := range config.STS.Providers {
 		if pc == nil || !pc.Enabled || pc.Type != sts.ProviderTypeOIDC {
 			continue
 		}
 		issuer, _ := pc.Config["issuer"].(string)
 		if issuer == "" {
-			glog.Warningf("OIDC provider %s in static config has empty issuer; skipping", pc.Name)
+			glog.Warningf("OIDC provider %s in static config has empty issuer; skipping mirror to store", pc.Name)
 			continue
 		}
-		arn, err := DeriveOIDCProviderARN(stsConfig.AccountId, issuer)
+		accountID := ""
+		if config.STS != nil {
+			accountID = config.STS.AccountId
+		}
+		arn, err := DeriveOIDCProviderARN(accountID, issuer)
 		if err != nil {
 			glog.Warningf("derive ARN for static OIDC provider %s: %v", pc.Name, err)
 			continue
 		}
-		out[arn] = &OIDCProviderRecord{
-			AccountID:               stsConfig.AccountId,
+		clientIDs := extractClientIDs(pc.Config)
+		ctx := context.Background()
+		// Preserve CreatedAt across reboots when a persistent store already
+		// has this provider — IAM's GetOpenIDConnectProvider response
+		// shouldn't shift its CreateDate every time the server restarts.
+		now := time.Now().UTC()
+		createdAt := now
+		if existing, err := store.GetProviderByARN(ctx, m.getFilerAddress(), arn); err == nil && existing != nil && !existing.CreatedAt.IsZero() {
+			createdAt = existing.CreatedAt
+		}
+		rec := &OIDCProviderRecord{
+			AccountID:               accountID,
 			ARN:                     arn,
 			URL:                     issuer,
-			ClientIDs:               extractClientIDs(pc.Config),
+			ClientIDs:               clientIDs,
 			Thumbprints:             extractStringList(pc.Config, "thumbprints"),
 			AllowedPrincipalTagKeys: extractStringList(pc.Config, "allowedPrincipalTagKeys"),
 			PolicyClaim:             extractString(pc.Config, "policyClaim"),
-			// No CreatedAt: a config-file provider has no creation time the
-			// server could report consistently across restarts.
+			CreatedAt:               createdAt,
+			UpdatedAt:               now,
 		}
-	}
-	return out
-}
-
-// oidcHydrateRetry bounds the backoff between startup load attempts.
-var oidcHydrateRetry = struct{ initial, max time.Duration }{initial: time.Second, max: 30 * time.Second}
-
-// retryOIDCProviderLoad retries loading store, the store it was started for,
-// until it succeeds or ctx is cancelled because another store was installed.
-// It calls refreshOIDCProvidersFrom, not RefreshOIDCProvidersFromStore, so it
-// never schedules a retry of its own.
-func (m *IAMManager) retryOIDCProviderLoad(ctx context.Context, store OIDCProviderStore, bounds struct{ initial, max time.Duration }) {
-	delay := bounds.initial
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(delay):
-		}
-		err := m.refreshOIDCProvidersFrom(ctx, store)
-		if err == nil {
-			glog.V(0).Infof("loaded OIDC providers from the store after retrying")
-			return
-		}
-		glog.V(1).Infof("load OIDC providers from the store: %v; retrying in %v", err, delay)
-		if delay *= 2; delay > bounds.max {
-			delay = bounds.max
+		if err := store.StoreProvider(ctx, m.getFilerAddress(), rec); err != nil {
+			glog.Warningf("mirror static OIDC provider %s into store: %v", pc.Name, err)
 		}
 	}
+	return nil
 }
 
 // refreshOIDCProvidersBestEffort calls RefreshOIDCProvidersFromStore and
@@ -902,46 +626,14 @@ func (m *IAMManager) refreshOIDCProvidersBestEffort(ctx context.Context, op, arn
 // the store is empty (clears the IAM-managed map). Records with empty URLs
 // or invalid configuration are logged and skipped so a single bad entry
 // does not stop the rest from refreshing.
-//
-// A refresh that fails keeps retrying in the background until the store
-// answers. Every caller needs that: a metadata-subscription event reports each
-// change ONCE, so a refresh that found the filer unreachable on it would leave
-// a peer's new provider untrusted, or a deleted one trusted, until an unrelated
-// later change; the refresh after a local IAM API mutation and the startup load
-// have the same shape. At most one retry runs.
 func (m *IAMManager) RefreshOIDCProvidersFromStore(ctx context.Context) error {
-	store := m.currentOIDCProviderStore()
-	err := m.refreshOIDCProvidersFrom(ctx, store)
-	if err != nil && store != nil {
-		m.startOIDCRetry(store)
-	}
-	return err
-}
-
-// refreshOIDCProvidersFrom is RefreshOIDCProvidersFromStore for a given store.
-func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCProviderStore) error {
-	if store == nil || m.stsService == nil {
+	if m.oidcProviderStore == nil || m.stsService == nil {
 		return nil
 	}
-	// Refreshes run concurrently: after an IAM API change, on a peer's change
-	// and in the startup retry. Unserialized, a refresh that read the store
-	// before a DeleteOIDCProvider could finish after that call's own refresh
-	// and keep the deleted provider trusted.
-	m.oidcRefreshMu.Lock()
-	defer m.oidcRefreshMu.Unlock()
-	records, err := store.ListProviders(ctx, m.getFilerAddress())
+	records, err := m.oidcProviderStore.ListProviders(ctx, m.getFilerAddress())
 	if err != nil {
 		return fmt.Errorf("list OIDC providers: %w", err)
 	}
-	// A snapshot of a store that has since been replaced must not replace the
-	// current store's providers: a retry is cancelled when another store is
-	// installed, and a refresh may have listed the old store just before.
-	if err := ctx.Err(); err != nil {
-		return err
-	}
-	if store != m.currentOIDCProviderStore() {
-		return fmt.Errorf("list OIDC providers: the store was replaced during the refresh")
-	}
 	byIssuer := make(map[string][]sts.ScopedOIDCProvider, len(records))
 	for _, rec := range records {
 		if rec == nil || rec.URL == "" {
@@ -972,11 +664,9 @@ func buildOIDCProviderFromRecord(rec *OIDCProviderRecord) (*oidc.OIDCProvider, e
 		return nil, fmt.Errorf("record cannot be nil")
 	}
 	cfg := &oidc.OIDCConfig{
-		Issuer:                  rec.URL,
-		ClientIDs:               append([]string(nil), rec.ClientIDs...),
-		Thumbprints:             append([]string(nil), rec.Thumbprints...),
-		AllowedPrincipalTagKeys: append([]string(nil), rec.AllowedPrincipalTagKeys...),
-		PolicyClaim:             rec.PolicyClaim,
+		Issuer:      rec.URL,
+		ClientIDs:   append([]string(nil), rec.ClientIDs...),
+		Thumbprints: append([]string(nil), rec.Thumbprints...),
 	}
 	provider := oidc.NewOIDCProvider(rec.ARN)
 	if err := provider.Initialize(cfg); err != nil {
@@ -1110,51 +800,14 @@ func (m *IAMManager) CreateRole(ctx context.Context, filerAddress string, roleNa
 	if !m.initialized {
 		return fmt.Errorf("IAM manager not initialized")
 	}
-	if err := PrepareRoleDefinition(roleName, roleDef); err != nil {
-		return err
-	}
-	if roleDef.RoleId == "" {
-		roleDef.RoleId = NewRoleID()
-	}
 
-	// Store role definition
-	return m.roleStore.StoreRole(ctx, "", roleName, roleDef)
-}
-
-// UpdateRole changes a role atomically in the role store (see
-// RoleStore.UpdateRole). update receives the role's current definition, nil
-// when it does not exist, and its result is validated like CreateRole's; a
-// nil result deletes the role.
-// The IAM API's role actions use it, so a change made on one S3 server is
-// neither lost to a concurrent change on another nor written over a delete.
-func (m *IAMManager) UpdateRole(ctx context.Context, roleName string, update RoleUpdate) error {
-	if !m.initialized {
-		return fmt.Errorf("IAM manager not initialized")
-	}
-	return m.roleStore.UpdateRole(ctx, "", roleName, func(current *RoleDefinition) (*RoleDefinition, error) {
-		next, err := update(current)
-		if err != nil || next == nil {
-			return next, err
-		}
-		if err := PrepareRoleDefinition(roleName, next); err != nil {
-			return nil, err
-		}
-		return next, nil
-	})
-}
-
-// PrepareRoleDefinition applies CreateRole's defaults and validation to a role
-// about to be stored or loaded.
-func PrepareRoleDefinition(roleName string, roleDef *RoleDefinition) error {
 	if roleName == "" {
 		return fmt.Errorf("role name cannot be empty")
 	}
+
 	if roleDef == nil {
 		return fmt.Errorf("role definition cannot be nil")
 	}
-	if roleDef.RoleName == "" {
-		roleDef.RoleName = roleName
-	}
 
 	// Set role ARN if not provided
 	if roleDef.RoleArn == "" {
@@ -1174,122 +827,12 @@ func PrepareRoleDefinition(roleName string, roleDef *RoleDefinition) error {
 			return fmt.Errorf("MaxSessionDuration must be between 3600 and 43200 seconds, got %d", roleDef.MaxSessionDuration)
 		}
 	}
-	return nil
-}
 
-// LoadStaticRoles installs the roles of the IAM config file.
-//
-// An in-memory store holds them as records, as it always has. A persistent
-// store never does: it outlives this process and may be shared by S3 servers
-// with different config files, so a record written from one file would
-// outlive its removal from that file and be honoured by every server. Those
-// roles are served from memory instead, ahead of the store, and cannot be
-// changed or deleted through the store (ErrRoleStatic). A role stored under
-// the same name takes precedence. They report no creation time.
-func (m *IAMManager) LoadStaticRoles(ctx context.Context, roles []*RoleDefinition) {
-	defs := make(map[string]*RoleDefinition, len(roles))
-	for _, role := range roles {
-		if role == nil {
-			continue
-		}
-		role.Source = RoleSourceStaticConfig
-		if role.RoleId == "" {
-			role.RoleId = StaticRoleID(role)
-		}
-		if err := PrepareRoleDefinition(role.RoleName, role); err != nil {
-			glog.Warningf("Failed to load role %s: %v", role.RoleName, err)
-			continue
-		}
-		defs[role.RoleName] = role
-	}
-	m.staticRoles = defs
-	m.installRoleStore(ctx, m.roleStore)
-}
-
-// installRoleStore makes store the role store, with the config-file roles
-// installed in it as LoadStaticRoles describes, so a store set after startup
-// behaves like the one set at startup.
-func (m *IAMManager) installRoleStore(ctx context.Context, store RoleStore) {
-	if overlay, ok := store.(*staticRoleOverlay); ok {
-		store = overlay.inner
-	}
-	if store == nil || m.staticRoles == nil {
-		m.roleStore = store
-		return
-	}
-	if _, inMemory := store.(*MemoryRoleStore); inMemory {
-		for name, role := range m.staticRoles {
-			if err := store.StoreRole(ctx, "", name, role); err != nil {
-				glog.Warningf("Failed to create role %s: %v", name, err)
-			}
-		}
-		m.roleStore = store
-		return
-	}
-	m.roleStore = &staticRoleOverlay{static: m.staticRoles, inner: store}
+	// Store role definition
+	return m.roleStore.StoreRole(ctx, "", roleName, roleDef)
 }
 
 // GetRole retrieves a role definition by name.
-// ListRoles returns every stored role definition.
-func (m *IAMManager) ListRoles(ctx context.Context) ([]*RoleDefinition, error) {
-	if !m.initialized {
-		return nil, fmt.Errorf("IAM manager not initialized")
-	}
-	names, err := m.roleStore.ListRoles(ctx, m.getFilerAddress())
-	if err != nil {
-		return nil, fmt.Errorf("list roles: %w", err)
-	}
-	roles := make([]*RoleDefinition, 0, len(names))
-	for _, name := range names {
-		role, err := m.roleStore.GetRole(ctx, m.getFilerAddress(), name)
-		if errors.Is(err, ErrRoleNotFound) {
-			continue // deleted between list and read
-		}
-		if err != nil {
-			return nil, fmt.Errorf("get role %s: %w", name, err)
-		}
-		roles = append(roles, role)
-	}
-	return roles, nil
-}
-
-// DeleteRole removes a role definition.
-func (m *IAMManager) DeleteRole(ctx context.Context, roleName string) error {
-	if !m.initialized {
-		return fmt.Errorf("IAM manager not initialized")
-	}
-	if roleName == "" {
-		return fmt.Errorf("role name cannot be empty")
-	}
-	return m.roleStore.DeleteRole(ctx, m.getFilerAddress(), roleName)
-}
-
-// InvalidateRoleCache drops any cached role definitions, so a change written
-// to the store by a peer is seen on the next lookup rather than after the
-// cache TTL.
-func (m *IAMManager) InvalidateRoleCache() {
-	if cached, ok := m.roleStore.(interface{ ClearCache() }); ok {
-		cached.ClearCache()
-	}
-}
-
-// RoleStoreDirectory is the filer directory the role store keeps roles in,
-// its configured basePath; empty when the store is not filer-backed. S3
-// servers watch it to drop cached roles when a peer changes one.
-func (m *IAMManager) RoleStoreDirectory() string {
-	store := m.roleStore
-	if overlay, ok := store.(*staticRoleOverlay); ok {
-		store = overlay.inner
-	}
-	if cached, ok := store.(*GenericCachedRoleStore); ok {
-		store = cached.adapter.store
-	}
-	if filerStore, ok := store.(*FilerRoleStore); ok {
-		return filerStore.basePath
-	}
-	return ""
-}
-
 func (m *IAMManager) GetRole(ctx context.Context, roleName string) (*RoleDefinition, error) {
 	if !m.initialized {
 		return nil, fmt.Errorf("IAM manager not initialized")
@@ -1399,8 +942,7 @@ func (m *IAMManager) AssumeRoleWithWebIdentity(ctx context.Context, request *sts
 	// Apply role-level MaxSessionDuration cap. The STS service still applies
 	// the global MaxSessionLength and the source-token-expiry cap on top of
 	// this; per-role takes precedence whenever it is the tightest bound.
-	request.DurationSeconds = capDurationByRole(request.DurationSeconds, roleDef.MaxSessionDuration, m.defaultTokenDurationSeconds(), m.maxSessionLengthSeconds())
-	request.RoleId = roleDef.RoleId
+	request.DurationSeconds = capDurationByRole(request.DurationSeconds, roleDef.MaxSessionDuration)
 
 	// Use STS service to assume the role
 	return m.stsService.AssumeRoleWithWebIdentity(ctx, request)
@@ -1463,54 +1005,22 @@ func extractIssuerFromJWT(token string) (string, error) {
 	return iss, nil
 }
 
-// capDurationByRole returns the session duration clamped to the role's
-// MaxSessionDuration. An omitted DurationSeconds resolves to the configured
-// default first, so the role bound caps defaults and explicit values alike.
-// A nil request is only materialized when something tightened the default
-// and the explicit value still passes the service's own input validation —
-// everything else is left nil so the service resolves the default and its
-// MaxSessionLength cap itself.
-func capDurationByRole(requested *int64, roleMax, defaultSec, serviceMaxSec int64) *int64 {
-	if requested != nil {
-		d := *requested
-		if roleMax > 0 && d > roleMax {
-			d = roleMax
-		}
-		return &d
+// capDurationByRole returns the requested duration clamped to the role's
+// MaxSessionDuration. A nil requested duration is left nil so the STS
+// service's calculateSessionDuration applies the global default (typically
+// 1 hour) — substituting the role's max here would silently mint a 12h
+// session for any caller who omitted DurationSeconds, which AWS does not
+// do. The role-max upper bound still applies in the downstream cap chain
+// once the request has a concrete duration.
+func capDurationByRole(requested *int64, roleMax int64) *int64 {
+	if roleMax <= 0 || requested == nil {
+		return requested
 	}
-	d := defaultSec
-	if roleMax > 0 && d > roleMax {
-		d = roleMax
+	if *requested > roleMax {
+		v := roleMax
+		return &v
 	}
-	if d > serviceMaxSec {
-		d = serviceMaxSec
-	}
-	if d < 900 && roleMax > 0 {
-		d = 900
-	}
-	if d >= defaultSec || d < 900 {
-		return nil
-	}
-	return &d
-}
-
-func (m *IAMManager) defaultTokenDurationSeconds() int64 {
-	if m.stsService == nil || m.stsService.Config == nil {
-		return sts.DefaultTokenDuration
-	}
-	return int64(m.stsService.Config.TokenDuration.Duration / time.Second)
-}
-
-// maxSessionLengthSeconds mirrors validateSessionDurationSeconds so a
-// materialized default stays inside the bound the service will enforce.
-func (m *IAMManager) maxSessionLengthSeconds() int64 {
-	maxSec := int64(sts.DefaultMaxSessionLength)
-	if m.stsService != nil && m.stsService.Config != nil && m.stsService.Config.MaxSessionLength.Duration > 0 {
-		if configured := int64(m.stsService.Config.MaxSessionLength.Duration / time.Second); configured >= 900 {
-			maxSec = configured
-		}
-	}
-	return maxSec
+	return requested
 }
 
 // AssumeRoleWithCredentials assumes a role using credentials (LDAP)
@@ -1534,10 +1044,9 @@ func (m *IAMManager) AssumeRoleWithCredentials(ctx context.Context, request *sts
 	}
 
 	// Apply role-level MaxSessionDuration cap.
-	request.DurationSeconds = capDurationByRole(request.DurationSeconds, roleDef.MaxSessionDuration, m.defaultTokenDurationSeconds(), m.maxSessionLengthSeconds())
+	request.DurationSeconds = capDurationByRole(request.DurationSeconds, roleDef.MaxSessionDuration)
 
 	// Use STS service to assume the role
-	request.RoleId = roleDef.RoleId
 	return m.stsService.AssumeRoleWithCredentials(ctx, request)
 }
 
@@ -1551,10 +1060,6 @@ func (m *IAMManager) IsActionAllowed(ctx context.Context, request *ActionRequest
 	// We always try to validate with the internal STS service first if it's a SeaweedFS token.
 	// This ensures that session policies embedded in the token are correctly extracted and enforced.
 	var sessionInfo *sts.SessionInfo
-	// boundRole is the role a session carrying a role ID was checked against;
-	// its policies are the ones evaluated, so the check and the evaluation
-	// see one definition even if the role is replaced in between.
-	var boundRole *RoleDefinition
 	if request.SessionToken != "" {
 		// Parse unverified to check issuer
 		parsed, _, err := new(jwt.Parser).ParseUnverified(request.SessionToken, jwt.MapClaims{})
@@ -1587,9 +1092,6 @@ func (m *IAMManager) IsActionAllowed(ctx context.Context, request *ActionRequest
 					return false, fmt.Errorf("session has been revoked")
 				}
 			}
-			if boundRole, err = m.checkSessionRoleBinding(ctx, sessionInfo); err != nil {
-				return false, err
-			}
 		}
 	}
 
@@ -1686,12 +1188,9 @@ func (m *IAMManager) IsActionAllowed(ctx context.Context, request *ActionRequest
 				policies = user.GetPolicyNames()
 			} else {
 				// Get role definition
-				roleDef := boundRole
-				if roleDef == nil || roleDef.RoleName != roleName {
-					roleDef, err = m.roleStore.GetRole(ctx, m.getFilerAddress(), roleName)
-					if err != nil {
-						return false, fmt.Errorf("role not found: %s", roleName)
-					}
+				roleDef, err := m.roleStore.GetRole(ctx, m.getFilerAddress(), roleName)
+				if err != nil {
+					return false, fmt.Errorf("role not found: %s", roleName)
 				}
 
 				hasManagedSubject = true

weed/iam/integration/iam_manager_trust.go

@@ -2,30 +2,16 @@ package integration
 
 import (
 	"context"
-	"errors"
 	"fmt"
 
 	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
 	"github.com/seaweedfs/seaweedfs/weed/iam/utils"
 )
 
-// ErrTrustPolicyDenied is wrapped when a role's trust policy does not admit
-// the principal.
-var ErrTrustPolicyDenied = errors.New("trust policy denies access to principal")
-
 // ValidateTrustPolicyForPrincipal validates if a principal is allowed to assume a role
 func (m *IAMManager) ValidateTrustPolicyForPrincipal(ctx context.Context, roleArn, principalArn string) error {
-	_, err := m.ResolveRoleForPrincipal(ctx, roleArn, principalArn)
-	return err
-}
-
-// ResolveRoleForPrincipal returns the role roleArn names if its trust policy
-// admits principalArn. Issuing a session from the definition returned binds
-// the session to the role whose trust was evaluated, not to a role that
-// replaced it under the same name in between.
-func (m *IAMManager) ResolveRoleForPrincipal(ctx context.Context, roleArn, principalArn string) (*RoleDefinition, error) {
 	if !m.initialized {
-		return nil, fmt.Errorf("IAM manager not initialized")
+		return fmt.Errorf("IAM manager not initialized")
 	}
 
 	// Extract role name from ARN
@@ -34,11 +20,11 @@ func (m *IAMManager) ResolveRoleForPrincipal(ctx context.Context, roleArn, princ
 	// Get role definition
 	roleDef, err := m.roleStore.GetRole(ctx, m.getFilerAddress(), roleName)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get role %s: %w", roleName, err)
+		return fmt.Errorf("failed to get role %s: %w", roleName, err)
 	}
 
 	if roleDef.TrustPolicy == nil {
-		return nil, fmt.Errorf("%w: role has no trust policy", ErrTrustPolicyDenied)
+		return fmt.Errorf("role has no trust policy")
 	}
 
 	// Create evaluation context with RequestContext populated so that
@@ -57,8 +43,8 @@ func (m *IAMManager) ResolveRoleForPrincipal(ctx context.Context, roleArn, princ
 
 	// Evaluate the trust policy
 	if !m.evaluateTrustPolicy(roleDef.TrustPolicy, evalCtx) {
-		return nil, fmt.Errorf("%w: %s", ErrTrustPolicyDenied, principalArn)
+		return fmt.Errorf("trust policy denies access to principal: %s", principalArn)
 	}
 
-	return roleDef, nil
+	return nil
 }

weed/iam/integration/oidc_provider_persist_test.go

@@ -1,561 +0,0 @@
-package integration
-
-import (
-	"context"
-	"errors"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/golang-jwt/jwt/v5"
-	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
-	"github.com/seaweedfs/seaweedfs/weed/iam/sts"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// persistentTestStore stands in for a store that outlives the process and may
-// be shared (the filer store): anything but *MemoryOIDCProviderStore is
-// treated as persistent.
-type persistentTestStore struct{ *MemoryOIDCProviderStore }
-
-const (
-	persistTestStaticIssuer = "https://static.example"
-	persistTestAPIIssuer    = "https://api.example"
-)
-
-// persistTestConfig is an IAM config whose file defines the given OIDC
-// issuers.
-func persistTestConfig(issuers ...string) *IAMConfig {
-	var providers []*sts.ProviderConfig
-	for i, issuer := range issuers {
-		providers = append(providers, &sts.ProviderConfig{
-			Name:    "static-" + string(rune('a'+i)),
-			Type:    sts.ProviderTypeOIDC,
-			Enabled: true,
-			Config:  map[string]interface{}{"issuer": issuer, "clientId": "aud"},
-		})
-	}
-	return &IAMConfig{
-		STS: &sts.STSConfig{
-			TokenDuration:    sts.FlexibleDuration{Duration: time.Hour},
-			MaxSessionLength: sts.FlexibleDuration{Duration: 12 * time.Hour},
-			Issuer:           "test-sts",
-			SigningKey:       []byte("test-signing-key-32-characters-long"),
-			Providers:        providers,
-		},
-		Policy: &policy.PolicyEngineConfig{DefaultEffect: "Deny", StoreType: "memory"},
-		Roles:  &RoleStoreConfig{StoreType: "memory"},
-	}
-}
-
-// startServer initializes a manager as an S3 server would at boot, with the
-// given config file providers and OIDC provider store.
-func startServer(t *testing.T, store OIDCProviderStore, issuers ...string) *IAMManager {
-	t.Helper()
-	cfg := persistTestConfig(issuers...)
-	mgr := NewIAMManager()
-	require.NoError(t, mgr.Initialize(cfg, func() string { return "localhost:8888" }))
-	mgr.installOIDCProviderStore(store, cfg.STS)
-	return mgr
-}
-
-func arnOf(t *testing.T, issuer string) string {
-	t.Helper()
-	arn, err := DeriveOIDCProviderARN("", issuer)
-	require.NoError(t, err)
-	return arn
-}
-
-func listedARNs(t *testing.T, mgr *IAMManager) map[string]bool {
-	t.Helper()
-	recs, err := mgr.ListOIDCProviders(context.Background())
-	require.NoError(t, err)
-	out := map[string]bool{}
-	for _, r := range recs {
-		out[r.ARN] = true
-	}
-	return out
-}
-
-func storedARNs(t *testing.T, store OIDCProviderStore) map[string]bool {
-	t.Helper()
-	recs, err := store.ListProviders(context.Background(), "")
-	require.NoError(t, err)
-	out := map[string]bool{}
-	for _, r := range recs {
-		out[r.ARN] = true
-	}
-	return out
-}
-
-func createAPIProvider(t *testing.T, mgr *IAMManager, issuer string) {
-	t.Helper()
-	require.NoError(t, mgr.CreateOIDCProvider(context.Background(), &OIDCProviderRecord{
-		ARN: arnOf(t, issuer), URL: issuer, ClientIDs: []string{"aud"},
-	}))
-}
-
-// stsKnowsIssuer reports whether STS resolves a provider for the issuer. The
-// token is not validly signed; the only question is which error comes back.
-func stsKnowsIssuer(t *testing.T, mgr *IAMManager, issuer string) bool {
-	t.Helper()
-	tok, err := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
-		"iss": issuer, "sub": "probe", "aud": "aud", "exp": time.Now().Add(time.Hour).Unix(),
-	}).SignedString([]byte("not-the-providers-key"))
-	require.NoError(t, err)
-	_, _, err = mgr.GetSTSService().ValidateWebIdentityToken(context.Background(), tok)
-	require.Error(t, err, "an unsigned-by-provider token for %s was accepted", issuer)
-	return !strings.Contains(err.Error(), "no identity provider registered")
-}
-
-// A persistent store never receives the config file's providers: it outlives
-// the file and may be shared with servers whose files differ.
-func TestPersistentStoreNeverHoldsConfigFileProviders(t *testing.T) {
-	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
-	mgr := startServer(t, store, persistTestStaticIssuer)
-
-	assert.Empty(t, storedARNs(t, store), "a config-file provider was written to the persistent store")
-	assert.True(t, listedARNs(t, mgr)[arnOf(t, persistTestStaticIssuer)], "the IAM API no longer lists the config-file provider")
-	rec, err := mgr.GetOIDCProvider(context.Background(), arnOf(t, persistTestStaticIssuer))
-	require.NoError(t, err)
-	assert.Equal(t, persistTestStaticIssuer, rec.URL)
-	assert.True(t, stsKnowsIssuer(t, mgr, persistTestStaticIssuer), "STS stopped trusting the config-file provider")
-}
-
-// Servers sharing a store, with different config files, must not remove or
-// replace each other's providers — including a zero-config server.
-func TestServersSharingAStoreKeepEachOthersProviders(t *testing.T) {
-	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
-	configured := startServer(t, store, persistTestStaticIssuer)
-	createAPIProvider(t, configured, persistTestAPIIssuer)
-
-	zeroConfig := startServer(t, store)
-
-	assert.True(t, storedARNs(t, store)[arnOf(t, persistTestAPIIssuer)], "a peer's start removed an API-created provider")
-	assert.True(t, listedARNs(t, configured)[arnOf(t, persistTestStaticIssuer)], "the configured server lost its config-file provider")
-	assert.False(t, listedARNs(t, zeroConfig)[arnOf(t, persistTestStaticIssuer)], "a server lists a provider only its peer's config file defines")
-	assert.True(t, stsKnowsIssuer(t, zeroConfig, persistTestAPIIssuer), "the peer does not trust the shared API-created provider")
-	assert.False(t, stsKnowsIssuer(t, zeroConfig, persistTestStaticIssuer), "the peer trusts a provider only another server's config file defines")
-}
-
-// Removing a provider from the config file revokes it at the next start.
-func TestRemovingAProviderFromTheConfigFileRevokesIt(t *testing.T) {
-	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
-	startServer(t, store, persistTestStaticIssuer)
-
-	restarted := startServer(t, store) // the file no longer lists it
-
-	assert.False(t, listedARNs(t, restarted)[arnOf(t, persistTestStaticIssuer)])
-	assert.False(t, stsKnowsIssuer(t, restarted, persistTestStaticIssuer), "a provider removed from the config file is still trusted")
-}
-
-// The API cannot change a provider the config file defines, nor create one
-// with its ARN; the file is where it changes.
-func TestConfigFileProvidersCannotBeChangedThroughTheAPI(t *testing.T) {
-	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
-	mgr := startServer(t, store, persistTestStaticIssuer)
-	ctx := context.Background()
-	arn := arnOf(t, persistTestStaticIssuer)
-
-	for name, err := range map[string]error{
-		"add client ID":     mgr.AddClientIDToOIDCProvider(ctx, arn, "other"),
-		"remove client ID":  mgr.RemoveClientIDFromOIDCProvider(ctx, arn, "aud"),
-		"update thumbprint": mgr.UpdateOIDCProviderThumbprints(ctx, arn, []string{"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"}),
-		"tag":               mgr.TagOIDCProvider(ctx, arn, map[string]string{"k": "v"}),
-		"untag":             mgr.UntagOIDCProvider(ctx, arn, []string{"k"}),
-		"delete":            mgr.DeleteOIDCProvider(ctx, arn),
-	} {
-		assert.ErrorIs(t, err, ErrOIDCProviderStatic, name)
-	}
-	err := mgr.CreateOIDCProvider(ctx, &OIDCProviderRecord{ARN: arn, URL: persistTestStaticIssuer, ClientIDs: []string{"aud"}})
-	assert.ErrorIs(t, err, ErrOIDCProviderAlreadyExists)
-	assert.Empty(t, storedARNs(t, store), "a refused change still wrote to the store")
-}
-
-// Providers created through the API on an earlier boot, or on a peer, are
-// trusted at startup rather than after the next change.
-func TestStartupLoadsStoredProvidersIntoSTS(t *testing.T) {
-	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
-	createAPIProvider(t, startServer(t, store), persistTestAPIIssuer)
-
-	restarted := startServer(t, store)
-	assert.True(t, stsKnowsIssuer(t, restarted, persistTestAPIIssuer))
-}
-
-// An in-memory store keeps its behaviour: the config file's providers are
-// records in it, as before.
-func TestInMemoryStoreStillHoldsConfigFileProviders(t *testing.T) {
-	store := NewMemoryOIDCProviderStore()
-	startServer(t, store, persistTestStaticIssuer)
-	assert.True(t, storedARNs(t, store)[arnOf(t, persistTestStaticIssuer)])
-}
-
-// unreachableThenReadyStore fails its first reads, as a filer that is not up
-// yet when the S3 server starts.
-type unreachableThenReadyStore struct {
-	*MemoryOIDCProviderStore
-	mu        sync.Mutex
-	failsLeft int
-}
-
-func (s *unreachableThenReadyStore) ListProviders(ctx context.Context, addr string) ([]*OIDCProviderRecord, error) {
-	s.mu.Lock()
-	if s.failsLeft > 0 {
-		s.failsLeft--
-		s.mu.Unlock()
-		return nil, errors.New("filer unavailable")
-	}
-	s.mu.Unlock()
-	return s.MemoryOIDCProviderStore.ListProviders(ctx, addr)
-}
-
-// A store that cannot be read at startup is retried: the metadata
-// subscription only reports later changes, so without a retry the providers
-// already stored would stay unknown to STS.
-func TestStartupLoadRetriesUntilTheStoreIsReadable(t *testing.T) {
-	saved := oidcHydrateRetry
-	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, 5*time.Millisecond
-	t.Cleanup(func() { oidcHydrateRetry = saved })
-
-	seeded := NewMemoryOIDCProviderStore()
-	require.NoError(t, seeded.StoreProvider(context.Background(), "", &OIDCProviderRecord{
-		ARN: arnOf(t, persistTestAPIIssuer), URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
-	}))
-	mgr := startServer(t, &unreachableThenReadyStore{MemoryOIDCProviderStore: seeded, failsLeft: 3})
-
-	deadline := time.Now().Add(2 * time.Second)
-	for !stsKnowsIssuer(t, mgr, persistTestAPIIssuer) {
-		if time.Now().After(deadline) {
-			t.Fatal("STS never learned the stored provider after the store became readable")
-		}
-		time.Sleep(5 * time.Millisecond)
-	}
-}
-
-// A provider stored under a config-file provider's ARN takes precedence, as it
-// does in STS; the API then changes the stored one, and deleting it brings the
-// config-file provider back.
-func TestStoredProviderTakesPrecedenceOverTheConfigFileOne(t *testing.T) {
-	ctx := context.Background()
-	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
-	configured := startServer(t, store, persistTestStaticIssuer)
-	peer := startServer(t, store)
-	arn := arnOf(t, persistTestStaticIssuer)
-	require.NoError(t, peer.CreateOIDCProvider(ctx, &OIDCProviderRecord{ARN: arn, URL: persistTestStaticIssuer, ClientIDs: []string{"stored"}}))
-
-	rec, err := configured.GetOIDCProvider(ctx, arn)
-	require.NoError(t, err)
-	assert.Equal(t, []string{"stored"}, rec.ClientIDs, "the config-file provider hides the stored one")
-	assert.NoError(t, configured.AddClientIDToOIDCProvider(ctx, arn, "more"), "the stored provider cannot be changed")
-
-	require.NoError(t, configured.DeleteOIDCProvider(ctx, arn))
-	rec, err = configured.GetOIDCProvider(ctx, arn)
-	require.NoError(t, err)
-	assert.Equal(t, []string{"aud"}, rec.ClientIDs, "deleting the stored provider did not bring the config-file one back")
-}
-
-// A store installed through SetOIDCProviderStore behaves like one installed
-// at startup: stored providers load into STS and config-file providers stay
-// visible to the IAM API.
-func TestSetOIDCProviderStoreInstallsLikeStartup(t *testing.T) {
-	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
-	require.NoError(t, store.StoreProvider(context.Background(), "", &OIDCProviderRecord{
-		ARN: arnOf(t, persistTestAPIIssuer), URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
-	}))
-
-	cfg := persistTestConfig(persistTestStaticIssuer)
-	mgr := NewIAMManager()
-	require.NoError(t, mgr.Initialize(cfg, func() string { return "localhost:8888" }))
-	mgr.SetOIDCProviderStore(store)
-
-	assert.True(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer), "a stored provider was not trusted after install")
-	assert.True(t, listedARNs(t, mgr)[arnOf(t, persistTestStaticIssuer)], "the IAM API no longer lists the config-file provider")
-	assert.ErrorIs(t, mgr.DeleteOIDCProvider(context.Background(), arnOf(t, persistTestStaticIssuer)), ErrOIDCProviderStatic)
-}
-
-// A config-file provider reports no creation time: a time taken at startup
-// would change with every restart.
-func TestConfigFileProvidersReportNoCreationTime(t *testing.T) {
-	store := &persistentTestStore{NewMemoryOIDCProviderStore()}
-	mgr := startServer(t, store, persistTestStaticIssuer)
-	rec, err := mgr.GetOIDCProvider(context.Background(), arnOf(t, persistTestStaticIssuer))
-	require.NoError(t, err)
-	assert.True(t, rec.CreatedAt.IsZero())
-}
-
-// countingUnreadableStore never becomes readable and counts the attempts.
-type countingUnreadableStore struct {
-	*MemoryOIDCProviderStore
-	mu    sync.Mutex
-	reads int
-}
-
-func (s *countingUnreadableStore) ListProviders(context.Context, string) ([]*OIDCProviderRecord, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.reads++
-	return nil, errors.New("filer unavailable")
-}
-
-func (s *countingUnreadableStore) readCount() int {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return s.reads
-}
-
-// Installing another store stops the previous one's startup retry.
-func TestInstallingAnotherStoreStopsThePreviousRetry(t *testing.T) {
-	saved := oidcHydrateRetry
-	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, time.Millisecond
-	t.Cleanup(func() { oidcHydrateRetry = saved })
-
-	first := &countingUnreadableStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore()}
-	mgr := startServer(t, first)
-	deadline := time.Now().Add(2 * time.Second)
-	for first.readCount() <= 1 {
-		if time.Now().After(deadline) {
-			t.Fatal("precondition: the first store is never retried")
-		}
-		time.Sleep(time.Millisecond)
-	}
-
-	mgr.installOIDCProviderStore(&persistentTestStore{NewMemoryOIDCProviderStore()}, persistTestConfig().STS)
-	time.Sleep(10 * time.Millisecond) // let an in-flight attempt finish
-	settled := first.readCount()
-	time.Sleep(30 * time.Millisecond)
-	assert.Equal(t, settled, first.readCount(), "the superseded store is still being retried")
-}
-
-// blockingListStore blocks the first ListProviders call after arm, having
-// already read its snapshot, until release is closed.
-type blockingListStore struct {
-	*MemoryOIDCProviderStore
-	mu      sync.Mutex
-	armed   bool
-	entered chan struct{}
-	release chan struct{}
-}
-
-func (s *blockingListStore) arm() {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.armed, s.entered, s.release = true, make(chan struct{}), make(chan struct{})
-}
-
-func (s *blockingListStore) ListProviders(ctx context.Context, addr string) ([]*OIDCProviderRecord, error) {
-	records, err := s.MemoryOIDCProviderStore.ListProviders(ctx, addr)
-	s.mu.Lock()
-	block := s.armed
-	s.armed = false
-	s.mu.Unlock()
-	if block {
-		close(s.entered)
-		<-s.release
-	}
-	return records, err
-}
-
-// A refresh that read the store before a provider was deleted cannot leave the
-// deleted provider trusted by finishing after the deletion's own refresh.
-func TestAnOlderRefreshCannotRestoreADeletedProvider(t *testing.T) {
-	store := &blockingListStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore()}
-	mgr := startServer(t, store)
-	createAPIProvider(t, mgr, persistTestAPIIssuer)
-	require.True(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer), "precondition: the provider is trusted")
-
-	store.arm()
-	stale := make(chan struct{})
-	go func() {
-		defer close(stale)
-		_ = mgr.RefreshOIDCProvidersFromStore(context.Background())
-	}()
-	<-store.entered // the stale refresh holds a snapshot with the provider
-
-	deleted := make(chan error, 1)
-	go func() { deleted <- mgr.DeleteOIDCProvider(context.Background(), arnOf(t, persistTestAPIIssuer)) }()
-	// Serialized, the deletion's refresh waits for the stale one; otherwise
-	// let it finish first, which is the ordering that went wrong.
-	var deleteErr error
-	select {
-	case deleteErr = <-deleted:
-		deleted = nil
-	case <-time.After(200 * time.Millisecond):
-	}
-	close(store.release)
-	<-stale
-	if deleted != nil {
-		deleteErr = <-deleted
-	}
-	require.NoError(t, deleteErr)
-	assert.False(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer), "a refresh older than the deletion left the deleted provider trusted")
-}
-
-// A refresh that fails is retried until the store answers. A change event reports each mutation once, so
-// a subscription refresh that found the filer unreachable — mid-restart, say — left the provider set stale
-// until some unrelated later change; a peer's new provider stayed untrusted and a deleted one trusted.
-func TestAFailedRefreshIsRetriedUntilTheStoreAnswers(t *testing.T) {
-	saved := oidcHydrateRetry
-	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, 5*time.Millisecond
-	t.Cleanup(func() { oidcHydrateRetry = saved })
-
-	store := &unreachableThenReadyStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore()}
-	mgr := startServer(t, store)
-	require.NoError(t, store.StoreProvider(context.Background(), "", &OIDCProviderRecord{
-		ARN: arnOf(t, persistTestAPIIssuer), URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
-	}))
-	store.mu.Lock()
-	store.failsLeft = 3
-	store.mu.Unlock()
-	require.Error(t, mgr.RefreshOIDCProvidersFromStore(context.Background()), "precondition: the refresh fails")
-
-	deadline := time.Now().Add(2 * time.Second)
-	for !stsKnowsIssuer(t, mgr, persistTestAPIIssuer) {
-		if time.Now().After(deadline) {
-			t.Fatal("a failed refresh was never retried: the stored provider stays untrusted until an unrelated change")
-		}
-		time.Sleep(5 * time.Millisecond)
-	}
-}
-
-// Failures during an outage start ONE retry, not one per event: a filer that is down for a while produces a
-// change event per mutation, and each would otherwise add a goroutine polling the same store. Once the store
-// answers and the retry ends, a later failure starts a new one.
-func TestFailedRefreshesShareOneRetry(t *testing.T) {
-	saved := oidcHydrateRetry
-	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, time.Millisecond
-	t.Cleanup(func() { oidcHydrateRetry = saved })
-
-	store := &unreachableThenReadyStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore()}
-	mgr := startServer(t, store)
-	store.mu.Lock()
-	store.failsLeft = 1 << 30
-	store.mu.Unlock()
-	for range 20 {
-		require.Error(t, mgr.RefreshOIDCProvidersFromStore(context.Background()))
-	}
-	mgr.oidcRetryMu.Lock()
-	started := mgr.oidcRetrySeq
-	mgr.oidcRetryMu.Unlock()
-	assert.Equal(t, uint64(1), started, "twenty failed refreshes started more than one retry")
-
-	store.mu.Lock()
-	store.failsLeft = 0
-	store.mu.Unlock()
-	deadline := time.Now().Add(2 * time.Second)
-	for {
-		mgr.oidcRetryMu.Lock()
-		running := mgr.oidcRetryGen != 0
-		mgr.oidcRetryMu.Unlock()
-		if !running {
-			break
-		}
-		if time.Now().After(deadline) {
-			t.Fatal("the retry never ended after the store answered")
-		}
-		time.Sleep(time.Millisecond)
-	}
-	store.mu.Lock()
-	store.failsLeft = 1
-	store.mu.Unlock()
-	require.Error(t, mgr.RefreshOIDCProvidersFromStore(context.Background()))
-	mgr.oidcRetryMu.Lock()
-	assert.Equal(t, uint64(2), mgr.oidcRetrySeq, "a failure after the retry ended starts a new one")
-	mgr.oidcRetryMu.Unlock()
-}
-
-// flakyCountingStore fails while fail is set and counts the lists that succeed.
-type flakyCountingStore struct {
-	*MemoryOIDCProviderStore
-	mu   sync.Mutex
-	fail bool
-	ok   int
-}
-
-func (s *flakyCountingStore) ListProviders(ctx context.Context, addr string) ([]*OIDCProviderRecord, error) {
-	s.mu.Lock()
-	if s.fail {
-		s.mu.Unlock()
-		return nil, errors.New("filer unavailable")
-	}
-	s.ok++
-	s.mu.Unlock()
-	return s.MemoryOIDCProviderStore.ListProviders(ctx, addr)
-}
-
-func (s *flakyCountingStore) set(fail bool) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.fail = fail
-}
-
-func (s *flakyCountingStore) successes() int {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return s.ok
-}
-
-// A refresh of store A that fails as store B is installed must not start a retry for A: nothing would cancel
-// it, and when A answered it would replace B's providers (a removed one trusted again, B's own untrusted).
-func TestAFailedRefreshOfASupersededStoreStartsNoRetry(t *testing.T) {
-	a := &persistentTestStore{NewMemoryOIDCProviderStore()}
-	mgr := startServer(t, a)
-	mgr.SetOIDCProviderStore(&persistentTestStore{NewMemoryOIDCProviderStore()})
-	mgr.oidcRetryMu.Lock()
-	before := mgr.oidcRetrySeq
-	mgr.oidcRetryMu.Unlock()
-
-	mgr.startOIDCRetry(a) // the late call of A's failed refresh, after B's install
-
-	mgr.oidcRetryMu.Lock()
-	defer mgr.oidcRetryMu.Unlock()
-	assert.Equal(t, before, mgr.oidcRetrySeq, "a retry was started for the store that was replaced")
-	assert.Zero(t, mgr.oidcRetryGen)
-}
-
-// A refresh that listed store A before B was installed must not hand STS A's providers afterwards.
-func TestASupersededSnapshotIsNotApplied(t *testing.T) {
-	a := &persistentTestStore{NewMemoryOIDCProviderStore()}
-	require.NoError(t, a.StoreProvider(context.Background(), "", &OIDCProviderRecord{
-		ARN: arnOf(t, persistTestAPIIssuer), URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
-	}))
-	b := &persistentTestStore{NewMemoryOIDCProviderStore()}
-	require.NoError(t, b.StoreProvider(context.Background(), "", &OIDCProviderRecord{
-		ARN: arnOf(t, persistTestStaticIssuer), URL: persistTestStaticIssuer, ClientIDs: []string{"aud"},
-	}))
-	mgr := startServer(t, a)
-	mgr.SetOIDCProviderStore(b)
-
-	assert.Error(t, mgr.refreshOIDCProvidersFrom(context.Background(), a), "a snapshot of the replaced store was applied")
-	assert.False(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer), "the replaced store's provider is trusted")
-	assert.True(t, stsKnowsIssuer(t, mgr, persistTestStaticIssuer), "the current store's provider is not")
-}
-
-// A refresh that fails while a retry runs is not dropped: the retry may already have listed an older snapshot,
-// so after its success it runs once more and picks up whatever the failed refresh would have seen.
-func TestAFailureDuringARetryIsNotDropped(t *testing.T) {
-	saved := oidcHydrateRetry
-	oidcHydrateRetry.initial, oidcHydrateRetry.max = time.Millisecond, 2*time.Millisecond
-	t.Cleanup(func() { oidcHydrateRetry = saved })
-
-	store := &flakyCountingStore{MemoryOIDCProviderStore: NewMemoryOIDCProviderStore(), fail: true}
-	mgr := startServer(t, store) // the startup load fails: a retry is running
-	require.Error(t, mgr.RefreshOIDCProvidersFromStore(context.Background()), "a later refresh fails while it runs")
-	store.set(false)
-
-	deadline := time.Now().Add(2 * time.Second)
-	for {
-		mgr.oidcRetryMu.Lock()
-		running := mgr.oidcRetryGen != 0
-		mgr.oidcRetryMu.Unlock()
-		if !running {
-			break
-		}
-		if time.Now().After(deadline) {
-			t.Fatal("the retry never ended")
-		}
-		time.Sleep(time.Millisecond)
-	}
-	assert.GreaterOrEqual(t, store.successes(), 2, "the retry ended on its first success, dropping the refresh that failed while it ran")
-}

weed/iam/integration/oidc_provider_store.go

@@ -27,9 +27,6 @@ import (
 var (
 	ErrOIDCProviderNotFound      = errors.New("OIDC provider not found")
 	ErrOIDCProviderAlreadyExists = errors.New("OIDC provider already exists")
-	// ErrOIDCProviderStatic refuses a change to a provider defined in the
-	// server's IAM config file: change it there instead.
-	ErrOIDCProviderStatic = errors.New("OIDC provider is defined in the IAM config file")
 )
 
 // OIDCProviderRecord is the persisted, IAM-managed view of an OIDC identity
@@ -270,20 +267,15 @@ func (f *FilerOIDCProviderStore) GetProviderByARN(ctx context.Context, filerAddr
 
 	var data []byte
 	err := f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
-		resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
+		resp, err := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
 			Directory: f.basePath,
 			Name:      f.fileName(arn),
 		})
-		// Only a confirmed absence is ErrOIDCProviderNotFound: callers create
-		// on it, so an unreachable filer must not read as "no such provider".
-		if errors.Is(err, filer_pb.ErrNotFound) {
-			return fmt.Errorf("%w: %s", ErrOIDCProviderNotFound, arn)
-		}
 		if err != nil {
-			return fmt.Errorf("lookup OIDC provider %s: %w", arn, err)
+			return fmt.Errorf("%w: %v", ErrOIDCProviderNotFound, err)
 		}
 		if resp.Entry == nil {
-			return fmt.Errorf("%w: %s", ErrOIDCProviderNotFound, arn)
+			return fmt.Errorf("OIDC provider not found: %s", arn)
 		}
 		data = resp.Entry.Content
 		return nil

weed/iam/integration/role_id_session_binding_test.go

@@ -1,165 +0,0 @@
-package integration
-
-import (
-	"context"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
-	"github.com/seaweedfs/seaweedfs/weed/iam/sts"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// A session is bound to the role it was issued for, not to the role's name:
-// deleting the role ends it, and a role created later under the same name —
-// by whoever may create roles — must not inherit it.
-func TestSessionIsBoundToTheRoleNotItsName(t *testing.T) {
-	ctx := context.Background()
-	m := setupIntegratedIAMSystem(t)
-
-	assume := func() (token, principal string) {
-		resp, err := m.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
-			RoleArn:          "arn:aws:iam::role/S3ReadOnlyRole",
-			WebIdentityToken: createTestJWT(t, "https://test-issuer.com", "test-user-123", "test-signing-key"),
-			RoleSessionName:  "binding-test",
-		})
-		require.NoError(t, err)
-		return resp.Credentials.SessionToken, resp.AssumedRoleUser.Arn
-	}
-	allowed := func(token, principal string) bool {
-		ok, _ := m.IsActionAllowed(ctx, &ActionRequest{
-			Principal:    principal,
-			Action:       "s3:GetObject",
-			Resource:     "arn:aws:s3:::test-bucket/file.txt",
-			SessionToken: token,
-		})
-		return ok
-	}
-
-	original, err := m.GetRole(ctx, "S3ReadOnlyRole")
-	require.NoError(t, err)
-	require.NotEmpty(t, original.RoleId, "a created role has an ID")
-
-	token, principal := assume()
-	require.True(t, allowed(token, principal), "precondition: the session works")
-
-	require.NoError(t, m.DeleteRole(ctx, "S3ReadOnlyRole"))
-	assert.False(t, allowed(token, principal), "the session outlived its role's deletion")
-
-	recreated := *original
-	recreated.RoleId = ""
-	require.NoError(t, m.CreateRole(ctx, "", "S3ReadOnlyRole", &recreated))
-	assert.NotEqual(t, original.RoleId, recreated.RoleId, "a role created again under the same name got the same ID")
-	assert.False(t, allowed(token, principal), "a session of the deleted role works again under a new role of the same name")
-
-	fresh, freshPrincipal := assume()
-	assert.True(t, allowed(fresh, freshPrincipal), "a session issued for the new role is refused")
-}
-
-// A session carries its role's attached policies, and S3 evaluates those
-// rather than looking the role up; the binding must hold on that path too.
-func TestASessionCarryingItsPoliciesIsStillBoundToItsRole(t *testing.T) {
-	ctx := context.Background()
-	m := setupIntegratedIAMSystem(t)
-
-	original, err := m.GetRole(ctx, "S3ReadOnlyRole")
-	require.NoError(t, err)
-	require.NotEmpty(t, original.AttachedPolicies, "precondition: the role attaches a policy")
-	resp, err := m.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
-		RoleArn:          "arn:aws:iam::role/S3ReadOnlyRole",
-		WebIdentityToken: createTestJWT(t, "https://test-issuer.com", "test-user-123", "test-signing-key"),
-		RoleSessionName:  "binding-test",
-	})
-	require.NoError(t, err)
-	allowed := func() bool {
-		ok, _ := m.IsActionAllowed(ctx, &ActionRequest{
-			Principal:    resp.AssumedRoleUser.Arn,
-			Action:       "s3:GetObject",
-			Resource:     "arn:aws:s3:::test-bucket/file.txt",
-			SessionToken: resp.Credentials.SessionToken,
-			PolicyNames:  original.AttachedPolicies,
-		})
-		return ok
-	}
-	require.True(t, allowed(), "precondition: the session works")
-
-	require.NoError(t, m.DeleteRole(ctx, "S3ReadOnlyRole"))
-	assert.False(t, allowed(), "the session's embedded policies outlived its role's deletion")
-
-	recreated := *original
-	recreated.RoleId = ""
-	require.NoError(t, m.CreateRole(ctx, "", "S3ReadOnlyRole", &recreated))
-	assert.False(t, allowed(), "the session's embedded policies work again under a new role of the same name")
-}
-
-func TestStaticRoleIDIsStableAndRuntimeIDsAreUnique(t *testing.T) {
-	trusting := func(principal string) *policy.PolicyDocument {
-		return &policy.PolicyDocument{Version: "2012-10-17", Statement: []policy.Statement{{
-			Effect: "Allow", Action: []string{"sts:AssumeRoleWithWebIdentity"},
-			Principal: map[string]interface{}{"Federated": principal},
-		}}}
-	}
-	app := &RoleDefinition{RoleName: "app", TrustPolicy: trusting("https://a.example")}
-	restarted := &RoleDefinition{RoleName: "app", TrustPolicy: trusting("https://a.example")}
-	replaced := &RoleDefinition{RoleName: "app", TrustPolicy: trusting("https://b.example")}
-
-	assert.Equal(t, StaticRoleID(app), StaticRoleID(restarted), "a config-file role must keep its ID across restarts")
-	assert.NotEqual(t, StaticRoleID(app), StaticRoleID(replaced), "a different role under the same name inherits the old one's ID")
-	assert.NotEqual(t, StaticRoleID(app), StaticRoleID(&RoleDefinition{RoleName: "other", TrustPolicy: trusting("https://a.example")}))
-	assert.Regexp(t, `^AROA[A-Z0-9]{17}$`, StaticRoleID(app))
-	assert.NotEqual(t, NewRoleID(), NewRoleID())
-	assert.Regexp(t, `^AROA[A-Z2-7]{17}$`, NewRoleID())
-}
-
-// replacedAfterFirstReadStore serves the stored role on its first read and a
-// replacement of the same name afterwards: a role replaced while a request
-// is being authorized.
-type replacedAfterFirstReadStore struct {
-	RoleStore
-	reads       int
-	replacement *RoleDefinition
-}
-
-func (s *replacedAfterFirstReadStore) GetRole(ctx context.Context, addr, name string) (*RoleDefinition, error) {
-	s.reads++
-	if s.reads > 1 && name == s.replacement.RoleName {
-		return copyRoleDefinition(s.replacement), nil
-	}
-	return s.RoleStore.GetRole(ctx, addr, name)
-}
-
-// The session's binding is checked against one definition of its role, and
-// that definition's policies are the ones evaluated: a replacement read in
-// between must not lend the session its permissions.
-func TestAuthorizationEvaluatesTheRoleTheBindingCheckSaw(t *testing.T) {
-	ctx := context.Background()
-	m := setupIntegratedIAMSystem(t)
-	require.NoError(t, m.CreatePolicy(ctx, "", "S3WritePolicy", &policy.PolicyDocument{
-		Version: "2012-10-17",
-		Statement: []policy.Statement{{
-			Effect: "Allow", Action: []string{"s3:PutObject"},
-			Resource: []string{"arn:aws:s3:::test-bucket/*"},
-		}},
-	}))
-	resp, err := m.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
-		RoleArn:          "arn:aws:iam::role/S3ReadOnlyRole",
-		WebIdentityToken: createTestJWT(t, "https://test-issuer.com", "test-user-123", "test-signing-key"),
-		RoleSessionName:  "snapshot-test",
-	})
-	require.NoError(t, err)
-
-	original, err := m.GetRole(ctx, "S3ReadOnlyRole")
-	require.NoError(t, err)
-	replacement := *original
-	replacement.RoleId = NewRoleID()
-	replacement.AttachedPolicies = []string{"S3WritePolicy"}
-	m.roleStore = &replacedAfterFirstReadStore{RoleStore: m.roleStore, replacement: &replacement}
-
-	allowed, _ := m.IsActionAllowed(ctx, &ActionRequest{
-		Principal:    resp.AssumedRoleUser.Arn,
-		Action:       "s3:PutObject",
-		Resource:     "arn:aws:s3:::test-bucket/file.txt",
-		SessionToken: resp.Credentials.SessionToken,
-	})
-	assert.False(t, allowed, "the session was authorized by the replacement role's policies")
-}

weed/iam/integration/role_max_session_test.go

@@ -6,27 +6,21 @@ func intPtr(v int64) *int64 { return &v }
 
 func TestCapDurationByRole(t *testing.T) {
 	cases := []struct {
-		name          string
-		requested     *int64
-		roleMax       int64
-		defaultSec    int64
-		serviceMaxSec int64
-		want          *int64
+		name      string
+		requested *int64
+		roleMax   int64
+		want      *int64
 	}{
-		{"no cap, no request -> nil keeps service default", nil, 0, 3600, 43200, nil},
-		{"no cap, with request", intPtr(7200), 0, 3600, 43200, intPtr(7200)},
-		{"cap below default, no request -> cap", nil, 1800, 3600, 43200, intPtr(1800)},
-		{"cap above default, no request -> nil", nil, 43200, 3600, 43200, nil},
-		{"request below cap -> request", intPtr(1800), 3600, 900, 43200, intPtr(1800)},
-		{"request equal cap -> request", intPtr(3600), 3600, 900, 43200, intPtr(3600)},
-		{"request above cap -> cap", intPtr(43200), 3600, 900, 43200, intPtr(3600)},
-		{"default above service max -> service cap materialized", nil, 0, 7200, 3600, intPtr(3600)},
-		{"role bound above service max -> service cap still applies", nil, 40000, 43200, 3600, intPtr(3600)},
-		{"role bound below service floor -> tightest issuable", nil, 500, 3600, 43200, intPtr(900)},
+		{"no cap, no request", nil, 0, nil},
+		{"no cap, with request", intPtr(7200), 0, intPtr(7200)},
+		{"cap only, no request -> nil so STS default applies", nil, 3600, nil},
+		{"request below cap -> request", intPtr(1800), 3600, intPtr(1800)},
+		{"request equal cap -> request", intPtr(3600), 3600, intPtr(3600)},
+		{"request above cap -> cap", intPtr(43200), 3600, intPtr(3600)},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
-			got := capDurationByRole(tc.requested, tc.roleMax, tc.defaultSec, tc.serviceMaxSec)
+			got := capDurationByRole(tc.requested, tc.roleMax)
 			switch {
 			case got == nil && tc.want == nil:
 				return

weed/iam/integration/role_store.go

@@ -3,10 +3,7 @@ package integration
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"io"
-	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -16,38 +13,9 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 )
 
-// ErrRoleNotFound is wrapped by every RoleStore's GetRole when the role does
-// not exist, so callers can tell a missing role from a store that could not
-// be read.
-var ErrRoleNotFound = errors.New("role not found")
-
-// ErrRoleStatic refuses a change to a role defined in the server's IAM config
-// file: change it there instead.
-var ErrRoleStatic = errors.New("role is defined in the IAM config file")
-
-// ErrRoleExists is returned by an UpdateRole whose update refuses a role that
-// already exists (CreateRole).
-var ErrRoleExists = errors.New("role already exists")
-
-// RoleUpdate computes a role's new definition from its current one, nil when
-// the role does not exist. It returns nil to delete the role, and an error to
-// leave it unchanged. It may run more than once: it is called again with the
-// fresh definition when another writer changed the role in between.
-type RoleUpdate func(current *RoleDefinition) (*RoleDefinition, error)
-
-// maxRoleUpdateAttempts bounds UpdateRole's retries under contention.
-const maxRoleUpdateAttempts = 10
-
-// errRoleUpdateContended is returned when the role kept changing under
-// UpdateRole for maxRoleUpdateAttempts reads.
-var errRoleUpdateContended = errors.New("role changed concurrently; retry")
-
 // RoleStore defines the interface for storing IAM role definitions
 type RoleStore interface {
 	// StoreRole stores a role definition (filerAddress ignored for memory stores)
@@ -61,14 +29,6 @@ type RoleStore interface {
 
 	// DeleteRole deletes a role definition (filerAddress ignored for memory stores)
 	DeleteRole(ctx context.Context, filerAddress string, roleName string) error
-
-	// UpdateRole replaces a role with update's result, atomically: the write
-	// lands only if the role is still as update saw it, absent included, and
-	// update is retried against the current role otherwise. Every change the
-	// IAM API and the filer IAM service make to a role goes through it, so
-	// writers on different servers neither lose each other's changes nor bring
-	// back a role deleted in between.
-	UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error
 }
 
 // MemoryRoleStore implements RoleStore using in-memory storage
@@ -112,7 +72,7 @@ func (m *MemoryRoleStore) GetRole(ctx context.Context, filerAddress string, role
 
 	role, exists := m.roles[roleName]
 	if !exists {
-		return nil, fmt.Errorf("%w: %s", ErrRoleNotFound, roleName)
+		return nil, fmt.Errorf("role not found: %s", roleName)
 	}
 
 	// Return a copy to prevent external modifications
@@ -145,26 +105,6 @@ func (m *MemoryRoleStore) DeleteRole(ctx context.Context, filerAddress string, r
 	return nil
 }
 
-// UpdateRole applies update under the store's lock (filerAddress ignored for
-// memory store).
-func (m *MemoryRoleStore) UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error {
-	if roleName == "" {
-		return fmt.Errorf("role name cannot be empty")
-	}
-	m.mutex.Lock()
-	defer m.mutex.Unlock()
-	next, err := update(copyRoleDefinition(m.roles[roleName]))
-	if err != nil {
-		return err
-	}
-	if next == nil {
-		delete(m.roles, roleName)
-		return nil
-	}
-	m.roles[roleName] = copyRoleDefinition(next)
-	return nil
-}
-
 // copyRoleDefinition creates a deep copy of a role definition
 func copyRoleDefinition(original *RoleDefinition) *RoleDefinition {
 	if original == nil {
@@ -172,13 +112,9 @@ func copyRoleDefinition(original *RoleDefinition) *RoleDefinition {
 	}
 
 	copied := &RoleDefinition{
-		RoleName:           original.RoleName,
-		RoleArn:            original.RoleArn,
-		Description:        original.Description,
-		MaxSessionDuration: original.MaxSessionDuration,
-		Source:             original.Source,
-		CreatedAt:          original.CreatedAt,
-		RoleId:             original.RoleId,
+		RoleName:    original.RoleName,
+		RoleArn:     original.RoleArn,
+		Description: original.Description,
 	}
 
 	// Deep copy trust policy if it exists
@@ -199,10 +135,6 @@ func copyRoleDefinition(original *RoleDefinition) *RoleDefinition {
 	return copied
 }
 
-// roleListPageSize is the number of entries FilerRoleStore.ListRoles asks the
-// filer for per page.
-const roleListPageSize = 1000
-
 // FilerRoleStore implements RoleStore using SeaweedFS filer
 type FilerRoleStore struct {
 	grpcDialOption       grpc.DialOption
@@ -281,139 +213,6 @@ func (f *FilerRoleStore) StoreRole(ctx context.Context, filerAddress string, rol
 	})
 }
 
-// UpdateRole reads the role's entry, applies update, and writes the result on
-// the condition that the entry is unchanged since the read — absent, when the
-// role did not exist — so the filer refuses a write racing another writer's
-// change or delete, and update is applied again to what that writer left. A
-// delete is made on the same condition, so it removes the role update saw and
-// not one written after it.
-func (f *FilerRoleStore) UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error {
-	if filerAddress == "" && f.filerAddressProvider != nil {
-		filerAddress = f.filerAddressProvider()
-	}
-	if filerAddress == "" {
-		return fmt.Errorf("filer address is required for FilerRoleStore")
-	}
-	if roleName == "" {
-		return fmt.Errorf("role name cannot be empty")
-	}
-	return f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
-		for attempt := 0; attempt < maxRoleUpdateAttempts; attempt++ {
-			var entry *filer_pb.Entry
-			var current *RoleDefinition
-			resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
-				Directory: f.basePath,
-				Name:      f.getRoleFileName(roleName),
-			})
-			switch {
-			case errors.Is(err, filer_pb.ErrNotFound):
-			case err != nil:
-				return fmt.Errorf("lookup role %s: %w", roleName, err)
-			case resp.Entry != nil:
-				entry = resp.Entry
-				current = &RoleDefinition{}
-				if err := json.Unmarshal(entry.Content, current); err != nil {
-					return fmt.Errorf("failed to deserialize role %s: %v", roleName, err)
-				}
-			}
-
-			next, err := update(current)
-			if err != nil {
-				return err
-			}
-			if next == nil {
-				if entry == nil {
-					return nil
-				}
-				deleted, err := f.deleteRoleEntryIfUnchanged(ctx, client, entry)
-				if err != nil {
-					return fmt.Errorf("failed to delete role %s: %w", roleName, err)
-				}
-				if !deleted {
-					glog.V(3).Infof("Role %s changed before its delete; retrying", roleName)
-					continue
-				}
-				return nil
-			}
-			roleData, err := json.MarshalIndent(next, "", "  ")
-			if err != nil {
-				return fmt.Errorf("failed to serialize role: %v", err)
-			}
-
-			clause := &filer_pb.WriteCondition_Clause{Kind: filer_pb.WriteCondition_IF_NOT_EXISTS}
-			if entry != nil {
-				clause = &filer_pb.WriteCondition_Clause{Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, ExpectedEntry: entry}
-			}
-			now := time.Now().Unix()
-			created, err := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
-				Directory: f.basePath,
-				Entry: &filer_pb.Entry{
-					Name: f.getRoleFileName(roleName),
-					Attributes: &filer_pb.FuseAttributes{
-						Mtime:    now,
-						Crtime:   now,
-						FileMode: uint32(0600),
-					},
-					Content: roleData,
-				},
-				Condition: &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{clause}},
-			})
-			if isRoleWriteConflict(created, err) {
-				glog.V(3).Infof("Role %s changed during update; retrying", roleName)
-				continue
-			}
-			if err != nil {
-				return fmt.Errorf("failed to store role %s: %v", roleName, err)
-			}
-			if created.Error != "" {
-				return fmt.Errorf("failed to store role %s: %s", roleName, created.Error)
-			}
-			return nil
-		}
-		return fmt.Errorf("update role %s: %w", roleName, errRoleUpdateContended)
-	})
-}
-
-// deleteRoleEntryIfUnchanged deletes the role's entry if it still equals
-// entry, reporting false when it changed. The delete is routed and locked as
-// the conditional CreateEntry of the same path is, so the two serialize.
-func (f *FilerRoleStore) deleteRoleEntryIfUnchanged(ctx context.Context, client filer_pb.SeaweedFilerClient, entry *filer_pb.Entry) (bool, error) {
-	fullPath := f.basePath + "/" + entry.Name
-	resp, err := client.ObjectTransaction(ctx, &filer_pb.ObjectTransactionRequest{
-		LockKey:  fullPath,
-		RouteKey: s3_constants.ObjectWriteRouteKeyPrefix + fullPath,
-		Condition: &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{
-			Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, ExpectedEntry: entry,
-		}}},
-		Mutations: []*filer_pb.ObjectMutation{{
-			Type: filer_pb.ObjectMutation_DELETE, Directory: f.basePath, Name: entry.Name, IsDeleteData: true,
-		}},
-	})
-	if err != nil {
-		if status.Code(err) == codes.FailedPrecondition {
-			return false, nil
-		}
-		return false, err
-	}
-	if resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED {
-		return false, nil
-	}
-	if resp.Error != "" {
-		return false, errors.New(resp.Error)
-	}
-	return true, nil
-}
-
-// isRoleWriteConflict reports a write the filer refused because its condition
-// no longer held: in the response, or as FailedPrecondition when the write
-// was forwarded to the entry's owner filer.
-func isRoleWriteConflict(resp *filer_pb.CreateEntryResponse, err error) bool {
-	if err != nil {
-		return status.Code(err) == codes.FailedPrecondition
-	}
-	return resp != nil && resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED
-}
-
 // GetRole retrieves a role definition from filer
 func (f *FilerRoleStore) GetRole(ctx context.Context, filerAddress string, roleName string) (*RoleDefinition, error) {
 	// Use provider function if filerAddress is not provided
@@ -435,16 +234,13 @@ func (f *FilerRoleStore) GetRole(ctx context.Context, filerAddress string, roleN
 		}
 
 		glog.V(3).Infof("Looking up role %s", roleName)
-		response, err := filer_pb.LookupEntry(ctx, client, request)
-		if errors.Is(err, filer_pb.ErrNotFound) {
-			return fmt.Errorf("%w: %s", ErrRoleNotFound, roleName)
-		}
+		response, err := client.LookupDirectoryEntry(ctx, request)
 		if err != nil {
-			return fmt.Errorf("lookup role %s: %w", roleName, err)
+			return fmt.Errorf("role not found: %v", err)
 		}
 
 		if response.Entry == nil {
-			return fmt.Errorf("%w: %s", ErrRoleNotFound, roleName)
+			return fmt.Errorf("role not found")
 		}
 
 		roleData = response.Entry.Content
@@ -474,50 +270,46 @@ func (f *FilerRoleStore) ListRoles(ctx context.Context, filerAddress string) ([]
 		return nil, fmt.Errorf("filer address is required for FilerRoleStore")
 	}
 
-	// Page through the directory, and fail on a broken stream rather than
-	// return what arrived: a truncated list would let DeletePolicy miss a role
-	// that still attaches the policy (RolesAttachingPolicy).
 	var roleNames []string
+
 	err := f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
+		request := &filer_pb.ListEntriesRequest{
+			Directory:          f.basePath,
+			Prefix:             "",
+			StartFromFileName:  "",
+			InclusiveStartFrom: false,
+			Limit:              1000, // Process in batches of 1000
+		}
+
 		glog.V(3).Infof("Listing roles in %s", f.basePath)
-		startFrom := ""
+		stream, err := client.ListEntries(ctx, request)
+		if err != nil {
+			return fmt.Errorf("failed to list roles: %v", err)
+		}
+
 		for {
-			stream, err := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
-				Directory:         f.basePath,
-				StartFromFileName: startFrom,
-				Limit:             roleListPageSize,
-			})
+			resp, err := stream.Recv()
 			if err != nil {
-				return err
+				break // End of stream or error
 			}
-			received := 0
-			for {
-				resp, err := stream.Recv()
-				if errors.Is(err, io.EOF) {
-					break
-				}
-				if err != nil {
-					return err
-				}
-				if resp.Entry == nil {
-					continue
-				}
-				received++
-				startFrom = resp.Entry.Name
-				if resp.Entry.IsDirectory {
-					continue
-				}
-				if name, ok := strings.CutSuffix(resp.Entry.Name, ".json"); ok {
-					roleNames = append(roleNames, name)
-				}
+
+			if resp.Entry == nil || resp.Entry.IsDirectory {
+				continue
 			}
-			if received < roleListPageSize {
-				return nil
+
+			// Extract role name from filename
+			filename := resp.Entry.Name
+			if strings.HasSuffix(filename, ".json") {
+				roleName := strings.TrimSuffix(filename, ".json")
+				roleNames = append(roleNames, roleName)
 			}
 		}
+
+		return nil
 	})
+
 	if err != nil {
-		return nil, fmt.Errorf("failed to list roles: %w", err)
+		return nil, err
 	}
 
 	return roleNames, nil
@@ -596,146 +388,3 @@ type CachedFilerRoleStoreConfig struct {
 	ListTTL      string `json:"listTtl,omitempty"`      // e.g., "1m", "30s"
 	MaxCacheSize int    `json:"maxCacheSize,omitempty"` // Maximum number of cached roles
 }
-
-// RolesAttachingPolicy returns the names of the roles that attach the policy.
-// A policy is attached to a role by name, so deleting it while attached would
-// let a policy created later under the same name take effect on the role;
-// callers refuse the delete instead, as AWS does (DeleteConflict).
-//
-// It sees the stored roles, which every server sharing the store sees, and
-// this server's config-file roles. A role defined only in another server's
-// config file is invisible here: servers sharing a role store may have
-// different config files, but a config-file role that attaches a managed
-// (API-created) policy is protected only on the servers whose file defines it.
-// Keep such roles in every server's file, or attach only config-file policies
-// to config-file roles.
-func RolesAttachingPolicy(ctx context.Context, store RoleStore, policyName string) ([]string, error) {
-	names, err := store.ListRoles(ctx, "")
-	if err != nil {
-		return nil, fmt.Errorf("list roles: %w", err)
-	}
-	var attaching []string
-	for _, name := range names {
-		role, err := store.GetRole(ctx, "", name)
-		if errors.Is(err, ErrRoleNotFound) {
-			continue // deleted between list and read
-		}
-		if err != nil {
-			return nil, fmt.Errorf("get role %s: %w", name, err)
-		}
-		for _, attached := range role.AttachedPolicies {
-			if attached == policyName {
-				attaching = append(attaching, name)
-				break
-			}
-		}
-	}
-	return attaching, nil
-}
-
-// MaxManagedPoliciesPerRole caps the managed policies attached to one role,
-// as AWS's default quota does (and as MaxManagedPoliciesPerUser does for users).
-const MaxManagedPoliciesPerRole = 10
-
-var roleNamePattern = regexp.MustCompile(`^[\w+=,.@-]{1,64}$`)
-
-// ValidateRoleName checks a role name against AWS's rules. A role is stored as
-// <name>.json in the filer, so the rules also keep a name from leaving the
-// role store's directory.
-func ValidateRoleName(name string) error {
-	if !roleNamePattern.MatchString(name) {
-		return fmt.Errorf("invalid role name %q: must be 1-64 characters of letters, digits and +=,.@_-", name)
-	}
-	return nil
-}
-
-// staticRoleOverlay serves the IAM config file's roles from memory beside a
-// persistent store, which never holds them (see IAMManager.LoadStaticRoles).
-// A role stored under the same name takes precedence, as a stored OIDC
-// provider does over a config-file one; deleting it brings the config-file
-// role back. A config-file role itself cannot be stored over or deleted.
-type staticRoleOverlay struct {
-	static map[string]*RoleDefinition
-	inner  RoleStore
-}
-
-// storedRoleExists reports whether the store itself holds the role.
-func (o *staticRoleOverlay) storedRoleExists(ctx context.Context, filerAddress, roleName string) (bool, error) {
-	_, err := o.inner.GetRole(ctx, filerAddress, roleName)
-	if errors.Is(err, ErrRoleNotFound) {
-		return false, nil
-	}
-	return err == nil, err
-}
-
-func (o *staticRoleOverlay) StoreRole(ctx context.Context, filerAddress string, roleName string, role *RoleDefinition) error {
-	if _, ok := o.static[roleName]; ok {
-		stored, err := o.storedRoleExists(ctx, filerAddress, roleName)
-		if err != nil {
-			return err
-		}
-		if !stored {
-			return fmt.Errorf("%w: %s", ErrRoleStatic, roleName)
-		}
-	}
-	return o.inner.StoreRole(ctx, filerAddress, roleName, role)
-}
-
-// UpdateRole refuses to create a stored role over a config-file role, as
-// StoreRole does; a stored role of that name may be changed.
-func (o *staticRoleOverlay) UpdateRole(ctx context.Context, filerAddress string, roleName string, update RoleUpdate) error {
-	_, static := o.static[roleName]
-	return o.inner.UpdateRole(ctx, filerAddress, roleName, func(current *RoleDefinition) (*RoleDefinition, error) {
-		if current == nil && static {
-			return nil, fmt.Errorf("%w: %s", ErrRoleStatic, roleName)
-		}
-		return update(current)
-	})
-}
-
-func (o *staticRoleOverlay) GetRole(ctx context.Context, filerAddress string, roleName string) (*RoleDefinition, error) {
-	role, err := o.inner.GetRole(ctx, filerAddress, roleName)
-	if errors.Is(err, ErrRoleNotFound) {
-		if static, ok := o.static[roleName]; ok {
-			return copyRoleDefinition(static), nil
-		}
-	}
-	return role, err
-}
-
-func (o *staticRoleOverlay) ListRoles(ctx context.Context, filerAddress string) ([]string, error) {
-	names, err := o.inner.ListRoles(ctx, filerAddress)
-	if err != nil {
-		return nil, err
-	}
-	seen := make(map[string]bool, len(names))
-	for _, name := range names {
-		seen[name] = true
-	}
-	for name := range o.static {
-		if !seen[name] {
-			names = append(names, name)
-		}
-	}
-	return names, nil
-}
-
-func (o *staticRoleOverlay) DeleteRole(ctx context.Context, filerAddress string, roleName string) error {
-	if _, ok := o.static[roleName]; ok {
-		stored, err := o.storedRoleExists(ctx, filerAddress, roleName)
-		if err != nil {
-			return err
-		}
-		if !stored {
-			return fmt.Errorf("%w: %s", ErrRoleStatic, roleName)
-		}
-	}
-	return o.inner.DeleteRole(ctx, filerAddress, roleName)
-}
-
-// ClearCache forwards cache invalidation to the store underneath.
-func (o *staticRoleOverlay) ClearCache() {
-	if cached, ok := o.inner.(interface{ ClearCache() }); ok {
-		cached.ClearCache()
-	}
-}

weed/iam/integration/role_store_filer_test.go

@@ -1,286 +0,0 @@
-package integration
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"net"
-	"slices"
-	"sort"
-	"strconv"
-	"sync"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/credentials/insecure"
-	"google.golang.org/grpc/status"
-	"google.golang.org/protobuf/proto"
-)
-
-// roleStoreTestFiler is a filer holding one directory. It evaluates the write
-// conditions FilerRoleStore sends as the filer does, and can run another
-// writer between a role's read and its write (afterLookup) or break a listing
-// stream partway (failListAfter).
-type roleStoreTestFiler struct {
-	filer_pb.UnimplementedSeaweedFilerServer
-	mu            sync.Mutex
-	entries       map[string]*filer_pb.Entry
-	afterLookup   func()
-	failListAfter int
-}
-
-func (s *roleStoreTestFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
-	s.mu.Lock()
-	entry, found := s.entries[req.Name]
-	hook := s.afterLookup
-	s.afterLookup = nil
-	s.mu.Unlock()
-	if hook != nil {
-		defer hook()
-	}
-	if !found {
-		return nil, status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())
-	}
-	return &filer_pb.LookupDirectoryEntryResponse{Entry: proto.Clone(entry).(*filer_pb.Entry)}, nil
-}
-
-func (s *roleStoreTestFiler) CreateEntry(_ context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	current, exists := s.entries[req.Entry.Name]
-	for _, c := range req.GetCondition().GetClauses() {
-		var ok bool
-		switch c.Kind {
-		case filer_pb.WriteCondition_IF_NOT_EXISTS:
-			ok = !exists
-		case filer_pb.WriteCondition_IF_ENTRY_EQUAL:
-			ok = exists && proto.Equal(current, c.ExpectedEntry)
-		default:
-			return nil, fmt.Errorf("unexpected condition %v", c.Kind)
-		}
-		if !ok {
-			return &filer_pb.CreateEntryResponse{Error: "precondition failed", ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
-		}
-	}
-	s.entries[req.Entry.Name] = proto.Clone(req.Entry).(*filer_pb.Entry)
-	return &filer_pb.CreateEntryResponse{}, nil
-}
-
-// ObjectTransaction applies the conditional delete FilerRoleStore sends.
-func (s *roleStoreTestFiler) ObjectTransaction(_ context.Context, req *filer_pb.ObjectTransactionRequest) (*filer_pb.ObjectTransactionResponse, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if len(req.Mutations) != 1 || req.Mutations[0].Type != filer_pb.ObjectMutation_DELETE {
-		return nil, fmt.Errorf("unexpected mutations %v", req.Mutations)
-	}
-	name := req.Mutations[0].Name
-	current, exists := s.entries[name]
-	for _, c := range req.GetCondition().GetClauses() {
-		if c.Kind != filer_pb.WriteCondition_IF_ENTRY_EQUAL {
-			return nil, fmt.Errorf("unexpected condition %v", c.Kind)
-		}
-		if !exists || !proto.Equal(current, c.ExpectedEntry) {
-			return &filer_pb.ObjectTransactionResponse{Error: "precondition failed", ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
-		}
-	}
-	delete(s.entries, name)
-	return &filer_pb.ObjectTransactionResponse{}, nil
-}
-
-func (s *roleStoreTestFiler) ListEntries(req *filer_pb.ListEntriesRequest, stream grpc.ServerStreamingServer[filer_pb.ListEntriesResponse]) error {
-	s.mu.Lock()
-	var names []string
-	for name := range s.entries {
-		if name > req.StartFromFileName {
-			names = append(names, name)
-		}
-	}
-	sort.Strings(names)
-	if req.Limit > 0 && len(names) > int(req.Limit) {
-		names = names[:req.Limit]
-	}
-	page := make([]*filer_pb.Entry, 0, len(names))
-	for _, name := range names {
-		page = append(page, proto.Clone(s.entries[name]).(*filer_pb.Entry))
-	}
-	failAfter := s.failListAfter
-	s.mu.Unlock()
-	for i, entry := range page {
-		if failAfter > 0 && i == failAfter {
-			return status.Error(codes.Unavailable, "filer went away")
-		}
-		if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: entry}); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (s *roleStoreTestFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	delete(s.entries, req.Name)
-	return &filer_pb.DeleteEntryResponse{}, nil
-}
-
-func newTestFilerRoleStore(t *testing.T) (*FilerRoleStore, *roleStoreTestFiler) {
-	t.Helper()
-	lis, err := net.Listen("tcp", "127.0.0.1:0")
-	require.NoError(t, err)
-	filer := &roleStoreTestFiler{entries: map[string]*filer_pb.Entry{}}
-	server := pb.NewGrpcServer()
-	filer_pb.RegisterSeaweedFilerServer(server, filer)
-	go func() { _ = server.Serve(lis) }()
-	t.Cleanup(func() {
-		server.Stop()
-		_ = lis.Close()
-	})
-	host, port, err := net.SplitHostPort(lis.Addr().String())
-	require.NoError(t, err)
-	grpcPort, err := strconv.Atoi(port)
-	require.NoError(t, err)
-	store, err := NewFilerRoleStore(nil, func() string { return string(pb.NewServerAddress(host, 1, grpcPort)) })
-	require.NoError(t, err)
-	store.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
-	return store, filer
-}
-
-func attachPolicy(policyName string) RoleUpdate {
-	return func(current *RoleDefinition) (*RoleDefinition, error) {
-		if current == nil {
-			return nil, ErrRoleNotFound
-		}
-		current.AttachedPolicies = append(current.AttachedPolicies, policyName)
-		return current, nil
-	}
-}
-
-func createRole(roleID string) RoleUpdate {
-	return func(current *RoleDefinition) (*RoleDefinition, error) {
-		if current != nil {
-			return nil, ErrRoleExists
-		}
-		return &RoleDefinition{RoleName: "app", RoleId: roleID}, nil
-	}
-}
-
-// Two servers changing one role: the change written second is applied to
-// what the first left, not to the role as it read it.
-func TestFilerRoleUpdateIsNotLostToAConcurrentChange(t *testing.T) {
-	ctx := context.Background()
-	store, filer := newTestFilerRoleStore(t)
-	require.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA1")))
-
-	filer.afterLookup = func() { assert.NoError(t, store.UpdateRole(ctx, "", "app", attachPolicy("peer"))) }
-	require.NoError(t, store.UpdateRole(ctx, "", "app", attachPolicy("mine")))
-
-	role, err := store.GetRole(ctx, "", "app")
-	require.NoError(t, err)
-	assert.ElementsMatch(t, []string{"peer", "mine"}, role.AttachedPolicies, "one server's change was lost")
-}
-
-// A change racing a delete must not write the role back.
-func TestFilerRoleUpdateDoesNotReviveADeletedRole(t *testing.T) {
-	ctx := context.Background()
-	store, filer := newTestFilerRoleStore(t)
-	require.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA1")))
-
-	filer.afterLookup = func() { assert.NoError(t, store.DeleteRole(ctx, "", "app")) }
-	err := store.UpdateRole(ctx, "", "app", attachPolicy("mine"))
-	assert.ErrorIs(t, err, ErrRoleNotFound)
-
-	_, err = store.GetRole(ctx, "", "app")
-	assert.ErrorIs(t, err, ErrRoleNotFound, "the deleted role was written back")
-}
-
-// Of two creates of one name, the second sees the first's role.
-func TestFilerRoleCreateRefusesARoleCreatedConcurrently(t *testing.T) {
-	ctx := context.Background()
-	store, filer := newTestFilerRoleStore(t)
-
-	filer.afterLookup = func() { assert.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA-FIRST"))) }
-	err := store.UpdateRole(ctx, "", "app", createRole("AROA-SECOND"))
-	assert.ErrorIs(t, err, ErrRoleExists)
-
-	role, err := store.GetRole(ctx, "", "app")
-	require.NoError(t, err)
-	assert.Equal(t, "AROA-FIRST", role.RoleId, "the second create replaced the first role")
-}
-
-var errRoleHasPolicies = errors.New("role has attached policies")
-
-// deleteUnattached deletes the role only if it attaches no policy, as
-// DeleteRole requires.
-func deleteUnattached(current *RoleDefinition) (*RoleDefinition, error) {
-	if current == nil {
-		return nil, ErrRoleNotFound
-	}
-	if len(current.AttachedPolicies) > 0 {
-		return nil, errRoleHasPolicies
-	}
-	return nil, nil
-}
-
-func TestFilerRoleUpdateCanDeleteTheRole(t *testing.T) {
-	ctx := context.Background()
-	store, _ := newTestFilerRoleStore(t)
-	require.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA1")))
-
-	require.NoError(t, store.UpdateRole(ctx, "", "app", deleteUnattached))
-	_, err := store.GetRole(ctx, "", "app")
-	assert.ErrorIs(t, err, ErrRoleNotFound)
-}
-
-// A delete is decided against the role it removes: a policy attached between
-// the check and the delete makes it refuse, not delete the role anyway.
-func TestFilerRoleDeleteIsDecidedAgainstTheRoleItRemoves(t *testing.T) {
-	ctx := context.Background()
-	store, filer := newTestFilerRoleStore(t)
-	require.NoError(t, store.UpdateRole(ctx, "", "app", createRole("AROA1")))
-
-	filer.afterLookup = func() { assert.NoError(t, store.UpdateRole(ctx, "", "app", attachPolicy("peer"))) }
-	err := store.UpdateRole(ctx, "", "app", deleteUnattached)
-	assert.ErrorIs(t, err, errRoleHasPolicies)
-
-	role, err := store.GetRole(ctx, "", "app")
-	require.NoError(t, err, "the role was deleted although a policy was attached to it")
-	assert.Equal(t, []string{"peer"}, role.AttachedPolicies)
-}
-
-func TestFilerRoleListingPagesPastTheFirstThousand(t *testing.T) {
-	store, filer := newTestFilerRoleStore(t)
-	for i := range roleListPageSize + 1 {
-		name := fmt.Sprintf("role-%04d.json", i)
-		filer.entries[name] = &filer_pb.Entry{Name: name}
-	}
-	filer.entries["role-0500"] = &filer_pb.Entry{Name: "role-0500", IsDirectory: true}
-
-	names, err := store.ListRoles(context.Background(), "")
-	require.NoError(t, err)
-	assert.Len(t, names, roleListPageSize+1)
-	assert.True(t, slices.Contains(names, fmt.Sprintf("role-%04d", roleListPageSize)), "the role past the first page is missing")
-}
-
-// A listing cut short must fail: DeletePolicy decides from it whether any
-// role still attaches the policy.
-func TestFilerRoleListingFailsOnABrokenStream(t *testing.T) {
-	store, filer := newTestFilerRoleStore(t)
-	for i := range 5 {
-		name := fmt.Sprintf("role-%d.json", i)
-		filer.entries[name] = &filer_pb.Entry{Name: name}
-	}
-	filer.failListAfter = 3
-
-	_, err := store.ListRoles(context.Background(), "")
-	require.Error(t, err, "a partial listing was returned as complete")
-
-	attaching, err := RolesAttachingPolicy(context.Background(), store, "read")
-	assert.Error(t, err)
-	assert.Nil(t, attaching)
-	assert.False(t, errors.Is(err, ErrRoleNotFound))
-}

weed/iam/integration/role_store_notfound_test.go

@@ -1,14 +0,0 @@
-package integration
-
-import (
-	"context"
-	"errors"
-	"testing"
-)
-
-func TestMemoryRoleStoreWrapsErrRoleNotFound(t *testing.T) {
-	_, err := NewMemoryRoleStore().GetRole(context.Background(), "", "missing")
-	if !errors.Is(err, ErrRoleNotFound) {
-		t.Fatalf("missing role error does not wrap ErrRoleNotFound: %v", err)
-	}
-}

weed/iam/integration/role_store_persist_test.go

@@ -1,196 +0,0 @@
-package integration
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// persistentTestRoleStore stands in for a role store that outlives the process
-// and may be shared (the filer store): anything but *MemoryRoleStore is
-// treated as persistent.
-type persistentTestRoleStore struct {
-	*MemoryRoleStore
-	cleared int
-}
-
-func (s *persistentTestRoleStore) ClearCache() { s.cleared++ }
-
-// startRoleServer initializes a manager as an S3 server would at boot, with the
-// given role store and config file roles.
-func startRoleServer(t *testing.T, store RoleStore, staticRoles ...string) *IAMManager {
-	t.Helper()
-	mgr := NewIAMManager()
-	require.NoError(t, mgr.Initialize(persistTestConfig(), func() string { return "localhost:8888" }))
-	mgr.roleStore = store
-	var defs []*RoleDefinition
-	for _, name := range staticRoles {
-		defs = append(defs, &RoleDefinition{RoleName: name})
-	}
-	mgr.LoadStaticRoles(context.Background(), defs)
-	return mgr
-}
-
-func roleNames(t *testing.T, store RoleStore) map[string]bool {
-	t.Helper()
-	names, err := store.ListRoles(context.Background(), "")
-	require.NoError(t, err)
-	out := map[string]bool{}
-	for _, n := range names {
-		out[n] = true
-	}
-	return out
-}
-
-// A persistent store never receives the config file's roles: it outlives the
-// file and may be shared with servers whose files differ.
-func TestPersistentRoleStoreNeverHoldsConfigFileRoles(t *testing.T) {
-	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
-	mgr := startRoleServer(t, store, "from-file")
-
-	assert.Empty(t, roleNames(t, store), "a config-file role was written to the persistent store")
-	assert.True(t, roleNames(t, mgr.GetRoleStore())["from-file"], "the config-file role is not listed")
-	role, err := mgr.GetRole(context.Background(), "from-file")
-	require.NoError(t, err)
-	assert.Equal(t, RoleSourceStaticConfig, role.Source)
-	assert.Equal(t, StaticRoleID(&RoleDefinition{RoleName: "from-file"}), role.RoleId)
-}
-
-// Servers sharing a store, with different config files, must not remove or
-// replace each other's roles — including a zero-config server.
-func TestServersSharingARoleStoreKeepEachOthersRoles(t *testing.T) {
-	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
-	configured := startRoleServer(t, store, "from-file")
-	require.NoError(t, configured.CreateRole(context.Background(), "", "created-at-runtime", &RoleDefinition{}))
-
-	zeroConfig := startRoleServer(t, store)
-
-	assert.True(t, roleNames(t, store)["created-at-runtime"], "a peer's start removed a role created at runtime")
-	assert.True(t, roleNames(t, configured.GetRoleStore())["from-file"], "the configured server lost its config-file role")
-	_, err := zeroConfig.GetRole(context.Background(), "from-file")
-	assert.ErrorIs(t, err, ErrRoleNotFound, "a server sees a role only its peer's config file defines")
-	_, err = zeroConfig.GetRole(context.Background(), "created-at-runtime")
-	assert.NoError(t, err, "the peer does not see the shared role created at runtime")
-}
-
-// Removing a role from the config file removes it at the next start.
-func TestRemovingARoleFromTheConfigFileRemovesIt(t *testing.T) {
-	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
-	startRoleServer(t, store, "from-file")
-
-	restarted := startRoleServer(t, store) // the file no longer lists it
-
-	_, err := restarted.GetRole(context.Background(), "from-file")
-	assert.ErrorIs(t, err, ErrRoleNotFound, "a role removed from the config file is still served")
-}
-
-// The store refuses to replace or delete a role the config file defines; the
-// file is where it changes.
-func TestConfigFileRolesCannotBeReplacedOrDeletedThroughTheStore(t *testing.T) {
-	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
-	mgr := startRoleServer(t, store, "from-file")
-
-	err := mgr.CreateRole(context.Background(), "", "from-file", &RoleDefinition{})
-	assert.ErrorIs(t, err, ErrRoleStatic)
-	assert.ErrorIs(t, mgr.DeleteRole(context.Background(), "from-file"), ErrRoleStatic)
-	assert.Empty(t, roleNames(t, store), "a refused change still wrote to the store")
-}
-
-// An in-memory store keeps its behaviour: the config file's roles are records
-// in it.
-func TestInMemoryRoleStoreStillHoldsConfigFileRoles(t *testing.T) {
-	store := NewMemoryRoleStore()
-	startRoleServer(t, store, "from-file")
-	assert.True(t, roleNames(t, store)["from-file"])
-}
-
-// Invalidating the role cache reaches the store beneath the config-file roles.
-func TestRoleCacheInvalidationReachesTheStoreBeneathTheOverlay(t *testing.T) {
-	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
-	mgr := startRoleServer(t, store, "from-file")
-	mgr.InvalidateRoleCache()
-	assert.Equal(t, 1, store.cleared)
-}
-
-func TestRoleCopiesKeepSource(t *testing.T) {
-	role := &RoleDefinition{RoleName: "r", Source: RoleSourceStaticConfig}
-	assert.Equal(t, RoleSourceStaticConfig, copyRoleDefinition(role).Source)
-	assert.Equal(t, RoleSourceStaticConfig, genericCopyRoleDefinition(role).Source)
-}
-
-// Config-file roles report no creation time: a time taken at load would
-// change with every restart.
-func TestConfigFileRolesReportNoCreationTime(t *testing.T) {
-	for name, store := range map[string]RoleStore{
-		"persistent": &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()},
-		"in-memory":  NewMemoryRoleStore(),
-	} {
-		t.Run(name, func(t *testing.T) {
-			mgr := startRoleServer(t, store, "from-file")
-			role, err := mgr.GetRole(context.Background(), "from-file")
-			require.NoError(t, err)
-			assert.True(t, role.CreatedAt.IsZero())
-		})
-	}
-}
-
-// A role stored under a config-file role's name takes precedence, as a stored
-// OIDC provider does; it can be changed, and deleting it brings the
-// config-file role back.
-func TestStoredRoleTakesPrecedenceOverTheConfigFileOne(t *testing.T) {
-	ctx := context.Background()
-	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
-	configured := startRoleServer(t, store, "shared-name")
-	peer := startRoleServer(t, store)
-	require.NoError(t, peer.CreateRole(ctx, "", "shared-name", &RoleDefinition{Description: "stored"}))
-
-	role, err := configured.GetRole(ctx, "shared-name")
-	require.NoError(t, err)
-	assert.Equal(t, "stored", role.Description, "the config-file role hides the stored one")
-
-	role.Description = "changed"
-	assert.NoError(t, configured.CreateRole(ctx, "", "shared-name", role), "the stored role cannot be changed")
-
-	require.NoError(t, configured.DeleteRole(ctx, "shared-name"))
-	role, err = configured.GetRole(ctx, "shared-name")
-	require.NoError(t, err)
-	assert.Equal(t, RoleSourceStaticConfig, role.Source, "deleting the stored role did not bring the config-file one back")
-}
-
-// A store installed through SetRoleStore behaves like one installed at
-// startup: the config-file roles stay visible and protected, and it serves
-// the roles it holds.
-func TestSetRoleStoreInstallsLikeStartup(t *testing.T) {
-	mgr := startRoleServer(t, &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}, "from-file")
-
-	store := &persistentTestRoleStore{MemoryRoleStore: NewMemoryRoleStore()}
-	require.NoError(t, store.StoreRole(context.Background(), "", "stored", &RoleDefinition{RoleName: "stored"}))
-	mgr.SetRoleStore(store)
-
-	names := roleNames(t, mgr.GetRoleStore())
-	assert.True(t, names["from-file"], "the config-file role is no longer listed")
-	assert.True(t, names["stored"], "the new store's role is not listed")
-	assert.ErrorIs(t, mgr.GetRoleStore().DeleteRole(context.Background(), "", "from-file"), ErrRoleStatic)
-	assert.False(t, roleNames(t, store)["from-file"], "the config-file role was written into the new store")
-}
-
-// S3 servers watch the directory a filer-backed role store keeps roles in, so
-// a store configured with its own basePath must report it, through the
-// cache and the config-file overlay alike.
-func TestRoleStoreDirectoryIsTheConfiguredBasePath(t *testing.T) {
-	provider := func() string { return "localhost:8888" }
-	cached, err := NewGenericCachedRoleStore(map[string]interface{}{"basePath": "/custom/roles"}, provider)
-	require.NoError(t, err)
-	mgr := startRoleServer(t, cached, "from-file")
-	assert.Equal(t, "/custom/roles", mgr.RoleStoreDirectory())
-
-	uncached, err := NewFilerRoleStore(nil, provider)
-	require.NoError(t, err)
-	mgr.SetRoleStore(uncached)
-	assert.Equal(t, "/etc/iam/roles", mgr.RoleStoreDirectory())
-
-	mgr.SetRoleStore(NewMemoryRoleStore())
-	assert.Empty(t, mgr.RoleStoreDirectory(), "a memory store has no directory")
-}

weed/iam/policy/policy_engine.go

@@ -473,11 +473,8 @@ func (e *PolicyEngine) EvaluateTrustPolicy(ctx context.Context, trustPolicy *Pol
 		return nil, fmt.Errorf("trust policy cannot be nil")
 	}
 
-	// A trust policy is deny-by-default, whatever the engine's DefaultEffect:
-	// a role may be assumed only by a principal a trust statement explicitly
-	// allows.
 	result := &EvaluationResult{
-		Effect: EffectDeny,
+		Effect: Effect(e.config.DefaultEffect),
 		EvaluationDetails: &EvaluationDetails{
 			Principal:         evalCtx.Principal,
 			Action:            evalCtx.Action,
@@ -514,7 +511,7 @@ func (e *PolicyEngine) EvaluateTrustPolicy(ctx context.Context, trustPolicy *Pol
 	// AWS IAM evaluation logic:
 	// 1. If there's an explicit Deny, the result is Deny
 	// 2. If there's an Allow and no Deny, the result is Allow
-	// 3. Otherwise, the implicit deny stands
+	// 3. Otherwise, use the default effect
 	if explicitDeny {
 		result.Effect = EffectDeny
 	} else if hasAllow {

weed/iam/policy/trust_policy_implicit_deny_test.go

@@ -1,80 +0,0 @@
-package policy
-
-import (
-	"context"
-	"encoding/json"
-	"testing"
-)
-
-const subjectBoundTrustPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow",
-	"Principal":{"Federated":"https://oidc.example"},"Action":["sts:AssumeRoleWithWebIdentity"],
-	"Condition":{"StringEquals":{"oidc:sub":"spiffe://example.org/ns/app/sa/app"}}}]}`
-
-func webIdentityContext(sub string) *EvaluationContext {
-	return &EvaluationContext{
-		Principal: "web-identity-user",
-		Action:    "sts:AssumeRoleWithWebIdentity",
-		Resource:  "arn:aws:iam::role/app",
-		RequestContext: map[string]interface{}{
-			"aws:FederatedProvider": "https://oidc.example",
-			"oidc:iss":              "https://oidc.example",
-			"oidc:sub":              sub,
-		},
-	}
-}
-
-// A trust policy is deny-by-default. The engine's DefaultEffect decides
-// requests no identity policy speaks to; it must not let a principal the trust
-// policy does not allow assume the role.
-func TestEvaluateTrustPolicyIsImplicitDenyWhateverTheDefaultEffect(t *testing.T) {
-	var trust PolicyDocument
-	if err := json.Unmarshal([]byte(subjectBoundTrustPolicy), &trust); err != nil {
-		t.Fatalf("parse test trust policy: %v", err)
-	}
-	for _, defaultEffect := range []string{"Allow", "Deny"} {
-		t.Run("defaultEffect="+defaultEffect, func(t *testing.T) {
-			engine := NewPolicyEngine()
-			if err := engine.Initialize(&PolicyEngineConfig{DefaultEffect: defaultEffect, StoreType: "memory"}); err != nil {
-				t.Fatalf("initialize policy engine: %v", err)
-			}
-
-			res, err := engine.EvaluateTrustPolicy(context.Background(), &trust, webIdentityContext("spiffe://example.org/ns/other/sa/x"))
-			if err != nil {
-				t.Fatalf("evaluate trust policy: %v", err)
-			}
-			if res.Effect != EffectDeny {
-				t.Errorf("a subject the trust policy does not allow got %s", res.Effect)
-			}
-
-			res, err = engine.EvaluateTrustPolicy(context.Background(), &trust, webIdentityContext("spiffe://example.org/ns/app/sa/app"))
-			if err != nil {
-				t.Fatalf("evaluate trust policy: %v", err)
-			}
-			if res.Effect != EffectAllow {
-				t.Errorf("the subject the trust policy allows got %s", res.Effect)
-			}
-		})
-	}
-}
-
-func TestEvaluateTrustPolicyExplicitDenyWins(t *testing.T) {
-	var trust PolicyDocument
-	doc := `{"Version":"2012-10-17","Statement":[
-		{"Effect":"Allow","Principal":{"Federated":"https://oidc.example"},"Action":["sts:AssumeRoleWithWebIdentity"]},
-		{"Effect":"Deny","Principal":{"Federated":"https://oidc.example"},"Action":["sts:AssumeRoleWithWebIdentity"],
-		 "Condition":{"StringEquals":{"oidc:sub":"spiffe://example.org/ns/app/sa/app"}}}]}`
-	if err := json.Unmarshal([]byte(doc), &trust); err != nil {
-		t.Fatalf("parse test trust policy: %v", err)
-	}
-	engine := NewPolicyEngine()
-	if err := engine.Initialize(&PolicyEngineConfig{DefaultEffect: "Allow", StoreType: "memory"}); err != nil {
-		t.Fatalf("initialize policy engine: %v", err)
-	}
-	res, err := engine.EvaluateTrustPolicy(context.Background(), &trust, webIdentityContext("spiffe://example.org/ns/app/sa/app"))
-	if err != nil {
-		t.Fatalf("evaluate trust policy: %v", err)
-	}
-	if res.Effect != EffectDeny {
-		t.Errorf("explicit Deny did not win: %s", res.Effect)
-	}
-}

weed/iam/responses.go

@@ -532,83 +532,3 @@ type ListUserTagsResponse struct {
 	} `xml:"ListUserTagsResult"`
 	CommonResponse
 }
-
-// IAMRole is the Role element returned by the role actions.
-type IAMRole struct {
-	Path                     string `xml:"Path"`
-	RoleName                 string `xml:"RoleName"`
-	RoleId                   string `xml:"RoleId"`
-	Arn                      string `xml:"Arn"`
-	CreateDate               string `xml:"CreateDate,omitempty"`
-	AssumeRolePolicyDocument string `xml:"AssumeRolePolicyDocument,omitempty"`
-	Description              string `xml:"Description,omitempty"`
-	MaxSessionDuration       int64  `xml:"MaxSessionDuration,omitempty"`
-}
-
-// IAMAttachedPolicy is one element of ListAttachedRolePolicies.
-type IAMAttachedPolicy struct {
-	PolicyName string `xml:"PolicyName"`
-	PolicyArn  string `xml:"PolicyArn"`
-}
-
-// CreateRoleResponse is the response for CreateRole.
-type CreateRoleResponse struct {
-	XMLName          xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateRoleResponse"`
-	CreateRoleResult struct {
-		Role IAMRole `xml:"Role"`
-	} `xml:"CreateRoleResult"`
-	CommonResponse
-}
-
-// GetRoleResponse is the response for GetRole.
-type GetRoleResponse struct {
-	XMLName       xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetRoleResponse"`
-	GetRoleResult struct {
-		Role IAMRole `xml:"Role"`
-	} `xml:"GetRoleResult"`
-	CommonResponse
-}
-
-// ListRolesResponse is the response for ListRoles.
-type ListRolesResponse struct {
-	XMLName         xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListRolesResponse"`
-	ListRolesResult struct {
-		Roles       []*IAMRole `xml:"Roles>member"`
-		IsTruncated bool       `xml:"IsTruncated"`
-	} `xml:"ListRolesResult"`
-	CommonResponse
-}
-
-// DeleteRoleResponse is the response for DeleteRole.
-type DeleteRoleResponse struct {
-	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteRoleResponse"`
-	CommonResponse
-}
-
-// UpdateAssumeRolePolicyResponse is the response for UpdateAssumeRolePolicy.
-type UpdateAssumeRolePolicyResponse struct {
-	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ UpdateAssumeRolePolicyResponse"`
-	CommonResponse
-}
-
-// AttachRolePolicyResponse is the response for AttachRolePolicy.
-type AttachRolePolicyResponse struct {
-	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ AttachRolePolicyResponse"`
-	CommonResponse
-}
-
-// DetachRolePolicyResponse is the response for DetachRolePolicy.
-type DetachRolePolicyResponse struct {
-	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DetachRolePolicyResponse"`
-	CommonResponse
-}
-
-// ListAttachedRolePoliciesResponse is the response for ListAttachedRolePolicies.
-type ListAttachedRolePoliciesResponse struct {
-	XMLName                        xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListAttachedRolePoliciesResponse"`
-	ListAttachedRolePoliciesResult struct {
-		AttachedPolicies []*IAMAttachedPolicy `xml:"AttachedPolicies>member"`
-		IsTruncated      bool                 `xml:"IsTruncated"`
-	} `xml:"ListAttachedRolePoliciesResult"`
-	CommonResponse
-}

weed/iam/sts/session_claims.go

@@ -65,10 +65,9 @@ type STSSessionClaims struct {
 	TokenType   string `json:"typ"`  // token_type
 
 	// Role information
-	RoleArn     string `json:"role"`          // role_arn
-	RoleId      string `json:"rid,omitempty"` // unique ID of the assumed role
-	AssumedRole string `json:"assumed"`       // assumed_role_user
-	Principal   string `json:"principal"`     // principal_arn
+	RoleArn     string `json:"role"`      // role_arn
+	AssumedRole string `json:"assumed"`   // assumed_role_user
+	Principal   string `json:"principal"` // principal_arn
 
 	// Authorization data
 	Policies []string `json:"pol,omitempty"` // policies (abbreviated)
@@ -137,7 +136,6 @@ func (c *STSSessionClaims) ToSessionInfo(credGen *CredentialGenerator) *SessionI
 		SessionId:        c.SessionId,
 		SessionName:      c.SessionName,
 		RoleArn:          c.RoleArn,
-		RoleId:           c.RoleId,
 		AssumedRoleUser:  c.AssumedRole,
 		Principal:        c.Principal,
 		Policies:         c.Policies,
@@ -198,13 +196,6 @@ func (c *STSSessionClaims) WithRoleInfo(roleArn, assumedRole, principal string)
 }
 
 // WithPolicies sets the policies associated with this session
-// WithRoleId binds the session to the assumed role's unique ID. An empty ID
-// leaves the session unbound (roles that predate role IDs).
-func (c *STSSessionClaims) WithRoleId(roleId string) *STSSessionClaims {
-	c.RoleId = roleId
-	return c
-}
-
 func (c *STSSessionClaims) WithPolicies(policies []string) *STSSessionClaims {
 	c.Policies = policies
 	return c

weed/iam/sts/sts_service.go

@@ -187,11 +187,6 @@ type AssumeRoleWithWebIdentityRequest struct {
 
 	// Policy is an optional session policy (optional)
 	Policy *string `json:"Policy,omitempty"`
-
-	// RoleId is the assumed role's unique ID, set by the IAM manager after it
-	// resolves the role. It is embedded in the session so the session is bound
-	// to this role and not to whatever role later holds the same name.
-	RoleId string `json:"-"`
 }
 
 // AssumeRoleWithCredentialsRequest represents a request to assume role with username/password
@@ -216,9 +211,6 @@ type AssumeRoleWithCredentialsRequest struct {
 
 	// Policy is an optional session policy (optional)
 	Policy *string `json:"Policy,omitempty"`
-
-	// RoleId is set by the IAM manager; see AssumeRoleWithWebIdentityRequest.
-	RoleId string `json:"-"`
 }
 
 // AssumeRoleResponse represents the response from assume role operations
@@ -312,10 +304,6 @@ type SessionInfo struct {
 
 	// ParentUser is the stable hashed identity (sub+iss) derived at federation time.
 	ParentUser string `json:"parentUser,omitempty"`
-
-	// RoleId is the unique ID of the role the session was issued for; empty
-	// for sessions issued before role IDs were recorded.
-	RoleId string `json:"roleId,omitempty"`
 }
 
 // NewSTSService creates a new STS service
@@ -616,7 +604,7 @@ func (s *STSService) AssumeRoleWithWebIdentity(ctx context.Context, request *Ass
 	}
 
 	// 4. Calculate session duration
-	sessionDuration := s.CalculateSessionDuration(request.DurationSeconds)
+	sessionDuration := s.calculateSessionDuration(request.DurationSeconds)
 	expiresAt := time.Now().Add(sessionDuration)
 
 	// 5. Generate session ID and credentials
@@ -700,7 +688,6 @@ func (s *STSService) AssumeRoleWithWebIdentity(ctx context.Context, request *Ass
 	sessionClaims := NewSTSSessionClaims(sessionId, s.Config.Issuer, expiresAt).
 		WithSessionName(request.RoleSessionName).
 		WithRoleInfo(effectiveRoleArn, assumedRoleUser.Arn, assumedRoleUser.Arn).
-		WithRoleId(request.RoleId).
 		WithIdentityProvider(provider.Name(), externalIdentity.UserID, externalIdentity.Issuer).
 		WithMaxDuration(sessionDuration).
 		WithRequestContext(requestContext)
@@ -773,16 +760,16 @@ func (s *STSService) AssumeRoleWithCredentials(ctx context.Context, request *Ass
 	}
 
 	// 4-7. Mint the session
-	return s.issueSession(request.RoleArn, request.RoleId, request.RoleSessionName, sessionPolicy,
+	return s.issueSession(request.RoleArn, request.RoleSessionName, sessionPolicy,
 		request.DurationSeconds, provider.Name(), externalIdentity.UserID)
 }
 
 // issueSession mints temporary credentials and the self-contained JWT that
 // carries the whole session, shared by every assume-role entry point.
-func (s *STSService) issueSession(roleArn, roleId, roleSessionName, sessionPolicy string,
+func (s *STSService) issueSession(roleArn, roleSessionName, sessionPolicy string,
 	durationSeconds *int64, providerName, subject string) (*AssumeRoleResponse, error) {
 
-	sessionDuration := s.CalculateSessionDuration(durationSeconds)
+	sessionDuration := s.calculateSessionDuration(durationSeconds)
 	expiresAt := time.Now().Add(sessionDuration)
 
 	sessionId, err := GenerateSessionId()
@@ -804,7 +791,6 @@ func (s *STSService) issueSession(roleArn, roleId, roleSessionName, sessionPolic
 	sessionClaims := NewSTSSessionClaims(sessionId, s.Config.Issuer, expiresAt).
 		WithSessionName(roleSessionName).
 		WithRoleInfo(roleArn, assumedRoleUser.Arn, assumedRoleUser.Arn).
-		WithRoleId(roleId).
 		WithIdentityProvider(providerName, subject, "").
 		WithMaxDuration(sessionDuration)
 	if sessionPolicy != "" {
@@ -905,9 +891,7 @@ func (s *STSService) AssumeRoleForPrincipal(ctx context.Context, request *Assume
 		return nil, fmt.Errorf("role assumption denied: %w", err)
 	}
 
-	// Iceberg credential vending resolves no role definition, so its sessions
-	// carry no role ID and stay bound by name.
-	return s.issueSession(request.RoleArn, "", request.RoleSessionName, sessionPolicy,
+	return s.issueSession(request.RoleArn, request.RoleSessionName, sessionPolicy,
 		request.DurationSeconds, request.ProviderName, request.Principal)
 }
 
@@ -1136,11 +1120,11 @@ func (s *STSService) validateRoleAssumptionForCredentials(ctx context.Context, r
 	return nil
 }
 
-// CalculateSessionDuration returns the requested DurationSeconds, or the
+// calculateSessionDuration returns the requested DurationSeconds, or the
 // configured TokenDuration default, capped at MaxSessionLength. The source
 // token's exp deliberately plays no part: per AWS semantics the session
 // outlives the (already verified) web identity token.
-func (s *STSService) CalculateSessionDuration(durationSeconds *int64) time.Duration {
+func (s *STSService) calculateSessionDuration(durationSeconds *int64) time.Duration {
 	var duration time.Duration
 	if durationSeconds != nil {
 		duration = time.Duration(*durationSeconds) * time.Second

weed/iam/sts/sts_service_test.go

@@ -30,8 +30,8 @@ func TestCalculateSessionDuration(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			if got := svc.CalculateSessionDuration(tc.durationSeconds); got != tc.want {
-				t.Errorf("CalculateSessionDuration(%v) = %v, want %v", tc.durationSeconds, got, tc.want)
+			if got := svc.calculateSessionDuration(tc.durationSeconds); got != tc.want {
+				t.Errorf("calculateSessionDuration(%v) = %v, want %v", tc.durationSeconds, got, tc.want)
 			}
 		})
 	}

weed/mount/filehandle.go

@@ -211,9 +211,6 @@ func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) {
 }
 
 func (fh *FileHandle) ReleaseHandle() {
-	fhActiveLock := fh.wfs.fhLockTable.AcquireLock("ReleaseHandle", fh.fh, util.ExclusiveLock)
-	defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
-
 	// Release distributed lock before cleaning up, so other mounts can
 	// proceed as soon as this handle is done flushing.
 	if fh.dlmLock != nil {
@@ -222,6 +219,9 @@ func (fh *FileHandle) ReleaseHandle() {
 		glog.V(1).Infof("DLM lock released for inode %d", fh.inode)
 	}
 
+	fhActiveLock := fh.wfs.fhLockTable.AcquireLock("ReleaseHandle", fh.fh, util.ExclusiveLock)
+	defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
+
 	if fh.entryChunkGroup != nil {
 		_ = fh.entryChunkGroup.Close()
 	}

weed/mount/weedfs_file_mkrm.go

@@ -9,7 +9,6 @@ import (
 	"github.com/seaweedfs/go-fuse/v2/fuse"
 	"google.golang.org/protobuf/proto"
 
-	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -73,26 +72,10 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
 		return code
 	}
 
-	// Acquire the DLM lock before the filer create: a lock failure after an
-	// eager create would return EAGAIN while the file stays persisted.
-	var dlmLock *cluster.LiveLock
-	if wfs.lockClient != nil {
-		owner := fmt.Sprintf("mount-%d", wfs.signature)
-		dlmLock = wfs.lockClient.NewBlockingLongLivedLock(
-			string(entryFullPath), owner, lock_manager.LiveLockTTL,
-		)
-		if dlmLock == nil {
-			return fuse.Status(syscall.EAGAIN)
-		}
-	}
-
 	inode, newEntry, code = wfs.createRegularFile(dirFullPath, name, in.Mode, in.Uid, in.Gid, 0, !wfs.option.EagerFilerCreate, true)
 	if code == fuse.Status(syscall.EEXIST) && in.Flags&syscall.O_EXCL == 0 {
 		// Race: another process created the file between our check and create.
-		// Reopen the winner's entry; AcquireHandle takes its own lock.
-		if dlmLock != nil {
-			dlmLock.Stop()
-		}
+		// Reopen the winner's entry.
 		newEntry, _, code = wfs.maybeLoadEntry(entryFullPath)
 		if code != fuse.OK {
 			return code
@@ -117,9 +100,6 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
 		out.OpenFlags = 0
 		return fuse.OK
 	} else if code != fuse.OK {
-		if dlmLock != nil {
-			dlmLock.Stop()
-		}
 		return code
 	} else {
 		inode = wfs.inodeToPath.Lookup(entryFullPath, newEntry.Attributes.Crtime, false, false, inode, true)
@@ -142,15 +122,15 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
 	// persisted the entry, so its handle starts clean.
 	fileHandle.dirtyMetadata = !wfs.option.EagerFilerCreate
 
-	// Create bypasses AcquireHandle, so attach the lock acquired above.
-	// A surviving handle may already hold one; ours is redundant then.
-	if dlmLock != nil {
-		if fileHandle.dlmLock == nil {
-			fileHandle.dlmLock = dlmLock
-			glog.V(1).Infof("DLM lock acquired for new file %s", entryFullPath)
-		} else {
-			dlmLock.Stop()
-		}
+	// Acquire DLM lock for new file creation (Create bypasses AcquireHandle
+	// so we must acquire the lock here). Always lock on Create since file
+	// creation is inherently a write operation.
+	if wfs.lockClient != nil && fileHandle.dlmLock == nil {
+		owner := fmt.Sprintf("mount-%d", wfs.signature)
+		fileHandle.dlmLock = wfs.lockClient.NewBlockingLongLivedLock(
+			string(entryFullPath), owner, lock_manager.LiveLockTTL,
+		)
+		glog.V(1).Infof("DLM lock acquired for new file %s", entryFullPath)
 	}
 
 	out.Fh = uint64(fileHandle.fh)

weed/mount/weedfs_filehandle.go

@@ -2,7 +2,6 @@ package mount
 
 import (
 	"fmt"
-	"syscall"
 
 	"github.com/seaweedfs/go-fuse/v2/fuse"
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
@@ -78,12 +77,6 @@ func (wfs *WFS) AcquireHandle(inode uint64, flags, uid, gid uint32) (fileHandle
 		fileHandle.dlmLock = wfs.lockClient.NewBlockingLongLivedLock(
 			string(path), owner, lock_manager.LiveLockTTL,
 		)
-		if fileHandle.dlmLock == nil {
-			// No lock server is reachable: proceeding would silently drop
-			// cross-mount write serialization, so fail the open instead.
-			wfs.fhMap.ReleaseByHandle(fileHandle.fh)
-			return nil, fuse.Status(syscall.EAGAIN)
-		}
 		glog.V(1).Infof("DLM lock acquired for %s", path)
 	}
 	return fileHandle, fuse.OK

weed/mount/weedfs_rename.go

@@ -9,7 +9,6 @@ import (
 	"syscall"
 
 	"github.com/seaweedfs/go-fuse/v2/fuse"
-	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -17,10 +16,10 @@ import (
 )
 
 // doRename tries the streaming mux first, falling back to unary on transport errors.
-func (wfs *WFS) doRename(ctx context.Context, request *filer_pb.StreamRenameEntryRequest, oldPath, newPath util.FullPath, newPathLock **cluster.LiveLock) error {
+func (wfs *WFS) doRename(ctx context.Context, request *filer_pb.StreamRenameEntryRequest, oldPath, newPath util.FullPath) error {
 	if wfs.streamMutate != nil && wfs.streamMutate.IsAvailable() {
 		err := wfs.streamMutate.Rename(ctx, request, func(resp *filer_pb.StreamRenameEntryResponse) error {
-			return wfs.handleRenameResponse(ctx, resp, newPath, newPathLock)
+			return wfs.handleRenameResponse(ctx, resp)
 		})
 		if err == nil || !errors.Is(err, ErrStreamTransport) {
 			return err // success or application error
@@ -40,7 +39,7 @@ func (wfs *WFS) doRename(ctx context.Context, request *filer_pb.StreamRenameEntr
 				}
 				return fmt.Errorf("dir Rename %s => %s receive: %v", oldPath, newPath, recvErr)
 			}
-			if err := wfs.handleRenameResponse(ctx, resp, newPath, newPathLock); err != nil {
+			if err := wfs.handleRenameResponse(ctx, resp); err != nil {
 				return err
 			}
 		}
@@ -241,102 +240,6 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
 
 	glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath)
 
-	// Acquire DLM locks on both old and new paths to prevent another mount
-	// from opening either path for writing during the rename. Lock in
-	// sorted order to prevent deadlocks when two mounts rename in opposite
-	// directions (A→B vs B→A).
-	//
-	// Skip the old-path lock if this mount already holds it via an open
-	// file handle (otherwise we'd deadlock trying to re-acquire our own lock).
-	// Acquiring before the handle marks below keeps a lock failure from
-	// leaving source handles flagged for a rename that never happened.
-	var heldLocks []*cluster.LiveLock
-	var newPathLock *cluster.LiveLock
-	if wfs.lockClient != nil {
-		// A handle open on the target holds its lock; claiming it up front
-		// keeps the path locked through the rename (its close would otherwise
-		// release it mid-flight) without waiting on our own ownership.
-		var newPathLockFh *FileHandle
-		defer func() {
-			for _, l := range heldLocks {
-				l.Stop()
-			}
-			if newPathLock != nil {
-				// Never adopted by a migrated handle — return it to the
-				// handle it came from if that is still open.
-				if newPathLockFh != nil {
-					if cur, ok := wfs.fhMap.FindFileHandle(newPathLockFh.inode); ok && cur == newPathLockFh {
-						lk := wfs.fhLockTable.AcquireLock("renameDLM", cur.fh, util.ExclusiveLock)
-						if cur.dlmLock == nil {
-							cur.dlmLock = newPathLock
-							newPathLock = nil
-						}
-						wfs.fhLockTable.ReleaseLock(cur.fh, lk)
-					}
-				}
-				if newPathLock != nil {
-					newPathLock.Stop()
-				}
-			}
-		}()
-		owner := fmt.Sprintf("mount-%d", wfs.signature)
-
-		// Check if the source file handle already holds a DLM lock on oldPath
-		oldPathAlreadyLocked := false
-		sourceInode, sourceMapped := wfs.inodeToPath.GetInode(oldPath)
-		if !sourceMapped && oldEntry != nil && oldEntry.Attributes != nil {
-			sourceInode = oldEntry.Attributes.Inode
-		}
-		if sourceInode != 0 {
-			if fh, ok := wfs.fhMap.FindFileHandle(sourceInode); ok {
-				lk := wfs.fhLockTable.AcquireLock("renameDLM", fh.fh, util.ExclusiveLock)
-				oldPathAlreadyLocked = fh.dlmLock != nil
-				wfs.fhLockTable.ReleaseLock(fh.fh, lk)
-			}
-		}
-		targetInode, targetMapped := wfs.inodeToPath.GetInode(newPath)
-		if !targetMapped && newEntry != nil && newEntry.Attributes != nil {
-			targetInode = newEntry.Attributes.Inode
-		}
-		if targetInode != 0 && targetInode != sourceInode {
-			if targetFh, ok := wfs.fhMap.FindFileHandle(targetInode); ok {
-				targetFhLock := wfs.fhLockTable.AcquireLock("renameDLM", targetFh.fh, util.ExclusiveLock)
-				if targetFh.dlmLock != nil {
-					newPathLock = targetFh.dlmLock
-					targetFh.dlmLock = nil
-					newPathLockFh = targetFh
-				}
-				wfs.fhLockTable.ReleaseLock(targetFh.fh, targetFhLock)
-			}
-		}
-
-		// Determine which paths need new DLM locks
-		pathsToLock := []string{}
-		if newPathLock == nil {
-			pathsToLock = append(pathsToLock, string(newPath))
-		}
-		if !oldPathAlreadyLocked && string(oldPath) != string(newPath) {
-			pathsToLock = append(pathsToLock, string(oldPath))
-		}
-		// Sort for consistent lock ordering
-		if len(pathsToLock) == 2 && pathsToLock[0] > pathsToLock[1] {
-			pathsToLock[0], pathsToLock[1] = pathsToLock[1], pathsToLock[0]
-		}
-
-		for _, p := range pathsToLock {
-			dlmLock := wfs.lockClient.NewBlockingLongLivedLock(p, owner, lock_manager.LiveLockTTL)
-			if dlmLock == nil {
-				return fuse.Status(syscall.EAGAIN)
-			}
-			if p == string(newPath) {
-				newPathLock = dlmLock
-			} else {
-				heldLocks = append(heldLocks, dlmLock)
-			}
-		}
-		glog.V(1).Infof("DLM locks acquired for rename %s => %s (oldPathAlreadyLocked=%v)", oldPath, newPath, oldPathAlreadyLocked)
-	}
-
 	// Ensure the source file's metadata exists on the filer before renaming.
 	// Two cases can leave the entry only in the local cache:
 	//   1. deferFilerCreate=true — file handle still open, dirtyMetadata set.
@@ -397,6 +300,41 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
 		}
 	}
 
+	// Acquire DLM locks on both old and new paths to prevent another mount
+	// from opening either path for writing during the rename. Lock in
+	// sorted order to prevent deadlocks when two mounts rename in opposite
+	// directions (A→B vs B→A).
+	//
+	// Skip the old-path lock if this mount already holds it via an open
+	// file handle (otherwise we'd deadlock trying to re-acquire our own lock).
+	if wfs.lockClient != nil {
+		owner := fmt.Sprintf("mount-%d", wfs.signature)
+
+		// Check if the source file handle already holds a DLM lock on oldPath
+		oldPathAlreadyLocked := false
+		if sourceInode, found := wfs.inodeToPath.GetInode(oldPath); found {
+			if fh, ok := wfs.fhMap.FindFileHandle(sourceInode); ok && fh.dlmLock != nil {
+				oldPathAlreadyLocked = true
+			}
+		}
+
+		// Determine which paths need new DLM locks
+		pathsToLock := []string{string(newPath)}
+		if !oldPathAlreadyLocked {
+			pathsToLock = append(pathsToLock, string(oldPath))
+		}
+		// Sort for consistent lock ordering
+		if len(pathsToLock) == 2 && pathsToLock[0] > pathsToLock[1] {
+			pathsToLock[0], pathsToLock[1] = pathsToLock[1], pathsToLock[0]
+		}
+
+		for _, p := range pathsToLock {
+			dlmLock := wfs.lockClient.NewBlockingLongLivedLock(p, owner, lock_manager.LiveLockTTL)
+			defer dlmLock.Stop()
+		}
+		glog.V(1).Infof("DLM locks acquired for rename %s => %s (oldPathAlreadyLocked=%v)", oldPath, newPath, oldPathAlreadyLocked)
+	}
+
 	// update remote filer
 	request := &filer_pb.StreamRenameEntryRequest{
 		OldDirectory: string(oldDir),
@@ -407,7 +345,7 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
 	}
 
 	ctx := context.Background()
-	err := wfs.doRename(ctx, request, oldPath, newPath, &newPathLock)
+	err := wfs.doRename(ctx, request, oldPath, newPath)
 	if err != nil {
 		glog.V(0).Infof("Rename %s => %s: %v", oldPath, newPath, err)
 		// Map error strings to FUSE status codes. String matching is used
@@ -439,7 +377,7 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
 
 }
 
-func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamRenameEntryResponse, renameNewPath util.FullPath, renameNewPathLock **cluster.LiveLock) error {
+func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamRenameEntryResponse) error {
 	// comes from filer StreamRenameEntry, can only be create or delete entry
 
 	glog.V(4).Infof("dir Rename %+v", resp.EventNotification)
@@ -473,32 +411,16 @@ func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamR
 				// Migrate the DLM lock from old path to new path so the
 				// lock key matches the current file location. Hold the
 				// fhLockTable to prevent ReleaseHandle from concurrently
-				// stopping the lock during migration. Acquire before
-				// releasing: a failed migration keeps the old lock rather
-				// than leaving the handle unlocked.
-				if wfs.lockClient != nil && oldPath != newPath {
+				// stopping the lock during migration.
+				if wfs.lockClient != nil {
 					fhActiveLock := wfs.fhLockTable.AcquireLock("renameDLM", fh.fh, util.ExclusiveLock)
 					if fh.dlmLock != nil {
-						var newLock *cluster.LiveLock
-						if newPath == renameNewPath && renameNewPathLock != nil && *renameNewPathLock != nil {
-							// The rename already holds a lock on the target;
-							// adopt it — re-acquiring our own lock would block
-							// until this handle is released.
-							newLock = *renameNewPathLock
-							*renameNewPathLock = nil
-						} else {
-							owner := fmt.Sprintf("mount-%d", wfs.signature)
-							newLock = wfs.lockClient.NewBlockingLongLivedLock(
-								string(newPath), owner, lock_manager.LiveLockTTL,
-							)
-						}
-						if newLock != nil {
-							fh.dlmLock.Stop()
-							fh.dlmLock = newLock
-							glog.V(1).Infof("DLM lock migrated from %s to %s", oldPath, newPath)
-						} else {
-							glog.Warningf("DLM lock migration to %s failed; keeping lock on %s", newPath, oldPath)
-						}
+						owner := fmt.Sprintf("mount-%d", wfs.signature)
+						fh.dlmLock.Stop()
+						fh.dlmLock = wfs.lockClient.NewBlockingLongLivedLock(
+							string(newPath), owner, lock_manager.LiveLockTTL,
+						)
+						glog.V(1).Infof("DLM lock migrated from %s to %s", oldPath, newPath)
 					}
 					wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
 				}