remote sync: keep the event entry intact for IF_ENTRY_EQUAL

A FailedPrecondition substring in an unrelated error would have been
swallowed as a skipped stamp; status.FromError already unwraps.

.github/dependabot.yml

@@ -10,5 +10,3 @@ updates:
   directory: "/"
   schedule:
     interval: weekly
-  ignore:
-    - dependency-name: "github.com/seaweedfs/goexif"

.github/scripts/helm_jwt_keys.py

@@ -1,240 +0,0 @@
-#!/usr/bin/env python3
-"""Check JWT extraction and, with --context, real Helm upgrade persistence."""
-
-import argparse
-import base64
-import json
-from pathlib import Path
-import shutil
-import subprocess
-import sys
-import tempfile
-import uuid
-
-try:
-    import tomllib
-except ModuleNotFoundError:  # CI also exercises Python 3.10.
-    import tomli as tomllib
-
-
-ROOT = Path(__file__).resolve().parents[2]
-CHART = ROOT / "k8s/charts/seaweedfs"
-SECTIONS = ("jwt.signing", "jwt.signing.read", "jwt.filer_signing", "jwt.filer_signing.read")
-KEYS = {section: f"active-{index}" for index, section in enumerate(SECTIONS)}
-ESCAPED_HEADER = '["jw\\u0074".signing]\nkey = "existing"'
-
-
-def run(*args):
-    return subprocess.run(args, check=True, text=True, capture_output=True).stdout
-
-
-def keys(raw):
-    document = tomllib.loads(raw)
-    result = {}
-    for section in SECTIONS:
-        table = document
-        for part in section.split("."):
-            table = table.get(part, {})
-        if "key" in table:
-            result[section] = table["key"]
-    return result
-
-
-def fixtures():
-    canonical = "\n".join(f'[{section}]\nkey = "{value}"' for section, value in KEYS.items())
-    yield "canonical / final line without newline", canonical
-    yield "commented stale keys", canonical.replace("key =", '# key = "stale"\nkey =')
-    yield "commented headers / mixed line endings", "\n".join(
-        f'# [{section}]\r\n# key = "stale"\r\n[{section}]\nkey = "{value}"'
-        for section, value in KEYS.items()
-    )
-    yield "indented headers and keys / trailing comments", "\n".join(
-        f' \t[ {section} ] \t# header [note]\n \tkey \t= \t"{value}" # key comment'
-        for section, value in KEYS.items()
-    )
-    yield "CRLF", canonical.replace("\n", "\r\n")
-    yield "quoted and spaced section names", "\n".join(
-        f'["{section.split(".")[0]}" . \'{section.split(".")[1]}\''
-        + (f' . "{section.split(".")[2]}"' if section.count(".") == 2 else "")
-        + f'] # original table\nkey = "{value}"'
-        for section, value in KEYS.items()
-    )
-    yield "unrelated quoted header before JWT keys", '["custom section"]\nkey = "other"\n' + canonical
-    yield "brackets in comments", canonical.replace("key =", "# consult [notes]\nkey =")
-    yield "quoted values and quoted key names", "\n".join((
-        '[jwt.signing]\n"key" = "brackets[inside]#value"',
-        "[jwt.signing.read]\n'key' = 'literal\\path[#value]'",
-        r'[jwt.filer_signing]' + '\n' + r'key = "escaped\"quote\\slash\u0041"',
-        '[jwt.filer_signing.read]\nkey = ""',
-    ))
-    yield "absent keys and sections / unrelated key", '\n'.join((
-        '[jwt.signing]\nexpires_after_seconds = 10',
-        '[jwt.signing.read]\nkey = "read-only"',
-        '[unrelated]\nkey = "not-a-jwt-key"',
-        '# [jwt.filer_signing]\n# key = "not-active"',
-    ))
-    yield "no existing security.toml", ""
-
-
-def check_generated(value):
-    decoded = base64.b64decode(value, validate=True).decode("ascii")
-    assert len(decoded) == 10 and decoded.isascii() and decoded.isalnum(), "invalid generated JWT key"
-
-
-def check_helpers(helm, reference_helm):
-    # Exercise the real helper, not a second implementation of its matching rules.
-    with tempfile.TemporaryDirectory(prefix="helm-jwt-helper-") as directory:
-        chart = Path(directory)
-        (chart / "templates").mkdir()
-        (chart / "Chart.yaml").write_text("apiVersion: v2\nname: jwt-regression\nversion: 0.0.0\n")
-        shutil.copyfile(CHART / "templates/shared/_helpers.tpl", chart / "templates/_helpers.tpl")
-        entries = [
-            json.dumps(section) + ': {{ include "seaweedfs.existingTomlKey" (list '
-            + json.dumps(section) + ' .Values.raw) | toJson }}'
-            for section in SECTIONS
-        ]
-        template = chart / "templates/keys.yaml"
-        prefix = '{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"keys"},"data":{'
-        helper_template = prefix + ",".join(entries) + "}}"
-        reference_entries = [
-            json.dumps(section) + ': {{ dig '
-            + " ".join(json.dumps(part) for part in section.split("."))
-            + ' "key" "__ABSENT__" (fromToml .Values.raw) | toJson }}'
-            for section in SECTIONS
-        ]
-        reference_template = prefix + ",".join(reference_entries) + "}}"
-
-        def render(binary, source, raw):
-            template.write_text(source)
-            values = chart / "input.json"
-            values.write_text(json.dumps({"raw": raw}))
-            output = run(binary, "template", "keys", str(chart), "-f", str(values))
-            return json.loads(output[output.index("{"):])["data"]
-
-        for name, raw in fixtures():
-            expected = keys(raw)
-            tokens = render(helm, helper_template, raw)
-            actual = {section: tomllib.loads("key = " + token)["key"]
-                      for section, token in tokens.items() if token != ""}
-            assert actual == expected, f"{name}: extracted keys differ from stored TOML"
-            if reference_helm:
-                reference = render(reference_helm, reference_template, raw)
-                reference = {section: value for section, value in reference.items() if value != "__ABSENT__"}
-                assert actual == reference, f"{name}: keys differ from fromToml/dig"
-            print(f"PASS helper: {name}")
-
-        # A present but unsupported value must not silently become a fresh key.
-        try:
-            render(helm, helper_template, '[jwt.signing]\nkey = """multi\nline"""')
-        except subprocess.CalledProcessError as error:
-            assert "refusing to replace an existing key" in error.stderr, error.stderr
-        else:
-            raise AssertionError("multiline existing key was silently accepted or replaced")
-        print("PASS helper: unsupported existing value fails without rotation")
-
-        try:
-            render(helm, helper_template, ESCAPED_HEADER)
-        except subprocess.CalledProcessError as error:
-            assert "unsupported quoted section header" in error.stderr, error.stderr
-        else:
-            raise AssertionError("unsupported quoted header silently rotated its key")
-        print("PASS helper: unsupported quoted header fails without rotation")
-
-
-def check_upgrades(helm, context):
-    namespace = "jwt-key-persist-" + uuid.uuid4().hex[:8]
-    current = "jk-seaweedfs-security-config"
-    legacy = "seaweedfs-security-config"
-    kubectl = ["kubectl", "--context", context, "-n", namespace]
-    release_args = ["jk", str(CHART), "--kube-context", context, "-n", namespace]
-    # No workload is needed to exercise Helm's real ConfigMap lookup and update.
-    for setting in (
-        "master.enabled=false", "volume.enabled=false", "filer.enabled=false",
-        "global.seaweedfs.createClusterRole=false",
-        "global.seaweedfs.securityConfig.jwtSigning.volumeWrite=true",
-        "global.seaweedfs.securityConfig.jwtSigning.volumeRead=true",
-        "global.seaweedfs.securityConfig.jwtSigning.filerWrite=true",
-        "global.seaweedfs.securityConfig.jwtSigning.filerRead=true",
-    ):
-        release_args += ["--set", setting]
-
-    def stored():
-        cm = json.loads(run(*kubectl, "get", "configmap", current, "-o", "json"))
-        return keys(cm["data"]["security.toml"])
-
-    def upgrade():
-        run(helm, "upgrade", *release_args)
-        return stored()
-
-    def patch(raw):
-        # Seed previous-release content without taking Helm 4's SSA ownership.
-        run(*kubectl, "patch", "configmap", current, "--type=merge", "--field-manager=helm", "-p",
-            json.dumps({"data": {"security.toml": raw}}))
-
-    run(*kubectl, "create", "namespace", namespace)
-    try:
-        run(helm, "install", *release_args)
-        initial = stored()
-        assert set(initial) == set(SECTIONS), "install omitted a JWT section"
-        for value in initial.values():
-            check_generated(value)
-        assert upgrade() == initial, "no-op upgrade changed an existing key"
-        print("PASS upgrade: all four generated keys persist")
-
-        for name, raw in fixtures():
-            patch(raw)
-            actual = upgrade()
-            expected = keys(raw)
-            assert set(actual) == set(SECTIONS), f"{name}: upgrade omitted a JWT section"
-            for section in SECTIONS:
-                if section in expected:
-                    assert actual[section] == expected[section], f"{name}: changed {section}"
-                else:
-                    check_generated(actual[section])
-            assert upgrade() == actual, f"{name}: subsequent upgrade changed a key"
-            print(f"PASS upgrade: {name}")
-
-        # Migration from the old chart name, followed by precedence of the current name.
-        legacy_raw = "\n".join(f'[{section}]\nkey = "legacy-{index}"'
-                               for index, section in enumerate(SECTIONS))
-        run(*kubectl, "create", "configmap", legacy, "--from-literal=security.toml=" + legacy_raw)
-        run(*kubectl, "delete", "configmap", current)
-        assert upgrade() == keys(legacy_raw), "legacy ConfigMap keys were not preserved"
-        print("PASS upgrade: legacy ConfigMap migration")
-        current_raw = next(fixtures())[1]
-        patch(current_raw)
-        assert upgrade() == keys(current_raw), "legacy ConfigMap overrode current ConfigMap"
-        print("PASS upgrade: current ConfigMap takes precedence")
-
-        patch(ESCAPED_HEADER)
-        try:
-            upgrade()
-        except subprocess.CalledProcessError as error:
-            assert "unsupported quoted section header" in error.stderr, error.stderr
-        else:
-            raise AssertionError("unsupported quoted header silently rotated its key")
-        assert keys(run(*kubectl, "get", "configmap", current, "-o",
-                        "jsonpath={.data.security\\.toml}"))["jwt.signing"] == "existing"
-        print("PASS upgrade: unsupported quoted header leaves stored key untouched")
-    finally:
-        run(*kubectl, "delete", "namespace", namespace, "--wait=false")
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--helm", default="helm")
-    parser.add_argument("--reference-helm", help="Helm >=3.17 binary for differential checks")
-    parser.add_argument("--context", help="explicit disposable Kubernetes context for live upgrade checks")
-    args = parser.parse_args()
-    print(run(args.helm, "version", "--short").strip())
-    check_helpers(args.helm, args.reference_helm)
-    if args.context:
-        check_upgrades(args.helm, args.context)
-
-
-if __name__ == "__main__":
-    try:
-        main()
-    except subprocess.CalledProcessError as error:
-        print(error.stderr, file=sys.stderr)
-        sys.exit(error.returncode)

.github/workflows/codeql.yml

@@ -27,7 +27,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v4.38.3
+        uses: github/codeql-action/init@v4.38.1
         # Override language selection by uncommenting this and choosing your languages
         with:
           languages: go
@@ -35,7 +35,7 @@ jobs:
       # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below).
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v4.38.3
+        uses: github/codeql-action/autobuild@v4.38.1
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -49,4 +49,4 @@ jobs:
       #     make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v4.38.3
+        uses: github/codeql-action/analyze@v4.38.1

.github/workflows/container_latest.yml

@@ -156,7 +156,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        platform: [amd64, arm64, arm, 386, ppc64le, s390x]
+        platform: [amd64, arm64, arm, 386]
         variant: ${{ fromJSON(needs.setup.outputs.variants) }}
 
     steps:
@@ -405,7 +405,7 @@ jobs:
           output: trivy-results.sarif
           exit-code: '0'
       - name: Upload Trivy scan results to GitHub Security
-        uses: github/codeql-action/upload-sarif@v4.38.3
+        uses: github/codeql-action/upload-sarif@v4.38.1
         if: always()
         with:
           sarif_file: trivy-results.sarif
@@ -505,9 +505,7 @@ jobs:
             ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
             ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
             ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
-            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 \
-            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-ppc64le \
-            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-s390x
+            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
           # The copy and the signature below use the digest this run pushed, not whatever the tag points at by then.
           DIGEST=$(jq -er '."containerimage.descriptor".digest' /tmp/manifest.json)
           echo "digest=${DIGEST}" >> "$GITHUB_OUTPUT"
@@ -551,15 +549,13 @@ jobs:
             echo "Using skopeo to copy..."
             retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs@${DIGEST} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
           else
-            echo "Using docker buildx imagetools (pulling 6 images from Docker Hub)..."
+            echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
             # Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
             retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
               ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
               ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
               ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
-              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 \
-              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-ppc64le \
-              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-s390x
+              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
           fi
 
       - name: Sign

.github/workflows/container_release_unified.yml

@@ -149,16 +149,12 @@ jobs:
           - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/arm64,  arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
           - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest,    qemu: true }
           - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/386,    arch: i386,  runner: ubuntu-latest,    qemu: false }
-          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/ppc64le, arch: ppc64le, runner: ubuntu-latest,  qemu: true }
-          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/s390x,   arch: s390x, runner: ubuntu-latest,    qemu: true }
 
           # Large disk - multi-arch
           - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64,  arch: amd64, runner: ubuntu-latest,    qemu: false }
           - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64,  arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
           - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest,    qemu: true }
           - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386,    arch: i386,  runner: ubuntu-latest,    qemu: false }
-          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/ppc64le, arch: ppc64le, runner: ubuntu-latest,  qemu: true }
-          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/s390x,   arch: s390x, runner: ubuntu-latest,    qemu: true }
 
           # Full tags - multi-arch
           - { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest,    qemu: false }
@@ -460,7 +456,7 @@ jobs:
 
       - name: Upload Trivy scan results to GitHub Security
         if: always()
-        uses: github/codeql-action/upload-sarif@v4.38.3
+        uses: github/codeql-action/upload-sarif@v4.38.1
         with:
           sarif_file: trivy-results.sarif
           category: trivy-${{ matrix.variant }}

.github/workflows/go.yml

@@ -148,21 +148,3 @@ jobs:
     # -short skips the e2e suites already covered on amd64.
     - name: Test linux/386
       run: cd weed; GOOS=linux GOARCH=386 go test -short ./...
-
-  build-azurekms:
-    name: Build and test with azurekms tag
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v7
-    - name: Set up Go
-      uses: actions/setup-go@v7
-      with:
-        go-version-file: 'go.mod'
-    # weed/kms/azure is excluded from every default build, so without this
-    # nothing compiles it and tag-gated code silently rots.
-    - name: Build and test with azurekms
-      run: |
-        cd weed
-        go build -tags azurekms ./...
-        go test -tags azurekms ./kms/...

.github/workflows/helm_ci.yml

@@ -3,10 +3,10 @@ name: "helm: lint and test charts"
 on:
   push:
     branches: [ master ]
-    paths: ['k8s/**', '.github/workflows/helm_ci.yml', '.github/scripts/helm_jwt_keys.py']
+    paths: ['k8s/**', '.github/workflows/helm_ci.yml']
   pull_request:
     branches: [ master ]
-    paths: ['k8s/**', '.github/workflows/helm_ci.yml', '.github/scripts/helm_jwt_keys.py']
+    paths: ['k8s/**', '.github/workflows/helm_ci.yml']
 
 permissions:
   contents: read
@@ -20,14 +20,6 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Helm before fromToml was available
-        uses: azure/setup-helm@v5
-        with:
-          version: v3.16.3
-
-      - name: Record legacy Helm binary
-        run: echo "HELM_LEGACY=$(command -v helm)" >> "$GITHUB_ENV"
-
       - name: Set up Helm
         uses: azure/setup-helm@v5
         with:
@@ -52,17 +44,6 @@ jobs:
       - name: Run chart-testing (lint)
         run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts
 
-      - name: Verify legacy Helm rendering
-        run: |
-          "$HELM_LEGACY" lint k8s/charts/seaweedfs
-          "$HELM_LEGACY" template test k8s/charts/seaweedfs > "$RUNNER_TEMP/legacy-default.yaml"
-          "$HELM_LEGACY" template test k8s/charts/seaweedfs \
-            --set global.seaweedfs.securityConfig.jwtSigning.volumeRead=true \
-            --set global.seaweedfs.securityConfig.jwtSigning.filerWrite=true \
-            --set global.seaweedfs.securityConfig.jwtSigning.filerRead=true \
-            > "$RUNNER_TEMP/legacy-jwt.yaml"
-
-
       - name: Verify template rendering
         run: |
           set -e
@@ -76,60 +57,7 @@ jobs:
           helm template test $CHART_DIR --set s3.enabled=true > /tmp/s3.yaml
           grep -q "kind: Deployment" /tmp/s3.yaml && grep -q "seaweedfs-s3" /tmp/s3.yaml
           echo "S3 deployment renders correctly"
-
-          echo "=== Testing S3 rollout settings ==="
-          helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
-            --set s3.enabled=true > /tmp/s3-rollout-defaults.yaml
-          grep -q "terminationGracePeriodSeconds: 10$" /tmp/s3-rollout-defaults.yaml
-          test "$(grep -cE '^  strategy:|^ +lifecycle:' /tmp/s3-rollout-defaults.yaml)" -eq 0
-          helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
-            --set s3.enabled=true \
-            --set s3.terminationGracePeriodSeconds=30 \
-            --set s3.updateStrategy.type=RollingUpdate \
-            --set s3.updateStrategy.rollingUpdate.maxUnavailable=0 \
-            --set-string 's3.lifecycle.preStop.exec.command={sleep,5}' \
-            > /tmp/s3-rollout.yaml
-          grep -q "terminationGracePeriodSeconds: 30$" /tmp/s3-rollout.yaml
-          grep -A 4 "^  strategy:" /tmp/s3-rollout.yaml | grep -q "maxUnavailable: 0"
-          grep -A 4 "^  strategy:" /tmp/s3-rollout.yaml | grep -q "type: RollingUpdate"
-          grep -A 5 "^          lifecycle:" /tmp/s3-rollout.yaml | grep -q -- "- sleep"
-          echo "S3 rollout settings render correctly"
-
-          echo "=== Testing the S3 config checksum is stable across renders ==="
-          # s3-secret.yaml keeps its generated keys with lookup. Hashing the rendered
-          # Secret drew new keys for the checksum whenever lookup found nothing (on
-          # install), so the first upgrade changed the checksum and restarted the
-          # filer and S3 pods although nothing had changed.
-          s3_checksum() {  # <template> <helm args...>: the checksum/s3config of one workload
-            local template=$1; shift
-            local sum
-            sum=$(helm template test $CHART_DIR --show-only "$template" "$@" \
-              | grep 'checksum/s3config:' | awk '{print $2}')
-            test -n "$sum" || { echo "no checksum/s3config in $template" >&2; return 1; }
-            echo "$sum"
-          }
-          s3_auth_args=(--set s3.enabled=true --set s3.enableAuth=true)
-          s3_sum=$(s3_checksum templates/s3/s3-deployment.yaml "${s3_auth_args[@]}")
-          filer_sum=$(s3_checksum templates/filer/filer-statefulset.yaml "${s3_auth_args[@]}")
-          test "$s3_sum" = "$filer_sum"   # one Secret, one checksum
-          test "$s3_sum" = "$(s3_checksum templates/s3/s3-deployment.yaml "${s3_auth_args[@]}")"
-          s3_creds_sum=$(s3_checksum templates/s3/s3-deployment.yaml "${s3_auth_args[@]}" \
-            --set s3.credentials.admin.accessKey=a --set s3.credentials.admin.secretKey=b)
-          test "$s3_sum" != "$s3_creds_sum"
-          filer_audit_sum=$(s3_checksum templates/filer/filer-statefulset.yaml "${s3_auth_args[@]}" \
-            --set-json 'filer.s3.auditLogConfig={"fluent_port":24224}')
-          test "$filer_sum" != "$filer_audit_sum"
-          # Settings the Secret does not read must not restart the S3 and filer pods.
-          test "$filer_sum" = "$(s3_checksum templates/filer/filer-statefulset.yaml "${s3_auth_args[@]}" \
-            --set s3.replicas=3 --set s3.resources.limits.cpu=1)"
-          all_in_one_args=(--set allInOne.enabled=true --set allInOne.s3.enabled=true --set allInOne.s3.enableAuth=true)
-          aio_sum=$(s3_checksum templates/all-in-one/all-in-one-deployment.yaml "${all_in_one_args[@]}")
-          test "$aio_sum" = "$(s3_checksum templates/all-in-one/all-in-one-deployment.yaml "${all_in_one_args[@]}")"
-          aio_creds_sum=$(s3_checksum templates/all-in-one/all-in-one-deployment.yaml "${all_in_one_args[@]}" \
-            --set s3.credentials.admin.accessKey=a --set s3.credentials.admin.secretKey=b)
-          test "$aio_sum" != "$aio_creds_sum"
-          echo "S3 config checksum is stable and follows only what the S3 Secret reads"
-
+          
           echo "=== Testing S3 credentials from an existing secret ==="
           credential_args=(
             --set s3.credentials.admin.existingSecret=minio-root
@@ -168,23 +96,6 @@ jobs:
           grep -A 12 "^kind: Ingress" /tmp/s3-ingress-labels.yaml | grep -q "^    external-dns: s3"
           echo "S3 ingress renders custom labels"
 
-          echo "=== Testing HTTPRoute ==="
-          helm template test $CHART_DIR --show-only templates/s3/s3-httproute.yaml \
-            --set s3.enabled=true,s3.httpRoute.enabled=true \
-            --set 's3.httpRoute.parentRefs[0].name=gateway' > /tmp/s3-httproute.yaml
-          grep -q "^kind: HTTPRoute" /tmp/s3-httproute.yaml
-          grep -A 4 -- '- group: ""' /tmp/s3-httproute.yaml | grep -q "name: test-seaweedfs-s3$"
-          grep -A 4 -- '- group: ""' /tmp/s3-httproute.yaml | grep -q "port: 8333$"
-          echo "S3 HTTPRoute routes to the s3 service by default"
-          helm template test $CHART_DIR --show-only templates/s3/s3-httproute.yaml \
-            --set filer.s3.enabled=true,filer.s3.port=8334,s3.httpRoute.enabled=true > /tmp/filer-s3-httproute.yaml
-          grep -A 4 -- '- group: ""' /tmp/filer-s3-httproute.yaml | grep -q "port: 8334$"
-          echo "S3 on filer HTTPRoute uses filer.s3.port"
-          helm template test $CHART_DIR --show-only templates/master/master-httproute.yaml \
-            --set allInOne.enabled=true,master.enabled=false,master.httpRoute.enabled=true > /tmp/allinone-master-httproute.yaml
-          grep -A 4 -- '- group: ""' /tmp/allinone-master-httproute.yaml | grep -q "name: test-seaweedfs-all-in-one$"
-          echo "All-in-one master HTTPRoute routes to the all-in-one service"
-
           echo "=== Testing with all-in-one mode ==="
           helm template test $CHART_DIR --set allInOne.enabled=true > /tmp/allinone.yaml
           grep -q "seaweedfs-all-in-one" /tmp/allinone.yaml
@@ -205,21 +116,6 @@ jobs:
           grep -q "security-config" /tmp/security.yaml
           echo "Security configuration renders correctly"
 
-          echo "=== Testing secure bucket-creation hook certificate mounts ==="
-          helm template test $CHART_DIR \
-            --show-only templates/shared/post-install-bucket-hook.yaml \
-            --set s3.enabled=true \
-            --set 's3.createBuckets[0].name=data' \
-            --set global.seaweedfs.enableSecurity=true \
-            > /tmp/security-bucket-hook.yaml
-          test "$(grep -cE '^[[:space:]]*- name: ca-cert$' /tmp/security-bucket-hook.yaml)" -eq 2
-          test "$(grep -cE '^[[:space:]]*- name: client-cert$' /tmp/security-bucket-hook.yaml)" -eq 2
-          grep -q 'mountPath: /usr/local/share/ca-certificates/ca/' /tmp/security-bucket-hook.yaml
-          grep -q 'mountPath: /usr/local/share/ca-certificates/client/' /tmp/security-bucket-hook.yaml
-          grep -q 'secretName: test-seaweedfs-ca-cert' /tmp/security-bucket-hook.yaml
-          grep -q 'secretName: test-seaweedfs-client-cert' /tmp/security-bucket-hook.yaml
-          echo "Secure bucket-creation hook mounts its CA and client certificate"
-
           echo ""
           echo "=== Testing admin.allowInsecureBind satisfies the admin auth render guard ==="
           helm template test $CHART_DIR --set admin.enabled=true --set admin.allowInsecureBind=true \
@@ -822,168 +718,6 @@ jobs:
           helm template test $CHART_DIR --set cosi.enabled=true > /tmp/cosi.yaml
           grep -q "seaweedfs-cosi" /tmp/cosi.yaml
           echo "COSI driver renders correctly"
-
-          echo ""
-          echo "=== Testing configurable pod and container security contexts ==="
-          helm template test $CHART_DIR > /tmp/security-context-defaults.yaml
-
-          security_context_args=()
-          for component in master volume filer s3 sftp admin worker allInOne cosi; do
-            security_context_args+=(
-              --set "$component.podSecurityContext.enabled=true"
-              --set "$component.podSecurityContext.seccompProfile.type=RuntimeDefault"
-              --set "$component.containerSecurityContext.enabled=true"
-              --set "$component.containerSecurityContext.privileged=false"
-              --set "$component.containerSecurityContext.allowPrivilegeEscalation=false"
-              --set "$component.containerSecurityContext.readOnlyRootFilesystem=true"
-              --set "$component.containerSecurityContext.capabilities.drop[0]=ALL"
-              --set "$component.containerSecurityContext.seccompProfile.type=RuntimeDefault"
-            )
-          done
-
-          helm template test $CHART_DIR \
-            "${security_context_args[@]}" \
-            --set s3.enabled=true \
-            --set s3.createBuckets[0].name=test \
-            --set sftp.enabled=true \
-            --set admin.enabled=true \
-            --set admin.secret.adminPassword=ci-admin-password \
-            --set worker.enabled=true \
-            --set volume.idx.type=hostPath \
-            --set volume.idx.hostPathPrefix=/tmp \
-            --set cosi.enabled=true \
-            --set global.seaweedfs.tmpDir.sizeLimit=64Mi > /tmp/security-contexts.yaml
-          helm template test $CHART_DIR \
-            "${security_context_args[@]}" \
-            --set allInOne.enabled=true \
-            --set master.enabled=false \
-            --set volume.enabled=false \
-            --set filer.enabled=false \
-            --set global.seaweedfs.tmpDir.sizeLimit=64Mi > /tmp/security-contexts-aio.yaml
-          python3 - /tmp/security-context-defaults.yaml /tmp/security-contexts.yaml /tmp/security-contexts-aio.yaml <<'PYEOF'
-          import sys
-
-          import yaml
-
-          errors = []
-          workloads = 0
-          containers = 0
-          chart_managed_init_containers = 0
-          components = set()
-
-          def validate_container(workload_name, container):
-              context = container.get("securityContext", {})
-              prefix = f"{workload_name}/{container['name']}"
-              if "enabled" in context:
-                  errors.append(f"{prefix}: internal enabled flag leaked into container securityContext")
-              if context.get("privileged") is not False:
-                  errors.append(f"{prefix}: privileged is not false")
-              if context.get("allowPrivilegeEscalation") is not False:
-                  errors.append(f"{prefix}: allowPrivilegeEscalation is not false")
-              if context.get("readOnlyRootFilesystem") is not True:
-                  errors.append(f"{prefix}: readOnlyRootFilesystem is not true")
-              if context.get("capabilities", {}).get("drop") != ["ALL"]:
-                  errors.append(f"{prefix}: capabilities.drop is not exactly [ALL]")
-              if context.get("seccompProfile", {}).get("type") != "RuntimeDefault":
-                  errors.append(f"{prefix}: seccompProfile is not RuntimeDefault")
-              mounts = {mount["name"]: mount for mount in container.get("volumeMounts", [])}
-              if mounts.get("seaweedfs-tmp", {}).get("mountPath") != "/tmp":
-                  errors.append(f"{prefix}: writable temporary volume is not mounted at /tmp")
-
-          with open(sys.argv[1]) as stream:
-              default_documents = [document for document in yaml.safe_load_all(stream) if document]
-          for document in default_documents:
-              if document.get("kind") not in ("Deployment", "StatefulSet", "Job"):
-                  continue
-              name = document["metadata"]["name"]
-              pod = document["spec"]["template"]["spec"]
-              if "securityContext" in pod:
-                  errors.append(f"{name}: pod securityContext should be absent by default")
-              if any(volume["name"] == "seaweedfs-tmp" for volume in pod.get("volumes", [])):
-                  errors.append(f"{name}: writable /tmp volume should be absent by default")
-              for container in pod.get("containers", []):
-                  if "securityContext" in container:
-                      errors.append(
-                          f"{name}/{container['name']}: container securityContext "
-                          f"should be absent by default"
-                      )
-                  if any(
-                      mount["name"] == "seaweedfs-tmp"
-                      for mount in container.get("volumeMounts", [])
-                  ):
-                      errors.append(
-                          f"{name}/{container['name']}: writable /tmp mount "
-                          f"should be absent by default"
-                      )
-
-          for path in sys.argv[2:]:
-              with open(path) as stream:
-                  documents = [document for document in yaml.safe_load_all(stream) if document]
-              for document in documents:
-                  if document.get("kind") not in ("Deployment", "StatefulSet", "Job"):
-                      continue
-                  workloads += 1
-                  name = document["metadata"]["name"]
-                  pod = document["spec"]["template"]["spec"]
-                  volumes = {volume["name"]: volume for volume in pod.get("volumes", [])}
-                  temporary = volumes.get("seaweedfs-tmp", {}).get("emptyDir")
-                  if temporary is None:
-                      errors.append(f"{name}: writable /tmp emptyDir is missing")
-                  elif temporary.get("sizeLimit") != "64Mi":
-                      errors.append(f"{name}: temporary volume sizeLimit is not 64Mi")
-                  component = document["spec"]["template"]["metadata"]["labels"].get("app.kubernetes.io/component")
-                  if component:
-                      components.add(component)
-                  else:
-                      errors.append(f"{name}: app.kubernetes.io/component label is missing")
-                  pod_context = pod.get("securityContext", {})
-                  if "enabled" in pod_context:
-                      errors.append(f"{name}: internal enabled flag leaked into pod securityContext")
-                  if pod_context.get("seccompProfile", {}).get("type") != "RuntimeDefault":
-                      errors.append(f"{name}: pod seccompProfile is not RuntimeDefault")
-                  for container in pod.get("containers", []):
-                      containers += 1
-                      validate_container(name, container)
-                  for container in pod.get("initContainers", []):
-                      if container["name"] != "seaweedfs-vol-move-idx":
-                          continue
-                      chart_managed_init_containers += 1
-                      validate_container(name, container)
-
-          expected_components = {
-              "master", "volume", "filer", "s3", "sftp", "admin", "worker",
-              "objectstorage-provisioner", "bucket-hook", "seaweedfs-all-in-one",
-          }
-          if components != expected_components:
-              errors.append(
-                  f"security context workload coverage is incomplete: "
-                  f"expected {sorted(expected_components)}, got {sorted(components)}"
-              )
-          if workloads != 10 or containers != 12:
-              errors.append(
-                  f"expected 10 workloads and 12 containers, got "
-                  f"{workloads} workloads and {containers} containers"
-              )
-          if chart_managed_init_containers != 1:
-              errors.append(
-                  f"expected one chart-managed seaweedfs-vol-move-idx init container, "
-                  f"got {chart_managed_init_containers}"
-              )
-
-          if errors:
-              print("\n".join(f"FAIL: {error}" for error in errors), file=sys.stderr)
-              sys.exit(1)
-          print(f"Validated security contexts on {workloads} workloads and {containers} containers")
-          PYEOF
-
-          # The resize hook depends on lookup finding a live StatefulSet and
-          # therefore cannot render during helm template. Keep static coverage
-          # for both security-context blocks.
-          grep -Fqx '      securityContext: {{- omit .Values.volume.podSecurityContext "enabled" | toYaml | nindent 8 }}' \
-            "$CHART_DIR/templates/volume/volume-resize-hook.yaml"
-          grep -Fqx '          securityContext: {{- omit .Values.volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}' \
-            "$CHART_DIR/templates/volume/volume-resize-hook.yaml"
-          echo "Volume resize hook security contexts are covered"
           
           echo ""
           echo "=== Testing long release name: service names match DNS references ==="
@@ -1871,78 +1605,6 @@ jobs:
       - name: Create kind cluster
         uses: helm/kind-action@v1.15.0
 
-      - name: Verify volume resize hook security contexts
-        run: |
-          set -e
-          CHART_DIR="k8s/charts/seaweedfs"
-          NS="resize-hook-security"
-          kubectl create namespace "$NS"
-          kubectl apply -n "$NS" -f - <<'EOF'
-          apiVersion: v1
-          kind: PersistentVolumeClaim
-          metadata:
-            name: data1-resize-seaweedfs-volume-0
-          spec:
-            accessModes:
-              - ReadWriteOnce
-            resources:
-              requests:
-                storage: 1Gi
-          EOF
-
-          helm template resize "$CHART_DIR" -n "$NS" --dry-run=server \
-            --set volume.dataDirs[0].name=data1 \
-            --set volume.dataDirs[0].type=persistentVolumeClaim \
-            --set volume.dataDirs[0].size=2Gi \
-            --set volume.podSecurityContext.enabled=true \
-            --set volume.podSecurityContext.seccompProfile.type=RuntimeDefault \
-            --set volume.containerSecurityContext.enabled=true \
-            --set volume.containerSecurityContext.privileged=false \
-            --set volume.containerSecurityContext.allowPrivilegeEscalation=false \
-            --set volume.containerSecurityContext.readOnlyRootFilesystem=true \
-            --set volume.containerSecurityContext.capabilities.drop[0]=ALL \
-            --set volume.containerSecurityContext.seccompProfile.type=RuntimeDefault \
-            --set global.seaweedfs.tmpDir.sizeLimit=64Mi \
-            > /tmp/security-context-resize-hook.yaml
-
-          python3 - /tmp/security-context-resize-hook.yaml <<'PYEOF'
-          import sys
-
-          import yaml
-
-          with open(sys.argv[1]) as stream:
-              documents = [document for document in yaml.safe_load_all(stream) if document]
-
-          jobs = [
-              document for document in documents
-              if document.get("kind") == "Job"
-              and document["metadata"]["name"].endswith("-volume-resize-hook")
-          ]
-          if len(jobs) != 1:
-              raise AssertionError(f"expected one volume resize hook Job, got {len(jobs)}")
-
-          pod = jobs[0]["spec"]["template"]["spec"]
-          assert pod["securityContext"] == {
-              "seccompProfile": {"type": "RuntimeDefault"},
-          }
-          assert len(pod["containers"]) == 1
-          assert pod["containers"][0]["securityContext"] == {
-              "allowPrivilegeEscalation": False,
-              "capabilities": {"drop": ["ALL"]},
-              "privileged": False,
-              "readOnlyRootFilesystem": True,
-              "seccompProfile": {"type": "RuntimeDefault"},
-          }
-          assert pod["containers"][0]["volumeMounts"] == [
-              {"mountPath": "/tmp", "name": "seaweedfs-tmp"},
-          ]
-          assert pod["volumes"] == [
-              {"emptyDir": {"sizeLimit": "64Mi"}, "name": "seaweedfs-tmp"},
-          ]
-          PYEOF
-          kubectl delete namespace "$NS"
-          echo "Volume resize hook security contexts render correctly"
-
       - name: Run chart-testing (install)
         run: |
           ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts \
@@ -1993,16 +1655,6 @@ jobs:
           kubectl delete namespace "$NS"
           echo "SFTP host key lifecycle tests passed"
 
-      - name: Verify JWT signing key persistence across upgrades
-        run: |
-          # chart-testing puts its pip-less venv first on PATH; use setup-python.
-          PYTHON="$pythonLocation/bin/python3"
-          "$PYTHON" -m pip install tomli==2.2.1
-          CONTEXT=$(kubectl config current-context)
-          "$PYTHON" .github/scripts/helm_jwt_keys.py --context "$CONTEXT"
-          "$PYTHON" .github/scripts/helm_jwt_keys.py --helm "$HELM_LEGACY" \
-            --reference-helm helm --context "$CONTEXT"
-
       - name: Verify install into a default-deny namespace
         run: |
           set -e

.github/workflows/s3-parquet-tests.yml

@@ -139,8 +139,7 @@ jobs:
   unit-tests:
     name: Go Unit Tests (Implicit Directory)
     runs-on: ubuntu-latest
-    # Leave time for setup and the focused run before the nine-minute suite.
-    timeout-minutes: 20
+    timeout-minutes: 10
     
     steps:
       - name: Checkout code
@@ -160,5 +159,5 @@ jobs:
       - name: Run all S3 API tests
         run: |
           cd weed/s3api
-          go test -v -timeout 9m
+          go test -v -timeout 5m
 

.github/workflows/s3-tables-tests.yml

@@ -439,117 +439,6 @@ jobs:
           path: test/s3tables/catalog_clickhouse/test-output.log
           retention-days: 3
 
-  olake-iceberg-catalog-tests:
-    name: OLake Iceberg Catalog Integration Tests (${{ matrix.tag }})
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Pinned baseline, and latest so new OLake releases are exercised
-          # without a code change. OLake's Iceberg writer is a Java sidecar
-          # whose Iceberg version moves independently of the Go release, so
-          # the latest leg is the one that catches library drift.
-          - olake-image: olakego/source-postgres:v0.10.1
-            tag: "v0.10.1"
-          - olake-image: olakego/source-postgres:latest
-            tag: latest
-
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v7
-
-      - name: Set up Go
-        uses: actions/setup-go@v7
-        with:
-          go-version-file: 'go.mod'
-        id: go
-
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
-
-      - name: Pre-pull images
-        run: |
-          pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
-          pull ${{ matrix.olake-image }}
-          pull postgres:16
-          pull python:3.11-slim
-
-      - name: Run go mod tidy
-        run: go mod tidy
-
-      - name: Install SeaweedFS
-        run: |
-          go install -buildvcs=false ./weed
-
-      - name: Run OLake Iceberg Catalog Integration Tests
-        timeout-minutes: 25
-        working-directory: test/s3tables/catalog_olake
-        env:
-          OLAKE_IMAGE: ${{ matrix.olake-image }}
-        run: |
-          set -x
-          set -o pipefail
-          echo "=== System Information ==="
-          uname -a
-          free -h
-          df -h
-          docker info
-          echo "=== Starting OLake Iceberg Catalog Tests ==="
-
-          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
-            echo "OLake Iceberg catalog integration tests failed"
-            exit 1
-          }
-
-      # The suite skips itself when Docker is unavailable, so a green job is not
-      # by itself evidence that anything ran. Assert execution explicitly.
-      - name: Assert the suite actually ran
-        working-directory: test/s3tables/catalog_olake
-        run: |
-          log=test-output.log
-          if [ ! -f "$log" ]; then
-            echo "::error::no test-output.log; the suite did not run"
-            exit 1
-          fi
-          passes=$(grep -c '^--- PASS' "$log" || true)
-          skips=$(grep -c '^--- SKIP' "$log" || true)
-          echo "top-level PASS=$passes SKIP=$skips"
-          if [ "$skips" -gt 0 ]; then
-            echo "::error::the OLake suite skipped $skips top-level test(s); the environment it needs was not provisioned, so this job proves nothing"
-            grep '^--- SKIP' "$log" | head -20
-            exit 1
-          fi
-          if [ "$passes" -lt 1 ]; then
-            echo "::error::the OLake suite recorded no passing top-level test"
-            exit 1
-          fi
-
-      - name: Show test output on failure
-        if: failure()
-        working-directory: test/s3tables/catalog_olake
-        run: |
-          echo "=== Test Output ==="
-          if [ -f test-output.log ]; then
-            tail -200 test-output.log
-          fi
-
-          echo "=== Process information ==="
-          ps aux | grep -E "(weed|test|docker|olake|postgres)" || true
-          echo "=== Containers ==="
-          docker ps -a | head -30 || true
-
-      - name: Upload test logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v7
-        with:
-          name: olake-iceberg-catalog-test-logs-${{ matrix.tag }}
-          path: test/s3tables/catalog_olake/test-output.log
-          retention-days: 3
-
   polaris-integration-tests:
     name: Polaris Integration Tests
     runs-on: ubuntu-22.04

.github/workflows/s3tests.yml

@@ -289,11 +289,6 @@ jobs:
           s3tests/functional/test_s3.py::test_object_write_check_etag \
           s3tests/functional/test_s3.py::test_object_write_cache_control \
           s3tests/functional/test_s3.py::test_object_write_expires \
-          s3tests/functional/test_s3.py::test_put_obj_enc_conflict_c_s3 \
-          s3tests/functional/test_s3.py::test_put_obj_enc_conflict_c_kms \
-          s3tests/functional/test_s3.py::test_put_obj_enc_conflict_s3_kms \
-          s3tests/functional/test_s3.py::test_put_obj_enc_conflict_bad_enc_kms \
-          s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \
           s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
           s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
           s3tests/functional/test_s3.py::test_object_write_file \
@@ -316,7 +311,6 @@ jobs:
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
           s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
-          s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
           s3tests/functional/test_s3.py::test_bucket_head \
           s3tests/functional/test_s3.py::test_bucket_head_notexist \
           s3tests/functional/test_s3.py::test_object_raw_authenticated \
@@ -1155,11 +1149,6 @@ jobs:
           s3tests/functional/test_s3.py::test_object_write_check_etag \
           s3tests/functional/test_s3.py::test_object_write_cache_control \
           s3tests/functional/test_s3.py::test_object_write_expires \
-          s3tests/functional/test_s3.py::test_put_obj_enc_conflict_c_s3 \
-          s3tests/functional/test_s3.py::test_put_obj_enc_conflict_c_kms \
-          s3tests/functional/test_s3.py::test_put_obj_enc_conflict_s3_kms \
-          s3tests/functional/test_s3.py::test_put_obj_enc_conflict_bad_enc_kms \
-          s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \
           s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
           s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
           s3tests/functional/test_s3.py::test_object_write_file \
@@ -1182,7 +1171,6 @@ jobs:
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
           s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
-          s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
           s3tests/functional/test_s3.py::test_bucket_head \
           s3tests/functional/test_s3.py::test_bucket_head_notexist \
           s3tests/functional/test_s3.py::test_object_raw_authenticated \

README.md

@@ -21,7 +21,6 @@ One `weed` binary serves an S3 object store, a POSIX file system, and a lakehous
 
 - [Download Binaries for different platforms](https://github.com/seaweedfs/seaweedfs/releases/latest)
 - [Wiki Documentation](https://github.com/seaweedfs/seaweedfs/wiki)
-- [HTTP REST API](REST_API.md) for the filer, master, and volume servers
 - Community: [Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY), [Twitter](https://twitter.com/SeaweedFS), [Telegram](https://t.me/Seaweedfs), [Reddit](https://www.reddit.com/r/SeaweedFS/), [Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [SeaweedFS White Paper](https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS_Architecture.pdf) and introduction slides: [2025.5](https://docs.google.com/presentation/d/1tdkp45J01oRV68dIm4yoTXKJDof-EhainlA0LMXexQE/edit?usp=sharing), [2021.5](https://docs.google.com/presentation/d/1DcxKWlINc-HNCjhYeERkpGXXm6nTCES8mi2W5G0Z4Ts/edit?usp=sharing), [2019.3](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
 
@@ -82,8 +81,6 @@ AWS_ACCESS_KEY_ID=admin AWS_SECRET_ACCESS_KEY=secret \
 
 The same process also runs the master, a volume server, the filer, WebDAV, the Iceberg REST catalog, and the Admin UI. Add `S3_TABLE_BUCKET=warehouse` to also create an Iceberg table bucket, or `warehouse:LANCE` for a Lance one. Drop the AWS keys to run without authentication for development.
 
-Without Admin authentication or mTLS, `weed mini` binds the Admin UI/API and its worker gRPC control plane to loopback rather than `-ip.bind`. Set `WEED_ADMIN_PASSWORD` or configure `https.admin` mTLS to keep the network bind. Remote workers must opt in with `-admin.worker.ip=<address>` and should configure `grpc.admin` mTLS. `-admin.allowInsecureBind` restores the legacy unauthenticated network bind and should only be used on an isolated network.
-
 > macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first.
 
 `weed mini` is auto-tuned for one node and is fine for single-node production, such as an S3 gateway that issues presigned URLs. See [Quick Start with weed mini][WeedMini].

REST_API.md

@@ -1,344 +0,0 @@
-# SeaweedFS HTTP REST API
-
-SeaweedFS exposes three HTTP surfaces:
-
-| Service | Default port | Addressing |
-|---------|--------------|------------|
-| Filer | 8888 | File system paths (`/dir/name`) |
-| Master | 9333 | File id assignment and cluster topology |
-| Volume server | 8080 | File content by file id (`vid,fid`) |
-
-Most clients only need the filer API (paths) or the S3 API. The master and
-volume APIs are the lower-level blob store interface.
-
-Conventions applying to all three:
-
-- Responses are JSON unless noted otherwise. Append `&pretty=y` to pretty-print.
-- A file id (`fid`) has the form `volumeId,fileKeyCookie`, e.g. `3,01637037d6`.
-  An optional suffix selects a reserved id from a `count` assignment
-  (`3,01637037d6_1`, `_2`, ...), and an optional extension
-  (`3,01637037d6.jpg`) sets the content type on reads.
-- `replication` is a 3-digit replica placement `xyz`: `x` copies in other
-  data centers, `y` on other racks in the same data center, `z` on other
-  volume servers on the same rack. `000` = no replication, `001` = one copy
-  on the same rack, `010` = one copy on a different rack, `100` = one copy in
-  another data center, `200` = two copies in two other data centers, `110` =
-  one copy in another data center plus one on another rack.
-- `ttl` units: `m` minute, `h` hour, `d` day, `w` week, `M` month, `y` year.
-
-## Filer API (port 8888)
-
-The filer presents a POSIX-like namespace over the volume servers.
-
-### Upload a file
-
-```bash
-# PUT the raw body to the target path
-curl -T /home/chris/myphoto.jpg "http://localhost:8888/dir/myphoto.jpg"
-
-# or POST as multipart form (the part filename becomes the entry name)
-curl -F file=@/home/chris/myphoto.jpg "http://localhost:8888/dir/"
-```
-
-Response `201 Created`:
-
-```json
-{"name":"myphoto.jpg","size":43234,"eTag":"0x6c656...","mtime":"...","chunks":[...]}
-```
-
-Query parameters:
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `collection` | collection name | empty |
-| `replication` | replica placement code | filer default |
-| `ttl` | file expiration, e.g. `3d` | never |
-| `disk` | disk type to store on | filer default |
-| `fsync` | `true` fsyncs on the volume server | false |
-| `dataCenter` | preferred data center | empty |
-| `rack` | preferred rack | empty |
-| `dataNode` | preferred volume server | empty |
-| `saveInside` | store small content inside the metadata instead of a volume | false |
-| `maxMB` | split the upload into chunks of this many MB | filer `-maxMB` |
-| `mode` | unix permission bits, e.g. `0644` | `0660` |
-| `op` | `append` appends to an existing file | overwrite |
-| `skipCheckParentDir` | `true` skips the parent-directory existence check | false |
-
-### Create a directory
-
-```bash
-curl -X POST "http://localhost:8888/dir/newdir/"
-```
-
-A POST to a path ending in `/` with no content creates the directory,
-including missing parents.
-
-### Read a file
-
-```bash
-curl "http://localhost:8888/dir/myphoto.jpg"
-```
-
-Supports `Range` requests (`Accept-Ranges: bytes`), `ETag`, and the
-`If-None-Match` / `If-Modified-Since` conditional headers. `HEAD` returns
-headers only. Entry headers stored as extended attributes are echoed back,
-minus internal `Seaweed-` and `xattr-` keys.
-
-Entry metadata instead of content:
-
-```bash
-curl "http://localhost:8888/dir/myphoto.jpg?metadata=true"
-```
-
-`metadata=true&resolveManifest=true` additionally resolves chunked-manifest
-entries into their real chunk list.
-
-### List a directory
-
-```bash
-curl -H "Accept: application/json" "http://localhost:8888/dir/?limit=10&lastFileName=a.jpg"
-```
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `limit` | max entries per page | filer `-dirListLimit` |
-| `lastFileName` | resume listing after this entry name | empty |
-| `namePattern` | include only names matching the wildcard | empty |
-| `namePatternExclude` | exclude names matching the wildcard | empty |
-
-The JSON response carries `Path`, `Entries`, `Limit`, `LastFileName`,
-`ShouldDisplayLoadMore`, and `EmptyFolder`. Without the `Accept` header the
-filer renders its HTML browser.
-
-### Move and copy
-
-```bash
-curl -X POST "http://localhost:8888/dir/newname.jpg?mv.from=/dir/myphoto.jpg"
-curl -X POST "http://localhost:8888/dir/copy.jpg?cp.from=/dir/myphoto.jpg"
-```
-
-`mv.from` renames or moves the source to the request path (`204 No Content`).
-`cp.from` copies it.
-
-### Append
-
-```bash
-curl -T chunk2.bin "http://localhost:8888/dir/file.bin?op=append"
-```
-
-### Delete
-
-```bash
-curl -X DELETE "http://localhost:8888/dir/myphoto.jpg"
-curl -X DELETE "http://localhost:8888/dir/?recursive=true"
-```
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `recursive` | delete a non-empty directory tree | false; when the filer runs with `filer.options.recursive_delete=true`, deletes are recursive unless `recursive=false` |
-| `ignoreRecursiveError` | keep deleting remaining entries after an error | false |
-| `skipChunkDeletion` | remove only the metadata, keep volume data | false |
-
-### Tagging
-
-Tags are carried as `Seaweed-`-prefixed request headers, not query
-parameters; `?tagging` selects the tagging handler and `?tagging=K1,K2`
-lists the keys to remove. Header names are canonicalized on write
-(`Seaweed-k1` is stored as `Seaweed-K1`), and the delete list is matched
-case-sensitively against the stored names.
-
-```bash
-curl -X PUT -H "Seaweed-k1: v1" -H "Seaweed-k2: v2" "http://localhost:8888/dir/file.jpg?tagging"
-curl -X DELETE "http://localhost:8888/dir/file.jpg?tagging=K1,K2"
-```
-
-### Read by file id
-
-```bash
-curl "http://localhost:8888/?proxyChunkId=3,01637037d6"
-```
-
-The filer proxies the chunk read to the right volume server, so only the
-filer port needs to be exposed.
-
-### Resumable uploads
-
-The filer serves the [TUS protocol](https://tus.io/) for resumable uploads
-(`POST`, `PATCH`, `HEAD` on upload URLs). It is enabled by default at
-`/.tus`; `-tusBasePath` changes the endpoint base path.
-
-### Health
-
-`GET /healthz` and `GET /readyz` return `200 OK`.
-
-## Master API (port 9333)
-
-Write-affecting endpoints are automatically proxied to the current leader, so
-any master in the quorum can serve them.
-
-### Assign a file id
-
-```bash
-curl "http://localhost:9333/dir/assign?count=1&replication=001&collection=turbo&dataCenter=dc1&ttl=3d&disk=ssd"
-{"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
-```
-
-Upload the file content to `http://<url>/<fid>` afterwards. With `count>1`,
-use `<fid>_1`, `<fid>_2`, ... for the additional ids.
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `count` | file ids to reserve | 1 |
-| `collection` | collection name | empty |
-| `dataCenter` | preferred data center | empty |
-| `rack` | preferred rack | empty |
-| `dataNode` | preferred volume server | empty |
-| `replication` | replica placement | master `-defaultReplication` |
-| `ttl` | file expiration, e.g. `3d` | never |
-| `disk` | disk type | empty |
-| `dataSize` | expected file size in bytes | 0 |
-| `preallocate` | bytes to preallocate for new volumes | master `-volumePreallocate` |
-| `writableVolumeCount` | grow this many volumes when none are writable | master default |
-| `memoryMapMaxSizeMb` | memory-mapped file size (Windows) | 0 |
-
-### Look up a volume or file id
-
-```bash
-curl "http://localhost:9333/dir/lookup?volumeId=3"
-{"locations":[{"url":"localhost:8080","publicUrl":"localhost:8080"}]}
-```
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `volumeId` | volume id; a full `vid,fid` is accepted too | required |
-| `fileId` | like `volumeId`, but also returns a write JWT when security is on | empty |
-| `collection` | speeds up the lookup | empty |
-| `read` | `yes` generates a read JWT instead of a write JWT | empty |
-
-### Store a file in one call
-
-```bash
-curl -F file=@/home/chris/report.pdf "http://localhost:9333/submit?collection=turbo&replication=001"
-{"fileName":"report.pdf","fid":"3,01637037d6","fileUrl":"localhost:8080/3,01637037d6","size":43234,"eTag":"0x6c656..."}
-```
-
-`POST /submit` accepts multipart file data plus the `dir/assign` placement
-parameters (`count`, `collection`, `dataCenter`, `rack`, `replication`,
-`ttl`, `disk`), assigns a file id, uploads to the volume server, and returns
-the result.
-
-### Redirect to a file
-
-```bash
-curl -v "http://localhost:9333/3,01637037d6"
-```
-
-`GET /{fileId}` answers `308 Permanent Redirect` to a volume server holding
-the file, preserving the query string (e.g. image-resize parameters).
-
-### Cluster status
-
-```bash
-curl "http://localhost:9333/dir/status?pretty=y"   # full topology tree
-curl "http://localhost:9333/vol/status?pretty=y"   # every volume on every node
-curl "http://localhost:9333/collection/info?collection=turbo"
-curl "http://localhost:9333/collection/info?collection=turbo&detail=true"
-```
-
-`collection/info` returns aggregated `TotalSize`, `FileCount`, `UsedSize`,
-`VolumeCount`; `detail=true` splits them per volume layout.
-
-### Grow volumes
-
-```bash
-curl "http://localhost:9333/vol/grow?count=4&replication=001&collection=turbo&ttl=5d&disk=ssd&dataCenter=dc1&rack=rack1"
-{"count":4}
-```
-
-`count` is required; the placement parameters match `dir/assign`. One volume
-serves one write at a time, so pre-allocated volumes raise write concurrency.
-
-### Vacuum deleted space
-
-```bash
-curl "http://localhost:9333/vol/vacuum?garbageThreshold=0.4"
-```
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `garbageThreshold` | minimum deleted-bytes ratio before a volume is compacted | master `-garbageThreshold` (0.3) |
-
-Vacuuming makes a volume read-only, copies live needles to a new volume, and
-swaps it in.
-
-### Delete a collection
-
-```bash
-curl "http://localhost:9333/col/delete?collection=benchmark"
-```
-
-Deletes all volumes of the collection, including erasure-coded shards.
-`204 No Content` on success.
-
-### Health
-
-```bash
-curl -I "http://localhost:9333/healthz"     # liveness
-curl -I "http://localhost:9333/readyz"      # readiness
-curl "http://localhost:9333/"               # web UI
-```
-
-## Volume server API (port 8080)
-
-The volume server stores file content by file id. Clients normally get the
-volume URL from `dir/assign` or `dir/lookup`.
-
-### Upload
-
-```bash
-curl -F file=@/home/chris/myphoto.jpg "http://127.0.0.1:8080/3,01637037d6"
-{"name":"myphoto.jpg","size":43234,"eTag":"0x6c656...","mime":"image/jpeg","contentMd5":"..."}
-```
-
-PUT or POST the body (or a multipart `file` part) to `/{vid},{fid}`.
-`204 No Content` is returned when the content is unchanged. `?ts=<unix>`
-sets the stored modification time.
-
-### Read
-
-```bash
-curl "http://127.0.0.1:8080/3,01637037d6"
-curl "http://127.0.0.1:8080/3,01637037d6.jpg"        # sets Content-Type from the extension
-```
-
-Supports `Range` and `HEAD`. Image files can be resized server-side:
-
-| Parameter | Description |
-|-----------|-------------|
-| `width`, `height` | resize bounds in pixels |
-| `mode` | `fit` (contain) or `fill` (cover); omitted resizes to `width`/`height` |
-| `crop_x1`, `crop_y1`, `crop_x2`, `crop_y2` | explicit crop rectangle |
-| `cm` | `false` returns the chunk-manifest blob instead of resolving it |
-| `readDeleted` | `true` reads soft-deleted needles |
-| `collection` | passed through redirects for the right volume |
-
-### Delete
-
-```bash
-curl -X DELETE "http://127.0.0.1:8080/3,01637037d6"
-{"size":43234}
-```
-
-`?ts=<unix>` sets the deletion timestamp. Replicated volumes propagate the
-delete to every replica.
-
-### Status
-
-```bash
-curl "http://localhost:8080/status?pretty=y"  # disk and volume inventory
-curl -I "http://localhost:8080/healthz"       # liveness/readiness
-```
-
-`OPTIONS` preflights answer CORS headers. When `-port.public` differs from
-`-port`, the volume server opens a separate read-only public listener on
-that port; `-publicUrl` sets the address it advertises to clients.

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dustin/go-humanize v1.1.0
+	github.com/dustin/go-humanize v1.0.1
 	github.com/eapache/go-resiliency v1.6.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
@@ -44,7 +44,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
-	github.com/klauspost/compress v1.20.1
+	github.com/klauspost/compress v1.19.2
 	github.com/klauspost/reedsolomon v1.14.2
 	github.com/kurin/blazer v0.5.3
 	github.com/linxGnu/grocksdb v1.10.8
@@ -60,11 +60,11 @@ require (
 	github.com/pquerna/cachecontrol v0.2.0
 	github.com/prometheus/client_golang v1.24.1
 	github.com/prometheus/client_model v0.6.3
-	github.com/prometheus/common v0.71.0 // indirect
+	github.com/prometheus/common v0.70.1 // indirect
 	github.com/prometheus/procfs v0.22.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	github.com/seaweedfs/goexif v1.0.3
+	github.com/seaweedfs/goexif v2.0.0+incompatible
 	github.com/seaweedfs/raft v1.2.1
 	github.com/sirupsen/logrus v1.9.4 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
@@ -73,7 +73,7 @@ require (
 	github.com/stretchr/testify v1.12.1
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tidwall/gjson v1.19.0
+	github.com/tidwall/gjson v1.18.0
 	github.com/tidwall/match v1.2.0
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
@@ -90,14 +90,14 @@ require (
 	gocloud.dev v0.46.0
 	gocloud.dev/pubsub/natspubsub v0.46.0
 	gocloud.dev/pubsub/rabbitpubsub v0.46.0
-	golang.org/x/crypto v0.58.0
+	golang.org/x/crypto v0.57.0
 	golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597
 	golang.org/x/image v0.46.0
-	golang.org/x/net v0.61.0
-	golang.org/x/oauth2 v0.37.0
-	golang.org/x/sys v0.49.0
-	golang.org/x/text v0.43.0 // indirect
-	golang.org/x/tools v0.51.0 // indirect
+	golang.org/x/net v0.58.0
+	golang.org/x/oauth2 v0.36.0
+	golang.org/x/sys v0.48.0
+	golang.org/x/text v0.42.0 // indirect
+	golang.org/x/tools v0.49.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.297.0
 	google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect
@@ -111,21 +111,21 @@ require (
 )
 
 require (
-	cloud.google.com/go/kms v1.35.0
+	cloud.google.com/go/kms v1.34.0
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/Jille/raft-grpc-transport v1.6.1
 	github.com/ThreeDotsLabs/watermill v1.5.2
 	github.com/a-h/templ v0.3.1020
 	github.com/apache/cassandra-gocql-driver/v2 v2.1.2
-	github.com/apache/iceberg-go v0.7.0
+	github.com/apache/iceberg-go v0.6.1-0.20260817192109-c2105090c9e2
 	github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
 	github.com/arangodb/go-driver v1.6.9
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.47.1
-	github.com/aws/aws-sdk-go-v2/config v1.33.6
-	github.com/aws/aws-sdk-go-v2/credentials v1.20.6
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1
+	github.com/aws/aws-sdk-go-v2 v1.47.0
+	github.com/aws/aws-sdk-go-v2/config v1.32.35
+	github.com/aws/aws-sdk-go-v2/credentials v1.20.4
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3
 	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/cognusion/imaging v1.0.4
 	github.com/fluent/fluent-logger-golang v1.10.1
@@ -134,16 +134,16 @@ require (
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
 	github.com/hashicorp/golang-lru/v2 v2.0.7
-	github.com/hashicorp/raft v1.8.0
-	github.com/hashicorp/raft-boltdb/v2 v2.4.2
+	github.com/hashicorp/raft v1.7.3
+	github.com/hashicorp/raft-boltdb/v2 v2.3.1
 	github.com/hashicorp/vault/api v1.23.0
-	github.com/jhump/protoreflect v1.18.1
+	github.com/jhump/protoreflect v1.18.0
 	github.com/linkedin/goavro/v2 v2.15.0
 	github.com/minio/crc64nvme v1.1.1
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/parquet-go/parquet-go v0.32.0
 	github.com/pkg/sftp v1.13.11
-	github.com/rabbitmq/amqp091-go v1.15.0
+	github.com/rabbitmq/amqp091-go v1.14.0
 	github.com/rclone/rclone v1.75.1
 	github.com/rdleal/intervalst v1.5.0
 	github.com/redis/go-redis/v9 v9.22.0
@@ -154,13 +154,13 @@ require (
 	github.com/tarantool/go-tarantool/v3 v3.0.2
 	github.com/testcontainers/testcontainers-go v0.44.0
 	github.com/tikv/client-go/v2 v2.0.7
-	github.com/twmb/avro v1.9.0
+	github.com/twmb/avro v1.8.0
 	github.com/xeipuuv/gojsonschema v1.2.0
-	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.3
+	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1
 	go.etcd.io/etcd/client/pkg/v3 v3.7.1
 	go.uber.org/atomic v1.12.0
-	golang.org/x/sync v0.24.0
+	golang.org/x/sync v0.23.0
 	golang.org/x/tools/godoc v0.1.0-deprecated
 	google.golang.org/grpc/security/advancedtls v1.0.0
 )
@@ -168,6 +168,9 @@ require (
 require github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
 
 require (
+	atomicgo.dev/cursor v0.2.0 // indirect
+	atomicgo.dev/keyboard v0.2.9 // indirect
+	atomicgo.dev/schedule v0.1.0 // indirect
 	cloud.google.com/go/longrunning v1.2.0 // indirect
 	cloud.google.com/go/pubsub/v2 v2.6.0 // indirect
 	dario.cat/mergo v1.0.2 // indirect
@@ -175,14 +178,14 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
 	github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect
 	github.com/ProtonMail/gopenpgp/v3 v3.4.1 // indirect
-	github.com/RoaringBitmap/roaring/v2 v2.26.0 // indirect
+	github.com/RoaringBitmap/roaring/v2 v2.24.0 // indirect
 	github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
 	github.com/adrg/xdg v0.5.3 // indirect
 	github.com/anchore/go-lzo v0.1.1 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
-	github.com/apache/arrow-go/v18 v18.8.0 // indirect
+	github.com/apache/arrow-go/v18 v18.7.0 // indirect
 	github.com/apache/thrift v0.24.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/bazelbuild/rules_go v0.46.0 // indirect
 	github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
@@ -197,6 +200,7 @@ require (
 	github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
 	github.com/cockroachdb/version v0.0.0-20250314144055-3860cd14adf2 // indirect
+	github.com/containerd/console v1.0.5 // indirect
 	github.com/containerd/errdefs v1.0.0 // indirect
 	github.com/containerd/errdefs/pkg v0.3.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
@@ -215,6 +219,7 @@ require (
 	github.com/goccy/go-yaml v1.18.0 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/gookit/color v1.6.0 // indirect
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
@@ -232,6 +237,7 @@ require (
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/lib/pq v1.12.0 // indirect
+	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
 	github.com/lpar/calendar v0.2.0 // indirect
 	github.com/magiconair/properties v1.8.10 // indirect
@@ -254,10 +260,12 @@ require (
 	github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect
 	github.com/pierrre/geohash v1.0.0 // indirect
 	github.com/pquerna/otp v1.5.0 // indirect
+	github.com/pterm/pterm v0.12.83 // indirect
 	github.com/quic-go/qpack v0.6.0 // indirect
 	github.com/rclone/Proton-API-Bridge v1.0.5 // indirect
 	github.com/rclone/go-proton-api v1.0.4 // indirect
 	github.com/rogpeppe/go-internal v1.15.0 // indirect
+	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sasha-s/go-deadlock v0.3.1 // indirect
 	github.com/smarty/assertions v1.15.0 // indirect
@@ -273,6 +281,7 @@ require (
 	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/zeebo/xxh3 v1.1.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 // indirect
@@ -293,13 +302,13 @@ require (
 	cloud.google.com/go/iam v1.12.0 // indirect
 	cloud.google.com/go/monitoring v1.30.0 // indirect
 	filippo.io/edwards25519 v1.2.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
 	github.com/Azure/go-ntlmssp v0.1.1 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
 	github.com/Files-com/files-sdk-go/v3 v3.3.194 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect
@@ -313,25 +322,25 @@ require (
 	github.com/ProtonMail/go-srp v0.0.7 // indirect
 	github.com/PuerkitoBio/goquery v1.12.0 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
-	github.com/andybalholm/brotli v1.2.3 // indirect
+	github.com/andybalholm/brotli v1.2.2 // indirect
 	github.com/andybalholm/cascadia v1.3.4 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.51.1
+	github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.51.0
 	github.com/aws/smithy-go v1.28.1
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
@@ -356,7 +365,7 @@ require (
 	github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 // indirect
 	github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
-	github.com/fatih/color v1.19.0 // indirect
+	github.com/fatih/color v1.18.0 // indirect
 	github.com/felixge/httpsnoop v1.1.0 // indirect
 	github.com/flynn/noise v1.1.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.13 // indirect
@@ -388,10 +397,10 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-metrics v0.7.0 // indirect
-	github.com/hashicorp/go-msgpack/v2 v2.1.5 // indirect
+	github.com/hashicorp/go-metrics v0.5.4 // indirect
+	github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
-	github.com/hashicorp/golang-lru v1.0.2 // indirect
+	github.com/hashicorp/golang-lru v0.6.0 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
@@ -469,10 +478,10 @@ require (
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
-	github.com/yandex-cloud/go-genproto v0.0.0-20240819112322-98a264d392f6 // indirect
+	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
 	github.com/ydb-platform/ydb-go-genproto v0.0.0-20260810122915-65bfd5c4b705 // indirect
-	github.com/ydb-platform/ydb-go-yc v0.12.5 // indirect
-	github.com/ydb-platform/ydb-go-yc-metadata v0.6.2 // indirect
+	github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
+	github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.4 // indirect
@@ -483,15 +492,15 @@ require (
 	go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect
-	go.opentelemetry.io/otel v1.46.0 // indirect
-	go.opentelemetry.io/otel/metric v1.46.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.46.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect
-	go.opentelemetry.io/otel/trace v1.46.0 // indirect
+	go.opentelemetry.io/otel v1.45.0 // indirect
+	go.opentelemetry.io/otel/metric v1.45.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.45.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect
+	go.opentelemetry.io/otel/trace v1.45.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
-	golang.org/x/term v0.47.0
-	golang.org/x/time v0.16.0
+	golang.org/x/term v0.46.0
+	golang.org/x/time v0.15.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.48"
+appVersion: "4.47"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.48.2
+version: 4.47.1

k8s/charts/seaweedfs/README.md

@@ -27,19 +27,6 @@ so your deployment will be spread/HA.
 * cert config exists and can be enabled, but not been tested, requires cert-manager to be installed.
 
 ## Prerequisites
-
-The chart's templates render with Helm 3.16.3 and newer versions tested in CI.
-Earlier Helm versions are not covered by this chart's compatibility checks.
-When JWT signing is enabled, a live upgrade reuses keys from the existing
-security ConfigMap (including the legacy ConfigMap name); a key absent from
-that ConfigMap is generated. Plain `helm template` does not read cluster state;
-use an install and upgrade against a cluster to check key persistence.
-
-The key reader supports single-line quoted TOML strings in JWT sections,
-including indented assignments, quoted key names, and bare or simply quoted
-section-name segments. An unsupported value or quoted header fails the upgrade
-rather than silently rotating a key.
-
 ### Database
 
 leveldb is the default database, this supports multiple filer replicas that will [sync automatically](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication), with some [limitations](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication#limitation).
@@ -542,26 +529,6 @@ helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum
 helm install seaweedfs-worker-balance seaweedfs/seaweedfs -f values-worker-balance.yaml
 ```
 
-## Gateway API
-
-Every component with an `ingress` block (master, volume, filer, s3 and admin) can also be exposed through a [Gateway API](https://gateway-api.sigs.k8s.io/) `HTTPRoute`, for clusters that route through a Gateway instead of an Ingress controller. `<component>.httpRoute` sits next to `<component>.ingress` and is disabled by default. The chart does not create the Gateway, so point `parentRefs` at one that already exists, and the `gateway.networking.k8s.io/v1` CRDs must be installed.
-
-```yaml
-s3:
-  httpRoute:
-    enabled: true
-    parentRefs:
-      - group: gateway.networking.k8s.io
-        kind: Gateway
-        name: my-gateway
-        namespace: gateway-system
-        sectionName: https
-    hostnames:
-      - s3.example.com
-```
-
-With `rules` left empty the route sends all traffic to the component's own Service and port. A rule may set `matches`, `filters`, `timeouts` and `backendRefs`, and a rule without `backendRefs` still routes to that Service. In all-in-one mode the master, volume, filer and s3 routes target the all-in-one Service. When S3 runs only on the filer (`filer.s3.enabled` without `s3.enabled`), the s3 route uses `filer.s3.port`.
-
 ## Network Policies
 
 In a namespace with a default-deny policy the install hangs: the components cannot resolve each other, and the post-install bucket hook waits on the master and filer until it gives up. `networkPolicy.enabled` renders one `NetworkPolicy` per component, selecting its pods by the standard `app.kubernetes.io/{name,instance,component}` labels and admitting traffic from the other pods of the release on the ports that component listens on.
@@ -603,23 +570,6 @@ Two things worth knowing before you turn this on:
 
 The DNS selectors default to CoreDNS as kubeadm, kind and the managed offerings from AWS, Google and Azure install it. On OpenShift, override `egress.dnsNamespaceSelector` and `egress.dnsPodSelector` to match `openshift-dns`; see the comment in `values.yaml`.
 
-## Pod and container security contexts
-
-Pod and container security contexts are configurable independently for every built-in workload and remain empty by default for backwards compatibility. The examples in `values.yaml` show how to enable a `RuntimeDefault` seccomp profile, disable privilege escalation and privileged mode, drop all Linux capabilities, and use a read-only root filesystem.
-
-SeaweedFS uses `/tmp` for Unix sockets, temporary uploads, worker task files, and other runtime data. When `readOnlyRootFilesystem` is enabled for a built-in component, the chart mounts a writable `emptyDir` at `/tmp` for its chart-managed containers. Its optional size limit can be configured globally:
-
-```yaml
-global:
-  seaweedfs:
-    tmpDir:
-      sizeLimit: 1Gi
-```
-
-The chart does not enable `runAsNonRoot` by default because its default `hostPath` storage may be owned by root. To enforce the Kubernetes `restricted` Pod Security Standard, use storage that is writable by a non-root user and configure `runAsNonRoot` or use the OpenShift overrides below.
-
-Security contexts configured for a component also apply to the chart-managed helper containers for that component. User-provided init containers and sidecars must define their own container security context and writable mounts.
-
 ## OpenShift Support
 
 SeaweedFS can be deployed on OpenShift or any cluster enforcing the Kubernetes "restricted" Pod Security Standard. By default, OpenShift blocks containers that run as root or use `hostPath` volumes.
@@ -628,7 +578,6 @@ To deploy on OpenShift, use the provided `openshift-values.yaml` which overrides
 1. Use `PersistentVolumeClaims` instead of `hostPath`.
 2. Enable `runAsNonRoot` and omit hardcoded UIDs to allow OpenShift to assign valid UIDs automatically.
 3. Apply appropriate `seccompProfile` and drop capabilities.
-4. Use a read-only root filesystem with writable temporary storage at `/tmp`.
 
 Usage:
 ```bash

k8s/charts/seaweedfs/openshift-values.yaml

@@ -15,7 +15,6 @@
 #      automatically assign a valid UID from the namespace's allocated range.
 #   3. Dropping all Linux capabilities and setting allowPrivilegeEscalation: false
 #   4. Enabling RuntimeDefault seccompProfile
-#   5. Using a read-only root filesystem with writable temporary storage
 #
 # Usage:
 #   helm install seaweedfs seaweedfs/seaweedfs \
@@ -50,7 +49,6 @@ master:
   containerSecurityContext:
     enabled: true
     allowPrivilegeEscalation: false
-    readOnlyRootFilesystem: true
     capabilities:
       drop: ["ALL"]
     runAsNonRoot: true
@@ -78,7 +76,6 @@ volume:
   containerSecurityContext:
     enabled: true
     allowPrivilegeEscalation: false
-    readOnlyRootFilesystem: true
     capabilities:
       drop: ["ALL"]
     runAsNonRoot: true
@@ -104,7 +101,6 @@ filer:
   containerSecurityContext:
     enabled: true
     allowPrivilegeEscalation: false
-    readOnlyRootFilesystem: true
     capabilities:
       drop: ["ALL"]
     runAsNonRoot: true
@@ -129,7 +125,6 @@ s3:
   containerSecurityContext:
     enabled: true
     allowPrivilegeEscalation: false
-    readOnlyRootFilesystem: true
     capabilities:
       drop: ["ALL"]
     runAsNonRoot: true

k8s/charts/seaweedfs/templates/admin/admin-httproute.yaml

@@ -1,55 +0,0 @@
-{{- if and .Values.admin.enabled .Values.admin.httpRoute.enabled }}
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: {{ include "seaweedfs.componentName" (list . "admin") }}
-  namespace: {{ .Release.Namespace }}
-  {{- with .Values.admin.httpRoute.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  labels:
-    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/component: admin
-    {{- with .Values.admin.httpRoute.labels }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
-spec:
-  {{- with .Values.admin.httpRoute.parentRefs }}
-  parentRefs:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  {{- with .Values.admin.httpRoute.hostnames }}
-  hostnames:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  rules:
-    {{- range .Values.admin.httpRoute.rules | default (list dict) }}
-    - backendRefs:
-        {{- with .backendRefs }}
-        {{- toYaml . | nindent 8 }}
-        {{- else }}
-        - group: ""
-          kind: Service
-          name: {{ include "seaweedfs.componentName" (list $ "admin") }}
-          namespace: {{ $.Release.Namespace }}
-          port: {{ $.Values.admin.port }}
-          weight: 1
-        {{- end }}
-      {{- with .matches }}
-      matches:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .filters }}
-      filters:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .timeouts }}
-      timeouts:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-    {{- end }}
-{{- end }}

k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml

@@ -192,7 +192,6 @@ spec:
               {{ $arg }}{{- if lt $index (sub (len $.Values.admin.extraArgs) 1) }} \{{ end }}
               {{- end }}
           volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.admin.containerSecurityContext (tpl (.Values.admin.extraVolumeMounts | default "") .) (tpl (.Values.admin.extraVolumes | default "") .)) | nindent 12 }}
             {{- if or (eq .Values.admin.data.type "hostPath") (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.data.type "emptyDir") (eq .Values.admin.data.type "existingClaim") }}
             - name: admin-data
               mountPath: /data
@@ -268,7 +267,6 @@ spec:
       {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
       {{- end }}
       volumes:
-        {{- include "seaweedfs.tmpDirVolume" (list . .Values.admin.containerSecurityContext (tpl (.Values.admin.extraVolumeMounts | default "") .) (tpl (.Values.admin.extraVolumes | default "") .) false) | nindent 8 }}
         {{- if eq .Values.admin.data.type "hostPath" }}
         - name: admin-data
           hostPath:

k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml

@@ -43,7 +43,7 @@ spec:
       {{- $configSecret := (lookup "v1" "Secret" .Release.Namespace $existingS3ConfigSecret) | default dict }}
       {{- $_ := set $podAnnotations "checksum/s3config" ($configSecret | toYaml | sha256sum) }}
       {{- else }}
-      {{- $_ := set $podAnnotations "checksum/s3config" (include "seaweedfs.s3.configChecksum" .) }}
+      {{- $_ := set $podAnnotations "checksum/s3config" (include (print .Template.BasePath "/s3/s3-secret.yaml") . | sha256sum) }}
       {{- end }}
       {{- $_ := set $podAnnotations "checksum/master-config" (include (print .Template.BasePath "/master/master-configmap.yaml") . | sha256sum) }}
       annotations:
@@ -306,7 +306,6 @@ spec:
               {{- end }}
 
           volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.allInOne.containerSecurityContext (tpl (.Values.allInOne.extraVolumeMounts | default "") .) (tpl (.Values.allInOne.extraVolumes | default "") .)) | nindent 12 }}
             - name: data
               mountPath: /data
             {{- if and .Values.allInOne.s3.enabled (or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth) }}
@@ -434,7 +433,6 @@ spec:
       {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
       {{- end }}
       volumes:
-        {{- include "seaweedfs.tmpDirVolume" (list . .Values.allInOne.containerSecurityContext (tpl (.Values.allInOne.extraVolumeMounts | default "") .) (tpl (.Values.allInOne.extraVolumes | default "") .) false) | nindent 8 }}
         {{- include "seaweedfs.licenseVolume" . | nindent 8 }}
         - name: data
           {{- if eq .Values.allInOne.data.type "hostPath" }}

k8s/charts/seaweedfs/templates/cert/admin-cert.yaml

@@ -25,10 +25,6 @@ spec:
     organizations:
     - "SeaweedFS CA"
   dnsNames:
-    - '{{ include "seaweedfs.fullname" . }}-admin'
-    - '{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}'
-    - '{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}.svc'
-    - '{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}.svc.cluster.local'
     - '*.{{ include "seaweedfs.fullname" . }}-admin'
     - '*.{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}'
     - '*.{{ include "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}.svc'

k8s/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml

@@ -28,13 +28,6 @@ rules:
     - "update"
     - "create"
     - "delete"
-- apiGroups: ["objectstorage.k8s.io"]
-  resources:
-    - "bucketclasses"
-  verbs:
-    - "get"
-    - "list"
-    - "watch"
 - apiGroups: ["coordination.k8s.io"]
   resources: ["leases"]
   verbs:
@@ -68,7 +61,7 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
 subjects:
   - kind: ServiceAccount
-    name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner
+    name: {{ .Values.global.seaweedfs.serviceAccountName }}-objectstorage-provisioner
     namespace: {{ .Release.Namespace }}
 roleRef:
   kind: ClusterRole

k8s/charts/seaweedfs/templates/cosi/cosi-deployment.yaml

@@ -61,7 +61,7 @@ spec:
       schedulerName: {{ .Values.cosi.schedulerName | quote }}
       {{- end }}
       enableServiceLinks: false
-      serviceAccountName: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner
+      serviceAccountName: {{ include "seaweedfs.componentName" (list . "objectstorage-provisioner") }}
       {{- if .Values.cosi.initContainers }}
       initContainers:
         {{ tpl .Values.cosi.initContainers . | nindent 8 | trim }}
@@ -113,7 +113,6 @@ spec:
             {{- end -}}
             {{- end }}
           volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.cosi.containerSecurityContext (tpl (.Values.cosi.extraVolumeMounts | default "") .) (tpl (.Values.cosi.extraVolumes | default "") .)) | nindent 12 }}
             - mountPath: /var/lib/cosi
               name: socket
             {{- if .Values.cosi.enableAuth }}
@@ -149,9 +148,6 @@ spec:
           resources:
             {{- toYaml . | nindent 12 }}
           {{- end }}
-          {{- if .Values.cosi.containerSecurityContext.enabled }}
-          securityContext: {{- omit .Values.cosi.containerSecurityContext "enabled" | toYaml | nindent 12 }}
-          {{- end }}
         - name: seaweedfs-cosi-sidecar
           image: "{{ .Values.cosi.sidecar.image }}"
           imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
@@ -163,7 +159,6 @@ spec:
                 fieldRef:
                   fieldPath: metadata.namespace
           volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.cosi.containerSecurityContext "" "") | nindent 12 }}
             - mountPath: /var/lib/cosi
               name: socket
           {{- with .Values.cosi.sidecar.resources }}
@@ -177,7 +172,6 @@ spec:
       {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
       {{- end }}
       volumes:
-        {{- include "seaweedfs.tmpDirVolume" (list . .Values.cosi.containerSecurityContext (tpl (.Values.cosi.extraVolumeMounts | default "") .) (tpl (.Values.cosi.extraVolumes | default "") .) true) | nindent 8 }}
         - name: socket
           emptyDir: {}
         {{- if .Values.cosi.enableAuth }}

k8s/charts/seaweedfs/templates/cosi/cosi-service-account.yaml

@@ -3,7 +3,7 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner
+  name: {{ .Values.global.seaweedfs.serviceAccountName }}-objectstorage-provisioner
   namespace: {{ .Release.Namespace }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}

k8s/charts/seaweedfs/templates/filer/filer-httproute.yaml

@@ -1,57 +0,0 @@
-{{- $filerEnabled := or .Values.filer.enabled .Values.allInOne.enabled }}
-{{- if and $filerEnabled .Values.filer.httpRoute.enabled }}
-{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "filer")) .Values.allInOne.enabled }}
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: {{ include "seaweedfs.componentName" (list . "filer") }}
-  namespace: {{ .Release.Namespace }}
-  {{- with .Values.filer.httpRoute.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  labels:
-    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/component: filer
-    {{- with .Values.filer.httpRoute.labels }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
-spec:
-  {{- with .Values.filer.httpRoute.parentRefs }}
-  parentRefs:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  {{- with .Values.filer.httpRoute.hostnames }}
-  hostnames:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  rules:
-    {{- range .Values.filer.httpRoute.rules | default (list dict) }}
-    - backendRefs:
-        {{- with .backendRefs }}
-        {{- toYaml . | nindent 8 }}
-        {{- else }}
-        - group: ""
-          kind: Service
-          name: {{ $serviceName }}
-          namespace: {{ $.Release.Namespace }}
-          port: {{ $.Values.filer.port }}
-          weight: 1
-        {{- end }}
-      {{- with .matches }}
-      matches:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .filters }}
-      filters:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .timeouts }}
-      timeouts:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-    {{- end }}
-{{- end }}

k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml

@@ -48,7 +48,7 @@ spec:
       {{- $configSecret := (lookup "v1" "Secret" .Release.Namespace .Values.filer.s3.existingConfigSecret) | default dict }}
       {{- $_ := set $podAnnotations "checksum/s3config" ($configSecret | toYaml | sha256sum) }}
       {{- else }}
-      {{- $_ := set $podAnnotations "checksum/s3config" (include "seaweedfs.s3.configChecksum" .) }}
+      {{- $_ := set $podAnnotations "checksum/s3config" (include (print .Template.BasePath "/s3/s3-secret.yaml") . | sha256sum) }}
       {{- end }}
       annotations:
         {{- toYaml $podAnnotations | nindent 8 }}
@@ -229,7 +229,6 @@ spec:
               {{ . }} \
               {{- end }}
           volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.filer.containerSecurityContext (tpl (.Values.filer.extraVolumeMounts | default "") .) (tpl (.Values.filer.extraVolumes | default "") .)) | nindent 12 }}
             {{- if (or (eq .Values.filer.logs.type "hostPath") (eq .Values.filer.logs.type "persistentVolumeClaim") (eq .Values.filer.logs.type "emptyDir")) }}
             - name: seaweedfs-filer-log-volume
               mountPath: "/logs/"
@@ -338,7 +337,6 @@ spec:
         {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
         {{- end }}
       volumes:
-        {{- include "seaweedfs.tmpDirVolume" (list . .Values.filer.containerSecurityContext (tpl (.Values.filer.extraVolumeMounts | default "") .) (tpl (.Values.filer.extraVolumes | default "") .) false) | nindent 8 }}
         {{- if eq .Values.filer.logs.type "hostPath" }}
         - name: seaweedfs-filer-log-volume
           hostPath:

k8s/charts/seaweedfs/templates/master/master-httproute.yaml

@@ -1,57 +0,0 @@
-{{- $masterEnabled := or .Values.master.enabled .Values.allInOne.enabled }}
-{{- if and $masterEnabled .Values.master.httpRoute.enabled }}
-{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "master")) .Values.allInOne.enabled }}
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: {{ include "seaweedfs.componentName" (list . "master") }}
-  namespace: {{ .Release.Namespace }}
-  {{- with .Values.master.httpRoute.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  labels:
-    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/component: master
-    {{- with .Values.master.httpRoute.labels }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
-spec:
-  {{- with .Values.master.httpRoute.parentRefs }}
-  parentRefs:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  {{- with .Values.master.httpRoute.hostnames }}
-  hostnames:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  rules:
-    {{- range .Values.master.httpRoute.rules | default (list dict) }}
-    - backendRefs:
-        {{- with .backendRefs }}
-        {{- toYaml . | nindent 8 }}
-        {{- else }}
-        - group: ""
-          kind: Service
-          name: {{ $serviceName }}
-          namespace: {{ $.Release.Namespace }}
-          port: {{ $.Values.master.port }}
-          weight: 1
-        {{- end }}
-      {{- with .matches }}
-      matches:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .filters }}
-      filters:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .timeouts }}
-      timeouts:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-    {{- end }}
-{{- end }}

k8s/charts/seaweedfs/templates/master/master-statefulset.yaml

@@ -179,7 +179,6 @@ spec:
               {{ . }} \
               {{- end }}
           volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.master.containerSecurityContext (tpl (.Values.master.extraVolumeMounts | default "") .) (tpl (.Values.master.extraVolumes | default "") .)) | nindent 12 }}
             - name : data-{{ .Release.Namespace }}
               mountPath: /data
             {{- if or (eq .Values.master.logs.type "hostPath") (eq .Values.master.logs.type "persistentVolumeClaim") (eq .Values.master.logs.type "emptyDir") }}
@@ -259,7 +258,6 @@ spec:
         {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
         {{- end }}
       volumes:
-        {{- include "seaweedfs.tmpDirVolume" (list . .Values.master.containerSecurityContext (tpl (.Values.master.extraVolumeMounts | default "") .) (tpl (.Values.master.extraVolumes | default "") .) false) | nindent 8 }}
         {{- include "seaweedfs.licenseVolume" . | nindent 8 }}
         {{- if eq .Values.master.logs.type "hostPath" }}
         - name: seaweedfs-master-log-volume

k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml

@@ -17,10 +17,6 @@ metadata:
 {{- end }}
 spec:
   replicas: {{ .Values.s3.replicas }}
-  {{- with .Values.s3.updateStrategy }}
-  strategy:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
@@ -44,7 +40,7 @@ spec:
       {{- $configSecret := (lookup "v1" "Secret" .Release.Namespace .Values.s3.existingConfigSecret) | default dict }}
       {{- $_ := set $podAnnotations "checksum/s3config" ($configSecret | toYaml | sha256sum) }}
       {{- else }}
-      {{- $_ := set $podAnnotations "checksum/s3config" (include "seaweedfs.s3.configChecksum" .) }}
+      {{- $_ := set $podAnnotations "checksum/s3config" (include (print .Template.BasePath "/s3/s3-secret.yaml") . | sha256sum) }}
       {{- end }}
       annotations:
         {{- toYaml $podAnnotations | nindent 8 }}
@@ -63,7 +59,7 @@ spec:
         {{ tpl .Values.s3.tolerations . | nindent 8 | trim }}
       {{- end }}
       {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
-      terminationGracePeriodSeconds: {{ .Values.s3.terminationGracePeriodSeconds }}
+      terminationGracePeriodSeconds: 10
       {{- if .Values.s3.priorityClassName }}
       priorityClassName: {{ .Values.s3.priorityClassName | quote }}
       {{- end }}
@@ -161,7 +157,6 @@ spec:
               {{ . }} \
               {{- end }}
           volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.s3.containerSecurityContext (tpl (.Values.s3.extraVolumeMounts | default "") .) (tpl (.Values.s3.extraVolumes | default "") .)) | nindent 12 }}
             {{- if or (eq .Values.s3.logs.type "hostPath") (eq .Values.s3.logs.type "emptyDir") }}
             - name: logs
               mountPath: "/logs/"
@@ -239,10 +234,6 @@ spec:
             failureThreshold: {{ .Values.s3.livenessProbe.failureThreshold }}
             timeoutSeconds: {{ .Values.s3.livenessProbe.timeoutSeconds }}
           {{- end }}
-          {{- with .Values.s3.lifecycle }}
-          lifecycle:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
           {{- with .Values.s3.resources }}
           resources:
             {{- toYaml . | nindent 12 }}
@@ -254,7 +245,6 @@ spec:
       {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
       {{- end }}
       volumes:
-        {{- include "seaweedfs.tmpDirVolume" (list . .Values.s3.containerSecurityContext (tpl (.Values.s3.extraVolumeMounts | default "") .) (tpl (.Values.s3.extraVolumes | default "") .) false) | nindent 8 }}
         {{- if .Values.s3.enableAuth }}
         - name: config-users
           secret:

k8s/charts/seaweedfs/templates/s3/s3-httproute.yaml

@@ -1,64 +0,0 @@
-{{- $s3Enabled := or .Values.s3.enabled (and .Values.filer.s3.enabled (not .Values.allInOne.enabled)) (and .Values.allInOne.enabled .Values.allInOne.s3.enabled) }}
-{{- if and $s3Enabled .Values.s3.httpRoute.enabled }}
-{{- $serviceName := include "seaweedfs.componentName" (list . "s3") }}
-{{- $s3Port := .Values.filer.s3.port }}
-{{- if and .Values.allInOne.enabled .Values.allInOne.s3.enabled }}
-{{- $serviceName = include "seaweedfs.componentName" (list . "all-in-one") }}
-{{- $s3Port = .Values.allInOne.s3.port | default .Values.s3.port }}
-{{- else if .Values.s3.enabled }}
-{{- $s3Port = .Values.s3.port }}
-{{- end }}
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: {{ include "seaweedfs.componentName" (list . "s3") }}
-  namespace: {{ .Release.Namespace }}
-  {{- with .Values.s3.httpRoute.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  labels:
-    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/component: s3
-    {{- with .Values.s3.httpRoute.labels }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
-spec:
-  {{- with .Values.s3.httpRoute.parentRefs }}
-  parentRefs:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  {{- with .Values.s3.httpRoute.hostnames }}
-  hostnames:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  rules:
-    {{- range .Values.s3.httpRoute.rules | default (list dict) }}
-    - backendRefs:
-        {{- with .backendRefs }}
-        {{- toYaml . | nindent 8 }}
-        {{- else }}
-        - group: ""
-          kind: Service
-          name: {{ $serviceName }}
-          namespace: {{ $.Release.Namespace }}
-          port: {{ $s3Port }}
-          weight: 1
-        {{- end }}
-      {{- with .matches }}
-      matches:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .filters }}
-      filters:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .timeouts }}
-      timeouts:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-    {{- end }}
-{{- end }}

k8s/charts/seaweedfs/templates/sftp/sftp-deployment.yaml

@@ -170,7 +170,6 @@ spec:
               -userStoreFile=/etc/sw/seaweedfs_sftp_config \
               -filer={{ include "seaweedfs.componentName" (list . "filer-client") }}.{{ .Release.Namespace }}:{{ .Values.filer.port }}
           volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.sftp.containerSecurityContext (tpl (.Values.sftp.extraVolumeMounts | default "") .) (tpl (.Values.sftp.extraVolumes | default "") .)) | nindent 12 }}
             {{- if or (eq .Values.sftp.logs.type "hostPath") (eq .Values.sftp.logs.type "emptyDir") }}
             - name: logs
               mountPath: "/logs/"
@@ -250,7 +249,6 @@ spec:
       {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
       {{- end }}
       volumes:
-        {{- include "seaweedfs.tmpDirVolume" (list . .Values.sftp.containerSecurityContext (tpl (.Values.sftp.extraVolumeMounts | default "") .) (tpl (.Values.sftp.extraVolumes | default "") .) false) | nindent 8 }}
         {{- if .Values.sftp.enableAuth }}
         - name: config-users
           secret:

k8s/charts/seaweedfs/templates/shared/_helpers.tpl

@@ -76,43 +76,6 @@ Inject extra environment vars in the format key:value, if populated
 {{- end }}
 {{- end -}}
 
-{{/*
-Writable temporary directory for containers using a read-only root filesystem.
-Input: list of the root context, the component container security context, the
-rendered extraVolumeMounts and extraVolumes, and whether the pod has secondary
-chart-managed containers that mount seaweedfs-tmp. A user-supplied /tmp mount
-only covers the main container, so the volume is still emitted for secondaries;
-a user-supplied seaweedfs-tmp volume is reused rather than duplicated.
-*/}}
-{{- define "seaweedfs.tmpDirCovered" -}}
-{{- regexMatch `(?m)^\s*-?\s*mountPath:\s*['"]?/tmp/?['"]?\s*(#.*)?$` (index . 2) -}}
-{{- end -}}
-
-{{- define "seaweedfs.tmpDirVolume" -}}
-{{- $root := index . 0 -}}
-{{- $securityContext := index . 1 -}}
-{{- if and $securityContext.enabled $securityContext.readOnlyRootFilesystem
-  (or (index . 4) (ne (include "seaweedfs.tmpDirCovered" .) "true"))
-  (not (regexMatch `(?m)^\s*-?\s*name:\s*['"]?seaweedfs-tmp['"]?\s*(#.*)?$` (index . 3))) }}
-- name: seaweedfs-tmp
-  {{- with $root.Values.global.seaweedfs.tmpDir.sizeLimit }}
-  emptyDir:
-    sizeLimit: {{ . | quote }}
-  {{- else }}
-  emptyDir: {}
-  {{- end }}
-{{- end }}
-{{- end -}}
-
-{{- define "seaweedfs.tmpDirVolumeMount" -}}
-{{- $securityContext := index . 1 -}}
-{{- if and $securityContext.enabled $securityContext.readOnlyRootFilesystem
-  (ne (include "seaweedfs.tmpDirCovered" .) "true") }}
-- name: seaweedfs-tmp
-  mountPath: /tmp
-{{- end }}
-{{- end -}}
-
 {{/* Whether the mysql filer store is selected; a flag the chart cannot read counts as selected. */}}
 {{- define "seaweedfs.filer.mysqlEnabled" -}}
 {{- $merged := dict -}}
@@ -444,46 +407,6 @@ true
 {{- end -}}
 {{- end -}}
 
-{{/* Read a JWT key from the chart's existing security.toml without fromToml
-     (which requires Helm >=3.17). Args: (list "<section>" $raw).
-     Return the single-line TOML string token, including its quotes, so escapes
-     and explicitly empty values survive unchanged. No output means absent.
-     Accept bare or simply quoted dotted header segments. Fail on other quoted
-     headers rather than risk rotating a key hidden by unsupported syntax.
-     This reads the chart's section/key layout, not arbitrary TOML syntax. */}}
-{{- define "seaweedfs.existingTomlKey" -}}
-{{- $section := index . 0 -}}
-{{- $raw := index . 1 -}}
-{{- $parts := list -}}
-{{- range $part := splitList "." $section -}}
-  {{- $escaped := regexQuoteMeta $part -}}
-  {{- $parts = append $parts (printf `(?:%s|"%s"|'%s')` $escaped $escaped $escaped) -}}
-{{- end -}}
-{{- $header := printf `^\[[ \t]*%s[ \t]*\][ \t]*(#.*)?$` (join `[ \t]*\.[ \t]*` $parts) -}}
-{{- $segment := `(?:[A-Za-z0-9_-]+|"[^"\\]*"|'[^']*')` -}}
-{{- $simpleHeader := printf `^\[[ \t]*%s(?:[ \t]*\.[ \t]*%s)*[ \t]*\][ \t]*(#.*)?$` $segment $segment -}}
-{{- $assignment := `^(key|"key"|'key')[ \t]*=[ \t]*` -}}
-{{- $string := `"([^"\\]|\\.)*"|'[^']*'` -}}
-{{- $active := false -}}
-{{- $key := "" -}}
-{{- range $rawLine := splitList "\n" $raw -}}
-  {{- $line := trim $rawLine -}}
-  {{- if hasPrefix "[" $line -}}
-    {{- if and (regexMatch `^\[[^]]*["']` $line) (not (regexMatch $simpleHeader $line)) (eq $key "") -}}
-      {{- fail (printf "security.toml has an unsupported quoted section header; refusing to replace [%s].key" $section) -}}
-    {{- end -}}
-    {{- $active = regexMatch $header $line -}}
-  {{- else if and $active (eq $key "") (regexMatch $assignment $line) -}}
-    {{- $value := regexReplaceAll $assignment $line "" -}}
-    {{- if not (regexMatch (printf `^(%s)[ \t]*(#.*)?$` $string) $value) -}}
-      {{- fail (printf "security.toml [%s].key must be a single-line quoted TOML string; refusing to replace an existing key" $section) -}}
-    {{- end -}}
-    {{- $key = regexFind $string $value -}}
-  {{- end -}}
-{{- end -}}
-{{- $key -}}
-{{- end -}}
-
 {{/* True when the post-install bucket hook Job renders: an S3 endpoint, plus
      buckets to create on it. Read by the Job itself and by its NetworkPolicy,
      which has to appear exactly when the Job does - a Job without its policy
@@ -544,9 +467,8 @@ true
 {{-             $pvcName := printf "%s-%s-%s-%d" $dir.name $seaweedfsName $volumeName $e }}
 {{-             $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) }}
 {{-             if $currentPVC }}
-{{- /* include returns a string such as "6.442450944e+10"; convert back to a number, or gt compares lexically */}}
-{{-               $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage | float64 }}
-{{-               $newSize := include "seaweedfs.resource-quantity" $desiredSize | float64 }}
+{{-               $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage }}
+{{-               $newSize := include "seaweedfs.resource-quantity" $desiredSize }}
 {{-               if gt $newSize $oldSize }}
 {{-                 $commands = append $commands (printf "kubectl patch pvc %s-%s-%s-%d -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"%s\"}}}}'" $dir.name $seaweedfsName $volumeName $e $desiredSize) }}
 {{-               end }}
@@ -692,28 +614,3 @@ Usage: {{ include "seaweedfs.service.loadBalancerFields" .Values.s3.service }}
 {{- end }}
 {{- end }}
 {{- end -}}
-
-{{/*
-Checksum that restarts the S3-serving pods when the generated S3 config changes. It hashes the
-values s3-secret.yaml is built from, not the rendered Secret: that template keeps its generated keys
-with lookup, so rendering it again for a checksum draws new random keys whenever lookup finds
-nothing (on install), and the first upgrade then changed the checksum and restarted the pods
-although nothing had changed. Only the values that template reads are hashed, so scaling the S3
-gateway or changing its resources does not restart the filer. Keep this list in step with
-s3-secret.yaml.
-Keys the chart generated are not part of it. Deleting the Secret to rotate them regenerates them on
-the next upgrade without restarting the pods; restart them afterwards (kubectl rollout restart).
-Folding the stored keys in would bring the first-upgrade restart back, since lookup finds nothing
-on install.
-*/}}
-{{- define "seaweedfs.s3.configChecksum" -}}
-{{- dict
-  "secretName" (include "seaweedfs.fullname" .)
-  "enabled" (list .Values.s3.enabled .Values.filer.s3.enabled .Values.allInOne.enabled .Values.allInOne.s3.enabled)
-  "enableAuth" (list .Values.s3.enableAuth .Values.filer.s3.enableAuth .Values.allInOne.s3.enableAuth)
-  "existingConfigSecret" (list .Values.s3.existingConfigSecret .Values.filer.s3.existingConfigSecret .Values.allInOne.s3.existingConfigSecret)
-  "credentials" .Values.s3.credentials
-  "legacySecret" (list .Values.s3.reuseLegacySecret .Values.s3.legacySecretName)
-  "auditLogConfig" (list .Values.s3.auditLogConfig .Values.filer.s3.auditLogConfig)
-  | toJson | sha256sum -}}
-{{- end -}}

k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml

@@ -16,12 +16,6 @@
 {{- $clusterUpper := upper $clusterAlias }}
 {{- $clusterMasterKey := printf "WEED_CLUSTER_%s_MASTER" $clusterUpper }}
 {{- $clusterFilerKey := printf "WEED_CLUSTER_%s_FILER" $clusterUpper }}
-{{- $podSecurityContext := .Values.filer.podSecurityContext }}
-{{- $containerSecurityContext := .Values.filer.containerSecurityContext }}
-{{- if .Values.allInOne.enabled }}
-  {{- $podSecurityContext = .Values.allInOne.podSecurityContext }}
-  {{- $containerSecurityContext = .Values.allInOne.containerSecurityContext }}
-{{- end }}
 
 {{- /* Check allInOne mode first */}}
 {{- if .Values.allInOne.enabled }}
@@ -77,8 +71,8 @@ spec:
         app.kubernetes.io/component: bucket-hook
     spec:
       restartPolicy: Never
-      {{- if $podSecurityContext.enabled }}
-      securityContext: {{- omit $podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- if .Values.filer.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.filer.podSecurityContext "enabled" | toYaml | nindent 8 }}
       {{- end }}
       {{- include "seaweedfs.imagePullSecrets" $ | nindent 6 }}
       containers:
@@ -208,9 +202,8 @@ spec:
                 /usr/bin/weed shell
           {{- end }}  
           {{- end }}
-        {{- if or (and $containerSecurityContext.enabled $containerSecurityContext.readOnlyRootFilesystem) $enableAuth (include "seaweedfs.securityConfigEnabled" .) }}
+        {{- if or $enableAuth (include "seaweedfs.securityConfigEnabled" .) }}
         volumeMounts:
-          {{- include "seaweedfs.tmpDirVolumeMount" (list . $containerSecurityContext "" "") | nindent 10 }}
           {{- if $enableAuth }}
           - name: config-users
             mountPath: /etc/sw
@@ -222,14 +215,6 @@ spec:
             mountPath: /etc/seaweedfs/security.toml
             subPath: security.toml
           {{- end }}
-          {{- if .Values.global.seaweedfs.enableSecurity }}
-          - name: ca-cert
-            readOnly: true
-            mountPath: /usr/local/share/ca-certificates/ca/
-          - name: client-cert
-            readOnly: true
-            mountPath: /usr/local/share/ca-certificates/client/
-          {{- end }}
         {{- end }}
         ports:
           - containerPort: {{ .Values.master.port }}
@@ -244,12 +229,11 @@ spec:
         resources:
           {{- toYaml . | nindent 10 }}
         {{- end }}
-        {{- if $containerSecurityContext.enabled }}
-        securityContext: {{- omit $containerSecurityContext "enabled" | toYaml | nindent 12 }}
+        {{- if .Values.filer.containerSecurityContext.enabled }}
+        securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
         {{- end }}
-    {{- if or (and $containerSecurityContext.enabled $containerSecurityContext.readOnlyRootFilesystem) $enableAuth (include "seaweedfs.securityConfigEnabled" .) }}
+    {{- if or $enableAuth (include "seaweedfs.securityConfigEnabled" .) }}
       volumes:
-        {{- include "seaweedfs.tmpDirVolume" (list . $containerSecurityContext "" "" false) | nindent 8 }}
         {{- if $enableAuth }}
         - name: config-users
           secret:
@@ -265,13 +249,5 @@ spec:
           configMap:
             name: {{ include "seaweedfs.fullname" . }}-security-config
         {{- end }}
-        {{- if .Values.global.seaweedfs.enableSecurity }}
-        - name: ca-cert
-          secret:
-            secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
-        - name: client-cert
-          secret:
-            secretName: {{ include "seaweedfs.fullname" . }}-client-cert
-        {{- end }}
     {{- end }}
 {{- end }}

k8s/charts/seaweedfs/templates/shared/security-configmap.yaml

@@ -18,7 +18,7 @@ data:
   {{- $legacyName := printf "%s-%s" (include "seaweedfs.name" .) "security-config" }}
   {{- $existing = lookup "v1" "ConfigMap" .Release.Namespace $legacyName }}
   {{- end }}
-  {{- $existingToml := dig "data" "security.toml" "" $existing }}
+  {{- $securityConfig := fromToml (dig "data" "security.toml" "" $existing) }}
   {{- $securityConfigValues := .Values.global.seaweedfs.securityConfig | default dict }}
   {{- $jwtSigning := $securityConfigValues.jwtSigning | default dict }}
   {{- $expiresAfterSeconds := $jwtSigning.expiresAfterSeconds | default dict }}
@@ -29,7 +29,7 @@ data:
     # the jwt signing key is read by master and volume server
     # the jwt defaults to expire after 10 seconds
     [jwt.signing]
-    key = {{ include "seaweedfs.existingTomlKey" (list "jwt.signing" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
+    key = "{{ dig "jwt" "signing" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
     {{- if gt (int $expiresAfterSeconds.volumeWrite) 0 }}
     expires_after_seconds = {{ int $expiresAfterSeconds.volumeWrite }}
     {{- end }}
@@ -41,7 +41,7 @@ data:
     # - the Volume server validates the JWT on reading
     # the jwt defaults to expire after 60 seconds
     [jwt.signing.read]
-    key = {{ include "seaweedfs.existingTomlKey" (list "jwt.signing.read" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
+    key = "{{ dig "jwt" "signing" "read" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
     {{- if gt (int $expiresAfterSeconds.volumeRead) 0 }}
     expires_after_seconds = {{ int $expiresAfterSeconds.volumeRead }}
     {{- end }}
@@ -53,7 +53,7 @@ data:
     # - the Filer server validates the JWT on writing
     # the jwt defaults to expire after 10 seconds
     [jwt.filer_signing]
-    key = {{ include "seaweedfs.existingTomlKey" (list "jwt.filer_signing" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
+    key = "{{ dig "jwt" "filer_signing" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
     {{- if gt (int $expiresAfterSeconds.filerWrite) 0 }}
     expires_after_seconds = {{ int $expiresAfterSeconds.filerWrite }}
     {{- end }}
@@ -65,7 +65,7 @@ data:
     # - the Filer server validates the JWT on reading
     # the jwt defaults to expire after 60 seconds
     [jwt.filer_signing.read]
-    key = {{ include "seaweedfs.existingTomlKey" (list "jwt.filer_signing.read" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
+    key = "{{ dig "jwt" "filer_signing" "read" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
     {{- if gt (int $expiresAfterSeconds.filerRead) 0 }}
     expires_after_seconds = {{ int $expiresAfterSeconds.filerRead }}
     {{- end }}

k8s/charts/seaweedfs/templates/volume/volume-httproute.yaml

@@ -1,57 +0,0 @@
-{{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }}
-{{- if and $volumeEnabled .Values.volume.httpRoute.enabled }}
-{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }}
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: {{ include "seaweedfs.componentName" (list . "volume") }}
-  namespace: {{ .Release.Namespace }}
-  {{- with .Values.volume.httpRoute.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  labels:
-    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/component: volume
-    {{- with .Values.volume.httpRoute.labels }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
-spec:
-  {{- with .Values.volume.httpRoute.parentRefs }}
-  parentRefs:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  {{- with .Values.volume.httpRoute.hostnames }}
-  hostnames:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  rules:
-    {{- range .Values.volume.httpRoute.rules | default (list dict) }}
-    - backendRefs:
-        {{- with .backendRefs }}
-        {{- toYaml . | nindent 8 }}
-        {{- else }}
-        - group: ""
-          kind: Service
-          name: {{ $serviceName }}
-          namespace: {{ $.Release.Namespace }}
-          port: {{ $.Values.volume.port }}
-          weight: 1
-        {{- end }}
-      {{- with .matches }}
-      matches:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .filters }}
-      filters:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .timeouts }}
-      timeouts:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-    {{- end }}
-{{- end }}

k8s/charts/seaweedfs/templates/volume/volume-resize-hook.yaml

@@ -32,32 +32,13 @@ spec:
     spec:
       serviceAccountName: {{ $seaweedfsName }}-volume-resize-hook
       restartPolicy: Never
-      {{- if .Values.volume.podSecurityContext.enabled }}
-      securityContext: {{- omit .Values.volume.podSecurityContext "enabled" | toYaml | nindent 8 }}
-      {{- end }}
       containers:
         - name: resize
           image: {{ .Values.volume.resizeHook.image }}
-          {{- if and .Values.volume.containerSecurityContext.enabled .Values.volume.containerSecurityContext.readOnlyRootFilesystem }}
-          env:
-            - name: HOME
-              value: /tmp
-          {{- end }}
           command: ["sh", "-xec"]
           args:
             - |
 {{ $commands | indent 14 }}
-          {{- if and .Values.volume.containerSecurityContext.enabled .Values.volume.containerSecurityContext.readOnlyRootFilesystem }}
-          volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.volume.containerSecurityContext "" "") | nindent 12 }}
-          {{- end }}
-          {{- if .Values.volume.containerSecurityContext.enabled }}
-          securityContext: {{- omit .Values.volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}
-          {{- end }}
-      {{- if and .Values.volume.containerSecurityContext.enabled .Values.volume.containerSecurityContext.readOnlyRootFilesystem }}
-      volumes:
-        {{- include "seaweedfs.tmpDirVolume" (list . .Values.volume.containerSecurityContext "" "" false) | nindent 8 }}
-      {{- end }}
 ---
 apiVersion: v1
 kind: ServiceAccount

k8s/charts/seaweedfs/templates/volume/volume-statefulset.yaml

@@ -104,7 +104,6 @@ spec:
                 fi
               done
           volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list $ $volume.containerSecurityContext "" "") | nindent 12 }}
             - name: idx
               mountPath: /idx
           {{- range $dir :=  $volume.dataDirs }}
@@ -225,7 +224,6 @@ spec:
                 {{ . }} \
                 {{- end }}
           volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list $ $volume.containerSecurityContext (tpl (printf "{{ $volumeName := \"%s\" }}%s" $volumeName ($volume.extraVolumeMounts | default "")) $) (tpl ($volume.extraVolumes | default "") $)) | nindent 12 }}
             {{- range $dir := $volume.dataDirs }}
             {{- if not ( eq $dir.type "custom" ) }}
             - name: {{ $dir.name }}
@@ -308,7 +306,6 @@ spec:
       {{- include "seaweedfs.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
       {{- end }}
       volumes:
-        {{- include "seaweedfs.tmpDirVolume" (list $ $volume.containerSecurityContext (tpl (printf "{{ $volumeName := \"%s\" }}%s" $volumeName ($volume.extraVolumeMounts | default "")) $) (tpl ($volume.extraVolumes | default "") $) (and $initContainers_exists $volume.idx)) | nindent 8 }}
 
      {{- range $dir := $volume.dataDirs }}
 

k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml

@@ -144,7 +144,6 @@ spec:
               {{ $arg }}{{- if lt $index (sub (len $.Values.worker.extraArgs) 1) }} \{{ end }}
               {{- end }}
           volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.worker.containerSecurityContext (tpl (.Values.worker.extraVolumeMounts | default "") .) (tpl (.Values.worker.extraVolumes | default "") .)) | nindent 12 }}
             {{- if or (eq .Values.worker.data.type "hostPath") (eq .Values.worker.data.type "emptyDir") (eq .Values.worker.data.type "existingClaim") }}
             - name: worker-data
               mountPath: {{ .Values.worker.workingDir }}
@@ -263,16 +262,15 @@ spec:
               --metrics-ip=0.0.0.0 \
               {{- end }}
               --max-concurrency={{ .Values.worker.maxExecute }}
+          {{- if .Values.global.seaweedfs.enableSecurity }}
           volumeMounts:
-            {{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.worker.containerSecurityContext "" "") | nindent 12 }}
-            {{- if .Values.global.seaweedfs.enableSecurity }}
             - name: ca-cert
               readOnly: true
               mountPath: /usr/local/share/ca-certificates/ca/
             - name: worker-cert
               readOnly: true
               mountPath: /usr/local/share/ca-certificates/worker/
-            {{- end }}
+          {{- end }}
           {{- if .Values.worker.lanceMetricsPort }}
           ports:
             - containerPort: {{ .Values.worker.lanceMetricsPort }}
@@ -304,7 +302,6 @@ spec:
       {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
       {{- end }}
       volumes:
-        {{- include "seaweedfs.tmpDirVolume" (list . .Values.worker.containerSecurityContext (tpl (.Values.worker.extraVolumeMounts | default "") .) (tpl (.Values.worker.extraVolumes | default "") .) (ne (include "seaweedfs.worker.lanceNamespaceUrl" .) "")) | nindent 8 }}
         {{- if eq .Values.worker.data.type "hostPath" }}
         - name: worker-data
           hostPath:

k8s/charts/seaweedfs/values.yaml

@@ -25,9 +25,6 @@ global:
     imagePullPolicy: IfNotPresent
     restartPolicy: Always
     loggingLevel: 1
-    # Writable temporary storage used when containers run with a read-only root filesystem.
-    tmpDir:
-      sizeLimit: ""
     enableSecurity: false
     masterServer: null
     # filerWrite: true mounts security.toml on filer + admin without needing
@@ -263,8 +260,6 @@ master:
   #   runAsUser: 1000
   #   runAsGroup: 3000
   #   fsGroup: 2000
-  #   seccompProfile:
-  #     type: RuntimeDefault
   podSecurityContext: {}
 
   # Configure security context for Container
@@ -273,16 +268,7 @@ master:
   # containerSecurityContext:
   #   enabled: true
   #   runAsUser: 2000
-  #   runAsGroup: 3000
-  #   runAsNonRoot: true
-  #   privileged: false
   #   allowPrivilegeEscalation: false
-  #   readOnlyRootFilesystem: true
-  #   capabilities:
-  #     drop:
-  #       - ALL
-  #   seccompProfile:
-  #     type: RuntimeDefault
   containerSecurityContext: {}
 
   ingress:
@@ -311,24 +297,6 @@ master:
       #   sub_filter_once off;
     tls: []
 
-  # Gateway API HTTPRoute, an alternative to the ingress above.
-  httpRoute:
-    enabled: false
-    labels: {}
-    annotations: {}
-    # Gateways this route attaches to, for example:
-    #   - group: gateway.networking.k8s.io
-    #     kind: Gateway
-    #     name: my-gateway
-    #     namespace: gateway-system
-    #     sectionName: https
-    parentRefs: []
-    hostnames: []
-    # Each rule may set matches, filters, timeouts and backendRefs.
-    # A rule without backendRefs routes to the master service.
-    # An empty list routes all traffic there.
-    rules: []
-
   extraEnvironmentVars:
     WEED_MASTER_VOLUME_GROWTH_COPY_1: "7"
     WEED_MASTER_VOLUME_GROWTH_COPY_2: "6"
@@ -592,8 +560,6 @@ volume:
   #   runAsUser: 1000
   #   runAsGroup: 3000
   #   fsGroup: 2000
-  #   seccompProfile:
-  #     type: RuntimeDefault
   podSecurityContext: {}
 
   # Configure security context for Container
@@ -602,16 +568,7 @@ volume:
   # containerSecurityContext:
   #   enabled: true
   #   runAsUser: 2000
-  #   runAsGroup: 3000
-  #   runAsNonRoot: true
-  #   privileged: false
   #   allowPrivilegeEscalation: false
-  #   readOnlyRootFilesystem: true
-  #   capabilities:
-  #     drop:
-  #       - ALL
-  #   seccompProfile:
-  #     type: RuntimeDefault
   containerSecurityContext: {}
 
   # used to configure livenessProbe on volume-server containers
@@ -665,24 +622,6 @@ volume:
       #   sub_filter '/seaweedfsstatic' './seaweedfsstatic';
       #   sub_filter_once off;
 
-  # Gateway API HTTPRoute, an alternative to the ingress above.
-  httpRoute:
-    enabled: false
-    labels: {}
-    annotations: {}
-    # Gateways this route attaches to, for example:
-    #   - group: gateway.networking.k8s.io
-    #     kind: Gateway
-    #     name: my-gateway
-    #     namespace: gateway-system
-    #     sectionName: https
-    parentRefs: []
-    hostnames: []
-    # Each rule may set matches, filters, timeouts and backendRefs.
-    # A rule without backendRefs routes to the volume server service.
-    # An empty list routes all traffic there.
-    rules: []
-
 # Map of named volume groups for topology-aware deployments.
 # Each key inherits all fields from the `volume` section but can override
 # them locally—for example, replicas, nodeSelector, dataCenter, etc.
@@ -906,8 +845,6 @@ filer:
   #   runAsUser: 1000
   #   runAsGroup: 3000
   #   fsGroup: 2000
-  #   seccompProfile:
-  #     type: RuntimeDefault
   podSecurityContext: {}
 
   # Configure security context for Container
@@ -916,16 +853,7 @@ filer:
   # containerSecurityContext:
   #   enabled: true
   #   runAsUser: 2000
-  #   runAsGroup: 3000
-  #   runAsNonRoot: true
-  #   privileged: false
   #   allowPrivilegeEscalation: false
-  #   readOnlyRootFilesystem: true
-  #   capabilities:
-  #     drop:
-  #       - ALL
-  #   seccompProfile:
-  #     type: RuntimeDefault
   containerSecurityContext: {}
 
   ingresses:
@@ -975,24 +903,6 @@ filer:
         # requires ingress-nginx to run with --enable-ssl-passthrough.
       tls: []
 
-  # Gateway API HTTPRoute, an alternative to the ingress above.
-  httpRoute:
-    enabled: false
-    labels: {}
-    annotations: {}
-    # Gateways this route attaches to, for example:
-    #   - group: gateway.networking.k8s.io
-    #     kind: Gateway
-    #     name: my-gateway
-    #     namespace: gateway-system
-    #     sectionName: https
-    parentRefs: []
-    hostnames: []
-    # Each rule may set matches, filters, timeouts and backendRefs.
-    # A rule without backendRefs routes to the filer service.
-    # An empty list routes all traffic there.
-    rules: []
-
   # extraEnvVars is a list of extra environment variables to set with the stateful set.
   extraEnvironmentVars:
     # the WEED_MYSQL_* keys and the db credential secret only render while this is "true"
@@ -1089,17 +999,6 @@ s3:
   imageOverride: null
   restartPolicy: null
   replicas: 1
-  # Deployment update strategy, rendered as the Deployment's strategy
-  # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy
-  # Example:
-  # updateStrategy:
-  #   type: RollingUpdate
-  #   rollingUpdate:
-  #     maxUnavailable: 0
-  #     maxSurge: 25%
-  updateStrategy: {}
-  # Time the s3 pod is given to shut down, including any preStop hook
-  terminationGracePeriodSeconds: 10
   bindAddress: 0.0.0.0
   port: 8333
   # add additional https port
@@ -1228,8 +1127,6 @@ s3:
   #   runAsUser: 1000
   #   runAsGroup: 3000
   #   fsGroup: 2000
-  #   seccompProfile:
-  #     type: RuntimeDefault
   podSecurityContext: {}
 
   # Configure security context for Container
@@ -1238,16 +1135,7 @@ s3:
   # containerSecurityContext:
   #   enabled: true
   #   runAsUser: 2000
-  #   runAsGroup: 3000
-  #   runAsNonRoot: true
-  #   privileged: false
   #   allowPrivilegeEscalation: false
-  #   readOnlyRootFilesystem: true
-  #   capabilities:
-  #     drop:
-  #       - ALL
-  #   seccompProfile:
-  #     type: RuntimeDefault
   containerSecurityContext: {}
 
   # You can also use emptyDir storage:
@@ -1292,17 +1180,6 @@ s3:
     failureThreshold: 100
     timeoutSeconds: 10
 
-  # Lifecycle hooks for the s3 container. A preStop sleep lets the pod leave
-  # the Service endpoints before it receives SIGTERM; keep it shorter than
-  # terminationGracePeriodSeconds.
-  # ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
-  # Example:
-  # lifecycle:
-  #   preStop:
-  #     exec:
-  #       command: ["sleep", "5"]
-  lifecycle: {}
-
   createBucketsHook:
     resources: {}
 
@@ -1318,24 +1195,6 @@ s3:
     annotations: {}
     tls: []
 
-  # Gateway API HTTPRoute, an alternative to the ingress above.
-  httpRoute:
-    enabled: false
-    labels: {}
-    annotations: {}
-    # Gateways this route attaches to, for example:
-    #   - group: gateway.networking.k8s.io
-    #     kind: Gateway
-    #     name: my-gateway
-    #     namespace: gateway-system
-    #     sectionName: https
-    parentRefs: []
-    hostnames: []
-    # Each rule may set matches, filters, timeouts and backendRefs.
-    # A rule without backendRefs routes to the s3 service.
-    # An empty list routes all traffic there.
-    rules: []
-
   # Service settings
   service:
     type: ClusterIP
@@ -1428,33 +1287,7 @@ sftp:
   priorityClassName: ""
   schedulerName: ""
   serviceAccountName: ""
-  # Configure security context for Pod
-  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
-  # Example:
-  # podSecurityContext:
-  #   enabled: true
-  #   runAsUser: 1000
-  #   runAsGroup: 3000
-  #   fsGroup: 2000
-  #   seccompProfile:
-  #     type: RuntimeDefault
   podSecurityContext: {}
-  # Configure security context for Container
-  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
-  # Example:
-  # containerSecurityContext:
-  #   enabled: true
-  #   runAsUser: 2000
-  #   runAsGroup: 3000
-  #   runAsNonRoot: true
-  #   privileged: false
-  #   allowPrivilegeEscalation: false
-  #   readOnlyRootFilesystem: true
-  #   capabilities:
-  #     drop:
-  #       - ALL
-  #   seccompProfile:
-  #     type: RuntimeDefault
   containerSecurityContext: {}
 
   logs:
@@ -1601,33 +1434,7 @@ admin:
   priorityClassName: ""
   schedulerName: ""
   serviceAccountName: ""
-  # Configure security context for Pod
-  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
-  # Example:
-  # podSecurityContext:
-  #   enabled: true
-  #   runAsUser: 1000
-  #   runAsGroup: 3000
-  #   fsGroup: 2000
-  #   seccompProfile:
-  #     type: RuntimeDefault
   podSecurityContext: {}
-  # Configure security context for Container
-  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
-  # Example:
-  # containerSecurityContext:
-  #   enabled: true
-  #   runAsUser: 2000
-  #   runAsGroup: 3000
-  #   runAsNonRoot: true
-  #   privileged: false
-  #   allowPrivilegeEscalation: false
-  #   readOnlyRootFilesystem: true
-  #   capabilities:
-  #     drop:
-  #       - ALL
-  #   seccompProfile:
-  #     type: RuntimeDefault
   containerSecurityContext: {}
 
   extraEnvironmentVars: {}
@@ -1673,24 +1480,6 @@ admin:
     annotations: {}
     tls: []
 
-  # Gateway API HTTPRoute, an alternative to the ingress above.
-  httpRoute:
-    enabled: false
-    labels: {}
-    annotations: {}
-    # Gateways this route attaches to, for example:
-    #   - group: gateway.networking.k8s.io
-    #     kind: Gateway
-    #     name: my-gateway
-    #     namespace: gateway-system
-    #     sectionName: https
-    parentRefs: []
-    hostnames: []
-    # Each rule may set matches, filters, timeouts and backendRefs.
-    # A rule without backendRefs routes to the admin service.
-    # An empty list routes all traffic there.
-    rules: []
-
   service:
     type: ClusterIP
     annotations: {}
@@ -1802,33 +1591,7 @@ worker:
   priorityClassName: ""
   schedulerName: ""
   serviceAccountName: ""
-  # Configure security context for Pod
-  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
-  # Example:
-  # podSecurityContext:
-  #   enabled: true
-  #   runAsUser: 1000
-  #   runAsGroup: 3000
-  #   fsGroup: 2000
-  #   seccompProfile:
-  #     type: RuntimeDefault
   podSecurityContext: {}
-  # Configure security context for Container
-  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
-  # Example:
-  # containerSecurityContext:
-  #   enabled: true
-  #   runAsUser: 2000
-  #   runAsGroup: 3000
-  #   runAsNonRoot: true
-  #   privileged: false
-  #   allowPrivilegeEscalation: false
-  #   readOnlyRootFilesystem: true
-  #   capabilities:
-  #     drop:
-  #       - ALL
-  #   seccompProfile:
-  #     type: RuntimeDefault
   containerSecurityContext: {}
 
   extraEnvironmentVars: {}
@@ -2088,8 +1851,6 @@ allInOne:
   #   runAsUser: 1000
   #   runAsGroup: 3000
   #   fsGroup: 2000
-  #   seccompProfile:
-  #     type: RuntimeDefault
   podSecurityContext: {}
 
   # Configure security context for Container
@@ -2098,16 +1859,7 @@ allInOne:
   # containerSecurityContext:
   #   enabled: true
   #   runAsUser: 2000
-  #   runAsGroup: 3000
-  #   runAsNonRoot: true
-  #   privileged: false
   #   allowPrivilegeEscalation: false
-  #   readOnlyRootFilesystem: true
-  #   capabilities:
-  #     drop:
-  #       - ALL
-  #   seccompProfile:
-  #     type: RuntimeDefault
   containerSecurityContext: {}
 
   # Resource management
@@ -2146,33 +1898,7 @@ cosi:
   # should have a secret key called seaweedfs_s3_config with an inline json configure
   existingConfigSecret: null
 
-  # Configure security context for Pod
-  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
-  # Example:
-  # podSecurityContext:
-  #   enabled: true
-  #   runAsUser: 1000
-  #   runAsGroup: 3000
-  #   fsGroup: 2000
-  #   seccompProfile:
-  #     type: RuntimeDefault
   podSecurityContext: {}
-  # Configure security context for Container
-  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
-  # Example:
-  # containerSecurityContext:
-  #   enabled: true
-  #   runAsUser: 2000
-  #   runAsGroup: 3000
-  #   runAsNonRoot: true
-  #   privileged: false
-  #   allowPrivilegeEscalation: false
-  #   readOnlyRootFilesystem: true
-  #   capabilities:
-  #     drop:
-  #       - ALL
-  #   seccompProfile:
-  #     type: RuntimeDefault
   containerSecurityContext: {}
 
   # used to assign a custom scheduler to cosi pods

seaweed-common/Cargo.lock

@@ -148,9 +148,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.45"
+version = "0.23.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
+checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06"
 dependencies = [
  "aws-lc-rs",
  "log",

seaweed-volume/Cargo.lock

@@ -234,9 +234,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-rs"
-version = "1.18.1"
+version = "1.16.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"
+checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
 dependencies = [
  "aws-lc-sys",
  "zeroize",
@@ -244,15 +244,14 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-sys"
-version = "0.45.0"
+version = "0.39.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"
+checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399"
 dependencies = [
  "cc",
  "cmake",
  "dunce",
  "fs_extra",
- "pkg-config",
 ]
 
 [[package]]
@@ -2678,12 +2677,6 @@ dependencies = [
  "spki 0.7.3",
 ]
 
-[[package]]
-name = "pkg-config"
-version = "0.3.34"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
-
 [[package]]
 name = "png"
 version = "0.18.1"
@@ -3396,9 +3389,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.45"
+version = "0.23.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
+checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
 dependencies = [
  "aws-lc-rs",
  "log",
@@ -3443,9 +3436,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.103.15"
+version = "0.103.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
+checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
 dependencies = [
  "aws-lc-rs",
  "ring",
@@ -4562,7 +4555,6 @@ dependencies = [
  "aws-config",
  "aws-credential-types",
  "aws-sdk-s3",
- "aws-smithy-runtime-api",
  "aws-types",
  "axum",
  "base64",

seaweed-volume/Cargo.toml

@@ -157,9 +157,6 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] }
 
 [dev-dependencies]
 tempfile = "3"
-# Already a transitive dependency of aws-sdk-s3 at a single locked version;
-# needed directly only for the canned HttpClient in remote_storage::s3 tests.
-aws-smithy-runtime-api = "1"
 
 [build-dependencies]
 tonic-prost-build = "0.14"

seaweed-volume/proto/master.proto

@@ -213,9 +213,6 @@ message KeepConnectedRequest {
   string filer_group = 5;
   string data_center = 6;
   string rack = 7;
-  // A draining filer leaves the lock ring but stays a cluster member, so peers
-  // keep following its metadata until the stream closes.
-  bool leave_lock_ring = 8;
 }
 
 message VolumeLocation {

seaweed-volume/proto/volume_server.proto

@@ -168,7 +168,6 @@ message VacuumVolumeCheckRequest {
 }
 message VacuumVolumeCheckResponse {
     double garbage_ratio = 1;
-    bool disk_space_low = 4; // the volume is read-only solely because its disk is low on space — a cause compaction itself reclaims
 }
 
 message VacuumVolumeCompactRequest {
@@ -260,10 +259,6 @@ message VolumeDeleteRequest {
     // when true, do not remove the cloud-tier object backing the volume.
     // used for moves where another server is taking over the same .vif.
     bool keep_remote_data = 3;
-    // when true, delete only if every needle is deleted: the volume held
-    // data once but nothing is live anymore. Passing either check,
-    // only_empty or this one, is enough to delete.
-    bool only_garbage = 4;
 }
 message VolumeDeleteResponse {
 }
@@ -476,7 +471,6 @@ message VolumeEcShardsDeleteRequest {
     repeated uint32 shard_ids = 3;
     bool full_teardown = 4; // pre-encode cleanup: wipe every EC artifact + generation for this volume, not just shard_ids
     int64 encode_ts_ns = 5; // full_teardown generation fence: delete only a disk whose .vif generation is strictly OLDER than this; preserve same-or-newer, generation 0, and an unreadable .vif. 0 => wipe-all (shell pre-encode / pre-upgrade)
-    uint32 delete_generations_older_than = 6; // post-commit cleanup: delete only staged <base>.*.v<N> artifacts with N strictly below this; 0 disables
 }
 message VolumeEcShardsDeleteResponse {
     bool full_teardown_done = 1; // set by a new server that performed full_teardown; absent from an old server lets the caller detect the silent no-op

seaweed-volume/src/config.rs

@@ -178,22 +178,12 @@ pub struct Cli {
     pub min_free_space: String,
 
     /// Inflight upload data wait timeout of volume servers.
-    /// Go `time.Duration` syntax: `300ms`, `1.5h`, `2h45m`; a bare number other than `0` is rejected.
-    #[arg(
-        long = "inflightUploadDataTimeout",
-        default_value = "60s",
-        value_parser = parse_go_duration
-    )]
-    pub inflight_upload_data_timeout: std::time::Duration,
+    #[arg(long = "inflightUploadDataTimeout", default_value = "60s")]
+    pub inflight_upload_data_timeout: String,
 
     /// Inflight download data wait timeout of volume servers.
-    /// Go `time.Duration` syntax: `300ms`, `1.5h`, `2h45m`; a bare number other than `0` is rejected.
-    #[arg(
-        long = "inflightDownloadDataTimeout",
-        default_value = "60s",
-        value_parser = parse_go_duration
-    )]
-    pub inflight_download_data_timeout: std::time::Duration,
+    #[arg(long = "inflightDownloadDataTimeout", default_value = "60s")]
+    pub inflight_download_data_timeout: String,
 
     /// <experimental> if true, prevents slow reads from blocking other requests,
     /// but large file read P99 latency will increase.
@@ -481,214 +471,99 @@ fn find_options_arg(args: &[String]) -> String {
     String::new()
 }
 
-/// Parse a duration with Go's `time.ParseDuration` grammar, as Go's
-/// `flag.Duration` flags do: `[0-9]*(\.[0-9]*)?<unit>` repeated, units
-/// `ns`/`us`/`µs`/`μs`/`ms`/`s`/`m`/`h`. A bare `0` is the only unit-less
-/// value; a leading sign parses in Go but a negative has no
-/// `std::time::Duration`, so it is an error, not a clamp.
-fn parse_go_duration(s: &str) -> Result<std::time::Duration, String> {
-    let orig = s;
-    let invalid = || format!("time: invalid duration {:?}", orig);
-
-    let mut s = s;
-    if let Some(rest) = s.strip_prefix('-') {
-        if rest.is_empty() {
-            return Err(invalid());
-        }
-        return Err(format!("negative duration {:?} is not supported", orig));
-    }
-    if let Some(rest) = s.strip_prefix('+') {
-        s = rest;
-    }
-    if s == "0" {
-        return Ok(std::time::Duration::ZERO);
-    }
-    if s.is_empty() {
-        return Err(invalid());
-    }
-
-    // Go bounds the total by 1<<63 (int64 nanoseconds): overflow is an error.
-    const MAX: u64 = 1 << 63;
-    let mut total: u64 = 0;
-    let mut bytes = s.as_bytes();
-    while !bytes.is_empty() {
-        if !(bytes[0] == b'.' || bytes[0].is_ascii_digit()) {
-            return Err(invalid());
-        }
-
-        // Consume [0-9]*
-        let mut i = 0;
-        let mut v: u64 = 0;
-        while i < bytes.len() && bytes[i].is_ascii_digit() {
-            v = v
-                .checked_mul(10)
-                .and_then(|v| v.checked_add(u64::from(bytes[i] - b'0')))
-                .filter(|v| *v <= MAX)
-                .ok_or_else(invalid)?;
-            i += 1;
-        }
-        let pre = i != 0;
-        bytes = &bytes[i..];
-
-        // Consume (\.[0-9]*)?
-        let mut f: u64 = 0;
-        let mut scale: f64 = 1.0;
-        let mut post = false;
-        if let Some((b'.', rest)) = bytes.split_first() {
-            bytes = rest;
-            let mut i = 0;
-            let mut overflow = false;
-            while i < bytes.len() && bytes[i].is_ascii_digit() {
-                if !overflow {
-                    // Like Go's leadingFraction, digits past ~19 are dropped.
-                    match f
-                        .checked_mul(10)
-                        .and_then(|f| f.checked_add(u64::from(bytes[i] - b'0')))
-                        .filter(|f| *f <= MAX)
-                    {
-                        Some(next) => {
-                            f = next;
-                            scale *= 10.0;
-                        }
-                        None => overflow = true,
-                    }
-                }
-                i += 1;
-            }
-            post = i != 0;
-            bytes = &bytes[i..];
-        }
-        if !pre && !post {
-            return Err(invalid());
-        }
-
-        // Consume the unit: everything up to the next digit or '.'.
-        let unit_len = bytes
-            .iter()
-            .position(|c| *c == b'.' || c.is_ascii_digit())
-            .unwrap_or(bytes.len());
-        if unit_len == 0 {
-            return Err(format!("time: missing unit in duration {:?}", orig));
-        }
-        // The boundary is an ASCII byte or the end, so it is a char boundary.
-        let unit = std::str::from_utf8(&bytes[..unit_len]).map_err(|_| invalid())?;
-        bytes = &bytes[unit_len..];
-        let unit_nanos: u64 = match unit {
-            "ns" => 1,
-            "us" | "\u{00b5}s" | "\u{03bc}s" => 1_000,
-            "ms" => 1_000_000,
-            "s" => 1_000_000_000,
-            "m" => 60 * 1_000_000_000,
-            "h" => 3600 * 1_000_000_000,
-            _ => {
-                return Err(format!(
-                    "time: unknown unit {:?} in duration {:?}",
-                    unit, orig
-                ));
-            }
-        };
-
-        if v > MAX / unit_nanos {
-            return Err(invalid());
-        }
-        v *= unit_nanos;
-        if f > 0 {
-            // Go's float math, so `1.5h` rounds the same way on both sides.
-            v += (f as f64 * (unit_nanos as f64 / scale)) as u64;
-            if v > MAX {
-                return Err(invalid());
-            }
-        }
-        total = total
-            .checked_add(v)
-            .filter(|t| *t <= MAX)
-            .ok_or_else(invalid)?;
-    }
-    if total > MAX - 1 {
-        return Err(invalid());
-    }
-    Ok(std::time::Duration::from_nanos(total))
-}
-
-/// Byte-size suffixes of Go's `util.ParseBytes`, lower-cased: SI (`kb`, `k`)
-/// and IEC (`kib`, `ki`) forms, `b`, and no suffix.
-fn byte_unit_multiplier(unit: &str) -> Option<u64> {
-    const K: u64 = 1_000;
-    const KI: u64 = 1 << 10;
-    Some(match unit {
-        "" | "b" => 1,
-        "kb" | "k" => K,
-        "kib" | "ki" => KI,
-        "mb" | "m" => K * K,
-        "mib" | "mi" => KI * KI,
-        "gb" | "g" => K * K * K,
-        "gib" | "gi" => KI * KI * KI,
-        "tb" | "t" => K * K * K * K,
-        "tib" | "ti" => KI * KI * KI * KI,
-        "pb" | "p" => K * K * K * K * K,
-        "pib" | "pi" => KI * KI * KI * KI * KI,
-        "eb" | "e" => K * K * K * K * K * K,
-        "eib" | "ei" => KI * KI * KI * KI * KI * KI,
-        _ => return None,
-    })
-}
-
-/// Parse a human-readable byte count with Go's `util.ParseBytes` grammar:
-/// decimal number, optional whitespace, then a [`byte_unit_multiplier`]
-/// suffix, case-insensitive. `10GiB` → 10737418240, `1.5TB` → 1500000000000.
-///
-/// Thousands commas parse as in Go but never arrive from `-minFreeSpace`,
-/// which is split on commas into per-directory entries first.
-fn parse_bytes(s: &str) -> Result<u64, String> {
-    let digits_end = s
-        .find(|c: char| !(c.is_ascii_digit() || c == '.' || c == ','))
-        .unwrap_or(s.len());
-    let num = s[..digits_end].replace(',', "");
-    let value: f64 = num
-        .parse()
-        .map_err(|e| format!("invalid byte size {:?}: {}", s, e))?;
-    let unit = s[digits_end..].trim().to_lowercase();
-    let multiplier =
-        byte_unit_multiplier(&unit).ok_or_else(|| format!("unhandled size name: {}", unit))?;
-    let bytes = value * multiplier as f64;
-    if bytes >= u64::MAX as f64 {
-        return Err(format!("too large: {}", s));
-    }
-    Ok(bytes as u64)
-}
-
-/// Parse one `-minFreeSpace` entry like Go's `util.ParseMinFreeSpace()`: a
-/// plain number is a percentage in `0..=100`, anything else a byte size
-/// above 100 bytes. `150` errors rather than reading as bytes.
-fn parse_min_free_space(s: &str) -> Result<MinFreeSpace, String> {
+/// Parse a duration string like "60s", "5m", "1h" into a std::time::Duration.
+fn parse_duration(s: &str) -> std::time::Duration {
     let s = s.trim();
-    if let Ok(percent) = s.parse::<f64>() {
-        // Go's range check lets NaN through; rejecting it is the one
-        // intentional difference.
-        if !(0.0..=100.0).contains(&percent) {
-            return Err(format!("minFreeSpace is invalid: {:?}", s));
-        }
-        return Ok(MinFreeSpace::Percent(percent));
+    if s.is_empty() {
+        return std::time::Duration::from_secs(60);
     }
-    match parse_bytes(s) {
-        Ok(bytes) if bytes > 100 => Ok(MinFreeSpace::Bytes(bytes)),
-        _ => Err(format!("minFreeSpace is invalid: {:?}", s)),
+    if let Some(secs) = s.strip_suffix('s')
+        && let Ok(v) = secs.parse::<u64>()
+    {
+        return std::time::Duration::from_secs(v);
     }
+    if let Some(mins) = s.strip_suffix('m')
+        && let Ok(v) = mins.parse::<u64>()
+        && let Some(seconds) = v.checked_mul(60)
+    {
+        return std::time::Duration::from_secs(seconds);
+    }
+    if let Some(hours) = s.strip_suffix('h')
+        && let Ok(v) = hours.parse::<u64>()
+        && let Some(seconds) = v.checked_mul(3600)
+    {
+        return std::time::Duration::from_secs(seconds);
+    }
+    // Fallback: try parsing as raw seconds
+    if let Ok(v) = s.parse::<u64>() {
+        return std::time::Duration::from_secs(v);
+    }
+    std::time::Duration::from_secs(60)
 }
 
-/// Parse minFreeSpace / minFreeSpacePercent like Go's
-/// `util.MustParseMinFreeSpace()`: `--minFreeSpace` wins when set, and any
-/// invalid comma-separated entry fails the whole list.
-fn parse_min_free_spaces(
-    min_free_space: &str,
-    min_free_space_percent: &str,
-) -> Result<Vec<MinFreeSpace>, String> {
+/// Parse minFreeSpace / minFreeSpacePercent into MinFreeSpace values.
+/// Mirrors Go's `util.MustParseMinFreeSpace()`.
+fn parse_min_free_spaces(min_free_space: &str, min_free_space_percent: &str) -> Vec<MinFreeSpace> {
+    // If --minFreeSpace is provided, use it (takes precedence).
     let source = if !min_free_space.is_empty() {
         min_free_space
     } else {
         min_free_space_percent
     };
-    source.split(',').map(parse_min_free_space).collect()
+
+    source
+        .split(',')
+        .map(|s| {
+            let s = s.trim();
+            // Try parsing as a percentage (value <= 100)
+            if let Ok(v) = s.parse::<f64>() {
+                if v <= 100.0 {
+                    return MinFreeSpace::Percent(v);
+                }
+                // Treat as bytes if > 100
+                return MinFreeSpace::Bytes(v as u64);
+            }
+            // Try parsing human-readable bytes: e.g. "10GiB", "500MiB", "1TiB"
+            let s_upper = s.to_uppercase();
+            if let Some(rest) = s_upper.strip_suffix("TIB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1024.0 * 1024.0 * 1024.0 * 1024.0) as u64);
+            }
+            if let Some(rest) = s_upper.strip_suffix("GIB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1024.0 * 1024.0 * 1024.0) as u64);
+            }
+            if let Some(rest) = s_upper.strip_suffix("MIB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1024.0 * 1024.0) as u64);
+            }
+            if let Some(rest) = s_upper.strip_suffix("KIB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1024.0) as u64);
+            }
+            if let Some(rest) = s_upper.strip_suffix("TB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1_000_000_000_000.0) as u64);
+            }
+            if let Some(rest) = s_upper.strip_suffix("GB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1_000_000_000.0) as u64);
+            }
+            if let Some(rest) = s_upper.strip_suffix("MB")
+                && let Ok(v) = rest.trim().parse::<f64>()
+            {
+                return MinFreeSpace::Bytes((v * 1_000_000.0) as u64);
+            }
+            // Default: 1%
+            MinFreeSpace::Percent(1.0)
+        })
+        .collect()
 }
 
 /// Parse comma-separated tag groups like "fast:ssd,archive" into per-folder tag vectors.
@@ -782,9 +657,7 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
 
     // Parse min free spaces
     let mut min_free_spaces =
-        parse_min_free_spaces(&cli.min_free_space, &cli.min_free_space_percent).unwrap_or_else(
-            |e| panic!("The value specified in -minFreeSpace not a valid value: {e}"),
-        );
+        parse_min_free_spaces(&cli.min_free_space, &cli.min_free_space_percent);
     if min_free_spaces.len() == 1 && folder_count > 1 {
         let v = min_free_spaces[0].clone();
         min_free_spaces.resize(folder_count, v);
@@ -899,6 +772,10 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
         .collect();
     white_list.extend(sec.guard_white_list.iter().cloned());
 
+    // Parse durations
+    let inflight_upload_data_timeout = parse_duration(&cli.inflight_upload_data_timeout);
+    let inflight_download_data_timeout = parse_duration(&cli.inflight_download_data_timeout);
+
     VolumeServerConfig {
         port: cli.port,
         grpc_port,
@@ -930,8 +807,8 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
         file_size_limit_bytes: cli.file_size_limit_mb as i64 * 1024 * 1024,
         concurrent_upload_limit: cli.concurrent_upload_limit_mb as i64 * 1024 * 1024,
         concurrent_download_limit: cli.concurrent_download_limit_mb as i64 * 1024 * 1024,
-        inflight_upload_data_timeout: cli.inflight_upload_data_timeout,
-        inflight_download_data_timeout: cli.inflight_download_data_timeout,
+        inflight_upload_data_timeout,
+        inflight_download_data_timeout,
         has_slow_read: cli.has_slow_read,
         read_buffer_size_mb: cli.read_buffer_size_mb,
         ldb_timeout: cli.ldb_timeout,
@@ -1383,224 +1260,25 @@ mod tests {
     }
 
     #[test]
-    fn test_parse_go_duration_accepts_go_grammar() {
-        use std::time::Duration;
-        let cases = [
-            ("60s", Duration::from_secs(60)),
-            ("5m", Duration::from_secs(300)),
-            ("1h", Duration::from_secs(3600)),
-            ("1.5h", Duration::from_secs(5400)),
-            ("2h45m", Duration::from_secs(2 * 3600 + 45 * 60)),
-            ("300ms", Duration::from_millis(300)),
-            ("1.5s", Duration::from_millis(1500)),
-            ("250us", Duration::from_micros(250)),
-            ("250\u{00b5}s", Duration::from_micros(250)),
-            ("250\u{03bc}s", Duration::from_micros(250)),
-            ("7ns", Duration::from_nanos(7)),
-            (".5s", Duration::from_millis(500)),
-            ("5.s", Duration::from_secs(5)),
-            ("+10s", Duration::from_secs(10)),
-            ("0", Duration::ZERO),
-            ("0s", Duration::ZERO),
-            // Go: fraction digits past the 19th are ignored, not an error.
-            ("1.00000000000000000001s", Duration::from_secs(1)),
-        ];
-        for (input, expected) in cases {
-            assert_eq!(parse_go_duration(input), Ok(expected), "input={:?}", input);
-        }
-    }
-
-    #[test]
-    fn test_parse_go_duration_rejects_what_go_rejects() {
-        // Each of these makes Go's flag.Duration fail flag parsing; the old
-        // Rust parser turned them all into the 60 s default.
-        let rejected = [
-            "30",                    // bare number other than 0
-            "30sec",                 // unknown unit
-            "abc",                   // no number at all
-            "",                      // empty
-            "1h ",                   // trailing space is not a unit
-            "s",                     // unit without number
-            ".s",                    // dot without digits
-            "1.5",                   // fraction without unit
-            "1h-30m",                // sign is only allowed at the front
-            "-",                     // sign alone
-            "5124095576030432h",     // overflows int64 nanoseconds
-            "307445734561825861m",   // overflows int64 nanoseconds
-            "9223372036854775808ns", // 1<<63, one past int64
-        ];
-        for input in rejected {
-            let result = parse_go_duration(input);
-            assert!(result.is_err(), "input={:?} parsed as {:?}", input, result);
-            let message = result.unwrap_err();
-            assert!(
-                message.contains(&format!("{:?}", input)) || input.is_empty(),
-                "error for {:?} does not name the value: {}",
-                input,
-                message
-            );
-        }
-        // Largest value Go accepts: 1<<63 - 1 nanoseconds.
+    fn test_parse_duration() {
+        assert_eq!(parse_duration("60s"), std::time::Duration::from_secs(60));
+        assert_eq!(parse_duration("5m"), std::time::Duration::from_secs(300));
+        assert_eq!(parse_duration("1h"), std::time::Duration::from_secs(3600));
+        assert_eq!(parse_duration("30"), std::time::Duration::from_secs(30));
+        assert_eq!(parse_duration(""), std::time::Duration::from_secs(60));
         assert_eq!(
-            parse_go_duration("9223372036854775807ns"),
-            Ok(std::time::Duration::from_nanos(i64::MAX as u64))
-        );
-        // Go accepts negatives; std::time::Duration cannot, so it errors.
-        let negative = parse_go_duration("-1s").unwrap_err();
-        assert!(negative.contains("negative"), "{}", negative);
-    }
-
-    /// Clap refuses the flag so the process exits with a usage error, exactly
-    /// where Go's flag package does.
-    #[test]
-    fn test_cli_rejects_invalid_duration_flags() {
-        for value in ["30sec", "abc", "30"] {
-            let result = Cli::try_parse_from(["bin", "--inflightUploadDataTimeout", value]);
-            assert!(
-                result.is_err(),
-                "--inflightUploadDataTimeout={} was accepted as {:?}",
-                value,
-                result.map(|cli| cli.inflight_upload_data_timeout)
-            );
-            let err = Cli::try_parse_from(["bin", "--inflightDownloadDataTimeout", value])
-                .err()
-                .unwrap_or_else(|| panic!("--inflightDownloadDataTimeout={} was accepted", value));
-            assert_eq!(err.kind(), clap::error::ErrorKind::ValueValidation);
-            assert!(err.to_string().contains(value), "{}", err);
-        }
-        let cli = Cli::parse_from(["bin", "--inflightUploadDataTimeout", "1.5h"]);
-        assert_eq!(
-            cli.inflight_upload_data_timeout,
-            std::time::Duration::from_secs(5400)
-        );
-        // The default still parses through the same grammar.
-        let cli = Cli::parse_from(["bin"]);
-        assert_eq!(
-            cli.inflight_upload_data_timeout,
+            parse_duration("307445734561825861m"),
             std::time::Duration::from_secs(60)
         );
         assert_eq!(
-            cli.inflight_download_data_timeout,
+            parse_duration("5124095576030432h"),
             std::time::Duration::from_secs(60)
         );
     }
 
-    #[test]
-    fn test_parse_bytes_matches_go_parse_bytes() {
-        // Expected values are what Go's util.ParseBytes returns (weed/util/bytes_test.go).
-        let cases: [(&str, u64); 14] = [
-            ("42", 42),
-            ("42MB", 42_000_000),
-            ("42 MB", 42_000_000),
-            ("42 mib", 44_040_192),
-            ("42M", 42_000_000),
-            ("100Ki", 100 * 1024),
-            ("10GiB", 10 * 1024 * 1024 * 1024),
-            ("10gib", 10 * 1024 * 1024 * 1024),
-            ("10GB", 10_000_000_000),
-            ("1.5TB", 1_500_000_000_000),
-            // Go's ParseBytes accepts thousands commas; the -minFreeSpace flag
-            // never delivers one (see test_min_free_space_flag_commas_are_list_separators).
-            ("1,024MB", 1_024_000_000),
-            ("1,024", 1024),
-            ("1PiB", 1 << 50),
-            ("1eb", 1_000_000_000_000_000_000),
-        ];
-        for (input, expected) in cases {
-            assert_eq!(parse_bytes(input), Ok(expected), "input={:?}", input);
-        }
-        for input in [
-            "10GiBx",
-            "GiB",
-            "",
-            "-5GB",
-            "1.2.3GB",
-            "10 Gi B",
-            "99999999999999999999EB",
-        ] {
-            assert!(
-                parse_bytes(input).is_err(),
-                "input={:?} was accepted",
-                input
-            );
-        }
-    }
-
-    fn expect_percent(result: Result<MinFreeSpace, String>, expected: f64, input: &str) {
-        match result {
-            Ok(MinFreeSpace::Percent(v)) => {
-                assert!(
-                    (v - expected).abs() < f64::EPSILON,
-                    "input={:?}: {}",
-                    input,
-                    v
-                )
-            }
-            other => panic!(
-                "input={:?}: expected Percent({}), got {:?}",
-                input, expected, other
-            ),
-        }
-    }
-
-    fn expect_bytes(result: Result<MinFreeSpace, String>, expected: u64, input: &str) {
-        match result {
-            Ok(MinFreeSpace::Bytes(v)) => assert_eq!(v, expected, "input={:?}", input),
-            other => panic!(
-                "input={:?}: expected Bytes({}), got {:?}",
-                input, expected, other
-            ),
-        }
-    }
-
-    #[test]
-    fn test_parse_min_free_space_matches_go() {
-        // Fixtures from weed/util/minfreespace_test.go plus the byte-unit table.
-        expect_percent(parse_min_free_space("42"), 42.0, "42");
-        expect_percent(parse_min_free_space(" 42 "), 42.0, " 42 ");
-        expect_percent(parse_min_free_space("50"), 50.0, "50");
-        expect_percent(parse_min_free_space("100"), 100.0, "100");
-        expect_percent(parse_min_free_space("0"), 0.0, "0");
-        expect_percent(parse_min_free_space("2.5"), 2.5, "2.5");
-        expect_bytes(parse_min_free_space("100Ki"), 100 * 1024, "100Ki");
-        expect_bytes(parse_min_free_space("100GiB"), 100 << 30, "100GiB");
-        expect_bytes(parse_min_free_space("42M"), 42_000_000, "42M");
-        expect_bytes(parse_min_free_space(" 42M "), 42_000_000, " 42M ");
-        expect_bytes(parse_min_free_space("10GiB"), 10 << 30, "10GiB");
-        expect_bytes(parse_min_free_space("10gib"), 10 << 30, "10gib");
-        expect_bytes(parse_min_free_space("1.5TB"), 1_500_000_000_000, "1.5TB");
-        // Comma inputs never arrive whole from the flag; see
-        // test_min_free_space_flag_commas_are_list_separators.
-        expect_bytes(parse_min_free_space("1,024MB"), 1_024_000_000, "1,024MB");
-        expect_bytes(parse_min_free_space("1,000"), 1000, "1,000");
-
-        // Go: percent outside 0..=100, or a byte size of at most 100 bytes.
-        for input in [
-            "150", "101", "-1", "1e3", "inf", "nan", "100B", "50B", "0B", "10GiBx", "", "abc",
-            "10 Gi B",
-        ] {
-            let result = parse_min_free_space(input);
-            assert!(
-                result.is_err(),
-                "input={:?} was accepted as {:?}",
-                input,
-                result
-            );
-            assert!(
-                result
-                    .as_ref()
-                    .unwrap_err()
-                    .contains(&format!("{:?}", input.trim())),
-                "error for {:?} does not name the value: {:?}",
-                input,
-                result
-            );
-        }
-    }
-
     #[test]
     fn test_parse_min_free_spaces_percent() {
-        let result = parse_min_free_spaces("", "1").unwrap();
+        let result = parse_min_free_spaces("", "1");
         assert_eq!(result.len(), 1);
         match &result[0] {
             MinFreeSpace::Percent(v) => assert!((v - 1.0).abs() < f64::EPSILON),
@@ -1610,7 +1288,7 @@ mod tests {
 
     #[test]
     fn test_parse_min_free_spaces_bytes() {
-        let result = parse_min_free_spaces("10GiB", "").unwrap();
+        let result = parse_min_free_spaces("10GiB", "");
         assert_eq!(result.len(), 1);
         match &result[0] {
             MinFreeSpace::Bytes(v) => assert_eq!(*v, 10 * 1024 * 1024 * 1024),
@@ -1618,87 +1296,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_parse_min_free_spaces_list_and_precedence() {
-        // --minFreeSpace wins over --minFreeSpacePercent, as in Go's EmptyTo().
-        let result = parse_min_free_spaces("5, 10GiB", "1").unwrap();
-        assert_eq!(result.len(), 2);
-        assert!(matches!(result[0], MinFreeSpace::Percent(v) if v == 5.0));
-        assert!(matches!(result[1], MinFreeSpace::Bytes(v) if v == 10 << 30));
-        // One bad entry fails the whole list and names that entry.
-        let err = parse_min_free_spaces("5,10GiBx", "1").unwrap_err();
-        assert!(err.contains("\"10GiBx\""), "{}", err);
-        // Both flags empty: Go fatals (ParseMinFreeSpace("") fails) rather than
-        // assuming 1%.
-        assert!(parse_min_free_spaces("", "").is_err());
-    }
-
-    /// A comma in `-minFreeSpace` separates per-directory entries, as in Go's
-    /// `MustParseMinFreeSpace`: `1,024MB` is `1` (percent) and `024MB`, and
-    /// one `-dir` fails the count check.
-    #[test]
-    fn test_min_free_space_flag_commas_are_list_separators() {
-        let result = parse_min_free_spaces("1,024MB", "").unwrap();
-        assert_eq!(result.len(), 2, "{result:?}");
-        assert!(
-            matches!(result[0], MinFreeSpace::Percent(v) if v == 1.0),
-            "{result:?}"
-        );
-        assert!(
-            matches!(result[1], MinFreeSpace::Bytes(v) if v == 24_000_000),
-            "{result:?}"
-        );
-
-        let _guard = process_state_lock();
-        let cli = Cli::parse_from(["bin", "--dir", "/tmp/a", "--minFreeSpace", "1,024MB"]);
-        let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
-            resolve_config_with_env(cli, &|_| None).min_free_spaces
-        }));
-        assert!(
-            outcome.is_err(),
-            "one -dir with two minFreeSpace entries must abort"
-        );
-    }
-
-    /// Negative control at the startup layer: resolving the config with an
-    /// invalid --minFreeSpace must abort, as Go's MustParseMinFreeSpace does.
-    #[test]
-    fn test_resolve_config_rejects_invalid_min_free_space() {
-        // The security-config search reads the working directory.
-        let _guard = process_state_lock();
-        for (flag, value) in [
-            ("--minFreeSpace", "10GiBx"),
-            ("--minFreeSpace", "150"),
-            ("--minFreeSpace", "50B"),
-            ("--minFreeSpacePercent", "150"),
-            ("--minFreeSpacePercent", "abc"),
-        ] {
-            let cli = Cli::parse_from(["bin", flag, value]);
-            let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
-                resolve_config_with_env(cli, &|_| None).min_free_spaces
-            }));
-            let payload = match outcome {
-                Ok(spaces) => panic!("{}={} was accepted as {:?}", flag, value, spaces),
-                Err(payload) => payload,
-            };
-            let message = payload
-                .downcast_ref::<String>()
-                .cloned()
-                .or_else(|| payload.downcast_ref::<&str>().map(|s| s.to_string()))
-                .unwrap_or_default();
-            assert!(
-                message.contains("-minFreeSpace") && message.contains(value),
-                "{}={}: unexpected panic message {:?}",
-                flag,
-                value,
-                message
-            );
-        }
-        let cfg =
-            resolve_config_with_env(Cli::parse_from(["bin", "--minFreeSpace", "100"]), &|_| None);
-        assert!(matches!(cfg.min_free_spaces[0], MinFreeSpace::Percent(v) if v == 100.0));
-    }
-
     #[test]
     fn test_parse_volume_tags_single() {
         let tags = parse_volume_tags("fast:ssd", 3);

seaweed-volume/src/main.rs

@@ -371,9 +371,6 @@ async fn run(
                 .to_string_lossy()
                 .into_owned()
         },
-        ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
-        ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
-        ec_decode_tail_notify: tokio::sync::Notify::new(),
     });
 
     // Load persisted state from disk if it exists (matches Go's State.Load on startup)

seaweed-volume/src/remote_storage/s3.rs

@@ -4,7 +4,6 @@
 
 use aws_sdk_s3::Client;
 use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
-use aws_sdk_s3::error::{DisplayErrorContext, SdkError};
 use aws_sdk_s3::primitives::ByteStream;
 
 use super::{RemoteEntry, RemoteStorageClient, RemoteStorageError};
@@ -26,23 +25,6 @@ impl S3RemoteStorageClient {
         endpoint: &str,
         force_path_style: bool,
     ) -> Self {
-        let client = Client::from_conf(
-            Self::config_builder(access_key, secret_key, region, endpoint, force_path_style)
-                .build(),
-        );
-
-        S3RemoteStorageClient { client, conf }
-    }
-
-    /// Build the SDK config for the given credentials and endpoint. Split out so
-    /// tests can attach a canned HTTP client before building the [`Client`].
-    fn config_builder(
-        access_key: &str,
-        secret_key: &str,
-        region: &str,
-        endpoint: &str,
-        force_path_style: bool,
-    ) -> aws_sdk_s3::config::Builder {
         let region = if region.is_empty() {
             "us-east-1"
         } else {
@@ -67,7 +49,9 @@ impl S3RemoteStorageClient {
             s3_config = s3_config.endpoint_url(endpoint);
         }
 
-        s3_config
+        let client = Client::from_conf(s3_config.build());
+
+        S3RemoteStorageClient { client, conf }
     }
 }
 
@@ -91,14 +75,13 @@ impl RemoteStorageClient for S3RemoteStorageClient {
             req = req.range(format!("bytes={}-", offset));
         }
 
-        let resp = req.send().await.map_err(|e| match e {
-            // Go compares `aerr.Code()` to NoSuchKey on GET
-            // (s3_storage_client.go:436): a bare 404 maps to "NotFound"
-            // and stays a generic error, as it does here.
-            SdkError::ServiceError(ref se) if se.err().is_no_such_key() => {
+        let resp = req.send().await.map_err(|e| {
+            let msg = format!("{}", e);
+            if msg.contains("NoSuchKey") || msg.contains("404") {
                 RemoteStorageError::ObjectNotFound(format!("{}/{}", loc.bucket, key))
+            } else {
+                RemoteStorageError::Other(format!("s3 get object: {}", e))
             }
-            e => RemoteStorageError::Other(format!("s3 get object: {}", DisplayErrorContext(&e))),
         })?;
 
         let data = resp
@@ -125,9 +108,7 @@ impl RemoteStorageClient for S3RemoteStorageClient {
             .body(ByteStream::from(data.to_vec()))
             .send()
             .await
-            .map_err(|e| {
-                RemoteStorageError::Other(format!("s3 put object: {}", DisplayErrorContext(&e)))
-            })?;
+            .map_err(|e| RemoteStorageError::Other(format!("s3 put object: {}", e)))?;
 
         Ok(RemoteEntry {
             size: data.len() as i64,
@@ -153,18 +134,13 @@ impl RemoteStorageClient for S3RemoteStorageClient {
             .key(key)
             .send()
             .await
-            .map_err(|e| match e {
-                // Go checks only the raw HTTP status on HEAD
-                // (s3_storage_client.go:373): a HEAD response carries no
-                // error body, so a 404 is not-found whatever code the SDK
-                // assigns, and a non-404 is not.
-                SdkError::ServiceError(ref se) if se.raw().status().as_u16() == 404 => {
+            .map_err(|e| {
+                let msg = format!("{}", e);
+                if msg.contains("404") || msg.contains("NotFound") {
                     RemoteStorageError::ObjectNotFound(format!("{}/{}", loc.bucket, key))
+                } else {
+                    RemoteStorageError::Other(format!("s3 head object: {}", e))
                 }
-                e => RemoteStorageError::Other(format!(
-                    "s3 head object: {}",
-                    DisplayErrorContext(&e)
-                )),
             })?;
 
         Ok(RemoteEntry {
@@ -184,17 +160,18 @@ impl RemoteStorageClient for S3RemoteStorageClient {
             .key(key)
             .send()
             .await
-            .map_err(|e| {
-                RemoteStorageError::Other(format!("s3 delete object: {}", DisplayErrorContext(&e)))
-            })?;
+            .map_err(|e| RemoteStorageError::Other(format!("s3 delete object: {}", e)))?;
 
         Ok(())
     }
 
     async fn list_buckets(&self) -> Result<Vec<String>, RemoteStorageError> {
-        let resp = self.client.list_buckets().send().await.map_err(|e| {
-            RemoteStorageError::Other(format!("s3 list buckets: {}", DisplayErrorContext(&e)))
-        })?;
+        let resp = self
+            .client
+            .list_buckets()
+            .send()
+            .await
+            .map_err(|e| RemoteStorageError::Other(format!("s3 list buckets: {}", e)))?;
 
         Ok(resp
             .buckets()
@@ -207,178 +184,3 @@ impl RemoteStorageClient for S3RemoteStorageClient {
         &self.conf
     }
 }
-
-#[cfg(test)]
-pub(crate) mod tests {
-    use super::*;
-    use aws_sdk_s3::config::http::{HttpRequest, HttpResponse};
-    use aws_sdk_s3::config::retry::RetryConfig;
-    use aws_sdk_s3::config::{HttpClient, RuntimeComponents};
-    use aws_sdk_s3::primitives::SdkBody;
-    use aws_smithy_runtime_api::client::http::{
-        HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector,
-    };
-    use aws_smithy_runtime_api::http::StatusCode;
-
-    /// An SDK HTTP client that answers every request with one canned response,
-    /// so the error-mapping paths can be exercised without a network or a
-    /// running S3 server.
-    #[derive(Debug, Clone)]
-    pub(crate) struct CannedResponse {
-        pub(crate) status: u16,
-        pub(crate) body: &'static str,
-    }
-
-    impl HttpConnector for CannedResponse {
-        fn call(&self, _request: HttpRequest) -> HttpConnectorFuture {
-            let status = StatusCode::try_from(self.status).expect("valid HTTP status");
-            HttpConnectorFuture::ready(Ok(HttpResponse::new(status, SdkBody::from(self.body))))
-        }
-    }
-
-    impl HttpClient for CannedResponse {
-        fn http_connector(
-            &self,
-            _settings: &HttpConnectorSettings,
-            _components: &RuntimeComponents,
-        ) -> SharedHttpConnector {
-            SharedHttpConnector::new(self.clone())
-        }
-    }
-
-    fn client_with(status: u16, body: &'static str) -> S3RemoteStorageClient {
-        let config = S3RemoteStorageClient::config_builder(
-            "AKIATEST",
-            "secret",
-            "us-east-1",
-            "http://127.0.0.1:1",
-            true,
-        )
-        .http_client(CannedResponse { status, body })
-        .retry_config(RetryConfig::disabled())
-        .build();
-        S3RemoteStorageClient {
-            client: Client::from_conf(config),
-            conf: RemoteConf::default(),
-        }
-    }
-
-    fn location() -> RemoteStorageLocation {
-        RemoteStorageLocation {
-            name: "remote".to_string(),
-            bucket: "bucket".to_string(),
-            path: "/dir/missing".to_string(),
-            ..Default::default()
-        }
-    }
-
-    pub(crate) const NO_SUCH_KEY: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
-<Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><Key>dir/missing</Key></Error>"#;
-
-    const NOT_FOUND_BODY: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
-<Error><Code>NotFound</Code><Message>Not Found</Message></Error>"#;
-
-    const ACCESS_DENIED: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
-<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>"#;
-
-    #[tokio::test]
-    async fn get_no_such_key_is_object_not_found() {
-        let err = client_with(404, NO_SUCH_KEY)
-            .read_file(&location(), 0, 0)
-            .await
-            .unwrap_err();
-        assert!(
-            matches!(&err, RemoteStorageError::ObjectNotFound(path) if path == "bucket/dir/missing"),
-            "expected ObjectNotFound, got {err:?}"
-        );
-    }
-
-    #[tokio::test]
-    async fn get_bare_404_is_not_object_not_found() {
-        // Go compares codes, not statuses, on GET: a body-less 404 stays generic.
-        let err = client_with(404, "")
-            .read_file(&location(), 0, 0)
-            .await
-            .unwrap_err();
-        assert!(
-            matches!(err, RemoteStorageError::Other(_)),
-            "expected Other, got {err:?}"
-        );
-    }
-
-    #[tokio::test]
-    async fn head_404_is_object_not_found() {
-        let err = client_with(404, "")
-            .stat_file(&location())
-            .await
-            .unwrap_err();
-        assert!(
-            matches!(&err, RemoteStorageError::ObjectNotFound(path) if path == "bucket/dir/missing"),
-            "expected ObjectNotFound, got {err:?}"
-        );
-    }
-
-    #[tokio::test]
-    async fn head_404_with_foreign_error_body_is_object_not_found() {
-        // The raw status check makes a 404 not-found whatever body it carries.
-        let err = client_with(404, NO_SUCH_KEY)
-            .stat_file(&location())
-            .await
-            .unwrap_err();
-        assert!(
-            matches!(err, RemoteStorageError::ObjectNotFound(_)),
-            "expected ObjectNotFound, got {err:?}"
-        );
-    }
-
-    #[tokio::test]
-    async fn head_not_found_code_on_a_non_404_status_is_not_object_not_found() {
-        // A NotFound body on a non-404 status stays an error, as in Go.
-        let err = client_with(400, NOT_FOUND_BODY)
-            .stat_file(&location())
-            .await
-            .unwrap_err();
-        assert!(
-            matches!(&err, RemoteStorageError::Other(msg) if msg.contains("NotFound")),
-            "expected Other naming the code, got {err:?}"
-        );
-    }
-
-    #[tokio::test]
-    async fn get_access_denied_keeps_service_error_code() {
-        let err = client_with(403, ACCESS_DENIED)
-            .read_file(&location(), 0, 0)
-            .await
-            .unwrap_err();
-        let msg = err.to_string();
-        assert!(
-            matches!(err, RemoteStorageError::Other(_)),
-            "expected Other, got {err:?}"
-        );
-        assert!(
-            msg.contains("AccessDenied"),
-            "message should carry the S3 error code, got: {msg}"
-        );
-        assert!(
-            !msg.ends_with("service error"),
-            "message should not be the bare SdkError Display, got: {msg}"
-        );
-    }
-
-    #[tokio::test]
-    async fn head_access_denied_keeps_service_error_code() {
-        let err = client_with(403, ACCESS_DENIED)
-            .stat_file(&location())
-            .await
-            .unwrap_err();
-        let msg = err.to_string();
-        assert!(
-            matches!(err, RemoteStorageError::Other(_)),
-            "expected Other, got {err:?}"
-        );
-        assert!(
-            msg.contains("AccessDenied"),
-            "message should carry the S3 error code, got: {msg}"
-        );
-    }
-}

seaweed-volume/src/remote_storage/s3_tier.rs

@@ -8,11 +8,7 @@ use std::future::Future;
 use std::sync::{Arc, OnceLock, RwLock};
 
 use aws_sdk_s3::Client;
-use aws_sdk_s3::config::http::HttpResponse;
 use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
-use aws_sdk_s3::error::{DisplayErrorContext, SdkError};
-use aws_sdk_s3::operation::get_object::GetObjectError;
-use aws_sdk_s3::operation::head_object::HeadObjectError;
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
 use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
 use tokio::sync::Semaphore;
@@ -20,53 +16,6 @@ use tokio::sync::Semaphore;
 /// Concurrency limit for multipart upload/download (matches Go's s3manager).
 const CONCURRENCY: usize = 5;
 
-/// A tier transfer failure. The variant is what callers match on; the
-/// message is the operator-facing text.
-#[derive(Debug, thiserror::Error)]
-pub enum TierError {
-    /// The remote object does not exist.
-    #[error("{0}")]
-    NotFound(String),
-    /// An S3 request or a local file operation failed.
-    #[error("{0}")]
-    Io(String),
-    /// The tier I/O runtime could not be built or dropped the task.
-    #[error("{0}")]
-    RuntimeUnavailable(String),
-    /// The progress callback asked to stop.
-    #[error("{0}")]
-    Aborted(String),
-}
-
-// Not-found rules as in remote_storage/s3.rs: HEAD by the raw 404 status,
-// GET by the NoSuchKey code only.
-fn head_object_error(key: &str, e: SdkError<HeadObjectError, HttpResponse>) -> TierError {
-    let message = format!("failed to head object {}: {}", key, DisplayErrorContext(&e));
-    match e {
-        SdkError::ServiceError(ref se) if se.raw().status().as_u16() == 404 => {
-            TierError::NotFound(message)
-        }
-        _ => TierError::Io(message),
-    }
-}
-
-fn get_object_error(
-    key: &str,
-    range: &str,
-    e: SdkError<GetObjectError, HttpResponse>,
-) -> TierError {
-    let message = format!(
-        "failed to get object {} range {}: {}",
-        key,
-        range,
-        DisplayErrorContext(&e)
-    );
-    match e {
-        SdkError::ServiceError(ref se) if se.err().is_no_such_key() => TierError::NotFound(message),
-        _ => TierError::Io(message),
-    }
-}
-
 /// Configuration for an S3 tier backend.
 #[derive(Debug, Clone)]
 pub struct S3TierConfig {
@@ -140,7 +89,7 @@ impl S3TierBackend {
         &self,
         file_path: &str,
         progress_fn: F,
-    ) -> Result<(String, u64), TierError>
+    ) -> Result<(String, u64), String>
     where
         F: FnMut(i64, f32) -> Result<(), String> + Send + Sync + 'static,
     {
@@ -148,7 +97,7 @@ impl S3TierBackend {
 
         let metadata = tokio::fs::metadata(file_path)
             .await
-            .map_err(|e| TierError::Io(format!("failed to stat file {}: {}", file_path, e)))?;
+            .map_err(|e| format!("failed to stat file {}: {}", file_path, e))?;
         let file_size = metadata.len();
 
         // Calculate part size: start at 64MB, scale up for very large files (matches Go)
@@ -170,16 +119,11 @@ impl S3TierBackend {
             )
             .send()
             .await
-            .map_err(|e| {
-                TierError::Io(format!(
-                    "failed to create multipart upload: {}",
-                    DisplayErrorContext(&e)
-                ))
-            })?;
+            .map_err(|e| format!("failed to create multipart upload: {}", e))?;
 
         let upload_id = create_resp
             .upload_id()
-            .ok_or_else(|| TierError::Io("no upload_id in multipart upload response".to_string()))?
+            .ok_or_else(|| "no upload_id in multipart upload response".to_string())?
             .to_string();
 
         // Build list of (part_number, offset, size) for all parts
@@ -215,21 +159,19 @@ impl S3TierBackend {
                 let _permit = sem
                     .acquire()
                     .await
-                    .map_err(|e| TierError::Io(format!("semaphore error: {}", e)))?;
+                    .map_err(|e| format!("semaphore error: {}", e))?;
 
                 // Read this part's data from the file at the correct offset
                 let mut file = tokio::fs::File::open(&fp)
                     .await
-                    .map_err(|e| TierError::Io(format!("failed to open file {}: {}", fp, e)))?;
+                    .map_err(|e| format!("failed to open file {}: {}", fp, e))?;
                 file.seek(std::io::SeekFrom::Start(off))
                     .await
-                    .map_err(|e| {
-                        TierError::Io(format!("failed to seek to offset {}: {}", off, e))
-                    })?;
+                    .map_err(|e| format!("failed to seek to offset {}: {}", off, e))?;
                 let mut buf = vec![0u8; size];
-                file.read_exact(&mut buf).await.map_err(|e| {
-                    TierError::Io(format!("failed to read file at offset {}: {}", off, e))
-                })?;
+                file.read_exact(&mut buf)
+                    .await
+                    .map_err(|e| format!("failed to read file at offset {}: {}", off, e))?;
 
                 let upload_part_resp = client
                     .upload_part()
@@ -241,12 +183,7 @@ impl S3TierBackend {
                     .send()
                     .await
                     .map_err(|e| {
-                        TierError::Io(format!(
-                            "failed to upload part {} at offset {}: {}",
-                            pn,
-                            off,
-                            DisplayErrorContext(&e)
-                        ))
+                        format!("failed to upload part {} at offset {}: {}", pn, off, e)
                     })?;
 
                 let e_tag = upload_part_resp.e_tag().unwrap_or_default().to_string();
@@ -265,9 +202,9 @@ impl S3TierBackend {
                     };
                     (guard.1)(uploaded as i64, pct)
                 };
-                progress_result.map_err(TierError::Aborted)?;
+                progress_result?;
 
-                Ok::<_, TierError>(
+                Ok::<_, String>(
                     CompletedPart::builder()
                         .e_tag(e_tag)
                         .part_number(pn)
@@ -282,7 +219,7 @@ impl S3TierBackend {
             for handle in handles {
                 let part = handle
                     .await
-                    .map_err(|e| TierError::Io(format!("upload task panicked: {}", e)))??;
+                    .map_err(|e| format!("upload task panicked: {}", e))??;
                 completed_parts.push(part);
             }
 
@@ -299,14 +236,9 @@ impl S3TierBackend {
                 .multipart_upload(completed_upload)
                 .send()
                 .await
-                .map_err(|e| {
-                    TierError::Io(format!(
-                        "failed to complete multipart upload: {}",
-                        DisplayErrorContext(&e)
-                    ))
-                })?;
+                .map_err(|e| format!("failed to complete multipart upload: {}", e))?;
 
-            Ok::<(), TierError>(())
+            Ok::<(), String>(())
         }
         .await;
 
@@ -349,7 +281,7 @@ impl S3TierBackend {
         dest_path: &str,
         key: &str,
         progress_fn: F,
-    ) -> Result<u64, TierError>
+    ) -> Result<u64, String>
     where
         F: FnMut(i64, f32) -> Result<(), String> + Send + Sync + 'static,
     {
@@ -361,7 +293,7 @@ impl S3TierBackend {
             .key(key)
             .send()
             .await
-            .map_err(|e| head_object_error(key, e))?;
+            .map_err(|e| format!("failed to head object {}: {}", key, e))?;
 
         let file_size = head_resp.content_length().unwrap_or(0) as u64;
 
@@ -373,12 +305,10 @@ impl S3TierBackend {
                 .truncate(true)
                 .open(dest_path)
                 .await
-                .map_err(|e| {
-                    TierError::Io(format!("failed to open dest file {}: {}", dest_path, e))
-                })?;
+                .map_err(|e| format!("failed to open dest file {}: {}", dest_path, e))?;
             file.set_len(file_size)
                 .await
-                .map_err(|e| TierError::Io(format!("failed to set file length: {}", e)))?;
+                .map_err(|e| format!("failed to set file length: {}", e))?;
         }
 
         let part_size: u64 = 64 * 1024 * 1024;
@@ -414,7 +344,7 @@ impl S3TierBackend {
                 let _permit = sem
                     .acquire()
                     .await
-                    .map_err(|e| TierError::Io(format!("semaphore error: {}", e)))?;
+                    .map_err(|e| format!("semaphore error: {}", e))?;
 
                 let end = off + size - 1;
                 let range = format!("bytes={}-{}", off, end);
@@ -426,13 +356,13 @@ impl S3TierBackend {
                     .range(&range)
                     .send()
                     .await
-                    .map_err(|e| get_object_error(&key, &range, e))?;
+                    .map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
 
                 let body = get_resp
                     .body
                     .collect()
                     .await
-                    .map_err(|e| TierError::Io(format!("failed to read body: {}", e)))?;
+                    .map_err(|e| format!("failed to read body: {}", e))?;
                 let bytes = body.into_bytes();
 
                 // Write at the correct offset (like Go's WriteAt)
@@ -440,17 +370,13 @@ impl S3TierBackend {
                     .write(true)
                     .open(&dp)
                     .await
-                    .map_err(|e| {
-                        TierError::Io(format!("failed to open dest file {}: {}", dp, e))
-                    })?;
+                    .map_err(|e| format!("failed to open dest file {}: {}", dp, e))?;
                 file.seek(std::io::SeekFrom::Start(off))
                     .await
-                    .map_err(|e| {
-                        TierError::Io(format!("failed to seek to offset {}: {}", off, e))
-                    })?;
+                    .map_err(|e| format!("failed to seek to offset {}: {}", off, e))?;
                 file.write_all(&bytes)
                     .await
-                    .map_err(|e| TierError::Io(format!("failed to write to {}: {}", dp, e)))?;
+                    .map_err(|e| format!("failed to write to {}: {}", dp, e))?;
 
                 // Report progress. The lock is released before the result is
                 // propagated so an aborting callback cannot poison the mutex
@@ -466,9 +392,9 @@ impl S3TierBackend {
                     };
                     (guard.1)(downloaded as i64, pct)
                 };
-                progress_result.map_err(TierError::Aborted)?;
+                progress_result?;
 
-                Ok::<_, TierError>(())
+                Ok::<_, String>(())
             }));
         }
 
@@ -476,7 +402,7 @@ impl S3TierBackend {
         for handle in handles {
             handle
                 .await
-                .map_err(|e| TierError::Io(format!("download task panicked: {}", e)))??;
+                .map_err(|e| format!("download task panicked: {}", e))??;
         }
 
         // fsync the file so its content is durable before the caller trims the .vif
@@ -485,21 +411,16 @@ impl S3TierBackend {
             .write(true)
             .open(dest_path)
             .await
-            .map_err(|e| TierError::Io(format!("failed to open {} for fsync: {}", dest_path, e)))?;
+            .map_err(|e| format!("failed to open {} for fsync: {}", dest_path, e))?;
         synced
             .sync_all()
             .await
-            .map_err(|e| TierError::Io(format!("failed to fsync {}: {}", dest_path, e)))?;
+            .map_err(|e| format!("failed to fsync {}: {}", dest_path, e))?;
 
         Ok(file_size)
     }
 
-    pub async fn read_range(
-        &self,
-        key: &str,
-        offset: u64,
-        size: usize,
-    ) -> Result<Vec<u8>, TierError> {
+    pub async fn read_range(&self, key: &str, offset: u64, size: usize) -> Result<Vec<u8>, String> {
         let end = offset + (size as u64).saturating_sub(1);
         let range = format!("bytes={}-{}", offset, end);
         let resp = self
@@ -510,35 +431,29 @@ impl S3TierBackend {
             .range(&range)
             .send()
             .await
-            .map_err(|e| get_object_error(key, &range, e))?;
+            .map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
 
         let body = resp
             .body
             .collect()
             .await
-            .map_err(|e| TierError::Io(format!("failed to read object {} body: {}", key, e)))?;
+            .map_err(|e| format!("failed to read object {} body: {}", key, e))?;
         Ok(body.into_bytes().to_vec())
     }
 
     /// Delete a file from S3.
-    pub async fn delete_file(&self, key: &str) -> Result<(), TierError> {
+    pub async fn delete_file(&self, key: &str) -> Result<(), String> {
         self.client
             .delete_object()
             .bucket(&self.bucket)
             .key(key)
             .send()
             .await
-            .map_err(|e| {
-                TierError::Io(format!(
-                    "failed to delete object {}: {}",
-                    key,
-                    DisplayErrorContext(&e)
-                ))
-            })?;
+            .map_err(|e| format!("failed to delete object {}: {}", key, e))?;
         Ok(())
     }
 
-    pub fn delete_file_blocking(&self, key: &str) -> Result<(), TierError> {
+    pub fn delete_file_blocking(&self, key: &str) -> Result<(), String> {
         let client = self.client.clone();
         let bucket = self.bucket.clone();
         let key = key.to_string();
@@ -549,13 +464,7 @@ impl S3TierBackend {
                 .key(&key)
                 .send()
                 .await
-                .map_err(|e| {
-                    TierError::Io(format!(
-                        "failed to delete object {}: {}",
-                        key,
-                        DisplayErrorContext(&e)
-                    ))
-                })?;
+                .map_err(|e| format!("failed to delete object {}: {}", key, e))?;
             Ok(())
         })
     }
@@ -565,7 +474,7 @@ impl S3TierBackend {
         key: &str,
         offset: u64,
         size: usize,
-    ) -> Result<Vec<u8>, TierError> {
+    ) -> Result<Vec<u8>, String> {
         let client = self.client.clone();
         let bucket = self.bucket.clone();
         let key = key.to_string();
@@ -579,12 +488,13 @@ impl S3TierBackend {
                 .range(&range)
                 .send()
                 .await
-                .map_err(|e| get_object_error(&key, &range, e))?;
+                .map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
 
-            let body =
-                resp.body.collect().await.map_err(|e| {
-                    TierError::Io(format!("failed to read object {} body: {}", key, e))
-                })?;
+            let body = resp
+                .body
+                .collect()
+                .await
+                .map_err(|e| format!("failed to read object {} body: {}", key, e))?;
             Ok(body.into_bytes().to_vec())
         })
     }
@@ -645,279 +555,18 @@ pub fn global_s3_tier_registry() -> &'static RwLock<S3TierRegistry> {
     GLOBAL_S3_TIER_REGISTRY.get_or_init(|| RwLock::new(S3TierRegistry::new()))
 }
 
-/// The one process-wide runtime for tiered-S3 I/O issued from synchronous
-/// storage code. A per-call runtime tore down the SDK's pooled connections
-/// after every 64 KiB chunk, re-dialing TLS per read; a long-lived runtime
-/// keeps the pool warm.
-///
-/// Built on first use. A build failure is returned, not cached or panicked:
-/// callers sit inside `Volume::destroy` and needle reads, whose own error
-/// paths must run, and a later call may succeed.
-static TIER_RUNTIME: std::sync::Mutex<Option<tokio::runtime::Runtime>> =
-    std::sync::Mutex::new(None);
-
-fn tier_handle() -> Result<tokio::runtime::Handle, TierError> {
-    let mut slot = TIER_RUNTIME
-        .lock()
-        .unwrap_or_else(|poisoned| poisoned.into_inner());
-    if slot.is_none() {
-        let runtime = tokio::runtime::Builder::new_multi_thread()
-            .worker_threads(2)
-            .thread_name("tier-io")
-            .enable_all()
-            .build()
-            .map_err(|e| {
-                TierError::RuntimeUnavailable(format!(
-                    "failed to build the tier I/O tokio runtime: {}",
-                    e
-                ))
-            })?;
-        *slot = Some(runtime);
-    }
-    Ok(slot.as_ref().expect("just initialised").handle().clone())
-}
-
-/// Run `future` on the tier runtime and block the calling thread until it
-/// finishes. The caller may be a worker of *another* tokio runtime, so this
-/// waits on a channel rather than `Handle::block_on`, which panics when
-/// called from inside any runtime context.
-fn block_on_tier_future<F, T>(future: F) -> Result<T, TierError>
+fn block_on_tier_future<F, T>(future: F) -> Result<T, String>
 where
-    F: Future<Output = Result<T, TierError>> + Send + 'static,
+    F: Future<Output = Result<T, String>> + Send + 'static,
     T: Send + 'static,
 {
-    let handle = tier_handle()?;
-    let task = handle.spawn(future);
-    let (tx, rx) = std::sync::mpsc::sync_channel(1);
-    handle.spawn(async move {
-        // The receiver only goes away if the caller was unwound; nothing to
-        // report then.
-        let _ = tx.send(task.await);
-    });
-    match rx.recv() {
-        Ok(Ok(result)) => result,
-        Ok(Err(join_error)) => Err(describe_join_error(join_error)),
-        Err(_) => Err(TierError::RuntimeUnavailable(
-            "tier I/O runtime dropped the task before it finished".to_string(),
-        )),
-    }
-}
-
-/// Turn a `JoinError` into a message that keeps the panic payload, so an
-/// SDK panic surfaces as "boom" rather than a fixed "thread panicked".
-fn describe_join_error(join_error: tokio::task::JoinError) -> TierError {
-    if join_error.is_panic() {
-        let payload = join_error.into_panic();
-        let message = if let Some(s) = payload.downcast_ref::<&str>() {
-            (*s).to_string()
-        } else if let Some(s) = payload.downcast_ref::<String>() {
-            s.clone()
-        } else {
-            "non-string panic payload".to_string()
-        };
-        TierError::Io(format!("tier I/O task panicked: {}", message))
-    } else {
-        TierError::RuntimeUnavailable(format!("tier I/O task failed: {}", join_error))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::remote_storage::s3::tests::{CannedResponse, NO_SUCH_KEY};
-    use std::collections::HashSet;
-    use tokio::runtime::Handle;
-
-    fn probe() -> Result<(tokio::runtime::Id, Option<String>), TierError> {
-        block_on_tier_future(async {
-            Ok((
-                Handle::current().id(),
-                std::thread::current().name().map(str::to_string),
-            ))
-        })
-    }
-
-    #[test]
-    fn block_on_tier_future_reuses_one_runtime() {
-        let (first_runtime, first_thread) = probe().expect("first call");
-        let (second_runtime, second_thread) = probe().expect("second call");
-        assert_eq!(
-            first_runtime, second_runtime,
-            "each call must run on the same long-lived tier runtime"
-        );
-        assert_eq!(first_thread.as_deref(), Some("tier-io"));
-        assert_eq!(second_thread.as_deref(), Some("tier-io"));
-
-        let mut runtimes = HashSet::new();
-        for _ in 0..20 {
-            let (id, _) = probe().expect("probe");
-            runtimes.insert(id);
-        }
-        assert_eq!(runtimes.len(), 1);
-    }
-
-    #[test]
-    fn block_on_tier_future_returns_the_value_and_the_error() {
-        assert_eq!(block_on_tier_future(async { Ok(7u32) }).unwrap(), 7);
-        let err = block_on_tier_future::<_, u32>(async { Err(TierError::NotFound("nope".into())) })
-            .unwrap_err();
-        assert!(
-            matches!(&err, TierError::NotFound(m) if m == "nope"),
-            "{err:?}"
-        );
-    }
-
-    #[test]
-    fn block_on_tier_future_works_from_a_std_thread() {
-        let (id, _) = std::thread::spawn(probe)
-            .join()
-            .expect("probe thread")
-            .expect("probe");
-        assert_eq!(id, tier_handle().expect("tier runtime").id());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn block_on_tier_future_works_from_spawn_blocking() {
-        let (id, _) = tokio::task::spawn_blocking(probe)
-            .await
-            .expect("spawn_blocking")
-            .expect("probe");
-        assert_eq!(id, tier_handle().expect("tier runtime").id());
-        assert_ne!(id, Handle::current().id());
-    }
-
-    // Called straight from another runtime's async context: the case that
-    // would panic with `Handle::block_on` ("Cannot start a runtime from
-    // within a runtime").
-    #[tokio::test]
-    async fn block_on_tier_future_works_from_a_current_thread_runtime() {
-        let (id, _) = probe().expect("probe");
-        assert_eq!(id, tier_handle().expect("tier runtime").id());
-        assert_ne!(id, Handle::current().id());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn block_on_tier_future_works_from_a_multi_thread_runtime_worker() {
-        let (id, _) = probe().expect("probe");
-        assert_eq!(id, tier_handle().expect("tier runtime").id());
-        assert_ne!(id, Handle::current().id());
-    }
-
-    #[test]
-    fn block_on_tier_future_reports_the_panic_payload() {
-        let err = block_on_tier_future::<_, ()>(async {
-            if std::hint::black_box(true) {
-                panic!("boom {}", 42);
-            }
-            Ok(())
-        })
-        .expect_err("a panicking future must be an error");
-        assert!(matches!(err, TierError::Io(_)), "got: {err:?}");
-        assert!(err.to_string().contains("boom 42"), "got: {err}");
-        assert!(err.to_string().contains("panicked"), "got: {err}");
-    }
-
-    #[test]
-    fn block_on_tier_future_reports_a_str_panic_payload() {
-        let err = block_on_tier_future::<_, ()>(async {
-            if std::hint::black_box(true) {
-                panic!("static boom");
-            }
-            Ok(())
-        })
-        .expect_err("a panicking future must be an error");
-        assert!(err.to_string().contains("static boom"), "got: {err}");
-    }
-
-    fn backend_answering(status: u16, body: &'static str) -> S3TierBackend {
-        let config = aws_sdk_s3::Config::builder()
-            .behavior_version(BehaviorVersion::latest())
-            .region(Region::new("us-east-1"))
-            .credentials_provider(Credentials::new("AKIATEST", "secret", None, None, "test"))
-            .endpoint_url("http://127.0.0.1:1")
-            .force_path_style(true)
-            .http_client(CannedResponse { status, body })
-            .retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
-            .build();
-        S3TierBackend {
-            client: Client::from_conf(config),
-            bucket: "bucket".to_string(),
-            storage_class: "STANDARD".to_string(),
-        }
-    }
-
-    #[tokio::test]
-    async fn download_head_404_is_not_found() {
-        let tmp = tempfile::tempdir().unwrap();
-        let dest = tmp.path().join("1.dat");
-        let err = backend_answering(404, "")
-            .download_file(dest.to_str().unwrap(), "missing", |_, _| Ok(()))
-            .await
-            .unwrap_err();
-        assert!(matches!(err, TierError::NotFound(_)), "{err:?}");
-        assert!(
-            err.to_string()
-                .starts_with("failed to head object missing: "),
-            "{err}"
-        );
-    }
-
-    #[tokio::test]
-    async fn download_head_403_is_io() {
-        let tmp = tempfile::tempdir().unwrap();
-        let dest = tmp.path().join("1.dat");
-        let err = backend_answering(403, "")
-            .download_file(dest.to_str().unwrap(), "denied", |_, _| Ok(()))
-            .await
-            .unwrap_err();
-        assert!(matches!(err, TierError::Io(_)), "{err:?}");
-    }
-
-    #[tokio::test]
-    async fn read_range_no_such_key_is_not_found() {
-        let err = backend_answering(404, NO_SUCH_KEY)
-            .read_range("missing", 0, 8)
-            .await
-            .unwrap_err();
-        assert!(matches!(err, TierError::NotFound(_)), "{err:?}");
-        assert!(
-            err.to_string()
-                .starts_with("failed to get object missing range bytes=0-7: "),
-            "{err}"
-        );
-    }
-
-    #[tokio::test]
-    async fn read_range_bare_404_is_io() {
-        // As in Go, GET is not-found by the NoSuchKey code, not the status.
-        let err = backend_answering(404, "")
-            .read_range("missing", 0, 8)
-            .await
-            .unwrap_err();
-        assert!(matches!(err, TierError::Io(_)), "{err:?}");
-    }
-
-    #[test]
-    fn read_range_blocking_no_such_key_is_not_found() {
-        let err = backend_answering(404, NO_SUCH_KEY)
-            .read_range_blocking("missing", 0, 8)
-            .unwrap_err();
-        assert!(matches!(err, TierError::NotFound(_)), "{err:?}");
-    }
-
-    #[test]
-    fn backend_name_to_type_id_splits_on_dot() {
-        assert_eq!(
-            backend_name_to_type_id("s3"),
-            ("s3".to_string(), "default".to_string())
-        );
-        assert_eq!(
-            backend_name_to_type_id("s3.eu"),
-            ("s3".to_string(), "eu".to_string())
-        );
-        assert_eq!(
-            backend_name_to_type_id("s3.a.b"),
-            (String::new(), String::new())
-        );
-    }
+    std::thread::spawn(move || {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .map_err(|e| format!("failed to build tokio runtime: {}", e))?;
+        runtime.block_on(future)
+    })
+    .join()
+    .map_err(|_| "tier runtime thread panicked".to_string())?
 }

seaweed-volume/src/server/mod.rs

@@ -1,6 +1,5 @@
 use tonic::Status;
 
-use crate::remote_storage::s3_tier::TierError;
 use crate::storage::volume::VolumeError;
 
 #[cfg(unix)]
@@ -24,12 +23,8 @@ impl From<VolumeError> for Status {
     fn from(err: VolumeError) -> Self {
         let message = err.to_string();
         match err {
-            VolumeError::NotFound
-            | VolumeError::VolumeNotFound(_)
-            | VolumeError::Tier(TierError::NotFound(_)) => Status::not_found(message),
-            VolumeError::ReadOnly(_) | VolumeError::NotEmpty => {
-                Status::failed_precondition(message)
-            }
+            VolumeError::NotFound | VolumeError::VolumeNotFound(_) => Status::not_found(message),
+            VolumeError::ReadOnly | VolumeError::NotEmpty => Status::failed_precondition(message),
             VolumeError::InsufficientSpace { .. } => Status::resource_exhausted(message),
             VolumeError::AlreadyExists => Status::already_exists(message),
             _ => Status::internal(message),
@@ -71,14 +66,7 @@ mod tests {
             Code::NotFound
         );
         assert_eq!(code(VolumeError::NotFound), Code::NotFound);
-        assert_eq!(
-            code(VolumeError::ReadOnly(VolumeId(7))),
-            Code::FailedPrecondition
-        );
-        assert_eq!(
-            VolumeError::ReadOnly(VolumeId(7)).to_string(),
-            "volume 7 is read only"
-        );
+        assert_eq!(code(VolumeError::ReadOnly), Code::FailedPrecondition);
         assert_eq!(
             code(VolumeError::InsufficientSpace {
                 vid: VolumeId(7),
@@ -89,27 +77,6 @@ mod tests {
         );
         assert_eq!(code(VolumeError::AlreadyExists), Code::AlreadyExists);
         assert_eq!(code(VolumeError::NotInitialized), Code::Internal);
-        assert_eq!(
-            code(TierError::NotFound("gone".into()).into()),
-            Code::NotFound
-        );
-        for tier in [
-            TierError::Io("io".into()),
-            TierError::RuntimeUnavailable("rt".into()),
-            TierError::Aborted("bye".into()),
-        ] {
-            assert_eq!(code(tier.into()), Code::Internal);
-        }
-
-        let status = status_with_context(
-            "backend s3.default copy file /data/1.dat",
-            TierError::NotFound("failed to head object k: NotFound".into()).into(),
-        );
-        assert_eq!(status.code(), Code::NotFound);
-        assert_eq!(
-            status.message(),
-            "backend s3.default copy file /data/1.dat: failed to head object k: NotFound"
-        );
 
         let status = status_with_context(
             "compact volume 7",

seaweed-volume/src/server/store_ec.rs

@@ -95,41 +95,20 @@ struct Snapshot {
     encode_ts_ns: i64,
 }
 
-/// Why a distributed EC read has no needle to return.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum EcMiss {
-    NotFound,
-    /// Tombstoned in the local `.ecx`/`.ecj`, or reported deleted by a peer.
-    Deleted,
-    VolumeNotFound,
-}
-
-/// Top-level entry point. Returns `Ok(None)` for any miss — absent,
-/// deleted, or volume gone; errors propagate as `io::Error`.
+/// Top-level entry point. Returns `Ok(None)` for "not found" (matches
+/// Go's `ReadEcShardNeedle`); errors propagate as `io::Error`.
 pub async fn read_ec_shard_needle_distributed(
     state: &Arc<VolumeServerState>,
     vid: VolumeId,
     needle_id: NeedleId,
 ) -> io::Result<Option<Needle>> {
-    Ok(read_ec_shard_needle_or_miss(state, vid, needle_id)
-        .await?
-        .ok())
-}
-
-/// Like `read_ec_shard_needle_distributed`, but says why there is no needle,
-/// as Go's `ReadEcShardNeedle` tells `ErrorDeleted` from not-found.
-pub async fn read_ec_shard_needle_or_miss(
-    state: &Arc<VolumeServerState>,
-    vid: VolumeId,
-    needle_id: NeedleId,
-) -> io::Result<Result<Needle, EcMiss>> {
     // Phase A — under the Store read lock, locate the needle, compute
     // intervals, and read any locally-mounted shard intervals. We must
     // not `.await` while holding this guard (std::sync::RwLockReadGuard
     // is !Send).
     let mut snapshot = match snapshot_under_lock(state, vid, needle_id)? {
-        Ok(s) => s,
-        Err(miss) => return Ok(Err(miss)),
+        Some(s) => s,
+        None => return Ok(None),
     };
 
     // Phase B — refresh the shard_locations cache from the master if
@@ -231,11 +210,17 @@ pub async fn read_ec_shard_needle_or_miss(
         .collect()
         .await;
 
-    // A peer reports the needle deleted (a cross-server window where the
-    // local index still shows it live): answer deleted rather than serving zeros.
-    let Some(assembled) = gather_intervals(fetched)? else {
-        return Ok(Err(EcMiss::Deleted));
-    };
+    let mut assembled: Vec<Vec<u8>> = Vec::with_capacity(fetched.len());
+    for res in fetched {
+        let (buf, is_deleted) = res?;
+        // A peer reports the needle deleted (a cross-server window where the
+        // local index still shows it live): treat as not-found rather than
+        // serving zeros, mirroring Go's ErrorDeleted.
+        if is_deleted {
+            return Ok(None);
+        }
+        assembled.push(buf);
+    }
 
     // Phase D — assemble and parse the Needle. Mirrors the tail of
     // `EcVolume::read_ec_shard_needle`.
@@ -267,20 +252,7 @@ pub async fn read_ec_shard_needle_or_miss(
         snapshot.version,
     )
     .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{}", e)))?;
-    Ok(Ok(n))
-}
-
-/// `None` when any holder reported the needle deleted. That outranks another
-/// interval's error: deletes are never invented, so the needle is gone either way.
-fn gather_intervals(fetched: Vec<io::Result<(Vec<u8>, bool)>>) -> io::Result<Option<Vec<Vec<u8>>>> {
-    if fetched.iter().any(|r| matches!(r, Ok((_, true)))) {
-        return Ok(None);
-    }
-    fetched
-        .into_iter()
-        .map(|r| r.map(|(buf, _)| buf))
-        .collect::<io::Result<_>>()
-        .map(Some)
+    Ok(Some(n))
 }
 
 /// What one EC delete RPC carries — `VolumeEcBlobDeleteRequest` minus tonic.
@@ -419,24 +391,13 @@ async fn delete_on_ec_shard_holders(
 
     let mut last_err = None;
     if local_shards.contains(&shard_id) {
-        // A decode in its publishing tail must not miss this delete. The
-        // tail membership is verified again under the store write lock
-        // inside journal_delete_local — WouldBlock means the decode claimed
-        // it in the gap after this wait — so wait and retry.
-        loop {
-            crate::server::grpc_server::wait_ec_decode_tail(state, target.vid).await;
-            match journal_delete_local(state, target.vid, target.needle_id) {
-                Ok(()) => return Ok(true),
-                Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue,
-                // Nothing was committed — the volume unmounted or remounted
-                // without the needle — so it is safe to fall back to other
-                // shard holders, unlike an RPC failure which may have landed.
-                Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(false),
-                Err(e) => {
-                    last_err = Some(e);
-                    break;
-                }
-            }
+        match journal_delete_local(state, target.vid, target.needle_id) {
+            Ok(()) => return Ok(true),
+            // Nothing was committed — the volume unmounted or remounted
+            // without the needle — so it is safe to fall back to other
+            // shard holders, unlike an RPC failure which may have landed.
+            Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(false),
+            Err(e) => last_err = Some(e),
         }
     }
     if let Some(addrs) = addrs {
@@ -498,15 +459,6 @@ fn journal_delete_local(
     needle_id: NeedleId,
 ) -> io::Result<()> {
     let mut store = state.store.write().unwrap();
-    // Membership is read under the write lock: a decode can claim the
-    // publishing tail while this call waited for the decoder's read lock,
-    // so a check taken earlier would be stale by commit time.
-    if crate::server::grpc_server::ec_decode_tail_contains(state, vid) {
-        return Err(io::Error::new(
-            io::ErrorKind::WouldBlock,
-            format!("ec volume {} is in decode publishing tail", vid.0),
-        ));
-    }
     let ecv = store.find_ec_volume_mut(vid).ok_or_else(|| {
         io::Error::new(
             io::ErrorKind::NotFound,
@@ -912,10 +864,11 @@ fn snapshot_under_lock(
     state: &Arc<VolumeServerState>,
     vid: VolumeId,
     needle_id: NeedleId,
-) -> io::Result<Result<Snapshot, EcMiss>> {
+) -> io::Result<Option<Snapshot>> {
     let store = state.store.read().unwrap();
-    let Some(ecv) = store.find_ec_volume(vid) else {
-        return Ok(Err(EcMiss::VolumeNotFound));
+    let ecv = match store.find_ec_volume(vid) {
+        Some(v) => v,
+        None => return Ok(None),
     };
 
     // Reuse EcVolume::locate_needle for offset/size resolution AND
@@ -923,17 +876,11 @@ fn snapshot_under_lock(
     // local-only read path uses, so we stay byte-identical on the
     // shard-size + interval boundaries. locate_needle applies the runtime
     // delete mask, which is correct for serving reads.
-    let Some((offset, size, intervals)) = ecv.locate_needle(needle_id)? else {
-        // locate_needle folds a tombstone into not-found.
-        let deleted =
-            matches!(ecv.find_needle_from_ecx(needle_id)?, Some((_, s)) if s.is_deleted());
-        return Ok(Err(if deleted {
-            EcMiss::Deleted
-        } else {
-            EcMiss::NotFound
-        }));
+    let (offset, size, intervals) = match ecv.locate_needle(needle_id)? {
+        Some(v) => v,
+        None => return Ok(None),
     };
-    build_snapshot(ecv, offset, size, &intervals).map(Ok)
+    build_snapshot(ecv, offset, size, &intervals).map(Some)
 }
 
 /// Like `snapshot_under_lock`, but locates intervals from the RAW .ecx
@@ -1773,7 +1720,7 @@ async fn fetch_ec_index_from_one_peer(
         .await
         .map_err(|e| io::Error::other(format!("copy .ecx: {}", e)))?
         .into_inner();
-    drain_copy_stream(stream, ecx_path).await?;
+    drain_copy_stream(stream, ecx_path, false).await?;
 
     let meta =
         fs::metadata(ecx_path).map_err(|e| io::Error::other(format!("stat copied .ecx: {}", e)))?;
@@ -1786,30 +1733,15 @@ async fn fetch_ec_index_from_one_peer(
         )));
     }
 
-    // .ecj is the source peer's deletion journal; .vif carries EC params. Both
-    // are best-effort: a missing .ecj is recreated at mount and a missing .vif
-    // falls back to default EC parameters. The journal is a *set*: merge the
-    // peer's ids into any local ones as a union instead of appending, so
-    // a volume bounced between servers cannot double its journal. The merge
-    // only appends whole records, so a failure leaves nothing to clean up.
+    // .ecj is the source peer's deletion journal (appended); .vif carries EC
+    // params. Both are best-effort: a missing .ecj is recreated at mount and a
+    // missing .vif falls back to default EC parameters. A failed .ecj append
+    // leaves a partial file, so drop it.
     match client.copy_file(copy_req(".ecj", true)).await {
         Ok(resp) => {
-            let mut stream = resp.into_inner();
-            let merged = match crate::server::grpc_server::receive_ecj_ids(&mut stream).await {
-                Ok((ids, true)) => crate::server::grpc_server::merge_ecj_ids(
-                    state,
-                    m.vid,
-                    m.data_dir.clone(),
-                    ecj_path.to_string(),
-                    ids,
-                )
-                .await
-                .map(|_| ()),
-                Ok((_, false)) => Ok(()),
-                Err(e) => Err(e),
-            };
-            if let Err(e) = merged {
+            if let Err(e) = drain_copy_stream(resp.into_inner(), ecj_path, true).await {
                 tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .ecj: {}", e);
+                let _ = fs::remove_file(ecj_path);
             }
         }
         Err(e) => tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .ecj: {}", e),
@@ -1817,7 +1749,7 @@ async fn fetch_ec_index_from_one_peer(
 
     match client.copy_file(copy_req(".vif", true)).await {
         Ok(resp) => {
-            if let Err(e) = drain_copy_stream(resp.into_inner(), vif_path).await {
+            if let Err(e) = drain_copy_stream(resp.into_inner(), vif_path, false).await {
                 tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .vif: {}", e);
             }
         }
@@ -1827,14 +1759,22 @@ async fn fetch_ec_index_from_one_peer(
     Ok(())
 }
 
-/// Drain a CopyFile stream into a local file, truncating it first.
+/// Drain a CopyFile stream into a local file, appending or truncating.
 async fn drain_copy_stream(
     mut stream: tonic::Streaming<crate::pb::volume_server_pb::CopyFileResponse>,
     dest_path: &str,
+    append: bool,
 ) -> io::Result<()> {
     use std::io::Write;
-    let mut file = fs::File::create(dest_path)
-        .map_err(|e| io::Error::other(format!("create {}: {}", dest_path, e)))?;
+    let mut file = if append {
+        fs::OpenOptions::new()
+            .create(true)
+            .append(true)
+            .open(dest_path)
+    } else {
+        fs::File::create(dest_path)
+    }
+    .map_err(|e| io::Error::other(format!("create {}: {}", dest_path, e)))?;
     while let Some(chunk) = stream
         .message()
         .await
@@ -1850,21 +1790,6 @@ async fn drain_copy_stream(
 mod tests {
     use super::*;
 
-    #[test]
-    fn gather_intervals_puts_a_reported_deletion_ahead_of_errors() {
-        let failed = || Err(io::Error::other("shard unreachable"));
-        assert!(
-            gather_intervals(vec![failed(), Ok((Vec::new(), true))])
-                .unwrap()
-                .is_none()
-        );
-        assert!(gather_intervals(vec![failed(), Ok((vec![1], false))]).is_err());
-        assert_eq!(
-            gather_intervals(vec![Ok((vec![1], false)), Ok((vec![2], false))]).unwrap(),
-            Some(vec![vec![1], vec![2]])
-        );
-    }
-
     fn locations(count: usize) -> HashMap<ShardId, Vec<String>> {
         (0..count)
             .map(|sid| (sid as ShardId, vec!["127.0.0.1:8080".to_string()]))

seaweed-volume/src/server/volume_server.rs

@@ -114,21 +114,6 @@ pub struct VolumeServerState {
     pub cli_white_list: Vec<String>,
     /// Path to state.pb file for persisting VolumeServerState across restarts.
     pub state_file_path: String,
-    /// Volumes with an EC decode in flight. A dropped request leaves the
-    /// blocking job running; this keeps a retry from racing it on the
-    /// same volume files.
-    pub ec_decodes_in_flight:
-        std::sync::Mutex<std::collections::HashSet<crate::storage::types::VolumeId>>,
-    /// Volumes whose EC decode is in its publishing tail (journal catch-up,
-    /// .idx write, compaction). Local .ecj appenders wait on
-    /// `ec_decode_tail_notify` while their vid is listed, so no committed
-    /// delete falls between the last catch_up and the .cpd/.cpx swap —
-    /// the per-volume slice of Go's EcVolume.ecjFileAccessLock.
-    pub ec_decode_tail:
-        std::sync::Mutex<std::collections::HashSet<crate::storage::types::VolumeId>>,
-    /// Wakes .ecj appenders waiting on `ec_decode_tail` when a decode's
-    /// publishing tail ends.
-    pub ec_decode_tail_notify: tokio::sync::Notify,
 }
 
 impl VolumeServerState {

seaweed-volume/src/server/write_queue.rs

@@ -3,8 +3,8 @@
 //! Instead of each upload handler directly calling `write_needle`, writes are
 //! submitted to a queue. A background worker drains the queue in batches (up to
 //! 128 entries), groups them by volume ID, and processes them together under a
-//! single store lock. Durable writes to a volume share their .dat and .idx
-//! flushes (see `Volume::write_needles_grouped`).
+//! single store lock. Requests that asked for `fsync` are flushed by
+//! `write_needle` itself, one flush per durable write.
 
 use std::sync::Arc;
 
@@ -159,12 +159,8 @@ fn process_batch(state: Arc<VolumeServerState>, batch: Vec<WriteRequest>) {
     let mut store = state.store.write().unwrap();
 
     for (vid, entries) in groups {
-        let (mut writes, senders): (Vec<_>, Vec<_>) = entries
-            .into_iter()
-            .map(|(needle, fsync, response_tx)| ((needle, fsync), response_tx))
-            .unzip();
-        let results = store.write_volume_needles(vid, &mut writes);
-        for (response_tx, result) in senders.into_iter().zip(results) {
+        for (mut needle, fsync, response_tx) in entries {
+            let result = store.write_volume_needle(vid, &mut needle, fsync);
             // Send result back; ignore error if receiver dropped.
             let _ = response_tx.send(result);
         }
@@ -229,9 +225,6 @@ mod tests {
             security_file: String::new(),
             cli_white_list: vec![],
             state_file_path: String::new(),
-            ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
-            ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
-            ec_decode_tail_notify: tokio::sync::Notify::new(),
         })
     }
 
@@ -319,63 +312,6 @@ mod tests {
         }
     }
 
-    /// The queue hands a volume's batch to the grouped path, so ten durable
-    /// writes cost one .dat sync and one .idx sync, not ten of each.
-    #[test]
-    fn test_process_batch_group_commits_fsync_writes() {
-        use crate::config::MinFreeSpace;
-        use crate::storage::types::{DiskType, NeedleId};
-        use crate::storage::volume::VolumeSpec;
-
-        let tmp = tempfile::TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-        let state = make_test_state();
-        {
-            let mut store = state.store.write().unwrap();
-            store
-                .add_location(
-                    dir,
-                    dir,
-                    10,
-                    DiskType::HardDrive,
-                    MinFreeSpace::Percent(1.0),
-                    Vec::new(),
-                )
-                .unwrap();
-            store
-                .add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
-                .unwrap();
-        }
-
-        let mut receivers = Vec::new();
-        let batch = (1..=10u64)
-            .map(|id| {
-                let (response_tx, response_rx) = oneshot::channel();
-                receivers.push(response_rx);
-                WriteRequest {
-                    volume_id: VolumeId(1),
-                    needle: Needle {
-                        id: NeedleId(id),
-                        cookie: 0x1111.into(),
-                        data: vec![id as u8; 8],
-                        data_size: 8,
-                        ..Needle::default()
-                    },
-                    fsync: true,
-                    response_tx,
-                }
-            })
-            .collect();
-        process_batch(state.clone(), batch);
-
-        for mut rx in receivers {
-            assert!(matches!(rx.try_recv().unwrap(), Ok((_, _, false))));
-        }
-        let store = state.store.read().unwrap();
-        let (_, vol) = store.find_volume(VolumeId(1)).unwrap();
-        assert_eq!(vol.sync_counts_for_test(), (1, 1));
-    }
-
     #[tokio::test]
     async fn test_write_queue_dropped_sender() {
         // When the queue is dropped, subsequent submits should fail gracefully.

seaweed-volume/src/storage/disk_location.rs

@@ -18,9 +18,7 @@ use crate::storage::erasure_coding::ec_shard::{
     DATA_SHARDS_COUNT, ERASURE_CODING_LARGE_BLOCK_SIZE, ERASURE_CODING_SMALL_BLOCK_SIZE,
     EcVolumeShard, ShardId,
 };
-use crate::storage::erasure_coding::ec_volume::{
-    ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file,
-};
+use crate::storage::erasure_coding::ec_volume::EcVolume;
 use crate::storage::needle_map::NeedleMapKind;
 use crate::storage::super_block::SUPER_BLOCK_SIZE;
 use crate::storage::types::*;
@@ -462,16 +460,13 @@ impl DiskLocation {
         let idx_base = volume_file_name(&self.idx_directory, collection, vid);
         const MAX_SHARD_COUNT: usize = 32;
 
-        // Remove index files from idx directory (.ecx, .ecj, and a compaction
-        // tmp a crash may have left beside the .ecj)
+        // Remove index files from idx directory (.ecx, .ecj)
         rm_if_present(format!("{}.ecx", idx_base))?;
         rm_if_present(format!("{}.ecj", idx_base))?;
-        rm_if_present(format!("{}{}", idx_base, ECJ_COMPACT_TMP_EXT))?;
         // Also try data directory in case .ecx/.ecj were created before -dir.idx was configured
         if self.idx_directory != self.directory {
             rm_if_present(format!("{}.ecx", base))?;
             rm_if_present(format!("{}.ecj", base))?;
-            rm_if_present(format!("{}{}", base, ECJ_COMPACT_TMP_EXT))?;
         }
 
         // Remove all EC shard files (.ec00 ~ .ec31)
@@ -484,13 +479,6 @@ impl DiskLocation {
         if self.idx_directory != self.directory {
             remove_bitrot_sidecars(&idx_base)?;
         }
-
-        // Staged 2PC generations (<base>.ecNN.v<N>, versioned .ecx/.ecj/.vif)
-        // belong to this volume's EC state too; leaving them orphans the files.
-        crate::storage::erasure_coding::ec_shard::remove_ec_generation_files(&base, 0)?;
-        if self.idx_directory != self.directory {
-            crate::storage::erasure_coding::ec_shard::remove_ec_generation_files(&idx_base, 0)?;
-        }
         Ok(())
     }
 
@@ -610,20 +598,13 @@ impl DiskLocation {
         &mut self,
         vid: VolumeId,
         only_empty: bool,
-        only_garbage: bool,
         keep_remote_data: bool,
     ) -> Result<(), VolumeError> {
-        // Refuse before removing: a refused destroy must leave it mounted.
-        if let Some(v) = self.volumes.get(&vid)
-            && v.is_compacting()
-        {
-            return Err(v.compacting_error());
-        }
         if let Some(mut v) = self.volumes.remove(&vid) {
             crate::metrics::VOLUME_GAUGE
                 .with_label_values(&[&v.collection, "volume"])
                 .dec();
-            v.destroy(only_empty, only_garbage, keep_remote_data)?;
+            v.destroy(only_empty, keep_remote_data)?;
             Ok(())
         } else {
             Err(VolumeError::NotFound)
@@ -644,7 +625,7 @@ impl DiskLocation {
                 crate::metrics::VOLUME_GAUGE
                     .with_label_values(&[&v.collection, "volume"])
                     .dec();
-                if let Err(e) = v.destroy(false, false, false) {
+                if let Err(e) = v.destroy(false, false) {
                     warn!(volume_id = vid.0, error = %e, "delete collection: failed to destroy volume");
                 }
             }
@@ -798,17 +779,21 @@ impl DiskLocation {
     /// Mirrors `DiskLocation.HasEcxFileOnDisk` in
     /// `weed/storage/disk_location_ec.go`. Skips entries that are
     /// directories so a stray dir named `<collection>_<vid>.ecx` doesn't
-    /// register as a present index file. A 0-byte `.ecx` is a corrupt stub
-    /// left by a failed EC distribute copy; it must not steer placement
-    /// toward this disk, so it counts as absent (Go requires `Size() > 0`).
+    /// register as a present index file.
     pub fn has_ecx_file_on_disk(&self, collection: &str, vid: VolumeId) -> bool {
         let idx_base = volume_file_name(&self.idx_directory, collection, vid);
-        if is_usable_ecx_file(&format!("{}.ecx", idx_base)) {
+        let idx_path = format!("{}.ecx", idx_base);
+        if let Ok(meta) = fs::metadata(&idx_path)
+            && !meta.is_dir()
+        {
             return true;
         }
         if self.idx_directory != self.directory {
             let data_base = volume_file_name(&self.directory, collection, vid);
-            if is_usable_ecx_file(&format!("{}.ecx", data_base)) {
+            let data_path = format!("{}.ecx", data_base);
+            if let Ok(meta) = fs::metadata(&data_path)
+                && !meta.is_dir()
+            {
                 return true;
             }
         }
@@ -820,20 +805,6 @@ impl DiskLocation {
         self.ec_volumes.remove(&vid)
     }
 
-    /// Drop the in-memory EC volume for vid and close its descriptors without
-    /// deleting files, so a following unlink frees the inodes instead of
-    /// leaving open fds serving the old bytes. Mirrors Go's unloadEcVolume.
-    pub fn unload_ec_volume(&mut self, vid: VolumeId) {
-        if let Some(mut ec_vol) = self.ec_volumes.remove(&vid) {
-            for _ in 0..ec_vol.shard_count() {
-                crate::metrics::VOLUME_GAUGE
-                    .with_label_values(&[&ec_vol.collection, "ec_shards"])
-                    .dec();
-            }
-            ec_vol.close();
-        }
-    }
-
     /// Mount EC shards for a volume on this location.
     ///
     /// `source_disk_type` is the source volume's disk type carried on the
@@ -1766,7 +1737,7 @@ mod tests {
             .unwrap();
         assert_eq!(loc.volumes_len(), 2);
 
-        loc.delete_volume(VolumeId(1), false, false, false).unwrap();
+        loc.delete_volume(VolumeId(1), false, false).unwrap();
         assert_eq!(loc.volumes_len(), 1);
         assert!(loc.find_volume(VolumeId(1)).is_none());
     }
@@ -1819,34 +1790,6 @@ mod tests {
         assert!(loc.find_volume(VolumeId(3)).is_some());
     }
 
-    /// A 0-byte `.ecx` is the stub a failed EC distribute copy leaves behind.
-    /// Go's HasEcxFileOnDisk requires Size() > 0 so the stub cannot pin
-    /// placement to a disk that has no usable index.
-    #[test]
-    fn test_has_ecx_file_on_disk_ignores_zero_byte_stub() {
-        let tmp = TempDir::new().unwrap();
-        let data = tmp.path().join("data");
-        let idx = tmp.path().join("idx");
-        fs::create_dir_all(&data).unwrap();
-        fs::create_dir_all(&idx).unwrap();
-        let loc = DiskLocation::new(
-            data.to_str().unwrap(),
-            idx.to_str().unwrap(),
-            10,
-            DiskType::HardDrive,
-            MinFreeSpace::Percent(1.0),
-            Vec::new(),
-        )
-        .unwrap();
-
-        fs::write(idx.join("pics_7.ecx"), b"").unwrap();
-        assert!(!loc.has_ecx_file_on_disk("pics", VolumeId(7)));
-
-        // A real index in the data dir still counts, stub or no stub.
-        fs::write(data.join("pics_7.ecx"), [0u8; 16]).unwrap();
-        assert!(loc.has_ecx_file_on_disk("pics", VolumeId(7)));
-    }
-
     #[test]
     fn test_disk_location_delete_collection_removes_ec_volumes() {
         let tmp = TempDir::new().unwrap();
@@ -1863,8 +1806,6 @@ mod tests {
 
         let shard_path = format!("{}/pics_7.ec00", dir);
         std::fs::write(&shard_path, b"ec-shard").unwrap();
-        // An EC volume needs its .ecx to mount.
-        std::fs::write(format!("{}/pics_7.ecx", dir), [0u8; 16]).unwrap();
 
         loc.mount_ec_shards(VolumeId(7), "pics", &[0], "").unwrap();
         assert!(loc.has_ec_volume(VolumeId(7)));
@@ -1902,7 +1843,6 @@ mod tests {
         // mount_ec_shards with source_disk_type="ssd" — simulating the
         // VolumeEcShardsMount RPC path.
         std::fs::write(format!("{}/pics_7.ec00", dir), b"ec-shard").unwrap();
-        std::fs::write(format!("{}/pics_7.ecx", dir), [0u8; 16]).unwrap();
         loc.mount_ec_shards(VolumeId(7), "pics", &[0], "ssd")
             .unwrap();
         {
@@ -2002,7 +1942,6 @@ mod tests {
         // A collection name unique to this test: the gauge is process-global
         // and sibling tests running in parallel touch other labels.
         std::fs::write(format!("{}/dupmount_11.ec00", dir), b"shard bytes").unwrap();
-        std::fs::write(format!("{}/dupmount_11.ecx", dir), [0u8; 16]).unwrap();
         let gauge = crate::metrics::VOLUME_GAUGE.with_label_values(&["dupmount", "ec_shards"]);
         let before = gauge.get();
 

seaweed-volume/src/storage/erasure_coding/ec_decoder.rs

@@ -3,12 +3,10 @@
 //! Rebuilds the original .dat + .idx files from data shards (.ec00-.ec09)
 //! and the sorted index (.ecx) + deletion journal (.ecj).
 
-use std::collections::HashSet;
 use std::fs::File;
 use std::io::{self, Read, Write};
 
 use crate::storage::erasure_coding::ec_shard::*;
-use crate::storage::erasure_coding::ec_volume::read_ecj_ids;
 use crate::storage::idx;
 use crate::storage::needle::needle::get_actual_size;
 use crate::storage::super_block::SUPER_BLOCK_SIZE;
@@ -22,21 +20,18 @@ use crate::storage::volume::{fsync_dir, volume_file_name};
 /// `dir` is used both for reading `.ec00` and `.ecx`. For split-disk
 /// reconciled volumes call [`find_dat_file_size_with_dirs`] instead.
 pub fn find_dat_file_size(dir: &str, collection: &str, volume_id: VolumeId) -> io::Result<i64> {
-    let deleted = read_ecj_deletions(&[dir], collection, volume_id)?;
-    find_dat_file_size_with_dirs(dir, dir, collection, volume_id, &deleted)
+    find_dat_file_size_with_dirs(dir, dir, collection, volume_id)
 }
 
 /// Like [`find_dat_file_size`] but lets the caller pass separate dirs
 /// for `.ec00` (the data shard) and `.ecx` (the sealed index). This
 /// is the form needed when shards are split across data dirs and the
-/// `.ecx` lives on a sibling disk's idx dir (#9252). Needles in `deleted`
-/// count as deleted.
+/// `.ecx` lives on a sibling disk's idx dir (#9252).
 pub fn find_dat_file_size_with_dirs(
     ec00_dir: &str,
     ecx_dir: &str,
     collection: &str,
     volume_id: VolumeId,
-    deleted: &HashSet<NeedleId>,
 ) -> io::Result<i64> {
     let ec00_base = volume_file_name(ec00_dir, collection, volume_id);
     let ecx_base = volume_file_name(ecx_dir, collection, volume_id);
@@ -58,9 +53,9 @@ pub fn find_dat_file_size_with_dirs(
 
     for i in 0..entry_count {
         let start = i * NEEDLE_MAP_ENTRY_SIZE;
-        let (key, offset, size) =
+        let (_, offset, size) =
             idx_entry_from_bytes(&ecx_data[start..start + NEEDLE_MAP_ENTRY_SIZE]);
-        if size.is_deleted() || deleted.contains(&key) {
+        if size.is_deleted() {
             continue;
         }
         let entry_stop = offset.to_actual_offset() + get_actual_size(size, version);
@@ -72,124 +67,6 @@ pub fn find_dat_file_size_with_dirs(
     Ok(dat_size)
 }
 
-/// Whether the `.ecx` in `ecx_dir` indexes a needle deleted neither there nor
-/// in `deleted`.
-pub fn has_live_needles(
-    ecx_dir: &str,
-    collection: &str,
-    volume_id: VolumeId,
-    deleted: &HashSet<NeedleId>,
-) -> io::Result<bool> {
-    let ecx_base = volume_file_name(ecx_dir, collection, volume_id);
-    let ecx_data = std::fs::read(format!("{}.ecx", ecx_base))?;
-    let (entries, _) = ecx_data.as_chunks::<NEEDLE_MAP_ENTRY_SIZE>();
-    Ok(entries.iter().any(|entry| {
-        let (key, _, size) = idx_entry_from_bytes(entry);
-        !size.is_deleted() && !deleted.contains(&key)
-    }))
-}
-
-/// Distinct needle ids journaled in the `.ecj` of any of `dirs`. Go folds the
-/// journal into the `.ecx` (RebuildEcxFile) before a decode; reading it leaves
-/// the sealed index untouched. Only NotFound means "no journal".
-pub fn read_ecj_deletions(
-    dirs: &[&str],
-    collection: &str,
-    volume_id: VolumeId,
-) -> io::Result<HashSet<NeedleId>> {
-    Ok(EcjDeletions::read(dirs, collection, volume_id)?.ids)
-}
-
-/// The ids [`read_ecj_deletions`] returns, plus how far each journal was read
-/// so that ids journaled later can be added.
-pub struct EcjDeletions {
-    pub ids: HashSet<NeedleId>,
-    /// Each distinct journal path and the whole-record length read so far.
-    journals: Vec<(String, u64)>,
-}
-
-impl EcjDeletions {
-    pub fn read(dirs: &[&str], collection: &str, volume_id: VolumeId) -> io::Result<Self> {
-        let mut journals: Vec<(String, u64)> = Vec::new();
-        for dir in dirs {
-            let path = format!("{}.ecj", volume_file_name(dir, collection, volume_id));
-            if !journals.iter().any(|(p, _)| *p == path) {
-                journals.push((path, 0));
-            }
-        }
-        let mut deletions = EcjDeletions {
-            ids: HashSet::new(),
-            journals,
-        };
-        deletions.catch_up()?;
-        Ok(deletions)
-    }
-
-    /// Adds the ids appended to each journal since the last read. A journal
-    /// that shrank is read again from the start — and the whole set rebuilt,
-    /// since ids already folded in from the truncated tail may have been a
-    /// rolled-back append. Non-regular journals (the FIFOs the tests stand
-    /// in for a blocking disk) stat empty and cannot be rolled back, so they
-    /// never count as shrunk.
-    pub fn catch_up(&mut self) -> io::Result<()> {
-        let mut shrank = false;
-        for (path, read_to) in &self.journals {
-            if *read_to == 0 {
-                continue;
-            }
-            match std::fs::metadata(path) {
-                Ok(m) => {
-                    if m.is_file() && m.len() < *read_to {
-                        shrank = true;
-                        break;
-                    }
-                }
-                // A journal that was read before and is now gone shrank to
-                // nothing (e.g. the volume was destroyed mid-scan) — the ids
-                // read from it no longer reflect committed content.
-                Err(e) if e.kind() == io::ErrorKind::NotFound => {
-                    shrank = true;
-                    break;
-                }
-                Err(e) => return Err(e),
-            }
-        }
-        if shrank {
-            self.ids.clear();
-            for (_, read_to) in &mut self.journals {
-                *read_to = 0;
-            }
-        }
-        for (path, read_to) in &mut self.journals {
-            let file = match File::open(&*path) {
-                Ok(file) => file,
-                Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
-                Err(e) => return Err(e),
-            };
-            let len = file.metadata()?.len();
-            if len < *read_to {
-                *read_to = 0;
-            }
-            read_ecj_ids(&file, *read_to, len, &mut self.ids)?;
-            *read_to = len - len % NEEDLE_ID_SIZE as u64;
-        }
-        Ok(())
-    }
-
-    /// Clears the set and re-reads every journal from the start. Run under
-    /// the caller's store read lock (no append in flight): the result is
-    /// then exactly the committed content — an earlier unlocked read may
-    /// have folded in bytes a rolled-back append later truncated, or missed
-    /// a record re-appended to the very offset a rollback freed.
-    pub fn rescan(&mut self) -> io::Result<()> {
-        self.ids.clear();
-        for (_, read_to) in &mut self.journals {
-            *read_to = 0;
-        }
-        self.catch_up()
-    }
-}
-
 /// What it takes to rebuild a volume's .dat from its EC data shards.
 ///
 /// Mirrors Go's `WriteDatFile(baseFileName, datFileSize,
@@ -388,87 +265,53 @@ pub fn write_dat_file_from_shards(spec: &DatRebuild<'_>) -> io::Result<()> {
     write_result
 }
 
-/// Fails when the decoded `.dat` in `dat_dir` is shorter than the
-/// `dat_file_size` bytes its EC index references: the caller deletes the
-/// shards next, and they are the only other copy of the needles past the cut.
-/// A longer file passes.
-pub fn verify_decoded_dat_file(
-    dat_dir: &str,
-    collection: &str,
-    volume_id: VolumeId,
-    dat_file_size: i64,
-) -> io::Result<()> {
-    let dat_path = format!("{}.dat", volume_file_name(dat_dir, collection, volume_id));
-    let size = std::fs::metadata(&dat_path)?.len();
-    if (size as i64) < dat_file_size {
-        return Err(io::Error::new(
-            io::ErrorKind::UnexpectedEof,
-            format!(
-                "decoded {} is {} bytes, short of the {} its ec index references",
-                dat_path, size, dat_file_size
-            ),
-        ));
-    }
-    Ok(())
-}
-
 /// Write .idx file from .ecx index + .ecj deletion journal.
 ///
-/// See [`write_idx_file_from_ec_index_with_dirs`]; everything lives in `dir`.
+/// Copies sorted .ecx entries to .idx, then appends tombstones for
+/// deleted needles from .ecj.
 pub fn write_idx_file_from_ec_index(
     dir: &str,
     collection: &str,
     volume_id: VolumeId,
 ) -> io::Result<()> {
-    let deleted = read_ecj_deletions(&[dir], collection, volume_id)?;
-    let dat_file_size = find_dat_file_size_with_dirs(dir, dir, collection, volume_id, &deleted)?;
-    write_idx_file_from_ec_index_with_dirs(dir, dir, collection, volume_id, &deleted, dat_file_size)
-}
-
-/// Write the `.idx` for a `.dat` decoded to `dat_file_size` bytes, from the
-/// `.ecx` in `ecx_dir`, into `idx_dir`.
-///
-/// Copies the `.ecx` rows, then appends one tombstone per row whose needle is
-/// in `deleted`. A deleted needle at or past `dat_file_size` was cut from the
-/// `.dat`, so its row is dropped: a row pointing past the end of the `.dat`
-/// makes the volume load read-only.
-pub fn write_idx_file_from_ec_index_with_dirs(
-    ecx_dir: &str,
-    idx_dir: &str,
-    collection: &str,
-    volume_id: VolumeId,
-    deleted: &HashSet<NeedleId>,
-    dat_file_size: i64,
-) -> io::Result<()> {
-    let ecx_path = format!("{}.ecx", volume_file_name(ecx_dir, collection, volume_id));
-    let idx_path = format!("{}.idx", volume_file_name(idx_dir, collection, volume_id));
+    let base = volume_file_name(dir, collection, volume_id);
+    let ecx_path = format!("{}.ecx", base);
+    let ecj_path = format!("{}.ecj", base);
+    let idx_path = format!("{}.idx", base);
     // Write to a temp file and atomically rename into place, so a crash
     // mid-write never leaves a partial .idx at the final name beside the
     // source shards.
     let tmp_path = format!("{}.tmp", idx_path);
 
     let write_result = (|| -> io::Result<()> {
-        let mut ecx_file = File::open(&ecx_path)?;
-        let mut idx_file = io::BufWriter::new(File::create(&tmp_path)?);
-        let mut tombstoned = Vec::new();
-        idx::walk_index_file(&mut ecx_file, 0, |key, offset, size| {
-            let is_deleted = size.is_deleted() || deleted.contains(&key);
-            if is_deleted && offset.to_actual_offset() >= dat_file_size {
-                return Ok(());
+        // Copy .ecx to the temp .idx
+        std::fs::copy(&ecx_path, &tmp_path)?;
+
+        // Append deletions from .ecj as tombstones. Read the journal directly
+        // and treat only NotFound as "no journal": Path::exists would also
+        // swallow a permission/IO error and silently skip deletions, which
+        // would resurrect deleted needles as live.
+        let mut idx_file = std::fs::OpenOptions::new().append(true).open(&tmp_path)?;
+        match std::fs::read(&ecj_path) {
+            Ok(ecj_data) => {
+                let count = ecj_data.len() / NEEDLE_ID_SIZE;
+                for i in 0..count {
+                    let start = i * NEEDLE_ID_SIZE;
+                    let needle_id = NeedleId::from_bytes(&ecj_data[start..start + NEEDLE_ID_SIZE]);
+                    idx::write_index_entry(
+                        &mut idx_file,
+                        needle_id,
+                        Offset::default(),
+                        TOMBSTONE_FILE_SIZE,
+                    )?;
+                }
             }
-            idx::write_index_entry(&mut idx_file, key, offset, size)?;
-            if !size.is_deleted() && deleted.contains(&key) {
-                tombstoned.push(key);
-            }
-            Ok(())
-        })?;
-        for key in tombstoned {
-            idx::write_index_entry(&mut idx_file, key, Offset::default(), TOMBSTONE_FILE_SIZE)?;
+            Err(e) if e.kind() == io::ErrorKind::NotFound => {}
+            Err(e) => return Err(e),
         }
 
         // fsync, rename, then fsync the dir so the decoded .idx is durable and
         // atomically published before the caller deletes the source shards.
-        let idx_file = idx_file.into_inner().map_err(|e| e.into_error())?;
         idx_file.sync_all()?;
         drop(idx_file);
         // Windows rename does not replace an existing file on every version;
@@ -851,116 +694,4 @@ mod tests {
             .is_err()
         );
     }
-
-    /// A journal many chunks long that repeats a few ids reads back as those
-    /// ids, from each dir once, whatever its length.
-    #[test]
-    fn test_read_ecj_deletions_collects_distinct_ids_across_dirs() {
-        let tmp = TempDir::new().unwrap();
-        let data = tmp.path().join("data");
-        let idx = tmp.path().join("idx");
-        let missing = tmp.path().join("missing");
-        std::fs::create_dir_all(&data).unwrap();
-        std::fs::create_dir_all(&idx).unwrap();
-        let (data, idx, missing) = (
-            data.to_str().unwrap(),
-            idx.to_str().unwrap(),
-            missing.to_str().unwrap(),
-        );
-
-        let entry = |id: u64| {
-            let mut buf = [0u8; NEEDLE_ID_SIZE];
-            NeedleId(id).to_bytes(&mut buf);
-            buf
-        };
-        // Past two load chunks of three repeating ids, then an id only in the
-        // last chunk and a torn trailing record.
-        let mut ecj = Vec::new();
-        while ecj.len() <= 2 * (1 << 20) {
-            for id in [1, 2, 3] {
-                ecj.extend_from_slice(&entry(id));
-            }
-        }
-        ecj.extend_from_slice(&entry(7));
-        ecj.extend_from_slice(&entry(8)[..3]);
-        std::fs::write(format!("{idx}/1.ecj"), &ecj).unwrap();
-        std::fs::write(format!("{data}/1.ecj"), entry(9)).unwrap();
-
-        let ids = read_ecj_deletions(&[data, idx, idx, missing], "", VolumeId(1)).unwrap();
-        let expected: HashSet<NeedleId> = [1, 2, 3, 7, 9].into_iter().map(NeedleId).collect();
-        assert_eq!(ids, expected);
-    }
-
-    #[test]
-    fn test_verify_decoded_dat_file_rejects_a_short_dat() {
-        let tmp = TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-        let dat_path = format!("{dir}/1.dat");
-
-        let err = verify_decoded_dat_file(dir, "", VolumeId(1), 100).unwrap_err();
-        assert_eq!(err.kind(), io::ErrorKind::NotFound);
-
-        std::fs::write(&dat_path, vec![0u8; 99]).unwrap();
-        let err = verify_decoded_dat_file(dir, "", VolumeId(1), 100).unwrap_err();
-        assert!(err.to_string().contains("short of the 100"), "{err}");
-
-        std::fs::write(&dat_path, vec![0u8; 100]).unwrap();
-        verify_decoded_dat_file(dir, "", VolumeId(1), 100).unwrap();
-        std::fs::write(&dat_path, vec![0u8; 101]).unwrap();
-        verify_decoded_dat_file(dir, "", VolumeId(1), 100).unwrap();
-    }
-
-    /// Ids appended after the first read, including the rest of a record torn
-    /// at that point, are picked up; a journal that shrank is read again.
-    #[test]
-    fn test_ecj_deletions_catch_up_reads_appended_ids() {
-        let tmp = TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-        let ecj_path = format!("{dir}/1.ecj");
-        let entry = |id: u64| {
-            let mut buf = [0u8; NEEDLE_ID_SIZE];
-            NeedleId(id).to_bytes(&mut buf);
-            buf
-        };
-        let append = |bytes: &[u8]| {
-            let mut f = std::fs::OpenOptions::new()
-                .create(true)
-                .append(true)
-                .open(&ecj_path)
-                .unwrap();
-            f.write_all(bytes).unwrap();
-        };
-        let ids = |d: &EcjDeletions| {
-            let mut ids: Vec<u64> = d.ids.iter().map(|id| id.0).collect();
-            ids.sort();
-            ids
-        };
-
-        // No journal yet.
-        let mut deletions = EcjDeletions::read(&[dir, dir], "", VolumeId(1)).unwrap();
-        assert!(deletions.ids.is_empty());
-
-        append(&entry(1));
-        append(&entry(2)[..3]);
-        deletions.catch_up().unwrap();
-        assert_eq!(ids(&deletions), [1]);
-
-        append(&entry(2)[3..]);
-        append(&entry(3));
-        deletions.catch_up().unwrap();
-        assert_eq!(ids(&deletions), [1, 2, 3]);
-
-        // A shrunk journal is rebuilt from its surviving content: ids folded
-        // in from the truncated tail may have been rolled back and must not
-        // linger as phantom tombstones.
-        std::fs::write(&ecj_path, entry(9)).unwrap();
-        deletions.catch_up().unwrap();
-        assert_eq!(ids(&deletions), [9]);
-
-        // A journal removed since it was read is the extreme shrink: its
-        // earlier ids must go with it, not linger.
-        std::fs::remove_file(&ecj_path).unwrap();
-        deletions.catch_up().unwrap();
-        assert!(deletions.ids.is_empty());
-    }
 }

seaweed-volume/src/storage/erasure_coding/ec_encoder.rs

@@ -813,7 +813,7 @@ impl EncodeRun<'_> {
         // explicitly.
         for (i, buf) in self.buffers[..self.data_shards].iter_mut().enumerate() {
             let read_offset = offset + (i * block_size) as u64;
-            let n = crate::storage::io::read_full_at(self.dat_file, buf, read_offset)?;
+            let n = read_at_most(self.dat_file, buf, read_offset)?;
             buf[n..].fill(0);
         }
 
@@ -834,6 +834,19 @@ impl EncodeRun<'_> {
     }
 }
 
+/// Read into `buf` at `offset` until it is full or EOF; returns bytes read.
+fn read_at_most(dat_file: &File, buf: &mut [u8], offset: u64) -> io::Result<usize> {
+    let mut n = 0;
+    while n < buf.len() {
+        let r = crate::storage::io::read_at(dat_file, &mut buf[n..], offset + n as u64)?;
+        if r == 0 {
+            break;
+        }
+        n += r;
+    }
+    Ok(n)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

seaweed-volume/src/storage/erasure_coding/ec_shard.rs

@@ -87,14 +87,14 @@ impl EcVolumeShard {
         Ok(())
     }
 
-    /// Read data at a specific offset, filling `buf` unless the shard ends first.
+    /// Read data at a specific offset.
     pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize> {
         let file = self
             .ecd_file
             .as_ref()
             .ok_or_else(|| io::Error::other("shard file not open"))?;
 
-        crate::storage::io::read_full_at(file, buf, offset)
+        crate::storage::io::read_at(file, buf, offset)
     }
 
     /// Write data to the shard file (appends).
@@ -195,104 +195,6 @@ impl ShardBits {
     }
 }
 
-/// Parses the generation of a 2PC-staged `<base>.v<N>` file: `None` means the
-/// name is not a generation file of `base`.
-pub fn ec_file_generation(name: &str, base: &str) -> Option<u32> {
-    let suffix = name.strip_prefix(&format!("{}.v", base))?;
-    match suffix.parse::<u32>() {
-        Ok(g) if g > 0 => Some(g),
-        _ => None,
-    }
-}
-
-/// Removes 2PC generation files staged under `base`:
-/// `<base>.ecNN.v<N>`, `<base>.ecx.v<N>`, `<base>.ecj.v<N>`, `<base>.ecsum.v<N>`
-/// and `<base>.vif.v<N>`. `generations_older_than == 0` removes every
-/// generation; otherwise only generations strictly below it. Returns the
-/// first real removal failure. Mirrors Go's `RemoveEcGenerationFiles`.
-pub fn remove_ec_generation_files(base: &str, generations_older_than: u32) -> io::Result<()> {
-    let path = std::path::Path::new(base);
-    let (Some(parent), Some(fname)) = (path.parent(), path.file_name()) else {
-        return Ok(());
-    };
-    let ec_prefix = format!("{}.ec", fname.to_string_lossy());
-    let vif_name = format!("{}.vif", fname.to_string_lossy());
-    let mut first_err: Option<io::Error> = None;
-    let mut record = |res: io::Result<()>| {
-        if let Err(e) = res
-            && first_err.is_none()
-        {
-            first_err = Some(e);
-        }
-    };
-    match fs::read_dir(parent) {
-        Ok(entries) => {
-            for entry in entries {
-                let entry = match entry {
-                    Ok(entry) => entry,
-                    Err(e) => {
-                        // A skipped entry means an incomplete sweep; report it
-                        // instead of pretending the cleanup finished.
-                        record(Err(e));
-                        continue;
-                    }
-                };
-                let name = entry.file_name().to_string_lossy().into_owned();
-                let Some((artifact, _)) = name.rsplit_once(".v") else {
-                    continue;
-                };
-                if artifact != vif_name && !artifact.starts_with(&ec_prefix) {
-                    continue;
-                }
-                let Some(generation) = ec_file_generation(&name, artifact) else {
-                    continue;
-                };
-                if generations_older_than > 0 && generation >= generations_older_than {
-                    continue;
-                }
-                record(match fs::remove_file(entry.path()) {
-                    Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e),
-                    _ => Ok(()),
-                });
-            }
-        }
-        Err(e) if e.kind() != io::ErrorKind::NotFound => record(Err(e)),
-        Err(_) => {}
-    }
-    match first_err {
-        Some(e) => Err(e),
-        None => Ok(()),
-    }
-}
-
-/// Removes every staged generation `<shard_file>.v<N>` of one shard file.
-/// Returns true when at least one generation file was removed.
-pub fn remove_ec_shard_generations(shard_file: &str) -> io::Result<bool> {
-    let path = std::path::Path::new(shard_file);
-    let (Some(parent), Some(fname)) = (path.parent(), path.file_name()) else {
-        return Ok(false);
-    };
-    let fname = fname.to_string_lossy().into_owned();
-    let mut removed = false;
-    match fs::read_dir(parent) {
-        Ok(entries) => {
-            for entry in entries {
-                let entry = entry?;
-                let name = entry.file_name().to_string_lossy().into_owned();
-                if ec_file_generation(&name, &fname).is_some() {
-                    match fs::remove_file(entry.path()) {
-                        Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e),
-                        _ => removed = true,
-                    }
-                }
-            }
-        }
-        Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e),
-        Err(_) => {}
-    }
-    Ok(removed)
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

seaweed-volume/src/storage/erasure_coding/ecj_merge.rs

@@ -1,290 +0,0 @@
-//! Set-union merge of `.ecj` deletion journals for EC shard copy / index
-//! recovery. Mirrors Go's `weed/storage/erasure_coding/ecj_merge.go`.
-//!
-//! An EC volume's deletion journal (`<vid>.ecj`) is a *set* of deleted needle
-//! ids stored as 8-byte big-endian records. Shard copy and index recovery fold
-//! a peer's journal into the local one; they must append only the ids the
-//! local journal lacks, or every `ec_balance` round trip doubles the file.
-//!
-//! The journal is only ever appended to, never replaced: a mounted `EcVolume`
-//! holds it open, and a rename would leave that handle writing to an unlinked
-//! inode, losing every later delete at the next mount. A mounted volume merges
-//! through [`EcVolume::merge_journal`](super::ec_volume::EcVolume::merge_journal);
-//! [`append_ecj_ids`] is for a journal no volume has open.
-
-use std::collections::HashSet;
-use std::fs::{self, OpenOptions};
-use std::io::{self, Read, Seek, SeekFrom, Write};
-use std::path::Path;
-
-use crate::storage::types::{NEEDLE_ID_SIZE, NeedleId};
-use crate::storage::volume::fsync_dir;
-use crate::storage::volume_open::open_volume_file;
-
-/// Bytes per read when scanning a journal; a multiple of `NEEDLE_ID_SIZE`.
-const ECJ_READ_CHUNK_BYTES: usize = 1 << 20;
-
-/// Decodes `.ecj` records from a byte stream split at arbitrary boundaries (a
-/// CopyFile stream, chunked reads), collecting the distinct ids. Memory
-/// follows the number of distinct ids, not the journal's length. A trailing
-/// partial record is never decoded.
-#[derive(Default)]
-pub(crate) struct EcjIdDecoder {
-    ids: HashSet<NeedleId>,
-    partial: [u8; NEEDLE_ID_SIZE],
-    pending: usize,
-}
-
-impl EcjIdDecoder {
-    pub(crate) fn push(&mut self, mut bytes: &[u8]) {
-        if self.pending > 0 {
-            let n = (NEEDLE_ID_SIZE - self.pending).min(bytes.len());
-            self.partial[self.pending..self.pending + n].copy_from_slice(&bytes[..n]);
-            self.pending += n;
-            bytes = &bytes[n..];
-            if self.pending < NEEDLE_ID_SIZE {
-                return;
-            }
-            self.ids.insert(NeedleId::from_bytes(&self.partial));
-            self.pending = 0;
-        }
-        let mut records = bytes.chunks_exact(NEEDLE_ID_SIZE);
-        for record in &mut records {
-            self.ids.insert(NeedleId::from_bytes(record));
-        }
-        let rest = records.remainder();
-        self.partial[..rest.len()].copy_from_slice(rest);
-        self.pending = rest.len();
-    }
-
-    pub(crate) fn into_ids(self) -> HashSet<NeedleId> {
-        self.ids
-    }
-}
-
-/// Read the distinct ids of the journal at `path` in bounded chunks. A missing
-/// file reads as empty. Also returns the whole-record length read; a torn
-/// trailing partial record is excluded from it.
-pub(crate) fn read_ecj_ids(path: &str) -> io::Result<(HashSet<NeedleId>, u64)> {
-    let mut file = match fs::File::open(path) {
-        Ok(f) => f,
-        Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok((HashSet::new(), 0)),
-        Err(e) => return Err(e),
-    };
-    let len = file.metadata()?.len();
-    let size = len - len % NEEDLE_ID_SIZE as u64;
-    let mut decoder = EcjIdDecoder::default();
-    let mut buf = vec![0u8; (ECJ_READ_CHUNK_BYTES as u64).min(size) as usize];
-    let mut off = 0u64;
-    while off < size {
-        let want = (buf.len() as u64).min(size - off) as usize;
-        file.read_exact(&mut buf[..want])?;
-        decoder.push(&buf[..want]);
-        off += want as u64;
-    }
-    Ok((decoder.into_ids(), size))
-}
-
-/// The ids of `incoming` that `has` does not report, ascending, so a merge
-/// appends deterministic output.
-pub(crate) fn ecj_delta(
-    incoming: &HashSet<NeedleId>,
-    has: impl Fn(&NeedleId) -> bool,
-) -> Vec<NeedleId> {
-    let mut delta: Vec<NeedleId> = incoming.iter().copied().filter(|id| !has(id)).collect();
-    delta.sort_unstable();
-    delta
-}
-
-pub(crate) fn encode_ecj_ids(ids: &[NeedleId]) -> Vec<u8> {
-    let mut buf = vec![0u8; ids.len() * NEEDLE_ID_SIZE];
-    for (record, id) in buf.chunks_exact_mut(NEEDLE_ID_SIZE).zip(ids) {
-        id.to_bytes(record);
-    }
-    buf
-}
-
-/// Append to the journal at `path` the ids of `incoming` that `local` lacks,
-/// in one write and one fsync, returning how many were added. `local` and
-/// `size` come from [`read_ecj_ids`] on the same path: if the journal's
-/// whole-record length is no longer `size`, returns `Ok(None)` so the caller
-/// re-reads. A torn tail past `size` is truncated first so the new records
-/// stay aligned.
-pub(crate) fn append_ecj_ids(
-    path: &str,
-    local: &HashSet<NeedleId>,
-    incoming: &HashSet<NeedleId>,
-    size: u64,
-) -> io::Result<Option<usize>> {
-    let delta = ecj_delta(incoming, |id| local.contains(id));
-    if delta.is_empty() {
-        return Ok(Some(0));
-    }
-    let created = !Path::new(path).exists();
-    let mut file = open_volume_file(OpenOptions::new().read(true).write(true).create(true), path)?;
-    let len = file.metadata()?.len();
-    if len - len % NEEDLE_ID_SIZE as u64 != size {
-        return Ok(None);
-    }
-    if len != size {
-        file.set_len(size)?;
-    }
-    let appended = file
-        .seek(SeekFrom::Start(size))
-        .and_then(|_| file.write_all(&encode_ecj_ids(&delta)))
-        .and_then(|_| file.sync_all());
-    if let Err(e) = appended {
-        let _ = file.set_len(size);
-        return Err(e);
-    }
-    if created {
-        fsync_dir(path)?;
-    }
-    Ok(Some(delta.len()))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn ids(v: &[u64]) -> HashSet<NeedleId> {
-        v.iter().map(|&id| NeedleId(id)).collect()
-    }
-
-    fn bytes(v: &[u64]) -> Vec<u8> {
-        encode_ecj_ids(&v.iter().map(|&id| NeedleId(id)).collect::<Vec<_>>())
-    }
-
-    fn records(path: &str) -> Vec<u64> {
-        let data = fs::read(path).expect("read ecj");
-        assert_eq!(data.len() % NEEDLE_ID_SIZE, 0, "journal must stay aligned");
-        data.chunks_exact(NEEDLE_ID_SIZE)
-            .map(|c| NeedleId::from_bytes(c).0)
-            .collect()
-    }
-
-    /// Run the unmounted merge the way the server does: read, then append.
-    fn merge_file(path: &str, incoming: &HashSet<NeedleId>) -> usize {
-        let (local, size) = read_ecj_ids(path).expect("read");
-        append_ecj_ids(path, &local, incoming, size)
-            .expect("append")
-            .expect("journal unchanged")
-    }
-
-    #[test]
-    fn decoder_handles_records_split_across_chunks() {
-        let mut stream = bytes(&[1, 2, 3, 2, 0x0102030405060708]);
-        stream.extend_from_slice(&[9, 9, 9]);
-        for chunk in 1..=stream.len() {
-            let mut d = EcjIdDecoder::default();
-            for piece in stream.chunks(chunk) {
-                d.push(piece);
-            }
-            assert_eq!(
-                d.into_ids(),
-                ids(&[1, 2, 3, 0x0102030405060708]),
-                "chunk {}",
-                chunk
-            );
-        }
-    }
-
-    #[test]
-    fn read_ecj_ids_dedups_and_ignores_torn_tail() {
-        let dir = tempfile::tempdir().expect("tempdir");
-        let missing = dir.path().join("missing.ecj");
-        let (got, size) = read_ecj_ids(missing.to_str().unwrap()).expect("read");
-        assert!(got.is_empty());
-        assert_eq!(size, 0);
-
-        let torn = dir.path().join("torn.ecj");
-        let mut data = bytes(&[1, 2, 1]);
-        data.extend_from_slice(&[7, 7, 7]);
-        fs::write(&torn, data).unwrap();
-        let (got, size) = read_ecj_ids(torn.to_str().unwrap()).expect("read");
-        assert_eq!(got, ids(&[1, 2]));
-        assert_eq!(size, 3 * NEEDLE_ID_SIZE as u64);
-
-        // A bloated journal repeating a few ids across several read chunks
-        // keeps only the distinct ids.
-        let bloated = dir.path().join("bloated.ecj");
-        let data: Vec<u8> = (0..300_000u64).flat_map(|i| bytes(&[i % 3])).collect();
-        fs::write(&bloated, &data).unwrap();
-        let (got, size) = read_ecj_ids(bloated.to_str().unwrap()).expect("read");
-        assert_eq!(got, ids(&[0, 1, 2]));
-        assert_eq!(size, data.len() as u64);
-    }
-
-    #[test]
-    fn appends_only_missing_ids() {
-        let dir = tempfile::tempdir().expect("tempdir");
-        let path = dir.path().join("vol.ecj");
-        let path = path.to_str().unwrap();
-        fs::write(path, bytes(&[1, 2, 3])).unwrap();
-        assert_eq!(merge_file(path, &ids(&[3, 4])), 1);
-        assert_eq!(records(path), vec![1, 2, 3, 4]);
-    }
-
-    #[test]
-    fn round_trip_stays_constant() {
-        // A->B->A->B 20 times: the old append path doubled the journal each trip.
-        let dir = tempfile::tempdir().expect("tempdir");
-        let a = dir.path().join("a.ecj");
-        let b = dir.path().join("b.ecj");
-        let (a, b) = (a.to_str().unwrap(), b.to_str().unwrap());
-        fs::write(a, bytes(&[1, 2])).unwrap();
-        fs::write(b, bytes(&[2, 3])).unwrap();
-        for i in 0..20 {
-            let (src, dst) = if i % 2 == 0 { (a, b) } else { (b, a) };
-            let (incoming, _) = read_ecj_ids(src).expect("read");
-            merge_file(dst, &incoming);
-        }
-        for path in [a, b] {
-            let mut got = records(path);
-            got.sort_unstable();
-            assert_eq!(got, vec![1, 2, 3]);
-        }
-    }
-
-    #[test]
-    fn nothing_new_leaves_journal_alone() {
-        let dir = tempfile::tempdir().expect("tempdir");
-        let missing = dir.path().join("missing.ecj");
-        assert_eq!(merge_file(missing.to_str().unwrap(), &ids(&[])), 0);
-        assert!(
-            !missing.exists(),
-            "an empty merge must not create a journal"
-        );
-
-        let path = dir.path().join("vol.ecj");
-        let path = path.to_str().unwrap();
-        fs::write(path, bytes(&[1, 2])).unwrap();
-        assert_eq!(merge_file(path, &ids(&[2, 1])), 0);
-        assert_eq!(records(path), vec![1, 2]);
-    }
-
-    #[test]
-    fn repairs_torn_tail() {
-        let dir = tempfile::tempdir().expect("tempdir");
-        let path = dir.path().join("vol.ecj");
-        let path = path.to_str().unwrap();
-        let mut data = bytes(&[1, 2]);
-        data.extend_from_slice(&[9, 9, 9]);
-        fs::write(path, data).unwrap();
-        assert_eq!(merge_file(path, &ids(&[3])), 1);
-        assert_eq!(records(path), vec![1, 2, 3]);
-    }
-
-    #[test]
-    fn rejects_changed_journal() {
-        let dir = tempfile::tempdir().expect("tempdir");
-        let path = dir.path().join("vol.ecj");
-        let path = path.to_str().unwrap();
-        fs::write(path, bytes(&[1])).unwrap();
-        let (local, size) = read_ecj_ids(path).expect("read");
-        fs::write(path, bytes(&[1, 5])).unwrap();
-        let outcome = append_ecj_ids(path, &local, &ids(&[2]), size).expect("append");
-        assert_eq!(outcome, None);
-        assert_eq!(records(path), vec![1, 5]);
-    }
-}

seaweed-volume/src/storage/erasure_coding/ecj_registry.rs

@@ -1,326 +0,0 @@
-//! Process-wide coordination of everything that touches one `.ecj` path.
-//!
-//! Mount-time compaction replaces a deletion journal with a new inode. That is
-//! only safe while nothing else in this process can write the old one:
-//!
-//! - **Holders** are mounted `EcVolume`s with an append handle on the path. A
-//!   store holds one `EcVolume` per disk location, and a shard mount or a
-//!   cross-disk reconcile can point one disk's volume at another disk's
-//!   `.ecj`, so several holders of one path are normal. A holder that keeps
-//!   appending to a replaced inode acknowledges deletes that are gone at the
-//!   next mount.
-//! - **Writers** append to or replace the path by name without holding it
-//!   open across calls: `ReceiveFile` of an EC `.ecj`, and the unmounted
-//!   append in `merge_ec_journal`, which `VolumeEcShardsCopy` and EC index
-//!   recovery funnel a peer's journal through. Bytes they write after the
-//!   compactor sized the journal would be dropped by the rename.
-//!
-//! Compaction therefore runs only while its caller is the sole holder and no
-//! writer is active, and while it runs no holder may open the path and no
-//! writer may start. Both wait instead; a compaction rewrites only the distinct
-//! id set, so the wait is short.
-//!
-//! No writer active at the reservation is not enough: one that ran while the
-//! holder loaded the journal, or after, and has finished may have rewritten it
-//! in place to the same length (`ReceiveFile` truncates and refills), which
-//! the inode-and-size re-check cannot see. So each writer bumps the path's
-//! write generation as it starts, and a holder may compact only if no writer
-//! was active when it registered and the generation has not moved since.
-//!
-//! Paths are keyed by their canonical parent directory, so two disk locations
-//! that spell one directory differently still meet here.
-
-use std::collections::HashMap;
-use std::path::{Path, PathBuf};
-use std::sync::{Condvar, LazyLock, Mutex, MutexGuard};
-
-#[derive(Default)]
-struct PathState {
-    holders: usize,
-    writers: usize,
-    compacting: bool,
-    /// Writers that have started on the path. Lives as long as the entry,
-    /// which a registered holder keeps.
-    write_gen: u64,
-}
-
-impl PathState {
-    fn idle(&self) -> bool {
-        self.holders == 0 && self.writers == 0 && !self.compacting
-    }
-}
-
-struct Registry {
-    paths: Mutex<HashMap<PathBuf, PathState>>,
-    changed: Condvar,
-}
-
-static REGISTRY: LazyLock<Registry> = LazyLock::new(|| Registry {
-    paths: Mutex::new(HashMap::new()),
-    changed: Condvar::new(),
-});
-
-fn lock() -> MutexGuard<'static, HashMap<PathBuf, PathState>> {
-    // The critical sections only adjust counters and cannot panic midway, so
-    // a poisoned lock still guards consistent state.
-    REGISTRY.paths.lock().unwrap_or_else(|e| e.into_inner())
-}
-
-/// Canonical key for `path`: its resolved parent directory joined with the
-/// file name. The file itself may not exist yet (a copy creates it), so only
-/// the directory is resolved.
-fn key_for(path: &str) -> PathBuf {
-    let p = Path::new(path);
-    let (Some(parent), Some(name)) = (p.parent(), p.file_name()) else {
-        return std::path::absolute(p).unwrap_or_else(|_| p.to_path_buf());
-    };
-    let parent = if parent.as_os_str().is_empty() {
-        Path::new(".")
-    } else {
-        parent
-    };
-    let dir = std::fs::canonicalize(parent)
-        .or_else(|_| std::path::absolute(parent))
-        .unwrap_or_else(|_| parent.to_path_buf());
-    dir.join(name)
-}
-
-/// Block until no compaction is running on `key`, then apply `f` to its state.
-fn update_when_not_compacting<R>(key: &Path, f: impl FnOnce(&mut PathState) -> R) -> R {
-    let mut paths = lock();
-    while paths.get(key).is_some_and(|s| s.compacting) {
-        paths = REGISTRY
-            .changed
-            .wait(paths)
-            .unwrap_or_else(|e| e.into_inner());
-    }
-    f(paths.entry(key.to_path_buf()).or_default())
-}
-
-fn release(key: &Path, f: impl FnOnce(&mut PathState)) {
-    let mut paths = lock();
-    if let Some(state) = paths.get_mut(key) {
-        f(state);
-        if state.idle() {
-            paths.remove(key);
-        }
-    }
-    drop(paths);
-    REGISTRY.changed.notify_all();
-}
-
-/// A mounted `EcVolume`'s registration as a holder of its `.ecj`. Taken before
-/// the journal is opened and released when dropped.
-pub(crate) struct EcjHold {
-    key: PathBuf,
-    /// The path's write generation when the hold was taken, and whether a
-    /// writer was active then. Taken before the journal is opened and loaded,
-    /// so they cover every write the load might have missed.
-    write_gen: u64,
-    writer_at_start: bool,
-}
-
-impl EcjHold {
-    /// Register as a holder of `ecj_path`, first waiting out any compaction in
-    /// progress so the handle opened afterwards is on the final inode.
-    pub(crate) fn acquire(ecj_path: &str) -> Self {
-        let key = key_for(ecj_path);
-        let (write_gen, writer_at_start) = update_when_not_compacting(&key, |s| {
-            s.holders += 1;
-            (s.write_gen, s.writers > 0)
-        });
-        EcjHold {
-            key,
-            write_gen,
-            writer_at_start,
-        }
-    }
-
-    /// Reserve the path for a compaction, or `None` when another holder or an
-    /// active writer could still reach the current inode, or when a writer
-    /// has run on the path since the hold was taken, so the journal may no
-    /// longer be what the holder loaded.
-    pub(crate) fn try_begin_compaction(&self) -> Option<EcjCompaction> {
-        let mut paths = lock();
-        let state = paths.get_mut(&self.key)?;
-        if state.holders != 1 || state.writers != 0 || state.compacting {
-            return None;
-        }
-        if self.writer_at_start || state.write_gen != self.write_gen {
-            return None;
-        }
-        state.compacting = true;
-        Some(EcjCompaction {
-            key: self.key.clone(),
-        })
-    }
-}
-
-impl Drop for EcjHold {
-    fn drop(&mut self) {
-        release(&self.key, |s| s.holders = s.holders.saturating_sub(1));
-    }
-}
-
-/// An exclusive reservation of a `.ecj` path for compaction. Holders and
-/// writers wait until it is dropped.
-pub(crate) struct EcjCompaction {
-    key: PathBuf,
-}
-
-impl Drop for EcjCompaction {
-    fn drop(&mut self) {
-        release(&self.key, |s| s.compacting = false);
-    }
-}
-
-/// An out-of-band writer (shard copy, index recovery, `ReceiveFile`) on a
-/// `.ecj` path.
-/// Compaction does not start while one is alive.
-pub(crate) struct EcjWrite {
-    key: PathBuf,
-}
-
-impl Drop for EcjWrite {
-    fn drop(&mut self) {
-        release(&self.key, |s| s.writers = s.writers.saturating_sub(1));
-    }
-}
-
-/// Register as a writer of `ecj_path`, waiting out any compaction in progress.
-/// Blocks; async callers use [`begin_ecj_write_async`].
-pub(crate) fn begin_ecj_write(ecj_path: &str) -> EcjWrite {
-    let key = key_for(ecj_path);
-    update_when_not_compacting(&key, |s| {
-        s.writers += 1;
-        s.write_gen += 1;
-    });
-    EcjWrite { key }
-}
-
-/// [`begin_ecj_write`] for async handlers: the wait runs on the blocking pool
-/// so a compaction in progress never stalls a runtime worker.
-pub(crate) async fn begin_ecj_write_async(ecj_path: &str) -> EcjWrite {
-    let path = ecj_path.to_string();
-    match tokio::task::spawn_blocking(move || begin_ecj_write(&path)).await {
-        Ok(write) => write,
-        // Only a panic inside the registry lands here, and it leaves no count
-        // behind; registering inline is still correct, merely blocking.
-        Err(_) => begin_ecj_write(ecj_path),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::sync::mpsc;
-    use std::time::Duration;
-    use tempfile::TempDir;
-
-    fn ecj(dir: &TempDir) -> String {
-        dir.path().join("1.ecj").to_str().unwrap().to_string()
-    }
-
-    #[test]
-    fn sole_holder_may_compact() {
-        let dir = TempDir::new().unwrap();
-        let hold = EcjHold::acquire(&ecj(&dir));
-        assert!(hold.try_begin_compaction().is_some());
-    }
-
-    #[test]
-    fn second_holder_blocks_compaction() {
-        let dir = TempDir::new().unwrap();
-        let a = EcjHold::acquire(&ecj(&dir));
-        let b = EcjHold::acquire(&ecj(&dir));
-        assert!(a.try_begin_compaction().is_none());
-        drop(b);
-        assert!(a.try_begin_compaction().is_some());
-    }
-
-    #[test]
-    fn active_writer_blocks_compaction() {
-        let dir = TempDir::new().unwrap();
-        let hold = EcjHold::acquire(&ecj(&dir));
-        let w = begin_ecj_write(&ecj(&dir));
-        assert!(hold.try_begin_compaction().is_none());
-        drop(w);
-        // This hold loaded before the write; a later one may compact.
-        drop(hold);
-        let hold = EcjHold::acquire(&ecj(&dir));
-        assert!(hold.try_begin_compaction().is_some());
-    }
-
-    /// A writer that ran after the hold was taken, or was already running
-    /// then, may have changed the journal the holder loaded, even though it
-    /// has finished by the time compaction asks.
-    #[test]
-    fn finished_writer_since_hold_blocks_compaction() {
-        let dir = TempDir::new().unwrap();
-        let path = ecj(&dir);
-
-        let hold = EcjHold::acquire(&path);
-        drop(begin_ecj_write(&path));
-        assert!(
-            hold.try_begin_compaction().is_none(),
-            "a writer that started after the hold",
-        );
-        drop(hold);
-
-        let w = begin_ecj_write(&path);
-        let hold = EcjHold::acquire(&path);
-        drop(w);
-        assert!(
-            hold.try_begin_compaction().is_none(),
-            "a writer active when the hold was taken",
-        );
-        drop(hold);
-
-        let hold = EcjHold::acquire(&path);
-        assert!(
-            hold.try_begin_compaction().is_some(),
-            "no writer since the hold",
-        );
-    }
-
-    #[test]
-    fn differently_spelled_paths_share_one_key() {
-        let dir = TempDir::new().unwrap();
-        std::fs::create_dir(dir.path().join("sub")).unwrap();
-        let plain = ecj(&dir);
-        let dotted = dir
-            .path()
-            .join("sub")
-            .join("..")
-            .join("1.ecj")
-            .to_str()
-            .unwrap()
-            .to_string();
-        let a = EcjHold::acquire(&plain);
-        let _b = EcjHold::acquire(&dotted);
-        assert!(a.try_begin_compaction().is_none());
-    }
-
-    #[test]
-    fn writer_waits_for_compaction_to_finish() {
-        let dir = TempDir::new().unwrap();
-        let path = ecj(&dir);
-        let hold = EcjHold::acquire(&path);
-        let compaction = hold.try_begin_compaction().unwrap();
-
-        let (tx, rx) = mpsc::channel();
-        let p = path.clone();
-        let t = std::thread::spawn(move || {
-            let _w = begin_ecj_write(&p);
-            tx.send(()).unwrap();
-        });
-        assert!(
-            rx.recv_timeout(Duration::from_millis(100)).is_err(),
-            "a writer must not start while a compaction holds the path",
-        );
-        drop(compaction);
-        rx.recv_timeout(Duration::from_secs(5))
-            .expect("writer must proceed once the compaction ends");
-        t.join().unwrap();
-    }
-}

seaweed-volume/src/storage/erasure_coding/mod.rs

@@ -9,8 +9,6 @@ pub mod ec_encoder;
 pub mod ec_locate;
 pub mod ec_shard;
 pub mod ec_volume;
-pub mod ecj_merge;
-pub(crate) mod ecj_registry;
 
 pub use ec_shard::{
     DATA_SHARDS_COUNT, EcVolumeShard, MAX_SHARD_COUNT, MIN_TOTAL_DISKS, PARITY_SHARDS_COUNT,

seaweed-volume/src/storage/idx/mod.rs

@@ -21,26 +21,12 @@ where
     let mut buf = vec![0u8; NEEDLE_MAP_ENTRY_SIZE * ROWS_TO_READ];
 
     loop {
-        // Fill the batch before decoding: `read` may return a count that is
-        // not a multiple of the entry size, and a split entry would misalign
-        // every later row. Go is immune: `ReadAt` fills or errors.
-        let mut count = 0;
-        let mut eof = false;
-        while count < buf.len() {
-            match reader.read(&mut buf[count..]) {
-                Ok(0) => {
-                    eof = true;
-                    break;
-                }
-                Ok(n) => count += n,
-                Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
-                Err(ref e) if e.kind() == io::ErrorKind::UnexpectedEof => {
-                    eof = true;
-                    break;
-                }
-                Err(e) => return Err(e),
-            }
-        }
+        let count = match reader.read(&mut buf) {
+            Ok(0) => return Ok(()),
+            Ok(n) => n,
+            Err(ref e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(()),
+            Err(e) => return Err(e),
+        };
 
         let mut i = 0;
         while i + NEEDLE_MAP_ENTRY_SIZE <= count {
@@ -48,11 +34,6 @@ where
             f(key, offset, size)?;
             i += NEEDLE_MAP_ENTRY_SIZE;
         }
-
-        // A trailing partial entry at EOF is ignored, as Go does on `io.EOF`.
-        if eof {
-            return Ok(());
-        }
     }
 }
 
@@ -196,111 +177,6 @@ mod tests {
         data
     }
 
-    /// Reader that hands back at most `chunk` bytes per `read`. 7 is coprime
-    /// with the 17-byte entry size, so nearly every read ends mid-entry. With
-    /// `interrupts`, every other call fails with `ErrorKind::Interrupted`.
-    struct ShortReader {
-        inner: Cursor<Vec<u8>>,
-        chunk: usize,
-        interrupts: bool,
-        interrupt_next: bool,
-    }
-
-    impl ShortReader {
-        fn new(data: Vec<u8>, interrupts: bool) -> Self {
-            ShortReader {
-                inner: Cursor::new(data),
-                chunk: 7,
-                interrupts,
-                interrupt_next: false,
-            }
-        }
-    }
-
-    impl Read for ShortReader {
-        fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-            if self.interrupt_next {
-                self.interrupt_next = false;
-                return Err(io::Error::from(io::ErrorKind::Interrupted));
-            }
-            self.interrupt_next = self.interrupts;
-            let n = buf.len().min(self.chunk);
-            self.inner.read(&mut buf[..n])
-        }
-    }
-
-    impl Seek for ShortReader {
-        fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
-            self.inner.seek(pos)
-        }
-    }
-
-    fn walk_all<R: Read + Seek>(reader: &mut R, start_from: u64) -> Vec<(NeedleId, i64, Size)> {
-        let mut collected = Vec::new();
-        walk_index_file(reader, start_from, |key, offset, size| {
-            collected.push((key, offset.to_actual_offset(), size));
-            Ok(())
-        })
-        .unwrap();
-        collected
-    }
-
-    /// More than one ROWS_TO_READ batch, so the walk crosses a buffer refill.
-    fn many_entries() -> Vec<(NeedleId, Offset, Size)> {
-        (0..(ROWS_TO_READ as u64 * 2 + 37))
-            .map(|i| {
-                (
-                    NeedleId(i * 7 + 1),
-                    Offset::from_actual_offset(i as i64 * 128),
-                    Size(i as i32 + 1),
-                )
-            })
-            .collect()
-    }
-
-    #[test]
-    fn test_walk_index_file_short_reads_keep_alignment() {
-        let data = idx_bytes(&many_entries());
-        let expected = walk_all(&mut Cursor::new(data.clone()), 0);
-        assert_eq!(expected.len(), ROWS_TO_READ * 2 + 37);
-
-        let mut short = ShortReader::new(data, false);
-        assert_eq!(walk_all(&mut short, 0), expected);
-    }
-
-    #[test]
-    fn test_walk_index_file_retries_interrupted_reads() {
-        let data = idx_bytes(&many_entries());
-        let expected = walk_all(&mut Cursor::new(data.clone()), 0);
-
-        let mut short = ShortReader::new(data, true);
-        assert_eq!(walk_all(&mut short, 0), expected);
-    }
-
-    #[test]
-    fn test_walk_index_file_short_reads_start_from() {
-        let data = idx_bytes(&many_entries());
-        let expected = walk_all(&mut Cursor::new(data.clone()), 0);
-
-        let start = ROWS_TO_READ as u64 + 5;
-        let mut short = ShortReader::new(data, false);
-        assert_eq!(walk_all(&mut short, start), expected[start as usize..]);
-    }
-
-    #[test]
-    fn test_walk_index_file_ignores_trailing_partial_entry() {
-        // A torn final entry is dropped without an error, as Go does on io.EOF.
-        let entries = many_entries();
-        let mut data = idx_bytes(&entries);
-        data.extend_from_slice(&[0xAB; NEEDLE_MAP_ENTRY_SIZE - 1]);
-
-        let expected = walk_all(&mut Cursor::new(idx_bytes(&entries)), 0);
-        assert_eq!(walk_all(&mut Cursor::new(data.clone()), 0), expected);
-
-        let mut short = ShortReader::new(data, false);
-        assert_eq!(walk_all(&mut short, 0), expected);
-    }
-
     #[test]
     fn test_check_index_file_clean() {
         let data = idx_bytes(&[

seaweed-volume/src/storage/io.rs

@@ -36,11 +36,23 @@ pub(crate) fn read_exact_at(file: &File, buf: &mut [u8], offset: u64) -> io::Res
     }
     #[cfg(windows)]
     {
-        if read_full_at(file, buf, offset)? < buf.len() {
-            return Err(io::Error::new(
-                io::ErrorKind::UnexpectedEof,
-                "unexpected EOF in seek_read",
-            ));
+        use std::os::windows::fs::FileExt;
+        let mut filled = 0;
+        let mut at = offset;
+        while filled < buf.len() {
+            let n = match file.seek_read(&mut buf[filled..], at) {
+                Ok(n) => n,
+                Err(err) if err.kind() == io::ErrorKind::Interrupted => continue,
+                Err(err) => return Err(err),
+            };
+            if n == 0 {
+                return Err(io::Error::new(
+                    io::ErrorKind::UnexpectedEof,
+                    "unexpected EOF in seek_read",
+                ));
+            }
+            filled += n;
+            at += n as u64;
         }
     }
     #[cfg(not(any(unix, windows)))]
@@ -72,34 +84,9 @@ pub(crate) fn read_at(file: &File, buf: &mut [u8], offset: u64) -> io::Result<us
     }
 }
 
-/// Reads into `buf` at `offset` until it is full or the file ends, retrying
-/// interrupted reads; returns how many bytes were read.
-///
-/// Unlike [`read_at`], a count below `buf.len()` always means end of file.
-pub(crate) fn read_full_at(file: &File, buf: &mut [u8], offset: u64) -> io::Result<usize> {
-    fill_at(|b, at| read_at(file, b, at), buf, offset)
-}
-
-fn fill_at(
-    mut read: impl FnMut(&mut [u8], u64) -> io::Result<usize>,
-    buf: &mut [u8],
-    offset: u64,
-) -> io::Result<usize> {
-    let mut filled = 0;
-    while filled < buf.len() {
-        match read(&mut buf[filled..], offset + filled as u64) {
-            Ok(0) => break,
-            Ok(n) => filled += n,
-            Err(err) if err.kind() == io::ErrorKind::Interrupted => {}
-            Err(err) => return Err(err),
-        }
-    }
-    Ok(filled)
-}
-
 #[cfg(test)]
 mod tests {
-    use super::{fill_at, read_at, read_exact_at, read_full_at};
+    use super::{read_at, read_exact_at};
     use std::io::{ErrorKind, Write};
 
     fn temp_file(bytes: &[u8]) -> tempfile::NamedTempFile {
@@ -152,57 +139,4 @@ mod tests {
         let n = read_at(f.as_file(), &mut buf, 10).expect("read");
         assert_eq!(n, 0);
     }
-
-    /// A source that returns at most `chunk` bytes per call and fails with
-    /// `Interrupted` on its first call, like a network mount under a signal.
-    fn chunked(src: &[u8], chunk: usize) -> impl FnMut(&mut [u8], u64) -> std::io::Result<usize> {
-        let mut interrupted = false;
-        move |buf, at| {
-            if !interrupted {
-                interrupted = true;
-                return Err(ErrorKind::Interrupted.into());
-            }
-            let at = (at as usize).min(src.len());
-            let n = buf.len().min(chunk).min(src.len() - at);
-            buf[..n].copy_from_slice(&src[at..at + n]);
-            Ok(n)
-        }
-    }
-
-    #[test]
-    fn fill_at_fills_across_short_and_interrupted_reads() {
-        let src: Vec<u8> = (0..=255).collect();
-        let mut buf = [0u8; 100];
-        let n = fill_at(chunked(&src, 7), &mut buf, 50).expect("read");
-        assert_eq!(n, buf.len());
-        assert_eq!(&buf[..], &src[50..150]);
-    }
-
-    #[test]
-    fn fill_at_stops_at_end_of_source() {
-        let src: Vec<u8> = (0..=255).collect();
-        let mut buf = [0u8; 100];
-        let n = fill_at(chunked(&src, 7), &mut buf, 200).expect("read");
-        assert_eq!(n, 56);
-        assert_eq!(&buf[..n], &src[200..]);
-    }
-
-    #[test]
-    fn fill_at_propagates_other_errors() {
-        let mut buf = [0u8; 8];
-        let err = fill_at(|_, _| Err(ErrorKind::PermissionDenied.into()), &mut buf, 0)
-            .expect_err("error");
-        assert_eq!(err.kind(), ErrorKind::PermissionDenied);
-    }
-
-    #[test]
-    fn read_full_at_returns_the_short_count_only_at_eof() {
-        let f = temp_file(b"0123456789");
-        let mut buf = [0u8; 8];
-        assert_eq!(read_full_at(f.as_file(), &mut buf, 0).expect("read"), 8);
-        assert_eq!(&buf, b"01234567");
-        assert_eq!(read_full_at(f.as_file(), &mut buf, 6).expect("read"), 4);
-        assert_eq!(&buf[..4], b"6789");
-        assert_eq!(read_full_at(f.as_file(), &mut buf, 10).expect("read"), 0);
-    }
 }

seaweed-volume/src/storage/io_error.rs

@@ -3,7 +3,7 @@
 
 use std::io;
 use std::sync::Mutex;
-use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicI32, Ordering};
 
 /// Consecutive storage-media errors allowed before the volume is quarantined.
 pub(crate) const IO_ERROR_TOLERANCE: i32 = 3;
@@ -34,30 +34,10 @@ pub(crate) fn is_storage_io_error(e: &io::Error) -> bool {
 #[derive(Default)]
 pub(crate) struct IoErrorTracker {
     last: Mutex<Option<String>>,
-    /// The consecutive error count in the low 32 bits and, in the high 32,
-    /// how many times it has been cleared. They share one word so that
-    /// `record_success_at` updates both in one step: reads record their
-    /// outcomes here without the volume's write lock.
-    streak: AtomicU64,
+    count: AtomicI32,
     quarantined: AtomicBool,
 }
 
-const STREAK_COUNT_BITS: u64 = 0xffff_ffff;
-
-fn streak_count(streak: u64) -> i32 {
-    (streak & STREAK_COUNT_BITS) as i32
-}
-
-/// `streak` with its count cleared and one more clear on record.
-fn streak_cleared(streak: u64) -> u64 {
-    (streak >> 32).wrapping_add(1) << 32
-}
-
-/// A point in the error streak, taken where a write landed whose success
-/// is only recorded later. See `IoErrorTracker::record_success_at`.
-#[derive(Clone, Copy)]
-pub(crate) struct StreakMark(u64);
-
 impl IoErrorTracker {
     /// `Some(e)` records a failure, `None` a success. Only storage-media
     /// failures count; every other outcome clears the count and last error.
@@ -65,22 +45,14 @@ impl IoErrorTracker {
         if let Some(e) = err
             && is_storage_io_error(e)
         {
-            self.streak.fetch_add(1, Ordering::Relaxed);
+            self.count.fetch_add(1, Ordering::Relaxed);
             if let Ok(mut guard) = self.last.lock() {
                 *guard = Some(e.to_string());
             }
             crate::metrics::STORAGE_IO_ERROR_COUNTER.inc();
             return;
         }
-        self.clear_count();
-        self.clear_last();
-    }
-
-    fn clear_count(&self) {
-        self.update_streak(|streak| Some(streak_cleared(streak)));
-    }
-
-    fn clear_last(&self) {
+        self.count.store(0, Ordering::Relaxed);
         if let Ok(mut guard) = self.last.lock()
             && guard.is_some()
         {
@@ -88,58 +60,17 @@ impl IoErrorTracker {
         }
     }
 
-    /// Apply `f` to the streak atomically; `None` leaves it as it is.
-    /// Returns the streak `f` produced, if any.
-    fn update_streak(&self, mut f: impl FnMut(u64) -> Option<u64>) -> Option<u64> {
-        let mut updated = None;
-        let _ = self
-            .streak
-            .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |streak| {
-                updated = f(streak);
-                updated
-            });
-        updated
-    }
-
-    pub(crate) fn mark(&self) -> StreakMark {
-        StreakMark(self.streak.load(Ordering::Relaxed))
-    }
-
-    /// Record a success as if it had come at `mark`: the errors counted
-    /// before the mark are cleared and the ones counted since still stand,
-    /// as they would had each outcome been recorded in order. A streak
-    /// cleared since the mark is left as it is.
-    pub(crate) fn record_success_at(&self, mark: StreakMark) {
-        let before = streak_count(mark.0);
-        let updated = self.update_streak(|streak| {
-            if streak >> 32 != mark.0 >> 32 {
-                return None;
-            }
-            if streak_count(streak) <= before {
-                return Some(streak_cleared(streak));
-            }
-            Some(streak - before as u64)
-        });
-        // The last error stays when one counted since the mark is left.
-        if updated.is_some_and(|streak| streak_count(streak) == 0) {
-            self.clear_last();
-        }
-    }
-
     /// The last recorded error, the consecutive count, and the quarantine flag.
     pub(crate) fn get_io_error_state(&self) -> (Option<String>, i32, bool) {
         let err = self.last.lock().ok().and_then(|g| g.clone());
-        let count = self.count();
+        let count = self.count.load(Ordering::Relaxed);
         let quarantined = self.quarantined.load(Ordering::Relaxed);
         (err, count, quarantined)
     }
 
-    fn count(&self) -> i32 {
-        streak_count(self.streak.load(Ordering::Relaxed))
-    }
-
     pub(crate) fn should_quarantine(&self) -> bool {
-        self.quarantined.load(Ordering::Relaxed) || self.count() >= IO_ERROR_TOLERANCE
+        self.quarantined.load(Ordering::Relaxed)
+            || self.count.load(Ordering::Relaxed) >= IO_ERROR_TOLERANCE
     }
 
     pub(crate) fn mark_io_quarantined(&self) {
@@ -147,7 +78,7 @@ impl IoErrorTracker {
     }
 
     pub(crate) fn reset_io_error_state(&self) {
-        self.clear_count();
+        self.count.store(0, Ordering::Relaxed);
         self.quarantined.store(false, Ordering::Relaxed);
         if let Ok(mut guard) = self.last.lock() {
             *guard = None;
@@ -160,11 +91,9 @@ impl IoErrorTracker {
             *guard = err.map(|value| value.to_string());
         }
         if err.is_some() {
-            self.update_streak(|streak| {
-                Some((streak & !STREAK_COUNT_BITS) | IO_ERROR_TOLERANCE as u64)
-            });
+            self.count.store(IO_ERROR_TOLERANCE, Ordering::Relaxed);
         } else {
-            self.clear_count();
+            self.count.store(0, Ordering::Relaxed);
         }
     }
 }
@@ -242,87 +171,6 @@ mod tests {
         assert!(tracker.should_quarantine());
     }
 
-    #[test]
-    fn success_at_a_mark_keeps_only_the_errors_after_it() {
-        let tracker = IoErrorTracker::default();
-        tracker.check_read_write_error(Some(&media_error()));
-        tracker.check_read_write_error(Some(&media_error()));
-        let mark = tracker.mark();
-        tracker.check_read_write_error(Some(&media_error()));
-        tracker.record_success_at(mark);
-
-        assert_eq!(
-            tracker.get_io_error_state(),
-            (Some(media_error().to_string()), 1, false)
-        );
-    }
-
-    #[test]
-    fn success_at_a_mark_with_nothing_after_it_clears_the_streak() {
-        let tracker = IoErrorTracker::default();
-        tracker.check_read_write_error(Some(&media_error()));
-        let mark = tracker.mark();
-        tracker.record_success_at(mark);
-
-        assert_eq!(tracker.get_io_error_state(), (None, 0, false));
-    }
-
-    #[test]
-    fn success_at_a_mark_leaves_a_streak_cleared_since() {
-        let tracker = IoErrorTracker::default();
-        tracker.check_read_write_error(Some(&media_error()));
-        tracker.check_read_write_error(Some(&media_error()));
-        let mark = tracker.mark();
-        tracker.check_read_write_error(None);
-        tracker.check_read_write_error(Some(&media_error()));
-        tracker.record_success_at(mark);
-
-        assert_eq!(
-            tracker.get_io_error_state(),
-            (Some(media_error().to_string()), 1, false)
-        );
-    }
-
-    /// Reads update the tracker without the volume's write lock, so a
-    /// success replayed at a mark must not lose the errors they record
-    /// while it runs.
-    #[test]
-    fn success_at_a_mark_keeps_concurrent_errors() {
-        use std::sync::{Arc, Barrier};
-        const READERS: i32 = 4;
-        const ERRORS: i32 = 200;
-
-        for _ in 0..500 {
-            let tracker = Arc::new(IoErrorTracker::default());
-            tracker.check_read_write_error(Some(&media_error()));
-            tracker.check_read_write_error(Some(&media_error()));
-            let mark = tracker.mark();
-            let start = Arc::new(Barrier::new(READERS as usize + 1));
-            let readers: Vec<_> = (0..READERS)
-                .map(|_| {
-                    let (tracker, start) = (tracker.clone(), start.clone());
-                    std::thread::spawn(move || {
-                        start.wait();
-                        for _ in 0..ERRORS {
-                            tracker.check_read_write_error(Some(&media_error()));
-                        }
-                    })
-                })
-                .collect();
-            start.wait();
-            while tracker.get_io_error_state().1 < 2 + READERS * ERRORS / 2 {
-                std::hint::spin_loop();
-            }
-            tracker.record_success_at(mark);
-            for reader in readers {
-                reader.join().unwrap();
-            }
-            // The two errors before the mark are cleared; every error the
-            // readers recorded after it stands.
-            assert_eq!(tracker.get_io_error_state().1, READERS * ERRORS);
-        }
-    }
-
     #[test]
     fn reset_io_error_state_lifts_the_quarantine() {
         let tracker = IoErrorTracker::default();

seaweed-volume/src/storage/mod.rs

@@ -6,7 +6,6 @@ pub(crate) mod io_error;
 pub mod needle;
 pub mod needle_map;
 pub mod store;
-pub mod store_ec_journal;
 pub mod store_ec_mirror;
 pub mod store_ec_reconcile;
 pub mod super_block;

seaweed-volume/src/storage/needle_map.rs

@@ -195,12 +195,7 @@ impl NeedleMapKind {
 // ============================================================================
 
 /// Trait for appending to an index file.
-///
-/// The file is opened without append mode and each row is written at the
-/// current `idx_file_offset` — the same positioned-write model the Go
-/// server uses — because an append-mode handle cannot truncate on Windows,
-/// where the std library keeps it strictly append-only.
-pub trait IdxFileWriter: Write + Seek + Send + Sync {
+pub trait IdxFileWriter: Write + Send + Sync {
     fn sync_all(&self) -> io::Result<()>;
     /// Truncate the file to `len` bytes. Used to remove an orphan .idx row
     /// left by a failed redb commit so `idx_file_offset` stays a contiguous
@@ -229,9 +224,6 @@ pub struct CompactNeedleMap {
     metric: NeedleMapMetric,
     idx_file: Option<Box<dyn IdxFileWriter>>,
     idx_file_offset: u64,
-    /// The file holds bytes past `idx_file_offset` that must be trimmed
-    /// before another row can land aligned.
-    idx_torn: bool,
 }
 
 impl Default for CompactNeedleMap {
@@ -248,7 +240,6 @@ impl CompactNeedleMap {
             metric: NeedleMapMetric::default(),
             idx_file: None,
             idx_file_offset: 0,
-            idx_torn: false,
         }
     }
 
@@ -256,8 +247,6 @@ impl CompactNeedleMap {
     pub fn load_from_idx<R: Read + Seek>(reader: &mut R, version: Version) -> io::Result<Self> {
         let mut nm = CompactNeedleMap::new();
         idx::walk_index_file(reader, 0, |key, offset, size| {
-            // A read-only load attaches no writer, so this is its only size.
-            nm.idx_file_offset += NEEDLE_MAP_ENTRY_SIZE as u64;
             nm.metric.maybe_set_max_needle_end(offset, size, version);
             if offset.is_zero() || size.is_deleted() {
                 nm.delete_from_map(key);
@@ -273,7 +262,6 @@ impl CompactNeedleMap {
     pub fn set_idx_file(&mut self, file: Box<dyn IdxFileWriter>, offset: u64) {
         self.idx_file = Some(file);
         self.idx_file_offset = offset;
-        self.idx_torn = false;
     }
 
     /// True when an .idx file writer is attached. A read-only load leaves
@@ -288,8 +276,8 @@ impl CompactNeedleMap {
     /// Insert or update an entry. Appends to .idx file if present.
     pub fn put(&mut self, key: NeedleId, offset: Offset, size: Size) -> io::Result<()> {
         // Persist to idx file BEFORE mutating in-memory state for crash consistency
-        self.append_to_index_file(key, offset, size)?;
-        if self.idx_file.is_some() {
+        if let Some(ref mut idx_file) = self.idx_file {
+            idx::write_index_entry(idx_file, key, offset, size)?;
             self.idx_file_offset += NEEDLE_MAP_ENTRY_SIZE as u64;
         }
 
@@ -299,41 +287,6 @@ impl CompactNeedleMap {
         Ok(())
     }
 
-    /// Write one row to the .idx file at `idx_file_offset`. A row left
-    /// half-written by a failed write is trimmed back to the offset so the
-    /// next row still lands aligned; while the trim keeps failing no row is
-    /// written at all, or it would sit off alignment and parse as garbage on
-    /// load. The offset itself is advanced by the caller once the row counts.
-    fn append_to_index_file(
-        &mut self,
-        key: NeedleId,
-        offset: Offset,
-        size: Size,
-    ) -> io::Result<()> {
-        let Some(idx_file) = self.idx_file.as_mut() else {
-            return Ok(());
-        };
-        if self.idx_torn {
-            match idx_file.truncate_to(self.idx_file_offset) {
-                Ok(()) => self.idx_torn = false,
-                Err(e) => {
-                    return Err(io::Error::other(format!(
-                        "index file still holds a torn row: {e}"
-                    )));
-                }
-            }
-        }
-        idx_file.seek(io::SeekFrom::Start(self.idx_file_offset))?;
-        if let Err(e) = idx::write_index_entry(idx_file, key, offset, size) {
-            if let Err(te) = idx_file.truncate_to(self.idx_file_offset) {
-                self.idx_torn = true;
-                tracing::warn!("failed to trim torn .idx row: {}", te);
-            }
-            return Err(e);
-        }
-        Ok(())
-    }
-
     /// Look up a needle.
     pub fn get(&self, key: NeedleId) -> Option<NeedleValue> {
         self.map.get(key)
@@ -358,8 +311,8 @@ impl CompactNeedleMap {
         }
 
         // Always write tombstone to idx file (matching Go)
-        self.append_to_index_file(key, offset, TOMBSTONE_FILE_SIZE)?;
-        if self.idx_file.is_some() {
+        if let Some(ref mut idx_file) = self.idx_file {
+            idx::write_index_entry(idx_file, key, offset, TOMBSTONE_FILE_SIZE)?;
             self.idx_file_offset += NEEDLE_MAP_ENTRY_SIZE as u64;
         }
 
@@ -509,9 +462,6 @@ pub struct RedbNeedleMap {
     metric: NeedleMapMetric,
     idx_file: Option<Box<dyn IdxFileWriter>>,
     idx_file_offset: u64,
-    /// The file holds bytes past `idx_file_offset` that must be trimmed
-    /// before another row can land aligned.
-    idx_torn: bool,
     /// Puts/deletes since the last durable checkpoint.
     writes_since_checkpoint: u32,
 }
@@ -618,7 +568,6 @@ impl RedbNeedleMap {
             metric: NeedleMapMetric::default(),
             idx_file: None,
             idx_file_offset: 0,
-            idx_torn: false,
             writes_since_checkpoint: 0,
         })
     }
@@ -716,7 +665,6 @@ impl RedbNeedleMap {
             metric: NeedleMapMetric::default(),
             idx_file: None,
             idx_file_offset: 0,
-            idx_torn: false,
             writes_since_checkpoint: 0,
         };
 
@@ -908,7 +856,6 @@ impl RedbNeedleMap {
     pub fn set_idx_file(&mut self, file: Box<dyn IdxFileWriter>, offset: u64) {
         self.idx_file = Some(file);
         self.idx_file_offset = offset;
-        self.idx_torn = false;
     }
 
     /// True when an .idx file writer is attached. See CompactNeedleMap.
@@ -925,7 +872,9 @@ impl RedbNeedleMap {
         // commit leaves an orphan row in .idx that redb doesn't reflect, and
         // advancing the offset here would let a later checkpoint record it as
         // reflected, making the reload skip it permanently.
-        self.append_to_index_file(key, offset, size)?;
+        if let Some(ref mut idx_file) = self.idx_file {
+            idx::write_index_entry(idx_file, key, offset, size)?;
+        }
 
         let key_u64: u64 = key.into();
         let packed = pack_needle_value(&NeedleValue { offset, size });
@@ -996,41 +945,6 @@ impl RedbNeedleMap {
         Ok(())
     }
 
-    /// Write one row to the .idx file at `idx_file_offset`. A row left
-    /// half-written by a failed write is trimmed back to the offset so the
-    /// next row still lands aligned; while the trim keeps failing no row is
-    /// written at all, or it would sit off alignment and parse as garbage on
-    /// load. The offset itself is advanced by the caller once the row counts.
-    fn append_to_index_file(
-        &mut self,
-        key: NeedleId,
-        offset: Offset,
-        size: Size,
-    ) -> io::Result<()> {
-        let Some(idx_file) = self.idx_file.as_mut() else {
-            return Ok(());
-        };
-        if self.idx_torn {
-            match idx_file.truncate_to(self.idx_file_offset) {
-                Ok(()) => self.idx_torn = false,
-                Err(e) => {
-                    return Err(io::Error::other(format!(
-                        "index file still holds a torn row: {e}"
-                    )));
-                }
-            }
-        }
-        idx_file.seek(io::SeekFrom::Start(self.idx_file_offset))?;
-        if let Err(e) = idx::write_index_entry(idx_file, key, offset, size) {
-            if let Err(te) = idx_file.truncate_to(self.idx_file_offset) {
-                self.idx_torn = true;
-                tracing::warn!("failed to trim torn .idx row: {}", te);
-            }
-            return Err(e);
-        }
-        Ok(())
-    }
-
     /// Look up a needle. A redb failure is an ERROR, not an absent needle:
     /// answering "not found" would turn a database problem into a read miss
     /// and let a delete report success without recording a tombstone.
@@ -1077,7 +991,9 @@ impl RedbNeedleMap {
             return Ok(None);
         };
 
-        self.append_to_index_file(key, offset, TOMBSTONE_FILE_SIZE)?;
+        if let Some(ref mut idx_file) = self.idx_file {
+            idx::write_index_entry(idx_file, key, offset, TOMBSTONE_FILE_SIZE)?;
+        }
 
         let deleted_nv = NeedleValue {
             offset: old.offset,
@@ -1167,13 +1083,10 @@ impl RedbNeedleMap {
     /// a failed redb commit. Without this the next successful write appends
     /// after the orphan, `idx_file_offset` advances past it, and a later
     /// checkpoint records an offset that makes the reload skip the orphan.
-    /// When the trim fails the file is latched torn so no later row lands
-    /// after bytes the map does not reflect.
     fn truncate_idx_to_offset(&mut self) {
         if let Some(ref mut idx_file) = self.idx_file
             && let Err(e) = idx_file.truncate_to(self.idx_file_offset)
         {
-            self.idx_torn = true;
             tracing::warn!("failed to truncate orphan .idx row: {}", e);
         }
     }
@@ -1224,7 +1137,6 @@ impl RedbNeedleMap {
         self.db = reopened.db;
         self.metric = reopened.metric;
         self.idx_file_offset = actual_idx_size;
-        self.idx_torn = reopened.idx_torn;
         // The reopen replayed all rows since the last durable checkpoint
         // non-durably; start the counter fresh.
         self.writes_since_checkpoint = 0;
@@ -1474,18 +1386,6 @@ impl NeedleMap {
         }
     }
 
-    /// Skew the live file count away from what the `.idx` holds, so tests
-    /// can build a volume whose reported count disagrees with a reload.
-    #[cfg(test)]
-    pub(crate) fn add_file_count_for_test(&self, delta: i64) {
-        let metric = match self {
-            NeedleMap::InMemory(nm) => &nm.metric,
-            NeedleMap::Redb(nm) => &nm.metric,
-            NeedleMap::SortedFile(_) => panic!("sorted-file needle maps are read-only"),
-        };
-        metric.file_count.fetch_add(delta, Ordering::Relaxed);
-    }
-
     /// Largest (offset + actual size) seen during the load walk; 0 if the
     /// map is empty. Used at volume load to detect .idx entries that
     /// reference past the end of .dat (issue #8928) without a second scan.
@@ -1777,7 +1677,7 @@ mod tests {
         )
         .unwrap();
         let writer = std::fs::OpenOptions::new()
-            .write(true)
+            .append(true)
             .open(&idx_path)
             .unwrap();
         nm.set_idx_file(Box::new(writer), idx_size);

seaweed-volume/src/storage/store_ec_journal.rs

@@ -1,374 +0,0 @@
-//! Merging a peer's `.ecj` deletion ids into a local EC journal. Mirrors Go's
-//! `Store.MergeEcJournal` (`weed/storage/store_ec_journal.go`).
-
-use std::collections::HashSet;
-use std::io;
-use std::path::Path;
-use std::sync::RwLock;
-
-use crate::storage::erasure_coding::ecj_merge::{append_ecj_ids, read_ecj_ids};
-use crate::storage::store::Store;
-use crate::storage::types::{NeedleId, VolumeId};
-
-/// How often an unmounted merge re-reads a journal that changed under it. Only
-/// a mount-delete-unmount or a concurrent merge between the read and the
-/// append changes it, so one retry is nearly always enough.
-const ECJ_MERGE_ATTEMPTS: usize = 5;
-
-/// Fold a peer's deletion `ids` into the local journal of EC volume `vid` on
-/// the receiving disk, the one whose data directory is `data_dir`; `ecj_path`
-/// is that journal's path in the disk's index directory. Appends only the ids
-/// the journal lacks and returns how many it added. Blocking: call it from
-/// `spawn_blocking`.
-///
-/// A mounted volume owns its journal: the merge goes through its open handle
-/// and in-memory set. That is the receiving disk's own runtime for `vid`,
-/// wherever its journal lives (it may sit in the data dir rather than
-/// `ecj_path`'s index dir), else a sibling runtime journaling into `ecj_path`
-/// itself: disks sharing one index directory, or reconciliation mounting `vid`
-/// on a disk that journals into another's (#9212). Otherwise `ecj_path` is
-/// appended to under the store write lock, which mounts take, so no mount can
-/// open it mid-append. The read that computes the delta runs outside the lock,
-/// and a journal that changed in between — including by a concurrent merge —
-/// is re-read.
-pub fn merge_ec_journal(
-    store: &RwLock<Store>,
-    vid: VolumeId,
-    data_dir: &str,
-    ecj_path: &str,
-    ids: &HashSet<NeedleId>,
-) -> io::Result<usize> {
-    merge_ec_journal_with(store, vid, data_dir, ecj_path, ids, read_ecj_ids)
-}
-
-/// `merge_ec_journal` with the unlocked journal read injected, so a test can
-/// mount the volume between that read and the append.
-fn merge_ec_journal_with(
-    store: &RwLock<Store>,
-    vid: VolumeId,
-    data_dir: &str,
-    ecj_path: &str,
-    ids: &HashSet<NeedleId>,
-    mut read: impl FnMut(&str) -> io::Result<(HashSet<NeedleId>, u64)>,
-) -> io::Result<usize> {
-    for _ in 0..ECJ_MERGE_ATTEMPTS {
-        {
-            let mut store = store
-                .write()
-                .map_err(|_| io::Error::other("store lock poisoned"))?;
-            if let Some(primary) = mounted_ec_journal(&store, vid, data_dir, ecj_path)? {
-                let ecv = store.locations[primary]
-                    .find_ec_volume_mut(vid)
-                    .expect("mounted journal runtime");
-                let journal_path = ecv.ecj_file_name();
-                let added = ecv.merge_journal(ids)?;
-                // Publish to the holders of the file the merge wrote to — the
-                // picked runtime's journal may live outside ecj_path, and a
-                // holder of a different file must not claim ids it lacks.
-                publish_to_journal_siblings(&store, primary, vid, &journal_path, ids);
-                return Ok(added);
-            }
-        }
-        let (local, size) = read(ecj_path)?;
-        let mut store = store
-            .write()
-            .map_err(|_| io::Error::other("store lock poisoned"))?;
-        if let Some(primary) = mounted_ec_journal(&store, vid, data_dir, ecj_path)? {
-            // Mounted since the read: its handle owns the journal now.
-            let ecv = store.locations[primary]
-                .find_ec_volume_mut(vid)
-                .expect("mounted journal runtime");
-            let journal_path = ecv.ecj_file_name();
-            let added = ecv.merge_journal(ids)?;
-            publish_to_journal_siblings(&store, primary, vid, &journal_path, ids);
-            return Ok(added);
-        }
-        // The path append registers as a writer so a mount compacting this
-        // journal cannot swap its inode underneath it (the write itself is
-        // already serialized with mounts by the store lock).
-        let _ecj_write =
-            crate::storage::erasure_coding::ecj_registry::begin_ecj_write(ecj_path);
-        if let Some(added) = append_ecj_ids(ecj_path, &local, ids, size)? {
-            return Ok(added);
-        }
-    }
-    Err(io::Error::other(format!(
-        "ec volume {}: journal {} kept changing during merge",
-        vid.0, ecj_path
-    )))
-}
-
-/// The disk index of the runtime holding `ecj_path` open, if any: the disk at
-/// `data_dir`'s own, else the first sibling journaling into it.
-fn mounted_ec_journal(
-    store: &Store,
-    vid: VolumeId,
-    data_dir: &str,
-    ecj_path: &str,
-) -> io::Result<Option<usize>> {
-    let owner = store
-        .locations
-        .iter()
-        .position(|loc| Path::new(&loc.directory) == Path::new(data_dir))
-        .ok_or_else(|| {
-            io::Error::other(format!(
-                "ec volume {}: no disk at {} owns journal {}",
-                vid.0, data_dir, ecj_path
-            ))
-        })?;
-    let runtime = if store.locations[owner].has_ec_volume(vid) {
-        Some(owner)
-    } else {
-        store.locations.iter().position(|loc| {
-            loc.find_ec_volume(vid)
-                .is_some_and(|ecv| Path::new(&ecv.ecj_file_name()) == Path::new(ecj_path))
-        })
-    };
-    Ok(runtime)
-}
-
-/// Publishes merged ids into every other runtime journaling into `ecj_path`.
-fn publish_to_journal_siblings(
-    store: &Store,
-    primary: usize,
-    vid: VolumeId,
-    ecj_path: &str,
-    ids: &HashSet<NeedleId>,
-) {
-    for (i, loc) in store.locations.iter().enumerate() {
-        if i == primary {
-            continue;
-        }
-        if let Some(ecv) = loc.find_ec_volume(vid) {
-            if Path::new(&ecv.ecj_file_name()) == Path::new(ecj_path) {
-                ecv.publish_merged_ids(ids);
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::config::MinFreeSpace;
-    use crate::storage::needle_map::NeedleMapKind;
-    use crate::storage::types::DiskType;
-    use crate::storage::volume::{VifEcShardConfig, VifVolumeInfo};
-    use tempfile::TempDir;
-
-    const COLLECTION: &str = "c";
-    const VID: VolumeId = VolumeId(9);
-
-    /// A store with one disk per entry of `data`, all sharing `idx` when given,
-    /// else each indexing into its own data dir.
-    fn make_store(tmp: &TempDir, data: &[&str], idx: Option<&str>) -> RwLock<Store> {
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        for d in data {
-            let dir = tmp.path().join(d).to_string_lossy().into_owned();
-            let idx_dir = idx
-                .map(|i| tmp.path().join(i).to_string_lossy().into_owned())
-                .unwrap_or_else(|| dir.clone());
-            std::fs::create_dir_all(&dir).unwrap();
-            std::fs::create_dir_all(&idx_dir).unwrap();
-            store
-                .add_location(
-                    &dir,
-                    &idx_dir,
-                    100,
-                    DiskType::HardDrive,
-                    MinFreeSpace::Percent(0.0),
-                    Vec::new(),
-                )
-                .unwrap();
-        }
-        RwLock::new(store)
-    }
-
-    fn dir(tmp: &TempDir, d: &str) -> String {
-        tmp.path().join(d).to_string_lossy().into_owned()
-    }
-
-    fn records(ids: &[u64]) -> Vec<u8> {
-        let ids: Vec<NeedleId> = ids.iter().copied().map(NeedleId).collect();
-        crate::storage::erasure_coding::ecj_merge::encode_ecj_ids(&ids)
-    }
-
-    fn id_set(ids: &[u64]) -> HashSet<NeedleId> {
-        ids.iter().copied().map(NeedleId).collect()
-    }
-
-    /// Shard 0 of `VID` and its `.vif` in `data_dir`.
-    fn write_shard0(data_dir: &str) {
-        let base = format!("{}/{}_{}", data_dir, COLLECTION, VID.0);
-        std::fs::write(format!("{}.ec00", base), b"shard data nonempty").unwrap();
-        let vif = VifVolumeInfo {
-            version: 3,
-            ec_shard_config: Some(VifEcShardConfig {
-                data_shards: 10,
-                parity_shards: 4,
-                ..Default::default()
-            }),
-            ..Default::default()
-        };
-        std::fs::write(
-            format!("{}.vif", base),
-            serde_json::to_string(&vif).unwrap(),
-        )
-        .unwrap();
-    }
-
-    /// `VID`'s `.ecx` and a `.ecj` holding `deleted` in `dir`; returns the
-    /// journal path.
-    fn write_index(dir: &str, deleted: &[u64]) -> String {
-        let base = format!("{}/{}_{}", dir, COLLECTION, VID.0);
-        std::fs::write(format!("{}.ecx", base), vec![0u8; 16]).unwrap();
-        let ecj = format!("{}.ecj", base);
-        std::fs::write(&ecj, records(deleted)).unwrap();
-        ecj
-    }
-
-    fn deleted_on(store: &RwLock<Store>, disk: usize, id: u64) -> bool {
-        store.read().unwrap().locations[disk]
-            .find_ec_volume(VID)
-            .expect("mounted")
-            .is_needle_deleted(NeedleId(id))
-    }
-
-    /// Disks sharing one index directory all hold the same journal path.
-    /// Copying shards onto a disk that has not mounted `vid` must still reach
-    /// the sibling runtime holding that journal open.
-    #[test]
-    fn shared_index_dir_reaches_sibling_mount() {
-        let tmp = TempDir::new().unwrap();
-        let store = make_store(&tmp, &["d0", "d1"], Some("idx"));
-        write_shard0(&dir(&tmp, "d0"));
-        let ecj = write_index(&dir(&tmp, "idx"), &[1]);
-        store.write().unwrap().locations[0]
-            .mount_ec_shards(VID, COLLECTION, &[0], "")
-            .unwrap();
-        assert_eq!(
-            store.read().unwrap().locations[0]
-                .find_ec_volume(VID)
-                .unwrap()
-                .ecj_file_name(),
-            ecj
-        );
-
-        let added =
-            merge_ec_journal(&store, VID, &dir(&tmp, "d1"), &ecj, &id_set(&[1, 2])).unwrap();
-        assert_eq!(added, 1);
-        assert!(
-            deleted_on(&store, 0, 2),
-            "the mounted sibling must see id 2"
-        );
-        assert_eq!(std::fs::read(&ecj).unwrap(), records(&[1, 2]));
-    }
-
-    /// Every runtime holding the journal open must see merged ids in memory.
-    #[test]
-    fn shared_journal_reaches_every_holder() {
-        let tmp = TempDir::new().unwrap();
-        let store = make_store(&tmp, &["d0", "d1"], Some("idx"));
-        write_shard0(&dir(&tmp, "d0"));
-        write_shard0(&dir(&tmp, "d1"));
-        let ecj = write_index(&dir(&tmp, "idx"), &[1]);
-        for i in 0..2 {
-            store.write().unwrap().locations[i]
-                .mount_ec_shards(VID, COLLECTION, &[0], "")
-                .unwrap();
-        }
-
-        let added =
-            merge_ec_journal(&store, VID, &dir(&tmp, "d1"), &ecj, &id_set(&[1, 2])).unwrap();
-        assert_eq!(added, 1);
-        assert!(
-            deleted_on(&store, 0, 2) && deleted_on(&store, 1, 2),
-            "every journal holder must see the merged id"
-        );
-        assert_eq!(std::fs::read(&ecj).unwrap(), records(&[1, 2]));
-    }
-
-    /// The picked runtime may journal to a different file than the copied
-    /// one — its index lives in its data directory while a sibling's lives
-    /// in the index directory. The ids must be published only to holders of
-    /// the file they were written to.
-    #[test]
-    fn publishes_to_actual_journal_holders() {
-        let tmp = TempDir::new().unwrap();
-        let store = make_store(&tmp, &["d0", "d1"], Some("idx"));
-        write_shard0(&dir(&tmp, "d0"));
-        write_shard0(&dir(&tmp, "d1"));
-        let data_ecj = write_index(&dir(&tmp, "d0"), &[1]);
-        let idx_ecj = write_index(&dir(&tmp, "idx"), &[1]);
-        for i in 0..2 {
-            store.write().unwrap().locations[i]
-                .mount_ec_shards(VID, COLLECTION, &[0], "")
-                .unwrap();
-        }
-        assert_eq!(
-            store.read().unwrap().locations[0]
-                .find_ec_volume(VID)
-                .unwrap()
-                .ecj_file_name(),
-            data_ecj
-        );
-
-        let added =
-            merge_ec_journal(&store, VID, &dir(&tmp, "d0"), &idx_ecj, &id_set(&[1, 2])).unwrap();
-        assert_eq!(added, 1);
-        assert_eq!(std::fs::read(&data_ecj).unwrap(), records(&[1, 2]));
-        assert_eq!(std::fs::read(&idx_ecj).unwrap(), records(&[1]));
-        assert!(deleted_on(&store, 0, 2));
-        assert!(
-            !deleted_on(&store, 1, 2),
-            "a different journal's holder must not claim the merged id"
-        );
-    }
-
-    /// A sibling disk can mount `vid` from the receiving disk's index (#9212)
-    /// while the merge reads the journal unlocked. The merge must go through
-    /// that mount and report what it added.
-    #[test]
-    fn mount_during_read_is_merged_through_and_counted() {
-        let tmp = TempDir::new().unwrap();
-        let store = make_store(&tmp, &["d0", "d1"], None);
-        let owner = dir(&tmp, "d0");
-        let ecj = write_index(&owner, &[1]);
-        write_shard0(&dir(&tmp, "d1"));
-
-        let mut mounted = false;
-        let read = |path: &str| {
-            let read = read_ecj_ids(path);
-            if !mounted {
-                store.write().unwrap().locations[1]
-                    .mount_ec_shards_with_idx_dir(VID, COLLECTION, &[0], &owner, "")
-                    .unwrap();
-                mounted = true;
-            }
-            read
-        };
-        let added =
-            merge_ec_journal_with(&store, VID, &owner, &ecj, &id_set(&[1, 2, 3]), read).unwrap();
-        assert_eq!(added, 2, "the ids merged through the new mount are counted");
-        assert!(deleted_on(&store, 1, 2) && deleted_on(&store, 1, 3));
-        assert_eq!(std::fs::read(&ecj).unwrap(), records(&[1, 2, 3]));
-    }
-
-    /// An unmounted journal is merged on disk and a repeat adds nothing; a
-    /// data dir that is no disk is refused.
-    #[test]
-    fn unmounted_journal_is_idempotent() {
-        let tmp = TempDir::new().unwrap();
-        let store = make_store(&tmp, &["d0"], Some("idx"));
-        let ecj = write_index(&dir(&tmp, "idx"), &[1, 2]);
-        let d0 = dir(&tmp, "d0");
-        for want in [2, 0, 0] {
-            let added = merge_ec_journal(&store, VID, &d0, &ecj, &id_set(&[2, 3, 4])).unwrap();
-            assert_eq!(added, want);
-        }
-        assert_eq!(std::fs::read(&ecj).unwrap(), records(&[1, 2, 3, 4]));
-        assert!(
-            merge_ec_journal(&store, VID, &dir(&tmp, "elsewhere"), &ecj, &id_set(&[1])).is_err()
-        );
-    }
-}

seaweed-volume/src/storage/store_ec_mirror.rs

@@ -131,12 +131,6 @@ impl Store {
                     let Some(base) = name.strip_suffix(".ecx") else {
                         continue;
                     };
-                    // A 0-byte .ecx is a corrupt stub from a failed copy, not a
-                    // credible owner — skip it so the scan keeps looking for a
-                    // real index on a sibling disk (Go's indexEcxOwners).
-                    if !ent.metadata().is_ok_and(|m| m.len() > 0) {
-                        continue;
-                    }
                     let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
                         continue;
                     };
@@ -423,33 +417,4 @@ mod tests {
         let post = fs::read(dir0.join(format!("{}_{}.ecx", collection, vid))).unwrap();
         assert_eq!(post, ecx_local, "mirror overwrote dir0's existing .ecx");
     }
-
-    /// The mirror shares Go's indexEcxOwners, which skips a 0-byte `.ecx`:
-    /// a stub must not be chosen as the source to mirror from.
-    #[test]
-    fn mirror_owner_index_skips_zero_byte_ecx() {
-        let tmp = TempDir::new().unwrap();
-        let dir0 = tmp.path().join("data0");
-        let dir1 = tmp.path().join("data1");
-        fs::create_dir_all(&dir0).unwrap();
-        fs::create_dir_all(&dir1).unwrap();
-
-        let collection = "video-recordings";
-        let vid = 4123u32;
-        plant_ecx(&dir0, collection, vid, b"");
-        plant_ecx(&dir1, collection, vid, &[0xA1u8; 20]);
-
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        add_loc(&mut store, &dir0);
-        add_loc(&mut store, &dir1);
-
-        let owners = store.index_ecx_owners_for_mirror();
-        let owner = owners
-            .get(&EcKey {
-                collection: collection.to_string(),
-                vid: VolumeId(vid),
-            })
-            .expect("the valid .ecx on disk 1 must be indexed");
-        assert_eq!(owner.location, 1);
-    }
 }

seaweed-volume/src/storage/store_ec_reconcile.rs

@@ -37,7 +37,6 @@ pub(crate) struct EcVolumeMissingIndex {
     pub data_dir: String,
 }
 
-#[cfg(test)]
 pub(crate) fn ec_local_ecx_path(dir: &str, collection: &str, vid: VolumeId) -> String {
     if collection.is_empty() {
         format!("{}/{}.ecx", dir, vid.0)
@@ -118,9 +117,10 @@ impl Store {
                     );
                     continue;
                 };
-                // A 0-byte local stub is not a mirrored index (Go gates this fast
-                // path on HasEcxFileOnDisk); mount against the owner instead.
-                let use_local_idx = loc.has_ecx_file_on_disk(&key.collection, key.vid);
+                let local_ecx = ec_local_ecx_path(&loc.idx_directory, &key.collection, key.vid);
+                let local_ecx_in_data = ec_local_ecx_path(&loc.directory, &key.collection, key.vid);
+                let use_local_idx = std::path::Path::new(&local_ecx).exists()
+                    || std::path::Path::new(&local_ecx_in_data).exists();
 
                 if !use_local_idx && owner.location == loc_idx && owner.idx_dir == loc.idx_directory
                 {
@@ -422,12 +422,6 @@ impl Store {
                     let Some(base) = name.strip_suffix(".ecx") else {
                         continue;
                     };
-                    // A 0-byte .ecx is a corrupt stub from a failed copy, not a
-                    // credible owner — skip it so the scan keeps looking for a
-                    // real index on a sibling disk (Go's indexEcxOwners).
-                    if !ent.metadata().is_ok_and(|m| m.len() > 0) {
-                        continue;
-                    }
                     let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
                         continue;
                     };
@@ -645,34 +639,6 @@ mod tests {
         .unwrap();
     }
 
-    /// A 0-byte `.ecx` is not a credible owner (Go's indexEcxOwners skips
-    /// it): picking the stub would hide the valid index on the sibling disk.
-    #[test]
-    fn test_index_ecx_owners_skips_zero_byte_stub() {
-        let (store, _tmp) = make_test_store(2, None);
-        let d0 = store.locations[0].directory.clone();
-        let d1 = store.locations[1].directory.clone();
-        std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(7)), b"").unwrap();
-        write_index_files(&d1, "pics", 7, 10, 4);
-
-        let owners = store.index_ecx_owners();
-        let owner = owners
-            .get(&EcKey {
-                collection: "pics".to_string(),
-                vid: VolumeId(7),
-            })
-            .expect("the valid .ecx on disk 1 must be indexed");
-        assert_eq!(owner.location, 1);
-        assert_eq!(owner.idx_dir, d1);
-
-        // A stub with no real index anywhere owns nothing.
-        std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(8)), b"").unwrap();
-        assert!(!store.index_ecx_owners().contains_key(&EcKey {
-            collection: "pics".to_string(),
-            vid: VolumeId(8),
-        }));
-    }
-
     /// An empty `.dat` (<= a superblock, i.e. zero needles) for an EC volume
     /// is a leftover stub from the pre-fix loader. It must be swept on startup,
     /// not loaded as a phantom empty volume. With the same vid's stub on two
@@ -1005,56 +971,6 @@ mod tests {
         );
     }
 
-    /// dir0 holds orphan shards next to a 0-byte `.ecx` stub from a failed
-    /// copy; the real index is on dir1. The stub must not count as a
-    /// locally-mirrored index (Go gates that fast path on HasEcxFileOnDisk),
-    /// or the shards get registered against an empty index.
-    #[test]
-    fn test_reconcile_ignores_zero_byte_local_ecx_stub() {
-        let tmp = TempDir::new().unwrap();
-        let dir0 = tmp.path().join("data0");
-        let dir1 = tmp.path().join("data1");
-        std::fs::create_dir_all(&dir0).unwrap();
-        std::fs::create_dir_all(&dir1).unwrap();
-
-        let collection = "grafana-loki";
-        let vid = 1094u32;
-
-        write_shard(dir0.to_str().unwrap(), collection, vid, 0);
-        write_shard(dir1.to_str().unwrap(), collection, vid, 1);
-        write_index_files(dir1.to_str().unwrap(), collection, vid, 10, 4);
-
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        for dir in [&dir0, &dir1] {
-            store
-                .add_location(
-                    dir.to_str().unwrap(),
-                    dir.to_str().unwrap(),
-                    100,
-                    DiskType::HardDrive,
-                    MinFreeSpace::Percent(0.0),
-                    Vec::new(),
-                )
-                .unwrap();
-        }
-        // Plant the stub after the startup scan so only the reconcile decision
-        // is under test, then drop dir0's mount and reconcile again.
-        store.locations[0].remove_ec_volume(VolumeId(vid));
-        std::fs::write(
-            ec_local_ecx_path(dir0.to_str().unwrap(), collection, VolumeId(vid)),
-            b"",
-        )
-        .unwrap();
-
-        store.reconcile_ec_shards_across_disks();
-
-        let ev0 = store.locations[0]
-            .find_ec_volume(VolumeId(vid))
-            .expect("dir0's shard must be mounted against the owner's index");
-        assert!(ev0.has_shard(0));
-        assert_eq!(ev0.ecx_actual_dir(), dir1.to_str().unwrap());
-    }
-
     /// PR 9244 review case: idx_directory is configured but the
     /// owner's .ecx / .ecj / .vif live in the owner's data dir
     /// (the legacy "written before -dir.idx was set" layout). The
@@ -1476,7 +1392,7 @@ mod tests {
         let vid = VolumeId(7004);
         let collection = "grafana-loki";
 
-        store.delete_ec_shards(vid, collection, &[1]).unwrap();
+        store.delete_ec_shards(vid, collection, &[1]);
 
         // Shard 1 file is gone on disk 1.
         let p1 = format!(

seaweed-volume/tests/http_integration.rs

@@ -108,9 +108,6 @@ fn build_test_state(
         file_size_limit_bytes: 0,
         maintenance_byte_per_second: 0,
         is_heartbeating: std::sync::atomic::AtomicBool::new(true),
-        ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
-        ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
-        ec_decode_tail_notify: tokio::sync::Notify::new(),
         has_master: false,
         pre_stop_seconds: 0,
         volume_state_notify: tokio::sync::Notify::new(),
@@ -480,74 +477,6 @@ async fn delete_then_get_returns_404() {
     );
 }
 
-// Go answers both with 500 and an error containing "volume N is read only"
-// (the write behind "failed to write to local disk: ").
-#[tokio::test]
-async fn write_and_delete_on_read_only_volume_say_is_read_only() {
-    let (state, _tmp) = test_state();
-    let uri = "/1,01637037d6";
-
-    let request = |method: &str, body: &[u8]| {
-        Request::builder()
-            .method(method)
-            .uri(uri)
-            .body(Body::from(body.to_vec()))
-            .unwrap()
-    };
-    let error_of = |body: Vec<u8>| -> String {
-        let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
-        json["error"].as_str().unwrap_or_default().to_string()
-    };
-
-    let response = build_admin_router(state.clone())
-        .oneshot(request("POST", b"written before read-only"))
-        .await
-        .unwrap();
-    assert_eq!(response.status(), StatusCode::CREATED);
-
-    state
-        .store
-        .write()
-        .unwrap()
-        .find_volume_mut(VolumeId(1))
-        .unwrap()
-        .1
-        .set_no_write_or_delete(true);
-
-    let response = build_admin_router(state.clone())
-        .oneshot(request("POST", b"refused"))
-        .await
-        .unwrap();
-    assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
-    let error = error_of(body_bytes(response).await);
-    assert!(error.contains("volume 1 is read only"), "{error}");
-
-    let response = build_admin_router(state.clone())
-        .oneshot(request("DELETE", b""))
-        .await
-        .unwrap();
-    assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
-    assert_eq!(
-        error_of(body_bytes(response).await),
-        "Deletion Failed: volume 1 is read only"
-    );
-
-    // The needle is still there to delete once the volume is writable again.
-    state
-        .store
-        .write()
-        .unwrap()
-        .find_volume_mut(VolumeId(1))
-        .unwrap()
-        .1
-        .set_no_write_or_delete(false);
-    let response = build_admin_router(state.clone())
-        .oneshot(request("DELETE", b""))
-        .await
-        .unwrap();
-    assert_eq!(response.status(), StatusCode::ACCEPTED);
-}
-
 // ============================================================================
 // 6. HEAD returns headers without body
 // ============================================================================
@@ -1326,102 +1255,3 @@ async fn non_ascii_fid_and_ttl_are_rejected_not_panicked() {
         "a non-ASCII ttl must behave like any other invalid ttl, not panic"
     );
 }
-
-// ============================================================================
-// The write queue answers an upload with the needle's real ETag
-//
-// The queue worker computes the CRC on the needle it was handed, so the handler
-// has to know the checksum before it submits. Without that every queued upload
-// came back as "00000000". The direct path is the reference: same payload, same
-// ETag, for a plain body and for one the handler gzips before storing.
-// ============================================================================
-
-#[tokio::test]
-async fn write_queue_upload_returns_same_etag_as_direct_write() {
-    use seaweed_volume::server::write_queue::WriteQueue;
-
-    let (direct_state, _direct_tmp) = test_state();
-    let (queued_state, _queued_tmp) = test_state();
-    let wq = WriteQueue::new(queued_state.clone(), 128);
-    let _ = queued_state.write_queue.set(wq);
-
-    let compressible = "seaweedfs ".repeat(200).into_bytes();
-    let uploads: [(&str, &[u8]); 2] = [
-        ("/1,01637037d6", b"hello, seaweedfs!"),
-        ("/1/02637037d6/notes.txt", &compressible),
-    ];
-
-    for (uri, payload) in uploads {
-        let mut etags = Vec::new();
-        for state in [&direct_state, &queued_state] {
-            let app = build_admin_router(state.clone());
-            let response = app
-                .oneshot(
-                    Request::builder()
-                        .method("POST")
-                        .uri(uri)
-                        .body(Body::from(payload.to_vec()))
-                        .unwrap(),
-                )
-                .await
-                .unwrap();
-            assert_eq!(response.status(), StatusCode::CREATED);
-
-            let header = response
-                .headers()
-                .get("ETag")
-                .expect("upload response has no ETag")
-                .to_str()
-                .unwrap()
-                .to_string();
-            let body = body_bytes(response).await;
-            let json: serde_json::Value =
-                serde_json::from_slice(&body).expect("POST response is not valid JSON");
-            let etag = json["eTag"].as_str().unwrap().to_string();
-            assert_eq!(header, format!("\"{}\"", etag));
-            etags.push(etag);
-        }
-        assert_ne!(etags[0], "00000000", "{}: direct ETag is the zero CRC", uri);
-        assert_eq!(
-            etags[1], etags[0],
-            "{}: queued upload must return the direct path's ETag",
-            uri
-        );
-    }
-
-    // The second upload really was stored gzipped, so its ETag is the CRC of
-    // the compressed bytes on both paths.
-    let app = build_admin_router(queued_state.clone());
-    let response = app
-        .oneshot(
-            Request::builder()
-                .uri(uploads[1].0)
-                .header("Accept-Encoding", "gzip")
-                .body(Body::empty())
-                .unwrap(),
-        )
-        .await
-        .unwrap();
-    assert_eq!(response.status(), StatusCode::OK);
-    assert_eq!(response.headers()["Content-Encoding"], "gzip");
-
-    // Re-uploading the same bytes is the unchanged path: 204 with the same ETag.
-    let (uri, payload) = uploads[0];
-    let mut etags = Vec::new();
-    for state in [&direct_state, &queued_state] {
-        let app = build_admin_router(state.clone());
-        let response = app
-            .oneshot(
-                Request::builder()
-                    .method("POST")
-                    .uri(uri)
-                    .body(Body::from(payload.to_vec()))
-                    .unwrap(),
-            )
-            .await
-            .unwrap();
-        assert_eq!(response.status(), StatusCode::NO_CONTENT);
-        etags.push(response.headers()["ETag"].to_str().unwrap().to_string());
-    }
-    assert_eq!(etags[1], etags[0], "unchanged upload ETag differs");
-}

seaweed-worker/Cargo.lock

@@ -485,7 +485,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e"
 dependencies = [
  "aws-lc-sys",
- "untrusted 0.7.1",
  "zeroize",
 ]
 
@@ -5125,7 +5124,7 @@ dependencies = [
  "cfg-if",
  "getrandom 0.2.17",
  "libc",
- "untrusted 0.9.0",
+ "untrusted",
  "windows-sys 0.52.0",
 ]
 
@@ -5189,9 +5188,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.45"
+version = "0.23.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
+checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06"
 dependencies = [
  "aws-lc-rs",
  "log",
@@ -5261,7 +5260,7 @@ dependencies = [
  "aws-lc-rs",
  "ring",
  "rustls-pki-types",
- "untrusted 0.9.0",
+ "untrusted",
 ]
 
 [[package]]
@@ -5343,9 +5342,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
- "aws-lc-rs",
  "axum",
- "base64 0.22.1",
  "prometheus",
  "prost",
  "prost-types",
@@ -5784,7 +5781,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
  "windows-sys 0.61.2",
@@ -6216,12 +6213,6 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
 
-[[package]]
-name = "untrusted"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
-
 [[package]]
 name = "untrusted"
 version = "0.9.0"

seaweed-worker/crates/core/Cargo.toml

@@ -12,10 +12,6 @@ name = "seaweed_worker_core"
 # Helpers the Rust volume server needs as well. A path dependency because
 # the two trees are separate cargo workspaces with no common root manifest.
 seaweed-common = { path = "../../../seaweed-common" }
-# Already in the graph through tonic's `tls-aws-lc` feature; named here so the
-# admin token can sign with the same HMAC the Go worker uses.
-aws-lc-rs = "1"
-base64 = "0.22"
 anyhow.workspace = true
 async-trait.workspace = true
 prost.workspace = true

seaweed-worker/crates/core/src/config.rs

@@ -17,15 +17,6 @@ pub struct WorkerOptions {
     /// section of security.toml. None means plaintext, which is the default the
     /// Go worker also takes when no certificates are configured.
     pub tls: Option<TlsOptions>,
-    /// The shared secret the admin stream requires when the cluster configures
-    /// `jwt.filer_signing.key`; mirrored from that config value. Empty means no
-    /// token is attached, which only works against admins that either bound
-    /// their worker listener to loopback or authenticate with mTLS instead.
-    pub signing_key: Option<String>,
-    /// Seconds the minted admin token lives, mirroring
-    /// `jwt.filer_signing.expires_after_seconds`. Zero mints a token without an
-    /// expiry, matching the Go default.
-    pub signing_key_expires_after_seconds: u64,
 }
 
 /// Certificates for the control stream. All three are required together: the
@@ -53,8 +44,6 @@ impl Default for WorkerOptions {
             max_detection_concurrency: 1,
             max_execution_concurrency: 1,
             tls: None,
-            signing_key: None,
-            signing_key_expires_after_seconds: 0,
         }
     }
 }

seaweed-worker/crates/core/src/lib.rs

@@ -13,7 +13,6 @@ pub mod metrics;
 pub mod registry;
 pub mod senders;
 pub mod stream;
-pub mod token;
 
 /// Generated plugin.proto types.
 pub mod pb {

seaweed-worker/crates/core/src/stream.rs

@@ -176,19 +176,10 @@ async fn serve_once(
         metadata: Default::default(),
     }))?;
 
-    let mut request = tonic::Request::new(UnboundedReceiverStream::new(rx));
-    if let Some(token) = crate::token::admin_token(
-        options.signing_key.as_deref().unwrap_or("").as_bytes(),
-        options.signing_key_expires_after_seconds,
-    ) {
-        request.metadata_mut().insert(
-            "authorization",
-            format!("Bearer {token}")
-                .parse()
-                .context("minted admin token is not valid metadata")?,
-        );
-    }
-    let mut inbound = client.worker_stream(request).await?.into_inner();
+    let mut inbound = client
+        .worker_stream(UnboundedReceiverStream::new(rx))
+        .await?
+        .into_inner();
 
     let heartbeat = spawn_heartbeat(
         sender.clone(),

seaweed-worker/crates/core/src/token.rs

@@ -1,61 +0,0 @@
-use std::time::{SystemTime, UNIX_EPOCH};
-
-use aws_lc_rs::hmac;
-use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
-
-/// Mints the HS256 Bearer token the admin control stream requires when
-/// jwt.filer_signing.key is configured, matching the SeaweedFilerAdminClaims
-/// the Go worker sends: registered claims only, no session id.
-pub fn admin_token(signing_key: &[u8], expires_after_secs: u64) -> Option<String> {
-    if signing_key.is_empty() {
-        return None;
-    }
-    let claims = if expires_after_secs > 0 {
-        let exp = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .map(|d| d.as_secs())
-            .unwrap_or_default()
-            + expires_after_secs;
-        format!("{{\"exp\":{exp}}}")
-    } else {
-        "{}".to_string()
-    };
-    let signing_input = format!(
-        "{}.{}",
-        URL_SAFE_NO_PAD.encode(b"{\"alg\":\"HS256\",\"typ\":\"JWT\"}"),
-        URL_SAFE_NO_PAD.encode(claims.as_bytes())
-    );
-    let key = hmac::Key::new(hmac::HMAC_SHA256, signing_key);
-    let signature = hmac::sign(&key, signing_input.as_bytes());
-    Some(format!(
-        "{signing_input}.{}",
-        URL_SAFE_NO_PAD.encode(signature.as_ref())
-    ))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::admin_token;
-
-    #[test]
-    fn no_key_no_token() {
-        assert!(admin_token(b"", 0).is_none());
-    }
-
-    #[test]
-    fn minted_token_verifies_shape() {
-        let token = admin_token(b"secret", 60).unwrap();
-        let parts: Vec<&str> = token.split('.').collect();
-        assert_eq!(parts.len(), 3);
-        assert!(!parts.iter().any(|p| p.is_empty()));
-    }
-
-    #[test]
-    fn no_expiry_has_empty_claims() {
-        let token = admin_token(b"secret", 0).unwrap();
-        let payload = token.split('.').nth(1).unwrap();
-        use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
-        let claims = URL_SAFE_NO_PAD.decode(payload).unwrap();
-        assert_eq!(claims, b"{}");
-    }
-}

seaweed-worker/crates/lance/src/main.rs

@@ -61,18 +61,6 @@ struct Args {
     #[arg(long)]
     tls_server_name: Option<String>,
 
-    /// Shared secret for the admin control stream, the `jwt.filer_signing.key`
-    /// value the Go worker reads from its config file. Required by admins that
-    /// authenticate the worker stream with a signing key.
-    #[arg(long, env = "WEED_FILER_SIGNING_KEY")]
-    filer_signing_key: Option<String>,
-
-    /// Lifetime of the token minted from the signing key, mirroring
-    /// `jwt.filer_signing.expires_after_seconds`. Zero mints a token with no
-    /// expiry.
-    #[arg(long, default_value = "0")]
-    filer_signing_key_expires_after_seconds: u64,
-
     /// Serve /health, /ready and /metrics on this port, the way
     /// `weed worker -metricsPort` does. Zero, the default, serves nothing.
     #[arg(long, default_value = "0", env = "WEED_METRICS_PORT")]
@@ -143,8 +131,6 @@ async fn main() -> Result<()> {
         max_detection_concurrency: args.max_concurrency,
         max_execution_concurrency: args.max_concurrency,
         tls,
-        signing_key: args.filer_signing_key,
-        signing_key_expires_after_seconds: args.filer_signing_key_expires_after_seconds,
         ..Default::default()
     };
 

test/fuse_dlm/dlm_concurrent_write_test.go

@@ -1,5 +1,3 @@
-//go:build linux || darwin
-
 package fuse_dlm
 
 import (

test/fuse_dlm/framework_test.go

@@ -1,5 +1,3 @@
-//go:build linux || darwin
-
 package fuse_dlm
 
 import (

test/fuse_dlm/posix_lock_ring_test.go

@@ -1,5 +1,3 @@
-//go:build linux || darwin
-
 package fuse_dlm
 
 import (

test/fuse_p2p/peer_chunk_sharing_test.go

@@ -1,5 +1,3 @@
-//go:build linux || darwin
-
 package fuse_p2p
 
 import (

test/java/spark/pom.xml

@@ -16,12 +16,12 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>11</maven.compiler.source>
         <maven.compiler.target>11</maven.compiler.target>
-        <spark.version>3.5.8</spark.version>
+        <spark.version>3.5.7</spark.version>
         <hadoop.version>3.3.6</hadoop.version>
         <scala.binary.version>2.12</scala.binary.version>
         <junit.version>4.13.2</junit.version>
         <seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
-        <jackson.version>2.22.3</jackson.version> <!-- Upgraded from 2.15.3 -->
+        <jackson.version>2.22.1</jackson.version> <!-- Upgraded from 2.15.3 -->
         <!-- jackson-annotations dropped the patch digit in 2.20; matches jackson.version's major.minor -->
         <jackson.annotations.version>2.22</jackson.annotations.version>
         <netty.version>4.2.16.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->

test/kafka/go.mod

@@ -7,20 +7,20 @@ require (
 	github.com/linkedin/goavro/v2 v2.15.0
 	github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
 	github.com/segmentio/kafka-go v0.4.49
-	github.com/stretchr/testify v1.12.1
+	github.com/stretchr/testify v1.11.1
 	google.golang.org/grpc v1.85.0-dev.0.20260915183914-4e49413dcab7
 )
 
 replace github.com/seaweedfs/seaweedfs => ../../
 
 require (
-	github.com/andybalholm/brotli v1.2.3 // indirect
+	github.com/andybalholm/brotli v1.2.2 // indirect
 	github.com/aws/aws-sdk-go v1.55.8 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cognusion/imaging v1.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dustin/go-humanize v1.1.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/eapache/go-resiliency v1.7.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
@@ -61,15 +61,17 @@ require (
 	github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect
 	github.com/pierrec/lz4/v4 v4.1.29 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/prometheus/client_golang v1.24.1 // indirect
 	github.com/prometheus/client_model v0.6.3 // indirect
-	github.com/prometheus/common v0.71.0 // indirect
+	github.com/prometheus/common v0.70.1 // indirect
 	github.com/prometheus/procfs v0.22.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
 	github.com/rdleal/intervalst v1.5.0 // indirect
+	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
 	github.com/sagikazarmark/locafero v0.11.0 // indirect
-	github.com/seaweedfs/goexif v1.0.3 // indirect
+	github.com/seaweedfs/goexif v2.0.0+incompatible // indirect
 	github.com/shirou/gopsutil/v4 v4.26.7 // indirect
 	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
@@ -89,8 +91,8 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	go.yaml.in/yaml/v3 v3.0.5 // indirect
-	golang.org/x/crypto v0.57.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.56.0 // indirect
 	golang.org/x/image v0.46.0 // indirect
 	golang.org/x/net v0.58.0 // indirect
 	golang.org/x/sync v0.23.0 // indirect
@@ -99,4 +101,5 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect
 	google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
 	google.golang.org/protobuf v1.36.12 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

test/kafka/go.sum

@@ -10,8 +10,8 @@ github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZ
 github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
 github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
 github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
-github.com/andybalholm/brotli v1.2.3 h1:8H1qwOkl2LPfjf3YezB90JnCliZb6SInJ/OJkEbA5NQ=
-github.com/andybalholm/brotli v1.2.3/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
+github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM=
+github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -28,8 +28,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
-github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA=
 github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
 github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws=
@@ -174,8 +174,8 @@ github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59u
 github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
 github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
 github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
-github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8=
-github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
+github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
+github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
 github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
 github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
 github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg=
@@ -184,10 +184,12 @@ github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6ha
 github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
 github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
 github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
+github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc=
+github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
 github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
 github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
-github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
-github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
+github.com/seaweedfs/goexif v2.0.0+incompatible h1:x8pckiT12QQhifwhDQpeISgDfsqmQ6VR4LFPQ64JRps=
+github.com/seaweedfs/goexif v2.0.0+incompatible/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
 github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
 github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
 github.com/shirou/gopsutil/v4 v4.26.7 h1:IXzpHz/dkMRYAhKkOXr1HB6SuzWU3eoyyeWe7g3bNZc=
@@ -214,8 +216,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
-github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
@@ -261,15 +263,15 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
 go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
-go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
-go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
-golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
+golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
+golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
 golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
 golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -286,8 +288,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
 golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
-golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
-golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -349,6 +351,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
 google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

test/s3/error_compat/Makefile

@@ -1,105 +0,0 @@
-# S3/IAM error response compatibility test harness
-
-.PHONY: all test help build-weed check-deps start-server stop-server test-errors test-with-server clean logs
-
-WEED_BINARY := ../../../weed/weed_binary
-S3_PORT := 8334
-TEST_TIMEOUT := 10m
-SERVER_DIR := ./test-volume-data
-S3_CONFIG := ./test_s3.json
-
-.DEFAULT_GOAL := help
-
-all: test
-
-test: test-with-server
-
-help:
-	@echo "S3/IAM error response compatibility test harness"
-	@echo ""
-	@echo "Available targets:"
-	@echo "  build-weed      - Build the SeaweedFS binary"
-	@echo "  start-server    - Start SeaweedFS mini with S3 and embedded IAM enabled"
-	@echo "  stop-server     - Stop the test server"
-	@echo "  test-errors     - Run the error compatibility checks"
-	@echo "  test-with-server - Start server, run checks, stop server"
-	@echo "  logs            - Show server logs"
-	@echo "  clean           - Remove test artifacts"
-
-build-weed:
-	@echo "Building SeaweedFS binary..."
-	@cd ../../../weed && go build -o weed_binary .
-	@chmod +x $(WEED_BINARY)
-
-check-deps: build-weed
-	@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
-	@command -v python3 >/dev/null 2>&1 || (echo "python3 is required but not installed" && exit 1)
-	@python3 -c "import botocore" 2>/dev/null || (echo "botocore is required: pip install botocore" && exit 1)
-	@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
-	@test -f $(S3_CONFIG) || (echo "S3 config not found at $(S3_CONFIG)" && exit 1)
-
-start-server: check-deps
-	@echo "Starting SeaweedFS mini for error compatibility tests..."
-	@rm -f weed-server.pid
-	@mkdir -p $(SERVER_DIR)
-	@if curl -sS -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \
-		echo "port $(S3_PORT) is already served by another process; refusing to test against it"; \
-		exit 1; \
-	fi
-	@$(WEED_BINARY) mini \
-		-dir=$(SERVER_DIR) \
-		-s3 \
-		-s3.port=$(S3_PORT) \
-		-s3.config=$(S3_CONFIG) \
-		-s3.iam.readOnly=false \
-		-master.peers=none \
-		> weed-test.log 2>&1 & echo $$! > weed-server.pid
-	@for i in $$(seq 1 60); do \
-		kill -0 $$(cat weed-server.pid) 2>/dev/null || { \
-			echo "SeaweedFS server exited during startup"; \
-			echo "=== Server logs ==="; \
-			test -f weed-test.log && cat weed-test.log || true; \
-			exit 1; \
-		}; \
-		if curl -sS -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \
-			echo "SeaweedFS S3 server is ready on port $(S3_PORT)"; \
-			sleep 3; \
-			exit 0; \
-		fi; \
-		sleep 1; \
-	done; \
-	echo "SeaweedFS S3 server failed to start"; \
-	echo "=== Server logs ==="; \
-	test -f weed-test.log && cat weed-test.log || true; \
-	$(MAKE) --no-print-directory stop-server; \
-	exit 1
-
-stop-server:
-	@if [ -f weed-server.pid ]; then \
-		pid=$$(cat weed-server.pid); \
-		if ps -p $$pid -o comm= 2>/dev/null | grep -q weed; then \
-			echo "Stopping SeaweedFS server..."; \
-			kill $$pid 2>/dev/null || true; \
-			sleep 2; \
-		fi; \
-		rm -f weed-server.pid; \
-	fi
-
-test-errors:
-	@echo "Running S3/IAM error compatibility checks..."
-	python3 s3_error_compat_test.py \
-		--endpoint http://127.0.0.1:$(S3_PORT) \
-		--access-key admin \
-		--secret-key admin
-
-test-with-server:
-	@$(MAKE) start-server; START_RESULT=$$?; \
-	if [ $$START_RESULT -eq 0 ]; then $(MAKE) test-errors; TEST_RESULT=$$?; else TEST_RESULT=$$START_RESULT; fi; \
-	$(MAKE) stop-server; exit $$TEST_RESULT
-
-logs:
-	@test -f weed-test.log && tail -100 weed-test.log || echo "No log file found"
-
-clean: stop-server
-	@rm -rf $(SERVER_DIR) weed-test.log weed-server.pid
-	@echo "Cleaned up test artifacts"

test/s3/error_compat/s3_error_compat_test.py

@@ -1,216 +0,0 @@
-#!/usr/bin/env python3
-"""Verify S3/IAM error responses match AWS for common client mistakes.
-
-Needs only botocore (pip install botocore). Sends raw SigV4 requests so the exact
-status and response body are visible, compares each with what AWS returns, then
-cleans up everything it created (one bucket, one IAM user).
-
-Usage:
-  python3 s3_error_compat_test.py --endpoint http://127.0.0.1:8333 \
-      --access-key <admin key> --secret-key <admin secret>
-
-The key needs Admin rights, and IAM writes must be enabled (-s3.iam.readOnly=false).
-"""
-import argparse, atexit, base64, datetime, hashlib, http.client, re, sys, time, urllib.parse, uuid
-
-from botocore.auth import S3SigV4Auth, SigV4Auth
-from botocore.awsrequest import AWSRequest
-from botocore.credentials import Credentials
-
-ap = argparse.ArgumentParser()
-ap.add_argument('--endpoint', default='http://127.0.0.1:8333')
-ap.add_argument('--access-key', required=True)
-ap.add_argument('--secret-key', required=True)
-ap.add_argument('--region', default='us-east-1')
-args = ap.parse_args()
-
-EP = args.endpoint.rstrip('/')
-U = urllib.parse.urlsplit(EP)
-ADMIN = Credentials(args.access_key, args.secret_key)
-BAD_KEY = Credentials('AKIAREPRONOSUCHKEY00', 'x' * 40)
-BAD_SECRET = Credentials(args.access_key, 'y' * 40)
-B = 'errorcompat-' + uuid.uuid4().hex[:8]
-S3DOC = 'https://docs.aws.amazon.com/AmazonS3/latest/API/'
-IAMDOC = 'https://docs.aws.amazon.com/IAM/latest/APIReference/'
-rows = []
-
-
-def send(method, path, body=b'', headers=None, creds=ADMIN, service='s3', mutate=None, sign=True,
-         keep_sha256_header=False):
-    headers = dict(headers or {})
-    url = EP + path
-    if sign:
-        req = AWSRequest(method=method, url=url, data=body, headers=headers)
-        if service == 's3':
-            if 'X-Amz-Content-SHA256' not in req.headers:
-                req.headers['X-Amz-Content-SHA256'] = hashlib.sha256(body).hexdigest()
-            # S3SigV4Auth always recomputes X-Amz-Content-SHA256 from the body; the
-            # generic SigV4Auth signs a caller-supplied value instead, which is how
-            # a mismatched declared hash can be produced on the wire.
-            signer = SigV4Auth if keep_sha256_header else S3SigV4Auth
-            signer(creds, 's3', args.region).add_auth(req)
-        else:
-            SigV4Auth(creds, service, args.region).add_auth(req)
-        headers = dict(req.headers.items())
-    if mutate:
-        headers = mutate(headers)
-    conn = http.client.HTTPConnection(U.hostname, U.port, timeout=30)
-    conn.request(method, path, body=body, headers=headers)
-    r = conn.getresponse()
-    data = r.read().decode('utf-8', 'replace')
-    conn.close()
-    return r.status, data
-
-
-def shape(data):
-    """Root element and <Code> of an XML error body."""
-    body = re.sub(r'<\?xml[^>]*\?>', '', data).strip()
-    root = re.match(r'<(\w+)', body)
-    code = re.search(r'<Code>([^<]*)</Code>', data)
-    return (root.group(1) if root else ('-' if not body else 'non-xml')), (code.group(1) if code else '')
-
-
-def iam(params, creds=ADMIN):
-    p = dict(params, Version='2010-05-08')
-    return dict(method='POST', path='/', body=urllib.parse.urlencode(p).encode(), creds=creds, service='iam',
-                headers={'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8'})
-
-
-def case(group, name, expect, ref, req, check=None):
-    """expect = (status, root element or None, code or None); check() adds a side-effect note."""
-    st, data = send(**req)
-    root, code = shape(data)
-    note = check() if check else ''
-    ok = st == expect[0] and expect[1] in (None, root) and expect[2] in (None, code) and not note
-    rows.append((group, name, '%s %s %s' % (st, root, code), '%s %s %s' % (expect[0], expect[1] or '', expect[2] or ''), note, ok, ref))
-
-
-def exists(key):
-    return lambda: 'object %r now exists' % key if send('HEAD', '/%s/%s' % (B, key))[0] == 200 else ''
-
-
-def S3(method, path, **kw):
-    return dict(method=method, path=path, **kw)
-
-
-# ---------------- cleanup (atexit so failed runs still clean up) ----------------
-user = USER = upload_id = None
-
-
-def cleanup():
-    try:
-        if user and USER:
-            send(**iam({'Action': 'DeleteAccessKey', 'UserName': user, 'AccessKeyId': USER.access_key}))
-        if user:
-            send(**iam({'Action': 'DeleteUser', 'UserName': user}))
-        if upload_id:
-            send('DELETE', '/%s/mp?uploadId=%s' % (B, upload_id))
-        st, d = send('GET', '/%s?list-type=2' % B)
-        for k in re.findall(r'<Key>([^<]+)</Key>', d):
-            send('DELETE', '/%s/%s' % (B, urllib.parse.quote(k)))
-        send('DELETE', '/' + B)
-    except Exception as e:
-        print('cleanup failed: %s' % e)
-
-
-atexit.register(cleanup)
-
-# ---------------- setup ----------------
-st, d = send('PUT', '/' + B)
-if st != 200:
-    sys.exit('cannot create bucket %s: %s %s' % (B, st, d[:200]))
-send('PUT', '/%s/obj' % B, body=b'hello')
-st, d = send('POST', '/%s/mp?uploads' % B)
-upload_id = re.search(r'<UploadId>([^<]+)', d).group(1)
-user = 'errorcompat-' + uuid.uuid4().hex[:6]
-send(**iam({'Action': 'CreateUser', 'UserName': user}))
-st, d = send(**iam({'Action': 'CreateAccessKey', 'UserName': user}))
-USER = Credentials(re.search(r'<AccessKeyId>([^<]+)', d).group(1), re.search(r'<SecretAccessKey>([^<]+)', d).group(1))
-for _ in range(15):  # wait until the new key is accepted
-    if 'InvalidAccessKeyId' not in send('GET', '/', creds=USER)[1]:
-        break
-    time.sleep(1)
-
-# ---------------- 1. requests that run a different operation ----------------
-for sub in ('logging', 'notification', 'accelerate', 'website', 'replication',
-            'analytics&id=a', 'inventory&id=a', 'metrics&id=a', 'intelligent-tiering&id=a'):
-    case(1, 'PUT ?%s' % sub, (501, 'Error', 'NotImplemented'), S3DOC + 'API_Error.html#:~:text=Code%3A%20NotImplemented',
-         S3('PUT', '/%s?%s' % (B, sub), body=b'<X/>'))
-case(1, 'DELETE ?logging', (501, 'Error', 'NotImplemented'), S3DOC + 'API_Error.html#:~:text=Code%3A%20NotImplemented',
-     S3('DELETE', '/%s?logging' % B),
-     lambda: '' if send('HEAD', '/' + B)[0] == 200 else 'BUCKET DELETED')
-case(1, 'CopyObject, x-amz-copy-source without "/"', (400, 'Error', 'InvalidArgument'),
-     S3DOC + 'API_CopyObject.html#AmazonS3-CopyObject-request-header-CopySource',
-     S3('PUT', '/%s/copy-dst' % B, headers={'x-amz-copy-source': 'nobucketonly'}), exists('copy-dst'))
-case(1, 'UploadPart partNumber=abc', (400, 'Error', 'InvalidArgument'),
-     S3DOC + 'API_UploadPart.html#AmazonS3-UploadPart-request-uri-querystring-PartNumber',
-     S3('PUT', '/%s/mp?partNumber=abc&uploadId=%s' % (B, upload_id), body=b'part-body'), exists('mp'))
-case(1, 'PutObject, x-amz-content-sha256 != body', (400, 'Error', 'XAmzContentSHA256Mismatch'),
-     'undocumented code; ' + S3DOC + 'API_UploadPart.html#API_UploadPart_RequestSyntax',
-     S3('PUT', '/%s/sha-mismatch' % B, body=b'abc',
-        headers={'X-Amz-Content-SHA256': hashlib.sha256(b'zzz').hexdigest()}, keep_sha256_header=True),
-     exists('sha-mismatch'))
-case(1, 'PutObject, x-amz-content-sha256 not hex', (400, 'Error', None), 'undocumented',
-     S3('PUT', '/%s/sha-nothex' % B, body=b'abc',
-        headers={'X-Amz-Content-SHA256': 'nothex'}, keep_sha256_header=True), exists('sha-nothex'))
-
-# ---------------- 2. IAM failures in the S3 <Error> envelope ----------------
-case(2, 'IAM ListUsers, unknown access key', (403, 'ErrorResponse', 'InvalidClientTokenId'),
-     'checked against iam.amazonaws.com (docs list UnrecognizedClientException); ' + IAMDOC + 'CommonErrors.html#CommonErrors-UnrecognizedClientException', iam({'Action': 'ListUsers'}, creds=BAD_KEY))
-case(2, 'IAM ListUsers, wrong secret', (403, 'ErrorResponse', None), 'undocumented for IAM',
-     iam({'Action': 'ListUsers'}, creds=BAD_SECRET))
-case(2, 'IAM ListUsers, non-admin user', (403, 'ErrorResponse', 'AccessDenied'),
-     IAMDOC + 'CommonErrors.html#CommonErrors-AccessDeniedException', iam({'Action': 'ListUsers'}, creds=USER))
-case(2, 'IAM CreateUser, non-admin user', (403, 'ErrorResponse', 'AccessDenied'),
-     IAMDOC + 'CommonErrors.html#CommonErrors-AccessDeniedException', iam({'Action': 'CreateUser', 'UserName': user + 'x'}, creds=USER))
-
-# ---------------- 3. wrong status or code ----------------
-tags = [('Action', 'TagUser'), ('UserName', user)] + [('Tags.member.%d.%s' % (i, k), 'k%d' % i if k == 'Key' else 'v')
-                                                       for i in range(1, 52) for k in ('Key', 'Value')]
-case(3, 'IAM TagUser with 51 tags', (409, 'ErrorResponse', 'LimitExceeded'), IAMDOC + 'API_TagUser.html#API_TagUser_Errors',
-     iam(dict(tags)))
-case(3, 'IAM unknown Action', (404, 'ErrorResponse', 'InvalidAction'), 'checked against iam.amazonaws.com',
-     iam({'Action': 'NoSuchActionZZ'}))
-
-
-def no_date(h):
-    return {k: v for k, v in h.items() if k.lower() != 'x-amz-date'}
-
-
-def drop(field):
-    return lambda h: dict(h, Authorization=re.sub(r',? *%s=[^,]*' % field, '', h['Authorization']))
-
-
-case(3, 'S3 request without x-amz-date', (403, 'Error', 'AccessDenied'), 'checked against s3.amazonaws.com; ' + S3DOC + 'API_Error.html#:~:text=Code%3A%20AccessDenied',
-     S3('GET', '/' + B, mutate=no_date))
-case(3, 'Authorization without Signature=', (400, 'Error', 'AuthorizationHeaderMalformed'),
-     'checked against s3.amazonaws.com; ' + S3DOC + 'API_Error.html#:~:text=Code%3A%20AuthorizationHeaderMalformed', S3('GET', '/' + B, mutate=drop('Signature')))
-case(3, 'Authorization without Credential=', (400, 'Error', 'InvalidArgument'),
-     'checked against s3.amazonaws.com ("Unsupported Authorization Type")', S3('GET', '/' + B, mutate=drop('Credential')))
-for m, exp, chk in (('GET', (400, 'Error', 'InvalidArgument'), 'checked against s3.amazonaws.com ("Invalid version id specified"); '),
-                    ('HEAD', (400, None, None), 'checked against s3.amazonaws.com; '),
-                    ('DELETE', (400, None, None), 'unverified (needs write access); ')):
-    case(3, '%s ?versionId=bogus, unversioned bucket' % m, exp,
-         chk + S3DOC + 'API_GetObject.html#AmazonS3-GetObject-request-uri-querystring-VersionId',
-         S3(m, '/%s/obj?versionId=bogus' % B))
-for pn in ('0', '10001'):
-    case(3, 'UploadPart partNumber=%s' % pn, (400, 'Error', 'InvalidArgument'),
-         S3DOC + 'API_UploadPart.html#AmazonS3-UploadPart-request-uri-querystring-PartNumber',
-         S3('PUT', '/%s/mp?partNumber=%s&uploadId=%s' % (B, pn, upload_id), body=b'x'))
-case(3, 'DeleteObjects with 1001 keys', (400, 'Error', 'MalformedXML'), S3DOC + 'API_DeleteObjects.html#API_DeleteObjects_RequestBody',
-     S3('POST', '/%s?delete' % B, body=('<Delete>' + ''.join('<Object><Key>k%d</Key></Object>' % i for i in range(1001)) + '</Delete>').encode()))
-# last: this one deletes the user when it should be refused
-case(3, 'IAM DeleteUser while it has an access key', (409, 'ErrorResponse', 'DeleteConflict'),
-     IAMDOC + 'API_DeleteUser.html#API_DeleteUser_Errors', iam({'Action': 'DeleteUser', 'UserName': user}))
-
-# ---------------- report ----------------
-w = max(len(r[1]) for r in rows)
-print('%-4s %-*s  %-46s %-40s %s' % ('', w, 'case', 'got (status root code)', 'expected (AWS)', 'side effect'))
-for g, name, got, exp, note, ok, ref in rows:
-    print('%-4s %-*s  %-46s %-40s %s' % ('ok' if ok else 'DIFF', w, name, got, exp, note))
-print('\nreferences:')
-for g, name, got, exp, note, ok, ref in rows:
-    print('  %-*s  %s' % (w, name, ref))
-diff = sum(not r[5] for r in rows)
-print('\n%d cases, %d differ from AWS' % (len(rows), diff))
-sys.exit(1 if diff else 0)

test/s3/error_compat/test_s3.json

@@ -1,20 +0,0 @@
-{
-  "identities": [
-    {
-      "name": "admin",
-      "credentials": [
-        {
-          "accessKey": "admin",
-          "secretKey": "admin"
-        }
-      ],
-      "actions": [
-        "Admin",
-        "Read",
-        "List",
-        "Tagging",
-        "Write"
-      ]
-    }
-  ]
-}

test/s3/iam/s3_sts_web_identity_api_test.go

@@ -1,258 +0,0 @@
-package iam
-
-import (
-	"crypto/rand"
-	"crypto/rsa"
-	"encoding/base64"
-	"encoding/json"
-	"encoding/xml"
-	"io"
-	"math/big"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/iam"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/golang-jwt/jwt/v5"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	webIdentityAudience = "web-identity-api-test"
-	webIdentitySubject  = "spiffe://example.org/ns/app/sa/app"
-)
-
-// webIdentityIssuer is an OIDC issuer served by the test: discovery document
-// and JWKS for one RSA key, so the server under test fetches real keys.
-type webIdentityIssuer struct {
-	server *httptest.Server
-	key    *rsa.PrivateKey
-}
-
-func newWebIdentityIssuer(t *testing.T) *webIdentityIssuer {
-	t.Helper()
-	key, err := rsa.GenerateKey(rand.Reader, 2048)
-	require.NoError(t, err)
-	issuer := &webIdentityIssuer{key: key}
-	mux := http.NewServeMux()
-	mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, _ *http.Request) {
-		_ = json.NewEncoder(w).Encode(map[string]any{
-			"issuer":                                issuer.server.URL,
-			"jwks_uri":                              issuer.server.URL + "/keys",
-			"id_token_signing_alg_values_supported": []string{"RS256"},
-		})
-	})
-	mux.HandleFunc("/keys", func(w http.ResponseWriter, _ *http.Request) {
-		b64 := func(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) }
-		_ = json.NewEncoder(w).Encode(map[string]any{"keys": []any{map[string]any{
-			"kty": "RSA", "kid": "k1", "use": "sig", "alg": "RS256",
-			"n": b64(key.PublicKey.N.Bytes()), "e": b64(big.NewInt(int64(key.PublicKey.E)).Bytes()),
-		}}})
-	})
-	issuer.server = httptest.NewServer(mux)
-	t.Cleanup(issuer.server.Close)
-	return issuer
-}
-
-func (i *webIdentityIssuer) claims(sub string) jwt.MapClaims {
-	now := time.Now()
-	return jwt.MapClaims{"iss": i.server.URL, "sub": sub, "aud": webIdentityAudience,
-		"iat": now.Unix(), "exp": now.Add(10 * time.Minute).Unix()}
-}
-
-func (i *webIdentityIssuer) token(t *testing.T, sub string, key *rsa.PrivateKey) string {
-	t.Helper()
-	tok := jwt.NewWithClaims(jwt.SigningMethodRS256, i.claims(sub))
-	tok.Header["kid"] = "k1"
-	signed, err := tok.SignedString(key)
-	require.NoError(t, err)
-	return signed
-}
-
-func (i *webIdentityIssuer) tokenForAudience(t *testing.T, sub, aud string) string {
-	t.Helper()
-	claims := i.claims(sub)
-	claims["aud"] = aud
-	tok := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)
-	tok.Header["kid"] = "k1"
-	signed, err := tok.SignedString(i.key)
-	require.NoError(t, err)
-	return signed
-}
-
-func (i *webIdentityIssuer) unsignedToken(t *testing.T, sub string) string {
-	t.Helper()
-	signed, err := jwt.NewWithClaims(jwt.SigningMethodNone, i.claims(sub)).SignedString(jwt.UnsafeAllowNoneSignatureType)
-	require.NoError(t, err)
-	return signed
-}
-
-func trustPolicyFor(issuerURL, sub string) string {
-	doc, _ := json.Marshal(map[string]any{"Version": "2012-10-17", "Statement": []any{map[string]any{
-		"Effect": "Allow", "Principal": map[string]any{"Federated": issuerURL},
-		"Action":    []string{"sts:AssumeRoleWithWebIdentity"},
-		"Condition": map[string]any{"StringEquals": map[string]any{"oidc:sub": sub}},
-	}}})
-	return string(doc)
-}
-
-// assumeWithWebIdentity returns session credentials, or nil and the error body.
-func assumeWithWebIdentity(t *testing.T, roleArn, token string) (*credentials.Credentials, string) {
-	t.Helper()
-	resp, err := callSTSAPI(t, url.Values{
-		"Action": {"AssumeRoleWithWebIdentity"}, "Version": {"2011-06-15"},
-		"RoleArn": {roleArn}, "RoleSessionName": {"web-identity-api"}, "WebIdentityToken": {token},
-	})
-	require.NoError(t, err)
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
-	require.NoError(t, err)
-	if resp.StatusCode != http.StatusOK {
-		return nil, string(body)
-	}
-	var out AssumeRoleWithWebIdentityTestResponse
-	require.NoError(t, xml.Unmarshal(body, &out), "body: %s", body)
-	c := out.Result.Credentials
-	return credentials.NewStaticCredentials(c.AccessKeyId, c.SecretAccessKey, c.SessionToken), ""
-}
-
-func s3ClientWith(t *testing.T, creds *credentials.Credentials) *s3.S3 {
-	t.Helper()
-	sess, err := session.NewSession(&aws.Config{
-		Region: aws.String(TestRegion), Endpoint: aws.String(TestS3Endpoint),
-		Credentials: creds, S3ForcePathStyle: aws.Bool(true), DisableSSL: aws.Bool(true),
-	})
-	require.NoError(t, err)
-	return s3.New(sess)
-}
-
-// TestWebIdentityWithProviderAndRoleManagedThroughIAMAPI configures STS
-// federation entirely at runtime — an OIDC provider, a managed policy and a role
-// created through the IAM API, with no static configuration — and checks that
-// the role admits exactly the subject its trust policy names, with exactly the
-// permissions of its attached policy.
-func TestWebIdentityWithProviderAndRoleManagedThroughIAMAPI(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-	if !isSTSEndpointRunning(t) {
-		t.Fatal("SeaweedFS STS endpoint is not running at", TestSTSEndpoint, "- please run 'make setup-all-tests' first")
-	}
-
-	framework := NewS3IAMTestFramework(t)
-	defer framework.Cleanup()
-	admin, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
-	require.NoError(t, err)
-	adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
-	require.NoError(t, err)
-
-	issuer := newWebIdentityIssuer(t)
-	bucket := framework.GenerateUniqueBucketName("web-identity")
-	require.NoError(t, framework.CreateBucketWithCleanup(adminS3, bucket))
-
-	provider, err := admin.CreateOpenIDConnectProvider(&iam.CreateOpenIDConnectProviderInput{
-		Url:          aws.String(issuer.server.URL),
-		ClientIDList: []*string{aws.String(webIdentityAudience)},
-		// Required by this SDK version; pinning only applies to a TLS issuer,
-		// and the test issuer is plain HTTP.
-		ThumbprintList: []*string{aws.String("0000000000000000000000000000000000000000")},
-	})
-	require.NoError(t, err)
-	defer admin.DeleteOpenIDConnectProvider(&iam.DeleteOpenIDConnectProviderInput{OpenIDConnectProviderArn: provider.OpenIDConnectProviderArn})
-
-	policyDoc, _ := json.Marshal(map[string]any{"Version": "2012-10-17", "Statement": []any{map[string]any{
-		"Effect": "Allow", "Action": []string{"s3:*"},
-		"Resource": []string{"arn:aws:s3:::" + bucket, "arn:aws:s3:::" + bucket + "/*"},
-	}}})
-	policy, err := admin.CreatePolicy(&iam.CreatePolicyInput{
-		PolicyName: aws.String(bucket + "-rw"), PolicyDocument: aws.String(string(policyDoc)),
-	})
-	require.NoError(t, err)
-	defer admin.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policy.Policy.Arn})
-
-	// Role names are at most 64 characters; the bucket name is longer.
-	roleName := "web-identity-" + strconv.FormatInt(time.Now().UnixNano(), 36)
-	role, err := admin.CreateRole(&iam.CreateRoleInput{
-		RoleName:                 aws.String(roleName),
-		AssumeRolePolicyDocument: aws.String(trustPolicyFor(issuer.server.URL, webIdentitySubject)),
-	})
-	require.NoError(t, err)
-	defer admin.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
-	_, err = admin.AttachRolePolicy(&iam.AttachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
-	require.NoError(t, err)
-	defer admin.DetachRolePolicy(&iam.DetachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
-	roleArn := aws.StringValue(role.Role.Arn)
-
-	t.Run("the trusted subject gets credentials scoped to the attached policy", func(t *testing.T) {
-		creds, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
-		require.NotNil(t, creds, "AssumeRoleWithWebIdentity refused the trusted subject: %s", failure)
-		client := s3ClientWith(t, creds)
-		_, err := client.PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("federated.txt")})
-		assert.NoError(t, err, "the session cannot write the bucket its policy grants")
-		_, err = client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket + "-other")})
-		assert.Error(t, err, "the session created a bucket its policy does not grant")
-	})
-
-	refusals := map[string]string{
-		"another subject": issuer.token(t, "spiffe://example.org/ns/other/sa/other", issuer.key),
-		"a token signed by another key": func() string {
-			other, err := rsa.GenerateKey(rand.Reader, 2048)
-			require.NoError(t, err)
-			return issuer.token(t, webIdentitySubject, other)
-		}(),
-		"an unsigned token": issuer.unsignedToken(t, webIdentitySubject),
-		// The provider accepts only its registered client IDs as audiences.
-		"a token issued for another audience": issuer.tokenForAudience(t, webIdentitySubject, "some-other-service"),
-	}
-	for name, token := range refusals {
-		t.Run("refuses "+name, func(t *testing.T) {
-			creds, _ := assumeWithWebIdentity(t, roleArn, token)
-			assert.Nil(t, creds, "AssumeRoleWithWebIdentity issued credentials for %s", name)
-		})
-	}
-
-	t.Run("an updated trust policy takes effect", func(t *testing.T) {
-		moved := "spiffe://example.org/ns/app/sa/moved"
-		_, err := admin.UpdateAssumeRolePolicy(&iam.UpdateAssumeRolePolicyInput{
-			RoleName: aws.String(roleName), PolicyDocument: aws.String(trustPolicyFor(issuer.server.URL, moved)),
-		})
-		require.NoError(t, err)
-		creds, _ := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
-		assert.Nil(t, creds, "the subject removed from the trust policy still assumes the role")
-		creds, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, moved, issuer.key))
-		assert.NotNil(t, creds, "the subject added to the trust policy was refused: %s", failure)
-	})
-
-	t.Run("a session does not survive its role being deleted and created again", func(t *testing.T) {
-		trust := trustPolicyFor(issuer.server.URL, webIdentitySubject)
-		_, err := admin.UpdateAssumeRolePolicy(&iam.UpdateAssumeRolePolicyInput{RoleName: aws.String(roleName), PolicyDocument: aws.String(trust)})
-		require.NoError(t, err)
-		before, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
-		require.NotNil(t, before, "precondition: %s", failure)
-
-		_, err = admin.DetachRolePolicy(&iam.DetachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
-		require.NoError(t, err)
-		_, err = admin.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
-		require.NoError(t, err)
-		recreated, err := admin.CreateRole(&iam.CreateRoleInput{RoleName: aws.String(roleName), AssumeRolePolicyDocument: aws.String(trust)})
-		require.NoError(t, err)
-		_, err = admin.AttachRolePolicy(&iam.AttachRolePolicyInput{RoleName: aws.String(roleName), PolicyArn: policy.Policy.Arn})
-		require.NoError(t, err)
-		assert.NotEqual(t, aws.StringValue(role.Role.RoleId), aws.StringValue(recreated.Role.RoleId), "the recreated role reuses the deleted role's ID")
-
-		_, err = s3ClientWith(t, before).PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("revived.txt")})
-		assert.Error(t, err, "a session of the deleted role works again under the new role of the same name")
-		after, failure := assumeWithWebIdentity(t, roleArn, issuer.token(t, webIdentitySubject, issuer.key))
-		require.NotNil(t, after, "the new role refuses its trusted subject: %s", failure)
-		_, err = s3ClientWith(t, after).PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("fresh.txt")})
-		assert.NoError(t, err, "a session of the new role cannot use its policy")
-	})
-}

test/s3/remote_cache/remote_cache_evict_test.go

@@ -1,231 +0,0 @@
-package remote_cache
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	evictRemoteS3      = "http://localhost:28334"
-	evictRemoteMaster  = "30334"
-	evictRemoteFiler   = "28889"
-	evictRemoteVolume  = "30341"
-	evictRemoteWebdav  = "27334"
-	evictRemoteMetrics = "30326"
-
-	evictPrimaryS3      = "http://localhost:28333"
-	evictPrimaryMaster  = "30333"
-	evictPrimaryFiler   = "28887"
-	evictPrimaryVolume  = "30340"
-	evictPrimaryWebdav  = "27333"
-	evictPrimaryMetrics = "30327"
-
-	evictBucket = "evictsrc"
-	evictMount  = "evictmnt"
-)
-
-var miniLogPaths []string
-
-func startMini(t *testing.T, dir string, args ...string) {
-	t.Helper()
-	require.NoError(t, os.MkdirAll(dir, 0755))
-	logPath := filepath.Join(dir, "weed.log")
-	logFile, err := os.Create(logPath)
-	require.NoError(t, err)
-	miniLogPaths = append(miniLogPaths, logPath)
-	cmd := exec.Command(weedBinary, append([]string{"mini",
-		"-dir=" + dir,
-		"-s3.config=s3_config.json",
-		"-s3.allowDeleteBucketNotEmpty=true",
-		"-ip=127.0.0.1", "-ip.bind=127.0.0.1",
-	}, args...)...)
-	cmd.Stdout = logFile
-	cmd.Stderr = logFile
-	require.NoError(t, cmd.Start())
-	t.Cleanup(func() {
-		cmd.Process.Kill()
-		cmd.Wait()
-		logFile.Close()
-	})
-}
-
-func waitForHTTP(t *testing.T, url string) {
-	t.Helper()
-	deadline := time.Now().Add(90 * time.Second)
-	for time.Now().Before(deadline) {
-		if resp, err := http.Get(url); err == nil {
-			resp.Body.Close()
-			return
-		}
-		time.Sleep(time.Second)
-	}
-	for _, p := range miniLogPaths {
-		if data, err := os.ReadFile(p); err == nil {
-			lines := strings.Split(string(data), "\n")
-			if len(lines) > 30 {
-				lines = lines[len(lines)-30:]
-			}
-			t.Logf("last lines of %s:\n%s", p, strings.Join(lines, "\n"))
-		}
-	}
-	t.Fatalf("timed out waiting for %s", url)
-}
-
-func shellOn(t *testing.T, masterPort, command string) string {
-	t.Helper()
-	cmd := exec.Command(weedBinary, "shell", "-master=localhost:"+masterPort)
-	cmd.Stdin = strings.NewReader(command + "\nexit\n")
-	out, err := cmd.CombinedOutput()
-	require.NoErrorf(t, err, "shell %q failed: %s", command, out)
-	return stripLogs(string(out))
-}
-
-func chunkCountOn(t *testing.T, masterPort, path string) string {
-	meta := shellOn(t, masterPort, "fs.meta.cat "+path)
-	idx := strings.LastIndex(meta, "chunks ")
-	require.GreaterOrEqualf(t, idx, 0, "no chunk count in %s", meta)
-	return strings.Fields(meta[idx+len("chunks "):])[0]
-}
-
-func volumeStats(t *testing.T, volumePort string) (size, garbage uint64) {
-	t.Helper()
-	resp, err := http.Get("http://localhost:" + volumePort + "/status")
-	require.NoError(t, err)
-	defer resp.Body.Close()
-	var status struct {
-		Volumes []struct {
-			Size             uint64 `json:"Size"`
-			DeletedByteCount uint64 `json:"DeletedByteCount"`
-		} `json:"Volumes"`
-	}
-	require.NoError(t, json.NewDecoder(resp.Body).Decode(&status))
-	for _, v := range status.Volumes {
-		size += v.Size
-		garbage += v.DeletedByteCount
-	}
-	return size, garbage
-}
-
-func readViaFiler(t *testing.T, filerPort, path string) []byte {
-	t.Helper()
-	resp, err := http.Get("http://localhost:" + filerPort + path)
-	require.NoError(t, err)
-	defer resp.Body.Close()
-	require.Equal(t, http.StatusOK, resp.StatusCode, "read %s", path)
-	data, err := io.ReadAll(resp.Body)
-	require.NoError(t, err)
-	return data
-}
-
-// TestRemoteCacheEvictUnderPressure fills a cache constrained to two small
-// volumes with remote-mounted objects until writes fail, then verifies the
-// filer evicts the oldest synced entry, vacuums the garbage, and a later read
-// caches again.
-func TestRemoteCacheEvictUnderPressure(t *testing.T) {
-	if testing.Short() {
-		t.Skip("spawns two weed mini clusters")
-	}
-	if _, err := os.Stat(weedBinary); err != nil {
-		t.Skipf("weed binary not found at %s; run make build-weed", weedBinary)
-	}
-	if isServerRunning(evictRemoteS3) || isServerRunning(evictPrimaryS3) {
-		t.Skip("eviction test ports are already in use")
-	}
-
-	tmp := t.TempDir()
-	startMini(t, filepath.Join(tmp, "remote"),
-		"-s3.port=28334", "-master.port="+evictRemoteMaster,
-		"-filer.port="+evictRemoteFiler, "-volume.port="+evictRemoteVolume,
-		"-webdav.port="+evictRemoteWebdav, "-metricsPort="+evictRemoteMetrics)
-	waitForHTTP(t, evictRemoteS3)
-	startMini(t, filepath.Join(tmp, "primary"),
-		"-s3.port=28333", "-master.port="+evictPrimaryMaster,
-		"-filer.port="+evictPrimaryFiler, "-volume.port="+evictPrimaryVolume,
-		"-webdav.port="+evictPrimaryWebdav, "-metricsPort="+evictPrimaryMetrics,
-		"-volume.allowUntrustedRemoteEndpoints", "-filer.allowUntrustedRemoteEndpoints",
-		"-s3.allowUntrustedRemoteEndpoints",
-		"-master.volumeSizeLimitMB=32", "-volume.max=2",
-		"-filer.remoteCacheEvictThreshold=0.99")
-	waitForHTTP(t, evictPrimaryS3)
-
-	remote := createS3Client(evictRemoteS3)
-	_, err := remote.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(evictBucket)})
-	require.NoError(t, err)
-	for i := 0; i < 5; i++ {
-		data := make([]byte, 16*1024*1024)
-		for j := range data {
-			data[j] = byte(i + j%251)
-		}
-		_, err = remote.PutObject(&s3.PutObjectInput{
-			Bucket: aws.String(evictBucket),
-			Key:    aws.String(fmt.Sprintf("obj%d.bin", i)),
-			Body:   bytes.NewReader(data),
-		})
-		require.NoError(t, err)
-	}
-
-	shellOn(t, evictPrimaryMaster, fmt.Sprintf(
-		"remote.configure -name=evictremote -type=s3 -s3.access_key=%s -s3.secret_key=%s -s3.endpoint=%s -s3.region=us-east-1",
-		accessKey, secretKey, evictRemoteS3))
-	shellOn(t, evictPrimaryMaster, fmt.Sprintf(
-		"remote.mount -dir=/buckets/%s -remote=evictremote/%s -nonempty", evictMount, evictBucket))
-	shellOn(t, evictPrimaryMaster, fmt.Sprintf("remote.meta.sync -dir=/buckets/%s", evictMount))
-	time.Sleep(2 * time.Second)
-
-	mount := "/buckets/" + evictMount
-
-	// obj0 caches alone so it is the oldest evictable entry.
-	first := readViaFiler(t, evictPrimaryFiler, mount+"/obj0.bin")
-	require.Len(t, first, 16*1024*1024)
-	require.NotEqual(t, "0", chunkCountOn(t, evictPrimaryMaster, mount+"/obj0.bin"), "obj0 should be cached")
-
-	// 4 x 16MB against ~64MB of capacity: the fills overrun, hit the
-	// capacity error path, and trigger eviction + vacuum.
-	var wg sync.WaitGroup
-	for i := 1; i <= 4; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			data := readViaFiler(t, evictPrimaryFiler, fmt.Sprintf("%s/obj%d.bin", mount, i))
-			assert.Len(t, data, 16*1024*1024)
-		}(i)
-	}
-	wg.Wait()
-
-	// Eviction must have dropped obj0's local chunks and vacuumed the
-	// tombstoned bytes, leaving only live data within the two volumes.
-	require.True(t, waitForCondition(t, func() bool {
-		size, garbage := volumeStats(t, evictPrimaryVolume)
-		return garbage == 0 && size <= 72*1024*1024
-	}, 2*time.Minute, "volumes to be reclaimed by eviction+vacuum"),
-		"evicted chunks were not reclaimed")
-
-	assert.Equal(t, "0", chunkCountOn(t, evictPrimaryMaster, mount+"/obj0.bin"),
-		"oldest cached entry should be evicted back to remote-only")
-
-	// The cache self-heals: reading the evicted object re-caches it. Fills
-	// still running from the concurrent wave may evict it again, so retry
-	// the read until a commit sticks.
-	var again []byte
-	require.True(t, waitForCondition(t, func() bool {
-		again = readViaFiler(t, evictPrimaryFiler, mount+"/obj0.bin")
-		return chunkCountOn(t, evictPrimaryMaster, mount+"/obj0.bin") != "0"
-	}, 2*time.Minute, "obj0 to re-cache"),
-		"evicted object did not re-cache on read")
-	assert.Equal(t, first, again)
-}

test/s3tables/catalog_olake/Dockerfile.reader

@@ -1,14 +0,0 @@
-# Strict Iceberg reader used by the OLake integration test. PyIceberg is
-# deliberate here: it rejects manifests that omit the spec's field ids and
-# parquet without either field ids or a name mapping, so a passing read proves
-# the catalog served something every engine can consume -- not just something
-# OLake itself can read back.
-FROM python:3.11-slim
-
-WORKDIR /app
-
-RUN pip install --no-cache-dir "pyiceberg[s3fs]==0.11.1" "pyarrow==25.0.0"
-
-COPY inspect_table.py /app/
-
-ENTRYPOINT ["python3", "/app/inspect_table.py"]

test/s3tables/catalog_olake/README.md

@@ -1,104 +0,0 @@
-# OLake Iceberg Catalog Integration Test
-
-An integration test for [OLake](https://github.com/datazip-inc/olake) against
-SeaweedFS's Iceberg REST Catalog, in the same shape as `catalog_clickhouse`.
-
-## Why OLake, given we already test five engines
-
-Two things here are covered by nothing else in this directory.
-
-**It is a strict Java Iceberg client.** OLake does not write Iceberg from Go —
-its Go process spawns a Java sidecar over gRPC and writes through the official
-Apache Iceberg library, because the Go library has no equality deletes and CDC
-needs them. So this test exercises the client class that
-`weed/s3api/iceberg/metadata_compliance.go` exists to serve: the one
-that fails with *"Cannot parse missing long current-snapshot-id"* when the
-catalog omits spec-required keys that `iceberg-go` strips via `omitempty`.
-
-**It produces equality deletes.** OLake is a CDC tool. Its upsert path commits
-`operation=overwrite` with an equality-delete file, and a delete manifest
-alongside the data manifests. ClickHouse, StarRocks, Doris and DuckDB all only
-append.
-
-## What it asserts
-
-`TestOLakeIcebergCatalog` runs six subtests against a `weed mini` cluster with
-a pre-created table bucket and a Postgres source:
-
-| Subtest | What a failure means |
-|---|---|
-| `CheckDestination` | `olake check` did not reach `SUCCEEDED`, or it passed without ever loading `org.apache.iceberg.rest.RESTSessionCatalog` — the second case means the destination was never actually contacted. |
-| `Discover` | OLake could not enumerate the source table, or wrote no `streams.json`. |
-| `FullSyncAppendsRows` | The sync read fewer than the three seeded rows, or committed no Iceberg snapshot. |
-| `StrictReaderSeesRows` | PyIceberg could not read back what the Java writer committed, or the values differ. This is the data path, not just metadata. |
-| `UpsertProducesEqualityDelete` | After an `UPDATE` and a re-sync, no snapshot recorded an overwrite carrying equality deletes, or the current snapshot has no delete manifest. |
-| `CompliantWriterNeedsNoRepair` | The catalog rewrote a manifest the official Iceberg Java writer produced. That is a regression in the repair gate, not a problem with OLake. |
-
-## The one thing this test deliberately does not check
-
-It does **not** assert that a reader sees the updated row and no duplicate.
-
-PyIceberg refuses to scan a table carrying equality deletes
-([apache/iceberg#6568](https://github.com/apache/iceberg/issues/6568)) while
-reading its metadata perfectly well — so a rows-mode read after the upsert would
-raise, not pass. The alternative is an engine that applies equality deletes,
-which for StarRocks means a 3 GB image and 12 GB of RAM in CI.
-
-The line drawn instead: **recording the commit correctly is the catalog's
-contract; applying deletes on read is the query engine's.** The metadata
-assertions cover our half.
-
-This was verified once by hand outside CI, on 2026-09-23, with StarRocks 4.1.4
-attached to the same catalog: after the upsert it read 3 rows / 3 distinct ids
-with `id=1` showing the updated value and `_op_type=u`. If someone later wants
-that inside the gate, add a reader that supports equality deletes — **do not**
-"upgrade" this test to a PyIceberg rows-mode read after the upsert. It would not
-pass; and if PyIceberg ever starts silently skipping deletes instead of raising,
-it would pass by not looking.
-
-## What the config proves
-
-Nothing in the destination config is SeaweedFS-specific:
-
-```json
-{
-  "catalog_type": "rest",
-  "rest_catalog_url": "http://HOST:ICEBERG_PORT",
-  "iceberg_s3_path": "s3://olake-tables",
-  "s3_endpoint": "http://HOST:S3_PORT",
-  "rest_auth_type": "oauth2",
-  "oauth2_uri": "http://HOST:ICEBERG_PORT/v1/oauth/tokens",
-  "credential": "ACCESS_KEY:SECRET_KEY"
-}
-```
-
-`catalog_type` is the generic `rest`, auth is the standard OAuth2
-client-credentials flow, and `s3_path_style` does not even need setting —
-OLake turns it on by itself whenever `s3_endpoint` is non-empty. As of
-OLake v0.10.1 this works with no change on either side.
-
-## Running it
-
-```sh
-go test ./test/s3tables/catalog_olake/ -run TestOLakeIcebergCatalog -v -timeout 25m
-```
-
-Needs Docker and a `weed` binary (at `weed/weed` under the repo root, or on
-`PATH`). Takes about 35 seconds. It skips rather than fails when Docker is
-absent, and `SEAWEEDFS_SKIP_OLAKE_TESTS=1` skips it outright.
-
-Overrides: `OLAKE_IMAGE` (default `olakego/source-postgres:latest`),
-`POSTGRES_IMAGE` (default `postgres:16`).
-
-## In CI
-
-Runs as `olake-iceberg-catalog-tests` in `.github/workflows/s3-tables-tests.yml`,
-on a matrix of a pinned image plus `latest` — the same shape the ClickHouse job
-uses, and for the same reason: OLake's Iceberg writer is a Java sidecar whose
-library version moves independently of the Go release, so the `latest` leg is
-what catches drift in the client rather than in OLake itself.
-
-The job asserts the suite actually ran — at least one top-level `--- PASS` and
-zero `--- SKIP` — rather than trusting a green exit. This suite skips itself
-when Docker is unavailable, and a skipped suite reporting success is how a gate
-quietly stops being one.

test/s3tables/catalog_olake/inspect_table.py

@@ -1,91 +0,0 @@
-#!/usr/bin/env python3
-"""Inspect an Iceberg table written by OLake, through the SeaweedFS REST catalog.
-
-Two modes, because a strict reader cannot do both:
-
-  rows      -- scan the table and print "id,region,amount" per row, ordered by
-               id. Only valid while the table has no equality deletes.
-  snapshots -- print one line per snapshot with its operation and delete-file
-               counters, plus the manifest content kinds of the current
-               snapshot.
-
-The split exists because PyIceberg refuses to scan a table carrying equality
-deletes (apache/iceberg#6568) while reading its metadata perfectly well. OLake
-is a CDC tool, so its upsert path produces exactly those deletes -- asserting
-the commit landed is the catalog's concern, and applying deletes on read is the
-query engine's.
-"""
-
-import argparse
-import sys
-
-from pyiceberg.catalog import load_catalog
-
-
-def main() -> int:
-    p = argparse.ArgumentParser()
-    p.add_argument("mode", choices=["rows", "snapshots"])
-    p.add_argument("--catalog-url", required=True)
-    p.add_argument("--warehouse", required=True)
-    p.add_argument("--prefix", required=True)
-    p.add_argument("--s3-endpoint", required=True)
-    p.add_argument("--access-key", required=True)
-    p.add_argument("--secret-key", required=True)
-    p.add_argument("--region", default="us-east-1")
-    p.add_argument("--namespace", action="append", required=True)
-    p.add_argument("--table", required=True)
-    args = p.parse_args()
-
-    catalog = load_catalog(
-        "rest",
-        **{
-            "type": "rest",
-            "uri": args.catalog_url,
-            "warehouse": args.warehouse,
-            "prefix": args.prefix,
-            "credential": f"{args.access_key}:{args.secret_key}",
-            "s3.access-key-id": args.access_key,
-            "s3.secret-access-key": args.secret_key,
-            "s3.endpoint": args.s3_endpoint,
-            "s3.region": args.region,
-            "s3.path-style-access": "true",
-        },
-    )
-
-    table = catalog.load_table(tuple(args.namespace) + (args.table,))
-
-    if args.mode == "rows":
-        data = table.scan().to_arrow().to_pydict()
-        # amount is decimal(10,2) in Postgres but arrives here as a float, whose
-        # repr drops trailing zeros (120.5, not 120.50). Format it to the source
-        # scale so the expected values in the Go test stay readable.
-        for row_id, region, amount in sorted(
-            zip(data["id"], data["region"], data["amount"])
-        ):
-            print("%s,%s,%.2f" % (row_id, region, float(amount)))
-        return 0
-
-    print("format-version=%d" % table.metadata.format_version)
-    ids = table.metadata.schemas[-1].identifier_field_ids
-    print("identifier-field-ids=%s" % ",".join(str(i) for i in ids))
-    for snap in table.metadata.snapshots:
-        s = snap.summary
-        print(
-            "snapshot operation=%s total-delete-files=%s added-delete-files=%s "
-            "added-equality-deletes=%s total-records=%s"
-            % (
-                s.operation,
-                s.get("total-delete-files", "0"),
-                s.get("added-delete-files", "0"),
-                s.get("added-equality-deletes", "0"),
-                s.get("total-records", "0"),
-            )
-        )
-    current = table.current_snapshot()
-    kinds = [str(m.content).rsplit(".", 1)[-1] for m in current.manifests(table.io)]
-    print("current-manifest-kinds=%s" % ",".join(kinds))
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

test/s3tables/catalog_olake/olake_catalog_test.go

@@ -1,623 +0,0 @@
-// Package catalog_olake provides an integration test for OLake
-// (github.com/datazip-inc/olake) against the SeaweedFS Iceberg REST Catalog.
-//
-// OLake matters here for two reasons that no other engine in this directory
-// covers. First, it does not write Iceberg from Go: it spawns a Java sidecar
-// over gRPC and writes through the official Apache Iceberg library, so this is
-// a strict Java client -- the class that iceberg/metadata_compliance.go exists
-// to serve. Second, it is a CDC tool, so its upsert path emits equality
-// deletes, which neither the ClickHouse nor the Doris test exercises.
-package catalog_olake
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/testutil"
-)
-
-const (
-	olakeDefaultImage    = "olakego/source-postgres:latest"
-	postgresDefaultImage = "postgres:16"
-	readerImage          = "seaweedfs-olake-reader:test"
-
-	tableBucketName = "olake-tables"
-	sourceDatabase  = "olakedb"
-	sourceUser      = "olake"
-	sourcePassword  = "olakepw"
-
-	// OLake derives the destination namespace from the source: it joins the
-	// connector name, database and schema. Asserting the derived name rather
-	// than configuring one keeps the test honest about what OLake actually
-	// does with our catalog.
-	expectedNamespace = "postgres_olakedb_public"
-	expectedTable     = "orders"
-
-	postgresStartTimeout = 90 * time.Second
-	olakeRunTimeout      = 6 * time.Minute
-)
-
-type TestEnvironment struct {
-	seaweedDir string
-	weedBinary string
-	dataDir    string
-	configDir  string
-	bindIP     string
-
-	masterPort     int
-	masterGrpcPort int
-	volumePort     int
-	volumeGrpcPort int
-	filerPort      int
-	filerGrpcPort  int
-	s3Port         int
-	s3GrpcPort     int
-	icebergPort    int
-	postgresPort   int
-
-	accessKey string
-	secretKey string
-
-	weedProcess       *exec.Cmd
-	weedCancel        func()
-	postgresContainer string
-}
-
-func TestOLakeIcebergCatalog(t *testing.T) {
-	requireOLakeRuntime(t)
-
-	env := NewTestEnvironment(t)
-	defer env.Cleanup(t)
-
-	env.StartSeaweedFS(t)
-	env.startPostgres(t)
-	env.seedSource(t)
-
-	buildReaderImage(t)
-	env.writeOLakeConfigs(t)
-
-	t.Run("CheckDestination", func(t *testing.T) {
-		out := env.runOLake(t, "check",
-			"--config", "/mnt/config/source.json",
-			"--destination", "/mnt/config/destination.json")
-		if !strings.Contains(out, `"status":"SUCCEEDED"`) {
-			t.Fatalf("olake check did not report SUCCEEDED.\n%s", tailLines(out, 40))
-		}
-		// The Java sidecar, not the Go process, is what talks to our catalog.
-		// If this line is missing the check passed without exercising the
-		// Iceberg REST path at all.
-		if !strings.Contains(out, "org.apache.iceberg.rest.RESTSessionCatalog") {
-			t.Errorf("check passed but never loaded the Iceberg REST catalog; "+
-				"the destination may not have been contacted.\n%s", tailLines(out, 40))
-		}
-	})
-
-	t.Run("Discover", func(t *testing.T) {
-		out := env.runOLake(t, "discover",
-			"--config", "/mnt/config/source.json",
-			"--destination", "/mnt/config/destination.json")
-		if !strings.Contains(out, `"stream_name":"`+expectedTable+`"`) {
-			t.Fatalf("discover did not report the %s stream.\n%s", expectedTable, tailLines(out, 20))
-		}
-		if _, err := os.Stat(filepath.Join(env.configDir, "streams.json")); err != nil {
-			t.Fatalf("discover did not write streams.json: %v", err)
-		}
-	})
-
-	t.Run("FullSyncAppendsRows", func(t *testing.T) {
-		out := env.runOLake(t, "sync",
-			"--config", "/mnt/config/source.json",
-			"--destination", "/mnt/config/destination.json",
-			"--streams", "/mnt/config/streams.json",
-			"--state", "/mnt/config/state.json")
-		if !strings.Contains(out, "Total records read: 3") {
-			t.Fatalf("sync did not read the three seeded rows.\n%s", tailLines(out, 30))
-		}
-		if !strings.Contains(out, "Committed snapshot") {
-			t.Fatalf("sync never committed an Iceberg snapshot.\n%s", tailLines(out, 30))
-		}
-	})
-
-	t.Run("StrictReaderSeesRows", func(t *testing.T) {
-		rows := env.readTable(t, "rows")
-		want := []string{"1,us-east,120.50", "2,us-west,87.20", "3,eu-west,210.00"}
-		got := nonEmptyLines(rows)
-		if len(got) != len(want) {
-			t.Fatalf("PyIceberg read %d rows, want %d.\n%s", len(got), len(want), rows)
-		}
-		for i := range want {
-			if got[i] != want[i] {
-				t.Errorf("row %d = %q, want %q", i, got[i], want[i])
-			}
-		}
-	})
-
-	t.Run("UpsertProducesEqualityDelete", func(t *testing.T) {
-		// Push the row past the incremental cursor so the next sync re-emits it
-		// as an update rather than skipping it.
-		env.execSQL(t, "UPDATE orders SET amount=999.99, region='ap-south' WHERE id=1")
-
-		out := env.runOLake(t, "sync",
-			"--config", "/mnt/config/source.json",
-			"--destination", "/mnt/config/destination.json",
-			"--streams", "/mnt/config/streams.json",
-			"--state", "/mnt/config/state.json")
-		if !strings.Contains(out, "delete files") {
-			t.Fatalf("re-sync committed no delete files.\n%s", tailLines(out, 30))
-		}
-
-		meta := env.readTable(t, "snapshots")
-		if !strings.Contains(meta, "format-version=2") {
-			t.Errorf("expected Iceberg format-version 2, got:\n%s", meta)
-		}
-		if !strings.Contains(meta, "identifier-field-ids=") ||
-			strings.Contains(meta, "identifier-field-ids=\n") {
-			t.Errorf("table carries no identifier fields, so OLake could not "+
-				"have upserted:\n%s", meta)
-		}
-
-		var sawOverwrite bool
-		for _, line := range nonEmptyLines(meta) {
-			if !strings.HasPrefix(line, "snapshot operation=") {
-				continue
-			}
-			if strings.Contains(line, "operation=Operation.OVERWRITE") &&
-				!strings.Contains(line, "added-equality-deletes=0") {
-				sawOverwrite = true
-			}
-		}
-		if !sawOverwrite {
-			t.Errorf("no snapshot recorded an overwrite carrying equality "+
-				"deletes:\n%s", meta)
-		}
-		if !strings.Contains(meta, "current-manifest-kinds=") ||
-			!strings.Contains(meta, "DELETES") {
-			t.Errorf("current snapshot has no delete manifest:\n%s", meta)
-		}
-		// Deliberately NOT asserted here: that a reader sees the updated value
-		// and no duplicate row. PyIceberg refuses to scan a table carrying
-		// equality deletes (apache/iceberg#6568), so a rows-mode read would
-		// fail rather than pass, and swapping in an engine that can apply them
-		// costs this test a multi-gigabyte image. Applying deletes on read is
-		// the engine's contract; recording the commit correctly is ours, and
-		// that is what the assertions above cover. See the README.
-	})
-
-	t.Run("CompliantWriterNeedsNoRepair", func(t *testing.T) {
-		names := env.listTableMetadata(t)
-		if len(names) == 0 {
-			t.Fatalf("no metadata files found for %s.%s", expectedNamespace, expectedTable)
-		}
-		for _, n := range names {
-			if strings.HasPrefix(n, "repaired-") {
-				t.Errorf("catalog repaired a manifest written by OLake (%s); "+
-					"the official Iceberg Java writer is expected to be "+
-					"spec-compliant, so this is a regression in the repair "+
-					"gate rather than in OLake", n)
-			}
-		}
-	})
-}
-
-func NewTestEnvironment(t *testing.T) *TestEnvironment {
-	t.Helper()
-
-	wd, err := os.Getwd()
-	if err != nil {
-		t.Fatalf("Failed to get working directory: %v", err)
-	}
-
-	seaweedDir := wd
-	for i := 0; i < 6; i++ {
-		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
-			break
-		}
-		seaweedDir = filepath.Dir(seaweedDir)
-	}
-
-	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
-	if info, err := os.Stat(weedBinary); err != nil || info.IsDir() {
-		weedBinary = filepath.Join(seaweedDir, "weed", "weed", "weed")
-		if info, err := os.Stat(weedBinary); err != nil || info.IsDir() {
-			weedBinary = "weed"
-			if _, err := exec.LookPath(weedBinary); err != nil {
-				t.Skip("weed binary not found, skipping integration test")
-			}
-		}
-	}
-
-	dataDir, err := os.MkdirTemp("", "seaweed-olake-test-*")
-	if err != nil {
-		t.Fatalf("Failed to create temp dir: %v", err)
-	}
-	configDir := filepath.Join(dataDir, "olake")
-	if err := os.MkdirAll(configDir, 0755); err != nil {
-		t.Fatalf("Failed to create config dir: %v", err)
-	}
-
-	// 9 for the mini cluster, 1 for the Postgres source mapped on the host.
-	ports := testutil.MustAllocatePorts(t, 10)
-
-	return &TestEnvironment{
-		seaweedDir:     seaweedDir,
-		weedBinary:     weedBinary,
-		dataDir:        dataDir,
-		configDir:      configDir,
-		bindIP:         testutil.FindBindIP(),
-		masterPort:     ports[0],
-		masterGrpcPort: ports[1],
-		volumePort:     ports[2],
-		volumeGrpcPort: ports[3],
-		filerPort:      ports[4],
-		filerGrpcPort:  ports[5],
-		s3Port:         ports[6],
-		s3GrpcPort:     ports[7],
-		icebergPort:    ports[8],
-		postgresPort:   ports[9],
-		accessKey:      "AKIAIOSFODNN7EXAMPLE",
-		secretKey:      "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
-	}
-}
-
-func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
-	t.Helper()
-
-	iamConfigPath, err := testutil.WriteIAMConfig(env.dataDir, env.accessKey, env.secretKey)
-	if err != nil {
-		t.Fatalf("Failed to create IAM config: %v", err)
-	}
-
-	cmd := exec.Command(env.weedBinary, "mini",
-		"-master.port", fmt.Sprintf("%d", env.masterPort),
-		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
-		"-volume.port", fmt.Sprintf("%d", env.volumePort),
-		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
-		"-filer.port", fmt.Sprintf("%d", env.filerPort),
-		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
-		"-s3.port", fmt.Sprintf("%d", env.s3Port),
-		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
-		"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergPort),
-		"-s3.config", iamConfigPath,
-		// Pre-create the table bucket the way an operator would, rather than
-		// reaching for the S3 Tables control plane from the test.
-		"-tableBucket", tableBucketName,
-		"-ip", env.bindIP,
-		"-ip.bind", "0.0.0.0",
-		"-dir", env.dataDir,
-	)
-	cmd.Dir = env.dataDir
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	cmd.Env = append(os.Environ(),
-		"AWS_ACCESS_KEY_ID="+env.accessKey,
-		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
-	)
-
-	if err := cmd.Start(); err != nil {
-		t.Fatalf("Failed to start SeaweedFS: %v", err)
-	}
-	env.weedProcess = cmd
-	env.weedCancel = func() {
-		if cmd.Process != nil {
-			_ = cmd.Process.Kill()
-		}
-	}
-
-	url := fmt.Sprintf("http://%s:%d/v1/config", env.bindIP, env.icebergPort)
-	if !waitForService(url, 45*time.Second) {
-		t.Fatalf("Iceberg REST API did not become ready at %s", url)
-	}
-}
-
-func (env *TestEnvironment) startPostgres(t *testing.T) {
-	t.Helper()
-
-	name := fmt.Sprintf("olake-pg-%d", env.postgresPort)
-	_ = exec.Command("docker", "rm", "-f", name).Run()
-
-	cmd := exec.Command("docker", "run", "-d", "--name", name,
-		"-e", "POSTGRES_USER="+sourceUser,
-		"-e", "POSTGRES_PASSWORD="+sourcePassword,
-		"-e", "POSTGRES_DB="+sourceDatabase,
-		"-p", fmt.Sprintf("%d:5432", env.postgresPort),
-		postgresImage(),
-	)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		t.Fatalf("Failed to start Postgres: %v\n%s", err, out)
-	}
-	env.postgresContainer = name
-
-	deadline := time.Now().Add(postgresStartTimeout)
-	for time.Now().Before(deadline) {
-		probe := exec.Command("docker", "exec", name,
-			"pg_isready", "-U", sourceUser, "-d", sourceDatabase)
-		if err := probe.Run(); err == nil {
-			return
-		}
-		time.Sleep(2 * time.Second)
-	}
-	t.Fatalf("Postgres did not become ready within %s", postgresStartTimeout)
-}
-
-func (env *TestEnvironment) seedSource(t *testing.T) {
-	t.Helper()
-	env.execSQL(t, `
-CREATE TABLE orders (
-    id       int PRIMARY KEY,
-    region   text,
-    amount   numeric(10,2),
-    order_ts timestamp
-);
-INSERT INTO orders VALUES
-    (1,'us-east',120.50,'2026-07-27 09:15'),
-    (2,'us-west', 87.20,'2026-07-27 09:20'),
-    (3,'eu-west',210.00,'2026-07-27 09:31');
-ALTER TABLE orders REPLICA IDENTITY FULL;`)
-}
-
-func (env *TestEnvironment) execSQL(t *testing.T, sqlText string) {
-	t.Helper()
-	cmd := exec.Command("docker", "exec", env.postgresContainer,
-		"psql", "-v", "ON_ERROR_STOP=1", "-U", sourceUser, "-d", sourceDatabase,
-		"-c", sqlText)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		t.Fatalf("psql failed: %v\n%s", err, out)
-	}
-}
-
-// writeOLakeConfigs writes the source and destination configs. The destination
-// is the point of this test: nothing in it is SeaweedFS-specific. catalog_type
-// is the generic "rest", auth is the standard OAuth2 client-credentials flow,
-// and path-style access is not even set here because OLake turns it on by
-// itself whenever s3_endpoint is non-empty.
-func (env *TestEnvironment) writeOLakeConfigs(t *testing.T) {
-	t.Helper()
-
-	source := map[string]any{
-		"host":            env.bindIP,
-		"port":            env.postgresPort,
-		"database":        sourceDatabase,
-		"username":        sourceUser,
-		"password":        sourcePassword,
-		"jdbc_url_params": map[string]any{},
-		"ssl":             map[string]any{"mode": "disable"},
-		"update_method":   map[string]any{"type": "Standalone"},
-		"max_threads":     2,
-		"retry_count":     0,
-	}
-
-	catalogURL := fmt.Sprintf("http://%s:%d", env.bindIP, env.icebergPort)
-	destination := map[string]any{
-		"type": "ICEBERG",
-		"writer": map[string]any{
-			"catalog_type":     "rest",
-			"rest_catalog_url": catalogURL,
-			"catalog_name":     "olake",
-			"iceberg_s3_path":  "s3://" + tableBucketName,
-			"s3_endpoint":      fmt.Sprintf("http://%s:%d", env.bindIP, env.s3Port),
-			"s3_use_ssl":       false,
-			"s3_path_style":    true,
-			"aws_region":       "us-east-1",
-			"aws_access_key":   env.accessKey,
-			"aws_secret_key":   env.secretKey,
-			"rest_auth_type":   "oauth2",
-			"oauth2_uri":       catalogURL + "/v1/oauth/tokens",
-			"credential":       env.accessKey + ":" + env.secretKey,
-		},
-	}
-
-	writeJSON(t, filepath.Join(env.configDir, "source.json"), source)
-	writeJSON(t, filepath.Join(env.configDir, "destination.json"), destination)
-	if err := os.WriteFile(filepath.Join(env.configDir, "state.json"), []byte("{}\n"), 0644); err != nil {
-		t.Fatalf("Failed to write state.json: %v", err)
-	}
-}
-
-func (env *TestEnvironment) runOLake(t *testing.T, args ...string) string {
-	t.Helper()
-
-	full := append([]string{
-		"run", "--rm",
-		"-v", dockerMount(env.configDir) + ":/mnt/config",
-		olakeImage(),
-	}, args...)
-
-	cmd := exec.Command("docker", full...)
-	done := make(chan struct{})
-	var out []byte
-	var err error
-	go func() {
-		out, err = cmd.CombinedOutput()
-		close(done)
-	}()
-	select {
-	case <-done:
-	case <-time.After(olakeRunTimeout):
-		if cmd.Process != nil {
-			_ = cmd.Process.Kill()
-		}
-		t.Fatalf("olake %s did not finish within %s", args[0], olakeRunTimeout)
-	}
-	if err != nil {
-		t.Fatalf("olake %s failed: %v\n%s", args[0], err, tailLines(string(out), 40))
-	}
-	return string(out)
-}
-
-func (env *TestEnvironment) readTable(t *testing.T, mode string) string {
-	t.Helper()
-
-	cmd := exec.Command("docker", "run", "--rm", readerImage, mode,
-		"--catalog-url", fmt.Sprintf("http://%s:%d", env.bindIP, env.icebergPort),
-		"--warehouse", "s3://"+tableBucketName,
-		"--prefix", tableBucketName,
-		"--s3-endpoint", fmt.Sprintf("http://%s:%d", env.bindIP, env.s3Port),
-		"--access-key", env.accessKey,
-		"--secret-key", env.secretKey,
-		"--namespace", expectedNamespace,
-		"--table", expectedTable,
-	)
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("reader (%s) failed: %v\n%s", mode, err, tailLines(string(out), 30))
-	}
-	return string(out)
-}
-
-// listTableMetadata lists the table's metadata directory through the filer so
-// the repair assertion looks at real objects rather than at what the catalog
-// reports about itself.
-func (env *TestEnvironment) listTableMetadata(t *testing.T) []string {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s:%d/buckets/%s/%s/%s/metadata/?limit=200",
-		env.bindIP, env.filerPort, tableBucketName, expectedNamespace, expectedTable)
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		t.Fatalf("Failed to build filer request: %v", err)
-	}
-	req.Header.Set("Accept", "application/json")
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		t.Fatalf("filer listing failed: %v", err)
-	}
-	defer resp.Body.Close()
-	body, _ := io.ReadAll(resp.Body)
-	if resp.StatusCode != http.StatusOK {
-		t.Fatalf("filer listing returned %d: %s", resp.StatusCode, body)
-	}
-
-	var parsed struct {
-		Entries []struct {
-			FullPath string `json:"FullPath"`
-		} `json:"Entries"`
-	}
-	if err := json.Unmarshal(body, &parsed); err != nil {
-		t.Fatalf("Failed to parse filer listing: %v\n%s", err, body)
-	}
-
-	names := make([]string, 0, len(parsed.Entries))
-	for _, e := range parsed.Entries {
-		names = append(names, e.FullPath[strings.LastIndex(e.FullPath, "/")+1:])
-	}
-	return names
-}
-
-func (env *TestEnvironment) Cleanup(t *testing.T) {
-	t.Helper()
-
-	if env.postgresContainer != "" {
-		_ = exec.Command("docker", "rm", "-f", env.postgresContainer).Run()
-	}
-	if env.weedCancel != nil {
-		env.weedCancel()
-	}
-	if env.weedProcess != nil {
-		_ = env.weedProcess.Wait()
-	}
-	if env.dataDir != "" {
-		_ = os.RemoveAll(env.dataDir)
-	}
-}
-
-func buildReaderImage(t *testing.T) {
-	t.Helper()
-
-	wd, err := os.Getwd()
-	if err != nil {
-		t.Fatalf("Failed to get working directory: %v", err)
-	}
-	cmd := exec.Command("docker", "build",
-		"-f", filepath.Join(wd, "Dockerfile.reader"),
-		"-t", readerImage, wd)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		t.Fatalf("Failed to build reader image: %v\n%s", err, tailLines(string(out), 30))
-	}
-}
-
-func olakeImage() string {
-	if v := os.Getenv("OLAKE_IMAGE"); v != "" {
-		return v
-	}
-	return olakeDefaultImage
-}
-
-func postgresImage() string {
-	if v := os.Getenv("POSTGRES_IMAGE"); v != "" {
-		return v
-	}
-	return postgresDefaultImage
-}
-
-func requireOLakeRuntime(t *testing.T) {
-	t.Helper()
-	if os.Getenv("SEAWEEDFS_SKIP_OLAKE_TESTS") != "" {
-		t.Skip("SEAWEEDFS_SKIP_OLAKE_TESTS set")
-	}
-	if _, err := exec.LookPath("docker"); err != nil {
-		t.Skip("docker not available, skipping OLake integration test")
-	}
-	if err := exec.Command("docker", "info").Run(); err != nil {
-		t.Skip("docker daemon not reachable, skipping OLake integration test")
-	}
-}
-
-func waitForService(url string, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	client := &http.Client{Timeout: 3 * time.Second}
-	for time.Now().Before(deadline) {
-		resp, err := client.Get(url)
-		if err == nil {
-			resp.Body.Close()
-			if resp.StatusCode < 500 {
-				return true
-			}
-		}
-		time.Sleep(time.Second)
-	}
-	return false
-}
-
-func writeJSON(t *testing.T, path string, v any) {
-	t.Helper()
-	body, err := json.MarshalIndent(v, "", "  ")
-	if err != nil {
-		t.Fatalf("Failed to marshal %s: %v", path, err)
-	}
-	if err := os.WriteFile(path, append(body, '\n'), 0644); err != nil {
-		t.Fatalf("Failed to write %s: %v", path, err)
-	}
-}
-
-// dockerMount normalises a host path for a -v bind mount. Docker Desktop
-// accepts forward slashes on Windows; the native separator it does not.
-func dockerMount(path string) string {
-	return strings.ReplaceAll(path, `\`, "/")
-}
-
-func nonEmptyLines(s string) []string {
-	var out []string
-	for _, line := range strings.Split(s, "\n") {
-		if trimmed := strings.TrimSpace(line); trimmed != "" {
-			out = append(out, trimmed)
-		}
-	}
-	return out
-}
-
-func tailLines(s string, n int) string {
-	lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
-	if len(lines) > n {
-		lines = lines[len(lines)-n:]
-	}
-	return strings.Join(lines, "\n")
-}

test/s3tables/unity_catalog/setup_test.go

@@ -26,10 +26,6 @@ const (
 	ucAPIBase      = "/api/2.1/unity-catalog"
 	ucWarehouse    = "lakehouse"
 	ucWarehouseKey = "warehouse"
-	// A non-empty s3.sessionToken.N puts UC's credential vendor on its
-	// StaticAwsCredentialGenerator path: it vends the configured access key,
-	// secret key and this token verbatim instead of calling AWS STS.
-	ucVendedSessionToken = "unity-catalog-vended-session-token"
 
 	// Role used by the master-role STS-vended variant of the test. The trust
 	// policy is wide open so any caller can assume it; in production UC
@@ -69,9 +65,6 @@ type ucServerOpts struct {
 	// MasterRoleArn populates aws.masterRoleArn. Empty means UC falls back to
 	// static aws.accessKey / aws.secretKey for storage operations.
 	MasterRoleArn string
-	// SessionToken populates s3.sessionToken.0, selecting UC's static
-	// credential generator for the per-bucket config.
-	SessionToken string
 	// ExtraEnv adds environment variables to the UC container, useful for
 	// AWS_ENDPOINT_URL_STS-style overrides.
 	ExtraEnv map[string]string
@@ -227,7 +220,6 @@ func (env *testEnv) startUnityCatalog(t *testing.T, ctx context.Context, opts uc
 		fmt.Sprintf("s3.awsRoleArn.0=%s", opts.MasterRoleArn),
 		fmt.Sprintf("s3.accessKey.0=%s", env.accessKey),
 		fmt.Sprintf("s3.secretKey.0=%s", env.secretKey),
-		fmt.Sprintf("s3.sessionToken.0=%s", opts.SessionToken),
 		fmt.Sprintf("s3.endpoint.0=%s", s3EndpointForContainer),
 		"",
 	}, "\n")

test/s3tables/unity_catalog/unity_catalog_test.go

@@ -8,7 +8,6 @@ package unity_catalog
 import (
 	"context"
 	"fmt"
-	"strings"
 	"testing"
 	"time"
 
@@ -47,7 +46,7 @@ func TestUnityCatalogDeltaIntegration(t *testing.T) {
 	}
 
 	t.Log(">>> starting Unity Catalog server (static keys)...")
-	env.startUnityCatalog(t, ctx, ucServerOpts{SessionToken: ucVendedSessionToken})
+	env.startUnityCatalog(t, ctx, ucServerOpts{})
 	t.Log(">>> Unity Catalog ready")
 
 	uc := newUCClient(fmt.Sprintf("http://127.0.0.1:%d", env.ucHostPort))
@@ -146,41 +145,28 @@ func TestUnityCatalogDeltaIntegration(t *testing.T) {
 		}
 	})
 
-	t.Run("TemporaryTableCredentials", func(t *testing.T) {
-		// s3.sessionToken.0 is set, so UC's AwsCredentialVendor takes the
-		// StaticAwsCredentialGenerator path and vends the configured keys
-		// verbatim, token included — no STS call at all. SeaweedFS authenticates
-		// the request on the static access key and tolerates the foreign token.
+	t.Run("TemporaryTableCredentialsRejected", func(t *testing.T) {
+		// With aws.masterRoleArn empty AND no s3.sessionToken.0 set, UC OSS
+		// always tries to AssumeRole via its internal StsClient (see
+		// AwsCredentialVendor.createPerBucketCredentialGenerator). Against a
+		// non-AWS endpoint, that call doesn't reach a real STS, so UC returns
+		// "S3 bucket configuration not found." or an STS-side error. This is
+		// the gap users hit at <https://github.com/data-engineering-helpers/mds-in-a-box/blob/main/unitycatalog-playground/etc/conf/server.properties#L45>:
+		// "with simple S3 access and secret keys, Unity Catalog does not seem
+		// to work."
+		//
+		// The assertion is therefore inverted: we expect a non-nil error from
+		// /temporary-table-credentials with this configuration. A future
+		// variant can pin s3.sessionToken.0 (UC's StaticAwsCredentialGenerator
+		// path) once SeaweedFS' SigV4 path tolerates the vended session token.
 		if createdTable.TableID == "" {
 			t.Fatalf("created table has empty table_id; cannot request temporary credentials")
 		}
-		creds, err := uc.generateTemporaryTableCredentials(ctx, createdTable.TableID, "READ_WRITE")
-		if err != nil {
-			t.Fatalf("temporary-table-credentials: %v", err)
-		}
-		awsCreds := creds.AwsTempCredentials
-		if awsCreds == nil || awsCreds.AccessKeyID == "" || awsCreds.SessionToken == "" {
-			t.Fatalf("expected aws_temp_credentials with a session_token, got %+v", creds)
-		}
-		if awsCreds.AccessKeyID != env.accessKey || awsCreds.SessionToken != ucVendedSessionToken {
-			t.Fatalf("expected UC to vend the configured static credential, got %+v", awsCreds)
-		}
-
-		s3v := env.newHostS3ClientWithCreds(t, ctx, awsCreds.AccessKeyID, awsCreds.SecretAccessKey, awsCreds.SessionToken)
-		probeKey := fmt.Sprintf("%s/%s/%s/vended.txt", ucWarehouseKey, schemaName, tableName)
-		if _, err := s3v.PutObject(ctx, &s3.PutObjectInput{
-			Bucket: aws.String(ucWarehouse),
-			Key:    aws.String(probeKey),
-			Body:   strings.NewReader("ok"),
-		}); err != nil {
-			t.Fatalf("PutObject with UC-vended credentials: %v", err)
-		}
-		if _, err := s3v.DeleteObject(ctx, &s3.DeleteObjectInput{
-			Bucket: aws.String(ucWarehouse),
-			Key:    aws.String(probeKey),
-		}); err != nil {
-			t.Fatalf("DeleteObject with UC-vended credentials: %v", err)
+		_, err := uc.generateTemporaryTableCredentials(ctx, createdTable.TableID, "READ_WRITE")
+		if err == nil {
+			t.Fatalf("expected /temporary-table-credentials to fail with the static-key playground configuration; it succeeded unexpectedly")
 		}
+		t.Logf("expected failure (UC static-key path requires AWS STS): %v", err)
 	})
 
 	t.Run("DeleteTableSchemaCatalog", func(t *testing.T) {