feat(admin): add profiling options for debugging high memory/CPU usage (#8923)

* feat(admin): add profiling options for debugging high memory/CPU usage

Add -debug, -debug.port, -cpuprofile, and -memprofile flags to the admin
command, matching the profiling support already available in master, volume,
and other server commands. This enables investigation of resource usage
issues like #8919.

* refactor(admin): move profiling flags into AdminOptions struct

Move cpuprofile and memprofile flags from global variables into the
AdminOptions struct and init() function for consistency with other flags.

* fix(debug): bind pprof server to localhost only and document profiling flags

StartDebugServer was binding to all interfaces (0.0.0.0), exposing
runtime profiling data to the network. Restrict to 127.0.0.1 since
this is a development/debugging tool.

Also add a "Debugging and Profiling" section to the admin command's
help text documenting the new flags.

.claude/scheduled_tasks.lock

@@ -0,0 +1 @@
+{"sessionId":"d6574c47-eafc-4a94-9dce-f9ffea22b53c","pid":10111,"acquiredAt":1775248373916}

.github/workflows/binaries_dev.yml

@@ -38,7 +38,7 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
@@ -87,7 +87,7 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}

.github/workflows/binaries_release0.yml

@@ -28,7 +28,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
       - name: Go Release Binaries Normal Volume Size
         uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:

.github/workflows/binaries_release1.yml

@@ -28,7 +28,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
       - name: Go Release Binaries Normal Volume Size
         uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:

.github/workflows/binaries_release2.yml

@@ -28,7 +28,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
       - name: Go Release Binaries Normal Volume Size
         uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:

.github/workflows/binaries_release3.yml

@@ -28,7 +28,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
       - name: Go Release Binaries Normal Volume Size
         uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:

.github/workflows/binaries_release4.yml

@@ -28,7 +28,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
       - name: Go Release Binaries Normal Volume Size
         uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:

.github/workflows/binaries_release5.yml

@@ -28,7 +28,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
       - name: Go Release Binaries Normal Volume Size
         uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:

.github/workflows/claude-code-review.yml

@@ -1,54 +0,0 @@
-name: Claude Code Review
-
-on:
-  pull_request:
-    types: [opened, synchronize, ready_for_review, reopened]
-    # Optional: Only run on specific file changes
-    # paths:
-    #   - "src/**/*.ts"
-    #   - "src/**/*.tsx"
-    #   - "src/**/*.js"
-    #   - "src/**/*.jsx"
-
-jobs:
-  claude-review:
-    # Optional: Filter by PR author
-    # if: |
-    #   github.event.pull_request.user.login == 'external-contributor' ||
-    #   github.event.pull_request.user.login == 'new-developer' ||
-    #   github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'
-
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      pull-requests: read
-      issues: read
-      id-token: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: '20'
-
-      - name: Install Claude Code
-        run: npm install -g @anthropic-ai/claude-code
-
-      - name: Run Claude Code Review
-        id: claude-review
-        uses: anthropics/claude-code-action@v1
-        with:
-          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
-          path_to_claude_code_executable: claude
-          allowed_bots: "*"
-          plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'
-          plugins: 'code-review@claude-code-plugins'
-          prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}'
-          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
-          # or https://code.claude.com/docs/en/cli-reference for available options
-

.github/workflows/claude.yml

@@ -1,50 +0,0 @@
-name: Claude Code
-
-on:
-  issue_comment:
-    types: [created]
-  pull_request_review_comment:
-    types: [created]
-  issues:
-    types: [opened, assigned]
-  pull_request_review:
-    types: [submitted]
-
-jobs:
-  claude:
-    if: |
-      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
-      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      pull-requests: read
-      issues: read
-      id-token: write
-      actions: read # Required for Claude to read CI results on PRs
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Run Claude Code
-        id: claude
-        uses: anthropics/claude-code-action@v1
-        with:
-          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
-
-          # This is an optional setting that allows Claude to read CI results on PRs
-          additional_permissions: |
-            actions: read
-
-          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
-          # prompt: 'Update the pull request description to include a summary of changes.'
-
-          # Optional: Add claude_args to customize behavior and configuration
-          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
-          # or https://code.claude.com/docs/en/cli-reference for available options
-          # claude_args: '--allowed-tools Bash(gh pr:*)'
-

.github/workflows/codeql.yml

@@ -18,7 +18,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

.github/workflows/container_dev.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
       -
         name: Docker meta
         id: docker_meta

.github/workflows/container_latest.yml

@@ -23,15 +23,22 @@ on:
           - all
           - standard
           - large_disk
+      publish:
+        description: 'Publish images and manifests'
+        required: true
+        type: boolean
+        default: false
 
 permissions:
   contents: read
+  security-events: write
 
 jobs:
   setup:
     runs-on: ubuntu-latest
     outputs:
       variants: ${{ steps.set-variants.outputs.variants }}
+      publish: ${{ steps.set-publish.outputs.publish }}
     steps:
       - name: Select variants for this run
         id: set-variants
@@ -42,6 +49,14 @@ jobs:
             variants='["standard","large_disk"]'
           fi
           echo "variants=$variants" >> "$GITHUB_OUTPUT"
+      - name: Select publish mode
+        id: set-publish
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
+          else
+            echo "publish=true" >> "$GITHUB_OUTPUT"
+          fi
 
   build:
     needs: [setup]
@@ -111,13 +126,13 @@ jobs:
           buildkitd-flags: "--debug"
           buildkitd-config: /tmp/buildkitd.toml
       - name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
+        if: needs.setup.outputs.publish == 'true'
         uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Login to GHCR
-        if: github.event_name != 'pull_request'
+        if: needs.setup.outputs.publish == 'true'
         uses: docker/login-action@v4
         with:
           registry: ghcr.io
@@ -129,7 +144,7 @@ jobs:
           DOCKER_BUILDKIT: 1
         with:
           context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ needs.setup.outputs.publish == 'true' }}
           file: ./docker/Dockerfile.go_build
           platforms: linux/${{ matrix.platform }}
           # Push to GHCR only during build to avoid Docker Hub rate limits
@@ -149,10 +164,128 @@ jobs:
           # Remove Go build cache
           sudo rm -rf /tmp/go-build*
 
-  create-manifest:
+  trivy-scan:
     runs-on: ubuntu-latest
     needs: [setup, build]
-    if: github.event_name != 'pull_request'
+    strategy:
+      matrix:
+        variant: ${{ fromJSON(needs.setup.outputs.variants) }}
+    steps:
+      - name: Configure variant
+        id: config
+        run: |
+          if [ "${{ matrix.variant }}" == "large_disk" ]; then
+            echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
+          else
+            echo "tag_suffix=" >> $GITHUB_OUTPUT
+          fi
+      - name: Login to GHCR
+        if: needs.setup.outputs.publish == 'true'
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.GHCR_USERNAME }}
+          password: ${{ secrets.GHCR_TOKEN }}
+      - name: Checkout for local scan build
+        if: needs.setup.outputs.publish != 'true'
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
+      - name: Create BuildKit config for local scan build
+        if: needs.setup.outputs.publish != 'true'
+        run: |
+          cat > /tmp/buildkitd.toml <<EOF
+          [registry."docker.io"]
+            mirrors = ["https://mirror.gcr.io"]
+          EOF
+      - name: Set up Docker Buildx for local scan build
+        if: needs.setup.outputs.publish != 'true'
+        uses: docker/setup-buildx-action@v4
+        with:
+          buildkitd-flags: "--debug"
+          buildkitd-config: /tmp/buildkitd.toml
+      - name: Build local scan image tarball
+        if: needs.setup.outputs.publish != 'true'
+        uses: docker/build-push-action@v7
+        env:
+          DOCKER_BUILDKIT: 1
+        with:
+          context: ./docker
+          file: ./docker/Dockerfile.go_build
+          platforms: linux/amd64
+          outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1
+            BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
+            ${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
+      - name: Trivy report (published image)
+        if: needs.setup.outputs.publish == 'true'
+        # Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          scan-type: image
+          # Scan amd64 only - OS packages are identical across architectures
+          # since they all use the same alpine base, so a single-arch scan
+          # provides sufficient coverage without multiplying CI time.
+          image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          limit-severities-for-sarif: true
+          format: sarif
+          output: trivy-results.sarif
+          exit-code: '0'
+      - name: Trivy report (local tarball)
+        if: needs.setup.outputs.publish != 'true'
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          limit-severities-for-sarif: true
+          format: sarif
+          output: trivy-results.sarif
+          exit-code: '0'
+      - name: Upload Trivy scan results to GitHub Security
+        uses: github/codeql-action/upload-sarif@v4
+        if: always()
+        with:
+          sarif_file: trivy-results.sarif
+      - name: Trivy gate (published image)
+        if: needs.setup.outputs.publish == 'true'
+        # Gate only on fixable high/critical vulnerabilities. Non-fixable
+        # findings are still visible in the SARIF upload above.
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          scan-type: image
+          image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          format: table
+          exit-code: '1'
+          skip-setup-trivy: true
+      - name: Trivy gate (local tarball)
+        if: needs.setup.outputs.publish != 'true'
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          format: table
+          exit-code: '1'
+          skip-setup-trivy: true
+
+  create-manifest:
+    runs-on: ubuntu-latest
+    needs: [setup, build, trivy-scan]
+    if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
     strategy:
       matrix:
         variant: ${{ fromJSON(needs.setup.outputs.variants) }}

.github/workflows/container_release_unified.yml

@@ -39,7 +39,80 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+
+  # ── Pre-build Rust volume server binaries natively ──────────────────
+  # Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
+  # emulated cargo build into ~15 minutes of native compilation.
+  build-rust-binaries:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-musl
+            arch: amd64
+          - target: aarch64-unknown-linux-musl
+            arch: arm64
+            cross: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Install musl tools (amd64)
+        if: ${{ !matrix.cross }}
+        run: sudo apt-get install -y musl-tools
+
+      - name: Install cross-compilation tools (arm64)
+        if: matrix.cross
+        run: |
+          sudo apt-get install -y gcc-aarch64-linux-gnu
+          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-docker-${{ matrix.target }}-
+
+      - name: Build large-disk variant
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }}
+          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
+
+      - name: Build normal variant
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }} --no-default-features
+          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: rust-volume-${{ matrix.arch }}
+          path: |
+            weed-volume-large-disk-${{ matrix.arch }}
+            weed-volume-normal-${{ matrix.arch }}
+
+  # ── Build Docker containers ─────────────────────────────────────────
   build:
+    needs: [build-rust-binaries]
     runs-on: ubuntu-latest
     strategy:
       # Build sequentially to avoid rate limits
@@ -52,20 +125,23 @@ jobs:
             dockerfile: ./docker/Dockerfile.go_build
             build_args: ""
             tag_suffix: ""
-            
-          # Large disk - multi-arch  
+            rust_variant: normal
+
+          # Large disk - multi-arch
           - variant: large_disk
             platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
             dockerfile: ./docker/Dockerfile.go_build
             build_args: TAGS=5BytesOffset
             tag_suffix: _large_disk
-            
+            rust_variant: large-disk
+
           # Full tags - multi-arch
           - variant: full
             platforms: linux/amd64,linux/arm64
             dockerfile: ./docker/Dockerfile.go_build
             build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
             tag_suffix: _full
+            rust_variant: normal
 
           # Large disk + full tags - multi-arch
           - variant: large_disk_full
@@ -73,19 +149,42 @@ jobs:
             dockerfile: ./docker/Dockerfile.go_build
             build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
             tag_suffix: _large_disk_full
-            
+            rust_variant: large-disk
+
           # RocksDB large disk - amd64 only
           - variant: rocksdb
             platforms: linux/amd64
             dockerfile: ./docker/Dockerfile.rocksdb_large
             build_args: ""
             tag_suffix: _large_disk_rocksdb
+            rust_variant: large-disk
             
     steps:
       - name: Checkout
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         uses: actions/checkout@v6
-        
+
+      - name: Download pre-built Rust binaries
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        uses: actions/download-artifact@v4
+        with:
+          pattern: rust-volume-*
+          merge-multiple: true
+          path: ./rust-bins
+
+      - name: Place Rust binaries in Docker context
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        run: |
+          mkdir -p docker/weed-volume-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
+              echo "Placed pre-built Rust binary for ${arch}"
+            fi
+          done
+          ls -la docker/weed-volume-prebuilt/
+
       - name: Free Disk Space
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         run: |

.github/workflows/container_rocksdb_version.yml

@@ -25,7 +25,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
 
       - name: Prepare Docker tag
         id: tag

.github/workflows/depsreview.yml

@@ -9,6 +9,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
       - name: 'Dependency Review'
         uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48

.github/workflows/fuse-integration.yml

@@ -77,4 +77,5 @@ jobs:
         name: fuse-integration-test-results
         path: |
           /tmp/fuse-test-output.log
+          /tmp/seaweedfs-fuse-logs/
         retention-days: 7

.github/workflows/helm_ci.yml

@@ -16,12 +16,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
         with:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v4
+        uses: azure/setup-helm@v5
         with:
           version: v3.18.4
 

.github/workflows/kms-tests.yml

@@ -0,0 +1,140 @@
+name: "KMS Tests"
+
+on:
+  pull_request:
+    paths:
+      - 'weed/kms/**'
+      - 'weed/s3api/s3_sse_*.go'
+      - 'weed/s3api/s3api_object_handlers.go'
+      - 'weed/s3api/s3api_object_handlers_put.go'
+      - 'test/kms/**'
+      - '.github/workflows/kms-tests.yml'
+  push:
+    branches: [ master, main ]
+    paths:
+      - 'weed/kms/**'
+      - 'weed/s3api/s3_sse_*.go'
+      - 'weed/s3api/s3api_object_handlers.go'
+      - 'weed/s3api/s3api_object_handlers_put.go'
+      - 'test/kms/**'
+
+concurrency:
+  group: ${{ github.head_ref || github.ref }}-kms-tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    working-directory: weed
+
+jobs:
+  kms-provider-tests:
+    name: KMS Provider Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false
+
+      - name: Run KMS provider integration tests
+        timeout-minutes: 15
+        working-directory: test/kms
+        run: |
+          set -x
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          docker --version
+
+          make test-provider-ci
+
+      - name: Show OpenBao logs on failure
+        if: failure()
+        run: |
+          echo "=== OpenBao Container Logs ==="
+          docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao container found"
+          echo "=== Setup Logs ==="
+          cat /tmp/openbao-ci-setup.log 2>/dev/null || echo "No setup log found"
+
+      - name: Cleanup
+        if: always()
+        working-directory: test/kms
+        run: |
+          make stop-openbao-ci || true
+
+  s3-kms-e2e-tests:
+    name: S3 KMS End-to-End Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 25
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false
+
+      - name: Run S3 KMS end-to-end tests
+        timeout-minutes: 20
+        working-directory: test/kms
+        run: |
+          set -x
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          docker --version
+          aws --version
+
+          make test-s3-kms-ci
+
+      - name: Show logs on failure
+        if: failure()
+        working-directory: test/kms
+        run: |
+          echo "=== OpenBao Container Logs ==="
+          cat /tmp/openbao-ci-container.log 2>/dev/null || docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao logs found"
+          echo "=== SeaweedFS Server Logs ==="
+          tail -100 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"
+          echo "=== Setup Logs ==="
+          cat /tmp/weed-kms-ci-setup.log 2>/dev/null || echo "No weed setup log"
+          echo "=== Process Information ==="
+          ps aux | grep -E "(weed|test)" || true
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: s3-kms-e2e-logs
+          path: |
+            /tmp/seaweedfs-kms-mini.log
+            /tmp/openbao-ci-container.log
+            /tmp/weed-kms-ci-setup.log
+          retention-days: 3
+
+      - name: Cleanup
+        if: always()
+        working-directory: test/kms
+        run: |
+          make stop-seaweedfs-ci || true
+          make stop-openbao-ci || true
+          make clean-ci || true

.github/workflows/rust-volume-server-tests.yml

@@ -0,0 +1,242 @@
+name: "Rust Volume Server Tests"
+
+on:
+  pull_request:
+    branches: [ master ]
+    paths:
+      - 'seaweed-volume/**'
+      - 'test/volume_server/**'
+      - 'weed/pb/volume_server.proto'
+      - 'weed/pb/volume_server_pb/**'
+      - '.github/workflows/rust-volume-server-tests.yml'
+  push:
+    branches: [ master, main ]
+    paths:
+      - 'seaweed-volume/**'
+      - 'test/volume_server/**'
+      - 'weed/pb/volume_server.proto'
+      - 'weed/pb/volume_server_pb/**'
+      - '.github/workflows/rust-volume-server-tests.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+
+jobs:
+  rust-unit-tests:
+    name: Rust Unit Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 15
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-
+
+      - name: Build Rust volume server
+        run: cd seaweed-volume && cargo build --release
+
+      - name: Run Rust unit tests
+        run: cd seaweed-volume && cargo test
+
+  rust-integration-tests:
+    name: Rust Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-
+
+      - name: Build Go weed binary
+        run: |
+          cd weed
+          go build -tags 5BytesOffset -o weed .
+          chmod +x weed
+          ./weed version
+
+      - name: Build Rust volume binary
+        run: cd seaweed-volume && cargo build --release
+
+      - name: Run integration tests
+        env:
+          WEED_BINARY: ${{ github.workspace }}/weed/weed
+          RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
+        run: |
+          echo "Running Rust volume server integration tests..."
+          go test -v -count=1 -timeout=15m ./test/volume_server/rust/...
+
+      - name: Collect logs on failure
+        if: failure()
+        run: |
+          mkdir -p /tmp/rust-volume-server-it-logs
+          find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-server-it-logs/ \; || true
+
+      - name: Archive logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-volume-server-integration-test-logs
+          path: /tmp/rust-volume-server-it-logs/
+          if-no-files-found: warn
+          retention-days: 7
+
+      - name: Test summary
+        if: always()
+        run: |
+          echo "## Rust Volume Server Integration Test Summary" >> "$GITHUB_STEP_SUMMARY"
+          echo "- Suite: test/volume_server/rust" >> "$GITHUB_STEP_SUMMARY"
+          echo "- Command: go test -v -count=1 -timeout=15m ./test/volume_server/rust/..." >> "$GITHUB_STEP_SUMMARY"
+
+  rust-volume-go-tests:
+    name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
+    runs-on: ubuntu-22.04
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        test-type: [grpc, http]
+        shard: [1, 2, 3]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-
+
+      - name: Build Go weed binary
+        run: |
+          cd weed
+          go build -tags 5BytesOffset -o weed .
+          chmod +x weed
+          ./weed version
+
+      - name: Build Rust volume binary
+        run: cd seaweed-volume && cargo build --release
+
+      - name: Run volume server integration tests with Rust volume
+        env:
+          WEED_BINARY: ${{ github.workspace }}/weed/weed
+          RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
+          VOLUME_SERVER_IMPL: rust
+        run: |
+          if [ "${{ matrix.test-type }}" == "grpc" ]; then
+            if [ "${{ matrix.shard }}" == "1" ]; then
+              TEST_PATTERN="^Test[A-H]"
+            elif [ "${{ matrix.shard }}" == "2" ]; then
+              TEST_PATTERN="^Test[I-S]"
+            else
+              TEST_PATTERN="^Test[T-Z]"
+            fi
+          else
+            if [ "${{ matrix.shard }}" == "1" ]; then
+              TEST_PATTERN="^Test[A-G]"
+            elif [ "${{ matrix.shard }}" == "2" ]; then
+              TEST_PATTERN="^Test[H-R]"
+            else
+              TEST_PATTERN="^Test[S-Z]"
+            fi
+          fi
+          echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
+          go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
+
+      - name: Collect logs on failure
+        if: failure()
+        run: |
+          mkdir -p /tmp/rust-volume-go-test-logs
+          find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-go-test-logs/ \; || true
+
+      - name: Archive logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-volume-go-test-logs-${{ matrix.test-type }}-shard${{ matrix.shard }}
+          path: /tmp/rust-volume-go-test-logs/
+          if-no-files-found: warn
+          retention-days: 7
+
+      - name: Test summary
+        if: always()
+        run: |
+          if [ "${{ matrix.test-type }}" == "grpc" ]; then
+            if [ "${{ matrix.shard }}" == "1" ]; then
+              TEST_PATTERN="^Test[A-H]"
+            elif [ "${{ matrix.shard }}" == "2" ]; then
+              TEST_PATTERN="^Test[I-S]"
+            else
+              TEST_PATTERN="^Test[T-Z]"
+            fi
+          else
+            if [ "${{ matrix.shard }}" == "1" ]; then
+              TEST_PATTERN="^Test[A-G]"
+            elif [ "${{ matrix.shard }}" == "2" ]; then
+              TEST_PATTERN="^Test[H-R]"
+            else
+              TEST_PATTERN="^Test[S-Z]"
+            fi
+          fi
+          echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
+          echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
+          echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/rust_binaries_dev.yml

@@ -0,0 +1,165 @@
+name: "rust: build dev volume server binaries"
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - 'seaweed-volume/**'
+      - '.github/workflows/rust_binaries_dev.yml'
+
+permissions:
+  contents: read
+
+jobs:
+
+  cleanup:
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Delete old Rust volume dev assets
+        uses: mknejp/delete-release-assets@v1
+        with:
+          token: ${{ github.token }}
+          tag: dev
+          fail-if-no-assets: false
+          assets: |
+            weed-volume-*
+
+  build-rust-volume-dev-linux:
+    permissions:
+      contents: write
+    needs: cleanup
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-gnu
+            asset_suffix: linux-amd64
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-dev-${{ matrix.target }}-
+
+      - name: Set BUILD_TIME
+        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
+
+      - name: Build Rust volume server (large disk)
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: cd seaweed-volume && cargo build --release
+
+      - name: Package large disk binary
+        run: |
+          cp seaweed-volume/target/release/weed-volume weed-volume-large-disk
+          tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
+          rm weed-volume-large-disk
+
+      - name: Build Rust volume server (normal)
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: cd seaweed-volume && cargo build --release --no-default-features
+
+      - name: Package normal binary
+        run: |
+          cp seaweed-volume/target/release/weed-volume weed-volume-normal
+          tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
+          rm weed-volume-normal
+
+      - name: Upload dev release assets
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: dev
+          prerelease: true
+          files: |
+            weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
+            weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  build-rust-volume-dev-darwin:
+    permissions:
+      contents: write
+    needs: build-rust-volume-dev-linux
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        include:
+          - target: aarch64-apple-darwin
+            asset_suffix: darwin-arm64
+          - target: x86_64-apple-darwin
+            asset_suffix: darwin-amd64
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install protobuf compiler
+        run: brew install protobuf
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-dev-${{ matrix.target }}-
+
+      - name: Set BUILD_TIME
+        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
+
+      - name: Build Rust volume server (large disk)
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }}
+
+      - name: Package large disk binary
+        run: |
+          cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
+          tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
+          rm weed-volume-large-disk
+
+      - name: Build Rust volume server (normal)
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }} --no-default-features
+
+      - name: Package normal binary
+        run: |
+          cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
+          tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
+          rm weed-volume-normal
+
+      - name: Upload dev release assets
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: dev
+          prerelease: true
+          files: |
+            weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
+            weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/rust_binaries_release.yml

@@ -0,0 +1,253 @@
+name: "rust: build versioned volume server binaries"
+
+on:
+  push:
+    tags:
+      - '*'
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+
+  build-rust-volume-linux:
+    permissions:
+      contents: write
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-gnu
+            asset_suffix: linux_amd64
+          - target: aarch64-unknown-linux-gnu
+            asset_suffix: linux_arm64
+            cross: true
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Install cross-compilation tools
+        if: matrix.cross
+        run: |
+          sudo dpkg --add-architecture arm64
+          sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
+          echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
+          echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
+          sudo apt-get update
+          sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
+          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
+          echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
+          echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
+          echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-release-${{ matrix.target }}-
+
+      - name: Build Rust volume server (large disk)
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }}
+
+      - name: Build Rust volume server (normal)
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }} --no-default-features
+
+      - name: Package binaries
+        run: |
+          # Large disk (default, 5bytes feature)
+          cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
+          tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
+          rm weed-volume-large-disk
+
+          # Normal volume size
+          cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
+          tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
+          rm weed-volume-normal
+
+      - name: Upload release assets
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
+            weed-volume_${{ matrix.asset_suffix }}.tar.gz
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload artifacts
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: rust-volume-${{ matrix.asset_suffix }}
+          path: |
+            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
+            weed-volume_${{ matrix.asset_suffix }}.tar.gz
+
+  build-rust-volume-darwin:
+    permissions:
+      contents: write
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-apple-darwin
+            asset_suffix: darwin_amd64
+          - target: aarch64-apple-darwin
+            asset_suffix: darwin_arm64
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install protobuf compiler
+        run: brew install protobuf
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-release-${{ matrix.target }}-
+
+      - name: Build Rust volume server (large disk)
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }}
+
+      - name: Build Rust volume server (normal)
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }} --no-default-features
+
+      - name: Package binaries
+        run: |
+          cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
+          tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
+          rm weed-volume-large-disk
+
+          cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
+          tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
+          rm weed-volume-normal
+
+      - name: Upload release assets
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
+            weed-volume_${{ matrix.asset_suffix }}.tar.gz
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload artifacts
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: rust-volume-${{ matrix.asset_suffix }}
+          path: |
+            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
+            weed-volume_${{ matrix.asset_suffix }}.tar.gz
+
+  build-rust-volume-windows:
+    permissions:
+      contents: write
+    runs-on: windows-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install protobuf compiler
+        run: choco install protoc -y
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-release-windows-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-release-windows-
+
+      - name: Build Rust volume server (large disk)
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release
+
+      - name: Build Rust volume server (normal)
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --no-default-features
+
+      - name: Package binaries
+        shell: bash
+        run: |
+          cp seaweed-volume/target/release/weed-volume.exe weed-volume-large-disk.exe
+          7z a weed-volume_large_disk_windows_amd64.zip weed-volume-large-disk.exe
+          rm weed-volume-large-disk.exe
+
+          cp seaweed-volume/target/release/weed-volume.exe weed-volume-normal.exe
+          7z a weed-volume_windows_amd64.zip weed-volume-normal.exe
+          rm weed-volume-normal.exe
+
+      - name: Upload release assets
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            weed-volume_large_disk_windows_amd64.zip
+            weed-volume_windows_amd64.zip
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload artifacts
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: rust-volume-windows_amd64
+          path: |
+            weed-volume_large_disk_windows_amd64.zip
+            weed-volume_windows_amd64.zip

.github/workflows/s3-mutation-regression-tests.yml

@@ -0,0 +1,133 @@
+name: "S3 Mutation Regression Tests"
+
+on:
+  pull_request:
+    paths:
+      - 'weed/s3api/**'
+      - 'test/s3/delete/**'
+      - 'test/s3/distributed_lock/**'
+      - 'test/s3/versioning/**'
+      - 'test/volume_server/framework/**'
+      - 'docker/compose/s3.json'
+      - '.github/workflows/s3-mutation-regression-tests.yml'
+
+concurrency:
+  group: ${{ github.head_ref }}/s3-mutation-regression-tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  s3-versioning-regressions:
+    name: S3 Versioning Regression Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 25
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Run S3 versioning regression tests
+        timeout-minutes: 20
+        working-directory: test/s3/versioning
+        run: |
+          set -x
+          make test-with-server TEST_PATTERN="TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker"
+
+      - name: Show server logs on failure
+        if: failure()
+        working-directory: test/s3/versioning
+        run: |
+          echo "=== Server Logs ==="
+          if [ -f weed-test.log ]; then
+            tail -100 weed-test.log
+          fi
+
+      - name: Upload versioning logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: s3-versioning-regression-logs
+          path: test/s3/versioning/weed-test*.log
+          retention-days: 3
+
+  s3-delete-regressions:
+    name: S3 Delete Regression Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Run S3 delete regression tests
+        timeout-minutes: 15
+        working-directory: test/s3/delete
+        run: |
+          set -x
+          make test-with-server
+
+      - name: Show server logs on failure
+        if: failure()
+        working-directory: test/s3/delete
+        run: |
+          echo "=== Server Logs ==="
+          if [ -f weed-test.log ]; then
+            tail -100 weed-test.log
+          fi
+
+      - name: Upload delete logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: s3-delete-regression-logs
+          path: test/s3/delete/weed-test*.log
+          retention-days: 3
+
+  s3-distributed-lock-regressions:
+    name: S3 Distributed Lock Regression Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Build SeaweedFS
+        run: |
+          go build -o weed/weed -buildvcs=false ./weed
+
+      - name: Run distributed lock regressions
+        timeout-minutes: 25
+        env:
+          TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp
+          S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1"
+          WEED_BINARY: ${{ github.workspace }}/weed/weed
+        run: |
+          set -x
+          mkdir -p "$TMPDIR"
+          go test -v -count=1 -timeout=20m ./test/s3/distributed_lock
+
+      - name: Upload distributed lock logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: s3-distributed-lock-regression-logs
+          path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_*
+          retention-days: 3

.github/workflows/tus-tests.yml

@@ -2,17 +2,6 @@ name: "TUS Protocol Tests"
 
 on:
   pull_request:
-    paths:
-      - 'weed/server/filer_server_tus*.go'
-      - 'weed/server/filer_server.go'
-      - 'test/tus/**'
-      - '.github/workflows/tus-tests.yml'
-  push:
-    branches: [ master, main ]
-    paths:
-      - 'weed/server/filer_server_tus*.go'
-      - 'weed/server/filer_server.go'
-      - 'test/tus/**'
 
 concurrency:
   group: ${{ github.head_ref || github.ref }}/tus-tests

.gitignore

@@ -143,3 +143,4 @@ test/s3/iam/.test_env
 weed_bin
 telemetry/server/telemetry-server
 .aider*
+/seaweed-volume/docs

.superset/config.json

@@ -0,0 +1,5 @@
+{
+  "setup": [],
+  "teardown": [],
+  "run": []
+}

VOLUME_SERVER_RUST_PLAN.md

@@ -0,0 +1,790 @@
+# Execution Plan: SeaweedFS Volume Server — Go to Rust Port
+
+## Scope Summary
+
+| Component | Go Source | Lines (non-test) | Description |
+|---|---|---|---|
+| CLI & startup | `weed/command/volume.go` | 476 | ~40 CLI flags, server bootstrap |
+| HTTP server + handlers | `weed/server/volume_server*.go` | 1,517 | Struct, routes, read/write/delete handlers |
+| gRPC handlers | `weed/server/volume_grpc_*.go` | 3,073 | 40 RPC method implementations |
+| Storage engine | `weed/storage/` | 15,271 | Volumes, needles, index, compaction, EC, backend |
+| Protobuf definitions | `weed/pb/volume_server.proto` | 759 | Service + message definitions |
+| Shared utilities | `weed/security/`, `weed/stats/`, `weed/util/` | ~2,000+ | JWT, TLS, metrics, helpers |
+| **Total** | | **~23,000+** | |
+
+## Rust Crate & Dependency Strategy
+
+```
+seaweed-volume/
+├── Cargo.toml
+├── build.rs                    # protobuf codegen
+├── proto/
+│   ├── volume_server.proto     # copied from Go, adapted
+│   └── remote.proto
+├── src/
+│   ├── main.rs                 # CLI entry point
+│   ├── config.rs               # CLI flags + config
+│   ├── server/
+│   │   ├── mod.rs
+│   │   ├── volume_server.rs    # VolumeServer struct + lifecycle
+│   │   ├── http_handlers.rs    # HTTP route dispatch
+│   │   ├── http_read.rs        # GET/HEAD handlers
+│   │   ├── http_write.rs       # POST/PUT handlers
+│   │   ├── http_delete.rs      # DELETE handler
+│   │   ├── http_admin.rs       # /status, /healthz, /ui
+│   │   ├── grpc_service.rs     # gRPC trait impl dispatch
+│   │   ├── grpc_vacuum.rs
+│   │   ├── grpc_copy.rs
+│   │   ├── grpc_erasure_coding.rs
+│   │   ├── grpc_tail.rs
+│   │   ├── grpc_admin.rs
+│   │   ├── grpc_read_write.rs
+│   │   ├── grpc_batch_delete.rs
+│   │   ├── grpc_scrub.rs
+│   │   ├── grpc_tier.rs
+│   │   ├── grpc_remote.rs
+│   │   ├── grpc_query.rs
+│   │   ├── grpc_state.rs
+│   │   └── grpc_client_to_master.rs  # heartbeat
+│   ├── storage/
+│   │   ├── mod.rs
+│   │   ├── store.rs            # Store (multi-disk manager)
+│   │   ├── volume.rs           # Volume struct + lifecycle
+│   │   ├── volume_read.rs
+│   │   ├── volume_write.rs
+│   │   ├── volume_compact.rs
+│   │   ├── volume_info.rs
+│   │   ├── needle/
+│   │   │   ├── mod.rs
+│   │   │   ├── needle.rs       # Needle struct + serialization
+│   │   │   ├── needle_read.rs
+│   │   │   ├── needle_write.rs
+│   │   │   ├── needle_map.rs   # in-memory NeedleMap
+│   │   │   ├── needle_value.rs
+│   │   │   └── crc.rs
+│   │   ├── super_block.rs
+│   │   ├── idx/
+│   │   │   ├── mod.rs
+│   │   │   └── idx.rs          # .idx file format read/write
+│   │   ├── needle_map_leveldb.rs
+│   │   ├── types.rs            # NeedleId, Offset, Size, DiskType
+│   │   ├── disk_location.rs    # DiskLocation per-directory
+│   │   ├── erasure_coding/
+│   │   │   ├── mod.rs
+│   │   │   ├── ec_volume.rs
+│   │   │   ├── ec_shard.rs
+│   │   │   ├── ec_encoder.rs   # Reed-Solomon encoding
+│   │   │   └── ec_decoder.rs
+│   │   └── backend/
+│   │       ├── mod.rs
+│   │       ├── disk.rs
+│   │       └── s3_backend.rs   # tiered storage to S3
+│   ├── topology/
+│   │   └── volume_layout.rs    # replication placement
+│   ├── security/
+│   │   ├── mod.rs
+│   │   ├── guard.rs            # whitelist + JWT gate
+│   │   ├── jwt.rs
+│   │   └── tls.rs
+│   ├── stats/
+│   │   ├── mod.rs
+│   │   └── metrics.rs          # Prometheus counters/gauges
+│   └── util/
+│       ├── mod.rs
+│       ├── grpc.rs
+│       ├── http.rs
+│       └── file.rs
+└── tests/
+    ├── integration/
+    │   ├── http_read_test.rs
+    │   ├── http_write_test.rs
+    │   ├── grpc_test.rs
+    │   └── storage_test.rs
+    └── unit/
+        ├── needle_test.rs
+        ├── idx_test.rs
+        ├── super_block_test.rs
+        └── ec_test.rs
+```
+
+### Key Rust dependencies
+
+| Purpose | Crate |
+|---|---|
+| Async runtime | `tokio` |
+| gRPC | `tonic` + `prost` |
+| HTTP server | `hyper` + `axum` |
+| CLI parsing | `clap` (derive) |
+| Prometheus metrics | `prometheus` |
+| JWT | `jsonwebtoken` |
+| TLS | `rustls` + `tokio-rustls` |
+| LevelDB | `rusty-leveldb` or `rocksdb` |
+| Reed-Solomon EC | `reed-solomon-erasure` |
+| Logging | `tracing` + `tracing-subscriber` |
+| Config (security.toml) | `toml` + `serde` |
+| CRC32 | `crc32fast` |
+| Memory-mapped files | `memmap2` |
+
+---
+
+## Phased Execution Plan
+
+### Phase 1: Project Skeleton & Protobuf Codegen
+**Goal:** Cargo project compiles, proto codegen works, CLI parses all flags.
+
+**Steps:**
+
+1.1. Create `seaweed-volume/Cargo.toml` with all dependencies listed above.
+
+1.2. Copy `volume_server.proto` and `remote.proto` into `proto/`. Adjust package paths for Rust codegen.
+
+1.3. Create `build.rs` using `tonic-build` to compile `.proto` files into Rust types.
+
+1.4. Create `src/main.rs` with `clap` derive structs mirroring all 40 CLI flags from `weed/command/volume.go`:
+   - `--port` (default 8080)
+   - `--port.grpc` (default 0 → 10000+port)
+   - `--port.public` (default 0 → same as port)
+   - `--ip` (auto-detect)
+   - `--id` (default empty → ip:port)
+   - `--publicUrl`
+   - `--ip.bind`
+   - `--master` (default "localhost:9333")
+   - `--mserver` (deprecated compat)
+   - `--preStopSeconds` (default 10)
+   - `--idleTimeout` (default 30)
+   - `--dataCenter`
+   - `--rack`
+   - `--index` [memory|leveldb|leveldbMedium|leveldbLarge]
+   - `--disk` [hdd|ssd|<tag>]
+   - `--tags`
+   - `--dir` (default temp dir)
+   - `--dir.idx`
+   - `--max` (default "8")
+   - `--whiteList`
+   - `--minFreeSpacePercent` (default "1")
+   - `--minFreeSpace`
+   - `--images.fix.orientation` (default false)
+   - `--readMode` [local|proxy|redirect] (default "proxy")
+   - `--cpuprofile`
+   - `--memprofile`
+   - `--compactionMBps` (default 0)
+   - `--maintenanceMBps` (default 0)
+   - `--fileSizeLimitMB` (default 256)
+   - `--concurrentUploadLimitMB` (default 0)
+   - `--concurrentDownloadLimitMB` (default 0)
+   - `--pprof` (default false)
+   - `--metricsPort` (default 0)
+   - `--metricsIp`
+   - `--inflightUploadDataTimeout` (default 60s)
+   - `--inflightDownloadDataTimeout` (default 60s)
+   - `--hasSlowRead` (default true)
+   - `--readBufferSizeMB` (default 4)
+   - `--index.leveldbTimeout` (default 0)
+   - `--debug` (default false)
+   - `--debug.port` (default 6060)
+
+1.5. Implement the same flag validation logic from `startVolumeServer()`:
+   - Parse comma-separated `--dir`, `--max`, `--minFreeSpace`, `--disk`, `--tags`
+   - Replicate single-value-to-all-dirs expansion
+   - Validate count matches between dirs and limits
+   - `--mserver` backward compat
+
+1.6. **Test:** `cargo build` succeeds. `cargo run -- --help` shows all flags. Proto types generated.
+
+**Verification:** Run with `--port 8080 --dir /tmp --master localhost:9333` — should parse without error and print config.
+
+---
+
+### Phase 2: Core Storage Types & On-Disk Format
+**Goal:** Read and write the SeaweedFS needle/volume binary format bit-for-bit compatible with Go.
+
+**Source files to port:**
+- `weed/storage/types/needle_types.go` → `src/storage/types.rs`
+- `weed/storage/needle/needle.go` → `src/storage/needle/needle.rs`
+- `weed/storage/needle/needle_read.go` → `src/storage/needle/needle_read.rs`
+- `weed/storage/needle/needle_write.go` (partial) → `src/storage/needle/needle_write.rs`
+- `weed/storage/needle/crc.go` → `src/storage/needle/crc.rs`
+- `weed/storage/needle/needle_value_map.go` → `src/storage/needle/needle_value.rs`
+- `weed/storage/super_block/super_block.go` → `src/storage/super_block.rs`
+- `weed/storage/idx/` → `src/storage/idx/`
+
+**Steps:**
+
+2.1. **Fundamental types** (`types.rs`):
+   - `NeedleId` (u64), `Offset` (u32 or u64 depending on version), `Size` (i32, negative = deleted)
+   - `Cookie` (u32)
+   - `DiskType` enum (HDD, SSD, Custom)
+   - Version constants (Version1=1, Version2=2, Version3=3, CurrentVersion=3)
+   - Byte serialization matching Go's `binary.BigEndian` encoding
+
+2.2. **SuperBlock** (`super_block.rs`):
+   - 8-byte header: Version(1) + ReplicaPlacement(1) + TTL(2) + CompactRevision(2) + Reserved(2)
+   - `ReplicaPlacement` struct with same/diff rack/dc counts
+   - `TTL` struct with count + unit
+   - Read/write from first 8 bytes of `.dat` file
+   - Match exact byte layout from `super_block.go`
+
+2.3. **Needle binary format** (`needle.rs`, `needle_read.rs`):
+   - Version 2/3 header: Cookie(4) + NeedleId(8) + Size(4)
+   - Body: Data, Flags, Name, Mime, PairsSize, Pairs, LastModified, TTL, Checksum, AppendAtNs, Padding
+   - CRC32 checksum (matching Go's `crc32.ChecksumIEEE`)
+   - Padding to 8-byte alignment
+   - Read path: read header → compute body length → read body → verify CRC
+
+2.4. **Idx file format** (`idx/`):
+   - Fixed 16-byte records: NeedleId(8) + Offset(4) + Size(4)
+   - Sequential append-only file
+   - Walk/iterate all entries
+   - Binary search not used (loaded into memory map)
+
+2.5. **NeedleMap (in-memory)** (`needle_map.rs`):
+   - HashMap<NeedleId, NeedleValue> where NeedleValue = {Offset, Size}
+   - Load from `.idx` file on volume mount
+   - Support Get, Set, Delete operations
+   - Track file count, deleted count, deleted byte count
+
+2.6. **Tests:**
+   - Unit test: write a needle to bytes → read it back → verify fields match
+   - Unit test: write/read SuperBlock round-trip
+   - Unit test: write/read idx entries round-trip
+   - **Cross-compat test:** Use Go volume server to create a small volume with known data. Read it from Rust and verify all needles decoded correctly. (Keep test fixture `.dat`/`.idx` files in `tests/fixtures/`)
+
+---
+
+### Phase 3: Volume Struct & Lifecycle
+**Goal:** Mount, read from, write to, and unmount a volume.
+
+**Source files to port:**
+- `weed/storage/volume.go` → `src/storage/volume.rs`
+- `weed/storage/volume_read.go` → `src/storage/volume_read.rs`
+- `weed/storage/volume_write.go` → `src/storage/volume_write.rs`
+- `weed/storage/volume_loading.go`
+- `weed/storage/volume_vacuum.go` → `src/storage/volume_compact.rs`
+- `weed/storage/volume_info/volume_info.go` → `src/storage/volume_info.rs`
+- `weed/storage/volume_super_block.go`
+
+**Steps:**
+
+3.1. **Volume struct** (`volume.rs`):
+   - Fields: Id, dir, dataFile, nm (NeedleMap), SuperBlock, readOnly, lastModifiedTs, lastCompactIndexOffset, lastCompactRevision
+   - `noWriteOrDelete` / `noWriteCanDelete` / `readOnly` state flags
+   - File handles for `.dat` file (read + append)
+   - Lock strategy: `RwLock` for concurrent reads, exclusive writes
+
+3.2. **Volume loading** — exact logic from `volume_loading.go`:
+   - Open `.dat` file, read SuperBlock from first 8 bytes
+   - Load `.idx` file into NeedleMap
+   - Handle `.vif` (VolumeInfo) JSON sidecar file
+   - Set volume state based on SuperBlock + VolumeInfo
+
+3.3. **Volume read** (`volume_read.rs`) — from `volume_read.go`:
+   - `ReadNeedle(needleId, cookie)`: lookup in NeedleMap → seek in .dat → read needle bytes → verify cookie + CRC → return data
+   - Handle deleted needles (Size < 0)
+   - `ReadNeedleBlob(offset, size)`: raw blob read
+   - `ReadNeedleMeta(needleId, offset, size)`: read metadata only
+
+3.4. **Volume write** (`volume_write.rs`) — from `volume_write.go`:
+   - `WriteNeedle(needle)`: serialize needle → append to .dat → update .idx → update NeedleMap
+   - `DeleteNeedle(needleId)`: mark as deleted in NeedleMap + append tombstone to .idx
+   - File size limit check
+   - Concurrent write serialization (mutex on write path)
+
+3.5. **Volume compaction** (`volume_compact.rs`) — from `volume_vacuum.go`:
+   - `CheckCompact()`: compute garbage ratio
+   - `Compact()`: create new .dat/.idx, copy only live needles, update compact revision
+   - `CommitCompact()`: rename compacted files over originals
+   - `CleanupCompact()`: remove temp files
+   - Throttle by `compactionBytePerSecond`
+
+3.6. **Volume info** (`volume_info.rs`):
+   - Read/write `.vif` JSON sidecar
+   - VolumeInfo protobuf struct mapping
+   - Remote file references for tiered storage
+
+3.7. **Tests:**
+   - Mount a volume, write 100 needles, read them all back, verify content
+   - Delete 50 needles, verify they return "deleted"
+   - Compact, verify only 50 remain, verify content
+   - Read Go-created volume fixtures
+
+---
+
+### Phase 4: Store (Multi-Volume, Multi-Disk Manager)
+**Goal:** Manage multiple volumes across multiple disk directories.
+
+**Source files to port:**
+- `weed/storage/store.go` → `src/storage/store.rs`
+- `weed/storage/disk_location.go` → `src/storage/disk_location.rs`
+- `weed/storage/store_ec.go`
+- `weed/storage/store_state.go`
+
+**Steps:**
+
+4.1. **DiskLocation** (`disk_location.rs`):
+   - Directory path, max volume count, min free space, disk type, tags
+   - Load all volumes from directory on startup
+   - Track free space, check writable
+
+4.2. **Store** (`store.rs`):
+   - Vector of `DiskLocation`s
+   - `GetVolume(volumeId)` → lookup across all locations
+   - `HasVolume(volumeId)` check
+   - `AllocateVolume(...)` — create new volume in appropriate location
+   - `DeleteVolume(...)`, `MountVolume(...)`, `UnmountVolume(...)`
+   - `DeleteCollection(collection)` — delete all volumes of a collection
+   - Collect volume status for heartbeat
+   - `SetStopping()`, `Close()`
+   - Persistent state (maintenance mode) via `store_state.go`
+
+4.3. **Store state** — `VolumeServerState` protobuf with maintenance flag, persisted to disk.
+
+4.4. **Tests:**
+   - Create store with 2 dirs, allocate volumes in each, verify load balancing
+   - Mount/unmount/delete lifecycle
+   - State persistence across restart
+
+---
+
+### Phase 5: Erasure Coding
+**Goal:** Full EC shard encode/decode/read/write/rebuild.
+
+**Source files to port:**
+- `weed/storage/erasure_coding/` (3,599 lines)
+
+**Steps:**
+
+5.1. **EC volume + shard structs** — `EcVolume`, `EcShard` with file handles for `.ec00`–`.ec13` shard files + `.ecx` index + `.ecj` journal.
+
+5.2. **EC encoder** — Reed-Solomon 10+4 (configurable) encoding using `reed-solomon-erasure` crate:
+   - `VolumeEcShardsGenerate`: read .dat → split into data shards → compute parity → write .ec00-.ec13 + .ecx
+
+5.3. **EC decoder/reader** — reconstruct data from any 10 of 14 shards:
+   - `EcShardRead`: read range from a specific shard
+   - Locate needle in EC volume via .ecx index
+   - Handle cross-shard needle reads
+
+5.4. **EC shard operations:**
+   - Copy, delete, mount, unmount shards
+   - `VolumeEcShardsRebuild`: rebuild missing shards from remaining
+   - `VolumeEcShardsToVolume`: reconstruct .dat from EC shards
+   - `VolumeEcBlobDelete`: mark deleted in EC journal
+   - `VolumeEcShardsInfo`: report shard metadata
+
+5.5. **Tests:**
+   - Encode a volume → verify 14 shards created
+   - Delete 4 shards → rebuild → verify data intact
+   - Read individual needles from EC volume
+   - Cross-compat with Go-generated EC shards
+
+---
+
+### Phase 6: Backend / Tiered Storage
+**Goal:** Support tiered storage to remote backends (S3, etc).
+
+**Source files to port:**
+- `weed/storage/backend/` (1,850 lines)
+
+**Steps:**
+
+6.1. **Backend trait** — abstract `BackendStorage` trait with `ReadAt`, `WriteAt`, `Truncate`, `Close`, `Name`.
+
+6.2. **Disk backend** — default local disk implementation.
+
+6.3. **S3 backend** — upload .dat to S3, read ranges via S3 range requests.
+
+6.4. **Tier move operations:**
+   - `VolumeTierMoveDatToRemote`: upload .dat to remote, optionally delete local
+   - `VolumeTierMoveDatFromRemote`: download .dat from remote
+
+6.5. **Tests:**
+   - Disk backend read/write round-trip
+   - S3 backend with mock/localstack
+
+---
+
+### Phase 7: Security Layer
+**Goal:** JWT authentication, whitelist guard, TLS configuration.
+
+**Source files to port:**
+- `weed/security/guard.go` → `src/security/guard.rs`
+- `weed/security/jwt.go` → `src/security/jwt.rs`
+- `weed/security/tls.go` → `src/security/tls.rs`
+
+**Steps:**
+
+7.1. **Guard** (`guard.rs`):
+   - Whitelist IP check (exact match on `r.RemoteAddr`)
+   - Wrap handlers with whitelist enforcement
+   - `UpdateWhiteList()` for live reload
+
+7.2. **JWT** (`jwt.rs`):
+   - `SeaweedFileIdClaims` with `fid` field
+   - Sign with HMAC-SHA256
+   - Verify + decode with expiry check
+   - Separate signing keys for read vs write
+   - `GetJwt(request)` — extract from `Authorization: Bearer` header or `jwt` query param
+
+7.3. **TLS** (`tls.rs`):
+   - Load server TLS cert/key for gRPC and HTTPS
+   - Load client TLS for mutual TLS
+   - Read from `security.toml` config (same format as Go's viper config)
+
+7.4. **Tests:**
+   - JWT sign → verify round-trip
+   - JWT with wrong key → reject
+   - JWT with expired token → reject
+   - JWT fid mismatch → reject
+   - Whitelist allow/deny
+
+---
+
+### Phase 8: Prometheus Metrics
+**Goal:** Export same metric names as Go for dashboard compatibility.
+
+**Source files to port:**
+- `weed/stats/metrics.go` (volume server counters/gauges/histograms)
+
+**Steps:**
+
+8.1. Define all Prometheus metrics matching Go names:
+   - `VolumeServerRequestCounter` (labels: method, status)
+   - `VolumeServerRequestHistogram` (labels: method)
+   - `VolumeServerInFlightRequestsGauge` (labels: method)
+   - `VolumeServerInFlightUploadSize`
+   - `VolumeServerInFlightDownloadSize`
+   - `VolumeServerConcurrentUploadLimit`
+   - `VolumeServerConcurrentDownloadLimit`
+   - `VolumeServerHandlerCounter` (labels: type — UploadLimitCond, DownloadLimitCond)
+   - Read/Write/Delete request counters
+
+8.2. Metrics HTTP endpoint on `--metricsPort`.
+
+8.3. Optional push-based metrics loop (`LoopPushingMetric`).
+
+8.4. **Test:** Verify metric names and labels match Go output.
+
+---
+
+### Phase 9: HTTP Server & Handlers
+**Goal:** All HTTP endpoints with exact same behavior as Go.
+
+**Source files to port:**
+- `weed/server/volume_server.go` → `src/server/volume_server.rs`
+- `weed/server/volume_server_handlers.go` → `src/server/http_handlers.rs`
+- `weed/server/volume_server_handlers_read.go` → `src/server/http_read.rs`
+- `weed/server/volume_server_handlers_write.go` → `src/server/http_write.rs`
+- `weed/server/volume_server_handlers_admin.go` → `src/server/http_admin.rs`
+- `weed/server/volume_server_handlers_helper.go` (URL parsing, proxy, JSON responses)
+- `weed/server/volume_server_handlers_ui.go` → `src/server/http_admin.rs`
+
+**Steps:**
+
+9.1. **URL path parsing** — from `handlers_helper.go`:
+   - Parse `/<vid>,<fid>` and `/<vid>/<fid>` patterns
+   - Extract volume ID, file ID, filename, ext
+
+9.2. **Route dispatch** — from `privateStoreHandler` and `publicReadOnlyHandler`:
+   - `GET /` → `GetOrHeadHandler`
+   - `HEAD /` → `GetOrHeadHandler`
+   - `POST /` → `PostHandler` (whitelist gated)
+   - `PUT /` → `PostHandler` (whitelist gated)
+   - `DELETE /` → `DeleteHandler` (whitelist gated)
+   - `OPTIONS /` → CORS preflight
+   - `GET /status` → JSON status
+   - `GET /healthz` → health check
+   - `GET /ui/index.html` → HTML UI page
+   - Static resources (CSS/JS for UI)
+
+9.3. **GET/HEAD handler** (`http_read.rs`) — from `handlers_read.go` (468 lines):
+   - JWT read authorization check
+   - Lookup needle by volume ID + needle ID + cookie
+   - ETag / If-None-Match / If-Modified-Since conditional responses
+   - Content-Type from stored MIME or filename extension
+   - Content-Disposition header
+   - Content-Encoding (gzip/zstd stored data)
+   - Range request support (HTTP 206 Partial Content)
+   - JPEG orientation fix (if configured)
+   - Proxy to replica on local miss (readMode=proxy)
+   - Redirect to replica (readMode=redirect)
+   - Download tracking (in-flight size accounting)
+
+9.4. **POST/PUT handler** (`http_write.rs`) — from `handlers_write.go` (170 lines):
+   - JWT write authorization check
+   - Multipart form parsing
+   - Extract file data, filename, content type, TTL, last-modified
+   - Optional gzip/zstd compression
+   - Write needle to volume
+   - Replicate to peers (same logic as Go's `DistributedOperation`)
+   - Return JSON: {name, size, eTag, error}
+
+9.5. **DELETE handler** — already in handlers.go:
+   - JWT authorization
+   - Delete from local volume
+   - Replicate delete to peers
+   - Return JSON result
+
+9.6. **Admin handlers** (`http_admin.rs`):
+   - `/status` → JSON with volumes, version, disk status
+   - `/healthz` → 200 OK if serving
+   - `/ui/index.html` → HTML dashboard
+
+9.7. **Concurrency limiting** — from `handlers.go`:
+   - Upload concurrency limit with `sync::Condvar` + timeout
+   - Download concurrency limit with proxy fallback to replicas
+   - HTTP 429 on timeout, 499 on client cancel
+   - Replication traffic bypasses upload limits
+
+9.8. **Public port** — if configured, separate listener with read-only routes (GET/HEAD/OPTIONS only).
+
+9.9. **Request ID middleware** — generate unique request ID per request.
+
+9.10. **Tests:**
+   - Integration: start server → upload file via POST → GET it back → verify content
+   - Integration: upload → DELETE → GET returns 404
+   - Integration: conditional GET with ETag → 304
+   - Integration: range request → 206 with correct bytes
+   - Integration: exceed upload limit → 429
+   - Integration: whitelist enforcement
+   - Integration: JWT enforcement
+
+---
+
+### Phase 10: gRPC Service Implementation
+**Goal:** All 40 gRPC methods with exact logic.
+
+**Source files to port:**
+- `weed/server/volume_grpc_admin.go` (380 lines)
+- `weed/server/volume_grpc_vacuum.go` (124 lines)
+- `weed/server/volume_grpc_copy.go` (636 lines)
+- `weed/server/volume_grpc_copy_incremental.go` (66 lines)
+- `weed/server/volume_grpc_read_write.go` (74 lines)
+- `weed/server/volume_grpc_batch_delete.go` (124 lines)
+- `weed/server/volume_grpc_tail.go` (140 lines)
+- `weed/server/volume_grpc_erasure_coding.go` (619 lines)
+- `weed/server/volume_grpc_scrub.go` (121 lines)
+- `weed/server/volume_grpc_tier_upload.go` (98 lines)
+- `weed/server/volume_grpc_tier_download.go` (85 lines)
+- `weed/server/volume_grpc_remote.go` (95 lines)
+- `weed/server/volume_grpc_query.go` (69 lines)
+- `weed/server/volume_grpc_state.go` (26 lines)
+- `weed/server/volume_grpc_read_all.go` (35 lines)
+- `weed/server/volume_grpc_client_to_master.go` (325 lines)
+
+**Steps (grouped by functional area):**
+
+10.1. **Implement `tonic::Service` for `VolumeServer`** — the generated trait from proto.
+
+10.2. **Admin RPCs** (`grpc_admin.rs`):
+   - `AllocateVolume` — create volume on appropriate disk location
+   - `VolumeMount` / `VolumeUnmount` / `VolumeDelete`
+   - `VolumeMarkReadonly` / `VolumeMarkWritable`
+   - `VolumeConfigure` — change replication
+   - `VolumeStatus` — return read-only, size, file counts
+   - `VolumeServerStatus` — disk statuses, memory, version, DC, rack
+   - `VolumeServerLeave` — deregister from master
+   - `DeleteCollection`
+   - `VolumeNeedleStatus` — get needle metadata by ID
+   - `Ping` — latency measurement
+   - `GetState` / `SetState` — maintenance mode
+
+10.3. **Vacuum RPCs** (`grpc_vacuum.rs`):
+   - `VacuumVolumeCheck` — return garbage ratio
+   - `VacuumVolumeCompact` — stream progress (streaming response)
+   - `VacuumVolumeCommit` — finalize compaction
+   - `VacuumVolumeCleanup` — remove temp files
+
+10.4. **Copy RPCs** (`grpc_copy.rs`):
+   - `VolumeCopy` — stream .dat/.idx from source to create local copy
+   - `VolumeSyncStatus` — return sync metadata
+   - `VolumeIncrementalCopy` — stream .dat delta since timestamp (streaming)
+   - `CopyFile` — generic file copy by extension (streaming)
+   - `ReceiveFile` — receive streamed file (client streaming)
+   - `ReadVolumeFileStatus` — return file timestamps and sizes
+
+10.5. **Read/Write RPCs** (`grpc_read_write.rs`):
+   - `ReadNeedleBlob` — raw needle blob read
+   - `ReadNeedleMeta` — needle metadata
+   - `WriteNeedleBlob` — raw needle blob write
+   - `ReadAllNeedles` — stream all needles from volume(s) (streaming)
+
+10.6. **Batch delete** (`grpc_batch_delete.rs`):
+   - `BatchDelete` — delete multiple file IDs, return per-ID results
+
+10.7. **Tail RPCs** (`grpc_tail.rs`):
+   - `VolumeTailSender` — stream new needles since timestamp (streaming)
+   - `VolumeTailReceiver` — connect to another volume server and tail its changes
+
+10.8. **Erasure coding RPCs** (`grpc_erasure_coding.rs`):
+   - `VolumeEcShardsGenerate` — generate EC shards from volume
+   - `VolumeEcShardsRebuild` — rebuild missing shards
+   - `VolumeEcShardsCopy` — copy shards from another server
+   - `VolumeEcShardsDelete` — delete EC shards
+   - `VolumeEcShardsMount` / `VolumeEcShardsUnmount`
+   - `VolumeEcShardRead` — read from EC shard (streaming)
+   - `VolumeEcBlobDelete` — mark blob deleted in EC volume
+   - `VolumeEcShardsToVolume` — reconstruct volume from EC shards
+   - `VolumeEcShardsInfo` — return shard metadata
+
+10.9. **Scrub RPCs** (`grpc_scrub.rs`):
+   - `ScrubVolume` — integrity check volumes (INDEX / FULL / LOCAL modes)
+   - `ScrubEcVolume` — integrity check EC volumes
+
+10.10. **Tier RPCs** (`grpc_tier.rs`):
+   - `VolumeTierMoveDatToRemote` — upload to remote backend (streaming progress)
+   - `VolumeTierMoveDatFromRemote` — download from remote (streaming progress)
+
+10.11. **Remote storage** (`grpc_remote.rs`):
+   - `FetchAndWriteNeedle` — fetch from remote storage, write locally, replicate
+
+10.12. **Query** (`grpc_query.rs`):
+   - `Query` — experimental CSV/JSON/Parquet select on stored data (streaming)
+
+10.13. **Master heartbeat** (`grpc_client_to_master.rs`):
+   - `heartbeat()` background task — periodic gRPC stream to master
+   - Send: volume info, EC shard info, disk stats, has-no-space flags, deleted volumes
+   - Receive: volume size limit, leader address, metrics config
+   - Reconnect on failure with backoff
+   - `StopHeartbeat()` for graceful shutdown
+
+10.14. **Tests:**
+   - Integration test per RPC: call via tonic client → verify response
+   - Streaming RPCs: verify all chunks received
+   - Error cases: invalid volume ID, non-existent volume, etc.
+   - Heartbeat: mock master gRPC server, verify registration
+
+---
+
+### Phase 11: Startup, Lifecycle & Graceful Shutdown
+**Goal:** Full server startup matching Go's `runVolume()` and `startVolumeServer()`.
+
+**Steps:**
+
+11.1. **Startup sequence** (match `volume.go` exactly):
+   1. Load security configuration from `security.toml`
+   2. Start metrics server on metrics port
+   3. Parse folder/max/minFreeSpace/diskType/tags
+   4. Validate all directory writable
+   5. Resolve IP, bind IP, public URL, gRPC port
+   6. Create `VolumeServer` struct
+   7. Check with master (initial handshake)
+   8. Create `Store` (loads all existing volumes from disk)
+   9. Create security `Guard`
+   10. Register HTTP routes on admin mux
+   11. Optionally register public mux
+   12. Start gRPC server on gRPC port
+   13. Start public HTTP server (if separated)
+   14. Start cluster HTTP server (with optional TLS)
+   15. Start heartbeat background task
+   16. Start metrics push loop
+   17. Register SIGHUP handler for config reload + new volume loading
+
+11.2. **Graceful shutdown** (match Go exactly):
+   1. On SIGINT/SIGTERM:
+   2. Stop heartbeat (notify master we're leaving)
+   3. Wait `preStopSeconds`
+   4. Stop public HTTP server
+   5. Stop cluster HTTP server
+   6. Graceful stop gRPC server
+   7. `volumeServer.Shutdown()` → `store.Close()` (flush all volumes)
+
+11.3. **Reload** (SIGHUP):
+   - Reload security config
+   - Update whitelist
+   - Load newly appeared volumes from disk
+
+11.4. **Tests:**
+   - Start server → send SIGTERM → verify clean shutdown
+   - Start server → SIGHUP → verify config reloaded
+
+---
+
+### Phase 12: Integration & Cross-Compatibility Testing
+**Goal:** Rust volume server is a drop-in replacement for Go volume server.
+
+**Steps:**
+
+12.1. **Binary compatibility tests:**
+   - Create volumes with Go volume server
+   - Start Rust volume server on same data directory
+   - Read all data → verify identical
+   - Write new data with Rust → read with Go → verify
+
+12.2. **API compatibility tests:**
+   - Run same HTTP requests against both Go and Rust servers
+   - Compare response bodies, headers, status codes
+   - Test all gRPC RPCs against both
+
+12.3. **Master interop test:**
+   - Start Go master server
+   - Register Rust volume server
+   - Verify heartbeat works
+   - Verify volume assignment works
+   - Upload via filer → stored on Rust volume server → read back
+
+12.4. **Performance benchmarks:**
+   - Throughput: sequential writes, sequential reads
+   - Latency: p50/p99 for read/write
+   - Concurrency: parallel reads/writes
+   - Compare Rust vs Go numbers
+
+12.5. **Edge cases:**
+   - Volume at max size
+   - Disk full handling
+   - Corrupt .dat file recovery
+   - Network partition during replication
+   - EC shard loss + rebuild
+
+---
+
+## Execution Order & Dependencies
+
+```
+Phase 1  (Skeleton + CLI)        ← no deps, start here
+   ↓
+Phase 2  (Storage types)         ← needs Phase 1 (types used everywhere)
+   ↓
+Phase 3  (Volume struct)         ← needs Phase 2
+   ↓
+Phase 4  (Store manager)         ← needs Phase 3
+   ↓
+Phase 7  (Security)              ← independent, can parallel with 3-4
+Phase 8  (Metrics)               ← independent, can parallel with 3-4
+   ↓
+Phase 9  (HTTP server)           ← needs Phase 4 + 7 + 8
+Phase 10 (gRPC server)           ← needs Phase 4 + 7 + 8
+   ↓
+Phase 5  (Erasure coding)        ← needs Phase 4, wire into Phase 10
+Phase 6  (Tiered storage)        ← needs Phase 4, wire into Phase 10
+   ↓
+Phase 11 (Startup + shutdown)    ← needs Phase 9 + 10
+   ↓
+Phase 12 (Integration tests)     ← needs all above
+```
+
+## Estimated Scope
+
+| Phase | Estimated Rust Lines | Complexity |
+|---|---|---|
+| 1. Skeleton + CLI | ~400 | Low |
+| 2. Storage types | ~2,000 | High (binary compat critical) |
+| 3. Volume struct | ~2,500 | High |
+| 4. Store manager | ~1,000 | Medium |
+| 5. Erasure coding | ~3,000 | High |
+| 6. Tiered storage | ~1,500 | Medium |
+| 7. Security | ~500 | Medium |
+| 8. Metrics | ~300 | Low |
+| 9. HTTP server | ~2,000 | High |
+| 10. gRPC server | ~3,500 | High |
+| 11. Startup/shutdown | ~500 | Medium |
+| 12. Integration tests | ~2,000 | Medium |
+| **Total** | **~19,000** | |
+
+## Critical Invariants to Preserve
+
+1. **Binary format compatibility** — Rust must read/write `.dat`, `.idx`, `.vif`, `.ecX` files identically to Go. A single byte off = data loss.
+2. **gRPC wire compatibility** — Same proto, same field semantics. Go master must talk to Rust volume server seamlessly.
+3. **HTTP API compatibility** — Same URL patterns, same JSON response shapes, same headers, same status codes.
+4. **Replication protocol** — Write replication between Go and Rust volume servers must work bidirectionally.
+5. **Heartbeat protocol** — Rust volume server must register with Go master and maintain heartbeat.
+6. **CRC32 algorithm** — Must use IEEE polynomial (same as Go's `crc32.ChecksumIEEE`).
+7. **JWT compatibility** — Tokens signed by Go filer/master must be verifiable by Rust volume server and vice versa.

docker/Dockerfile.go_build

@@ -16,9 +16,38 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
   && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
   && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
 
+# Rust volume server: use pre-built binary from CI when available (placed in
+# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
+# from source.  Pre-building avoids a multi-hour QEMU-emulated cargo build
+# for non-native architectures.
+FROM alpine:3.23 as rust_builder
+ARG TARGETARCH
+ARG TAGS
+COPY weed-volume-prebuilt/ /prebuilt/
+COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
+COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
+WORKDIR /build/seaweed-volume
+RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
+      echo "Using pre-built Rust binary for ${TARGETARCH}" && \
+      cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \
+    elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
+      apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
+      if [ "$TAGS" = "5BytesOffset" ]; then \
+        cargo build --release; \
+      else \
+        cargo build --release --no-default-features; \
+      fi && \
+      cp target/release/weed-volume /weed-volume; \
+    else \
+      echo "Skipping Rust build for $TARGETARCH (unsupported)" && \
+      touch /weed-volume; \
+    fi
+
 FROM alpine AS final
 LABEL author="Chris Lu"
 COPY --from=builder /go/bin/weed /usr/bin/
+# Copy Rust volume server binary (real binary on amd64/arm64, empty placeholder on other platforms)
+COPY --from=rust_builder /weed-volume /usr/bin/weed-volume
 RUN mkdir -p /etc/seaweedfs
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
@@ -27,7 +56,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
 # To disable: docker run -e GODEBUG=fips140=off ...
 
 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse curl su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse curl su-exec libgcc && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 

docker/Dockerfile.local

@@ -7,7 +7,8 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
 COPY ./entrypoint.sh /entrypoint.sh
 
 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse curl su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse curl su-exec && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 

docker/Dockerfile.rocksdb_large

@@ -34,7 +34,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
 
 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse snappy gflags curl su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse snappy gflags curl su-exec && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 

docker/Dockerfile.rocksdb_large_local

@@ -17,7 +17,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
 
 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse snappy gflags curl tmux su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse snappy gflags curl tmux su-exec && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 

docker/compose/s3_static_users_example.json

@@ -0,0 +1,53 @@
+{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "AKIAIOSFODNN7EXAMPLE",
+          "secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    },
+    {
+      "name": "steward",
+      "credentials": [
+        {
+          "accessKey": "steward-key",
+          "secretKey": "steward-secret"
+        }
+      ],
+      "actions": [
+        "Read",
+        "List",
+        "Write"
+      ]
+    },
+    {
+      "name": "le001",
+      "credentials": [
+        {
+          "accessKey": "le001-key",
+          "secretKey": "le001-secret"
+        }
+      ],
+      "actions": [
+        "Read",
+        "List"
+      ]
+    },
+    {
+      "name": "anonymous",
+      "actions": [
+        "Read"
+      ]
+    }
+  ]
+}

docker/compose/test-git-on-mount.sh

@@ -0,0 +1,260 @@
+#!/usr/bin/env bash
+#
+# Integration test: git clone & pull on a SeaweedFS FUSE mount.
+#
+# Verifies that the mount correctly supports git's file operations by:
+#   1. Creating a bare repo on the mount (acts as a remote)
+#   2. Cloning it, making commits, and pushing back to the mount
+#   3. Cloning from the mount into a working directory also on the mount
+#   4. Pushing additional commits to the bare repo
+#   5. Checking out an older revision in the on-mount clone
+#   6. Running git pull to fast-forward with real changes
+#   7. Verifying file content integrity at each step
+#
+# Usage:
+#   bash test-git-on-mount.sh /path/to/mount/point
+#
+# The mount must already be running. All test artifacts are created under
+# <mount>/git-test-<pid> and cleaned up on exit (unless TEST_KEEP=1).
+#
+set -euo pipefail
+
+MOUNT_DIR="${1:?Usage: $0 <mount-dir>}"
+TEST_DIR="$MOUNT_DIR/git-test-$$"
+LOCAL_DIR=$(mktemp -d)
+PASS=0
+FAIL=0
+
+cleanup() {
+    if [[ "${TEST_KEEP:-}" == "1" ]]; then
+        echo "TEST_KEEP=1 — leaving artifacts:"
+        echo "  mount: $TEST_DIR"
+        echo "  local: $LOCAL_DIR"
+    else
+        rm -rf "$TEST_DIR" 2>/dev/null || true
+        rm -rf "$LOCAL_DIR" 2>/dev/null || true
+    fi
+}
+trap cleanup EXIT
+
+# --- helpers ---------------------------------------------------------------
+
+pass() { PASS=$((PASS + 1)); echo "  PASS: $1"; }
+fail() { FAIL=$((FAIL + 1)); echo "  FAIL: $1"; }
+
+assert_file_contains() {
+    local file=$1 expected=$2 label=$3
+    if [[ -f "$file" ]] && grep -qF "$expected" "$file" 2>/dev/null; then
+        pass "$label"
+    else
+        fail "$label (expected '$expected' in $file)"
+    fi
+}
+
+assert_file_exists() {
+    local file=$1 label=$2
+    if [[ -f "$file" ]]; then
+        pass "$label"
+    else
+        fail "$label ($file not found)"
+    fi
+}
+
+assert_file_not_exists() {
+    local file=$1 label=$2
+    if [[ ! -f "$file" ]]; then
+        pass "$label"
+    else
+        fail "$label ($file should not exist)"
+    fi
+}
+
+assert_eq() {
+    local actual=$1 expected=$2 label=$3
+    if [[ "$actual" == "$expected" ]]; then
+        pass "$label"
+    else
+        fail "$label (expected '$expected', got '$actual')"
+    fi
+}
+
+# --- setup -----------------------------------------------------------------
+
+echo "========================================"
+echo "  Git-on-mount integration test"
+echo "========================================"
+echo "Mount:  $MOUNT_DIR"
+echo "Test:   $TEST_DIR"
+echo "Local:  $LOCAL_DIR"
+echo ""
+
+if ! mountpoint -q "$MOUNT_DIR" 2>/dev/null && [[ ! -d "$MOUNT_DIR" ]]; then
+    echo "ERROR: $MOUNT_DIR is not a valid directory"
+    exit 1
+fi
+
+mkdir -p "$TEST_DIR"
+
+# --- Phase 1: Create bare repo on mount -----------------------------------
+
+echo "--- Phase 1: Create bare repo on mount ---"
+
+BARE_REPO="$TEST_DIR/repo.git"
+git init --bare "$BARE_REPO" >/dev/null 2>&1
+pass "bare repo created on mount"
+
+# --- Phase 2: Clone locally, make initial commits, push -------------------
+
+echo "--- Phase 2: Clone locally, make initial commits, push ---"
+
+LOCAL_CLONE="$LOCAL_DIR/clone1"
+git clone "$BARE_REPO" "$LOCAL_CLONE" >/dev/null 2>&1
+cd "$LOCAL_CLONE"
+git config user.email "test@seaweedfs.test"
+git config user.name "SeaweedFS Test"
+
+# Commit 1: initial files
+echo "hello world" > README.md
+mkdir -p src
+echo 'package main; import "fmt"; func main() { fmt.Println("v1") }' > src/main.go
+git add -A && git commit -m "initial commit" >/dev/null 2>&1
+COMMIT1=$(git rev-parse HEAD)
+
+# Commit 2: add more files
+mkdir -p data
+for i in $(seq 1 20); do
+    printf "file-%03d: %s\n" "$i" "$(head -c 64 /dev/urandom | base64)" > "data/file-$(printf '%03d' $i).txt"
+done
+git add -A && git commit -m "add data files" >/dev/null 2>&1
+COMMIT2=$(git rev-parse HEAD)
+
+# Commit 3: modify and add
+echo 'package main; import "fmt"; func main() { fmt.Println("v2") }' > src/main.go
+echo "# Updated readme" >> README.md
+mkdir -p docs
+echo "documentation content" > docs/guide.md
+git add -A && git commit -m "update src and add docs" >/dev/null 2>&1
+COMMIT3=$(git rev-parse HEAD)
+
+git push origin master >/dev/null 2>&1 || git push origin main >/dev/null 2>&1
+BRANCH=$(git rev-parse --abbrev-ref HEAD)
+pass "3 commits pushed to mount bare repo (branch=$BRANCH)"
+
+# --- Phase 3: Clone from mount bare repo to mount working dir -------------
+
+echo "--- Phase 3: Clone from mount bare repo to on-mount working dir ---"
+
+MOUNT_CLONE="$TEST_DIR/working"
+git clone "$BARE_REPO" "$MOUNT_CLONE" >/dev/null 2>&1
+
+# Verify clone integrity
+assert_file_exists "$MOUNT_CLONE/README.md" "README.md exists after clone"
+assert_file_contains "$MOUNT_CLONE/README.md" "# Updated readme" "README.md has latest content"
+assert_file_contains "$MOUNT_CLONE/src/main.go" 'v2' "src/main.go has v2"
+assert_file_exists "$MOUNT_CLONE/docs/guide.md" "docs/guide.md exists"
+assert_file_exists "$MOUNT_CLONE/data/file-001.txt" "data files exist"
+assert_file_exists "$MOUNT_CLONE/data/file-020.txt" "data/file-020.txt exists"
+
+CLONE_HEAD=$(cd "$MOUNT_CLONE" && git rev-parse HEAD)
+assert_eq "$CLONE_HEAD" "$COMMIT3" "on-mount clone HEAD matches commit 3"
+
+# Count files
+FILE_COUNT=$(find "$MOUNT_CLONE/data" -name '*.txt' | wc -l | tr -d ' ')
+assert_eq "$FILE_COUNT" "20" "data/ has 20 files"
+
+# --- Phase 4: Push more commits from local clone --------------------------
+
+echo "--- Phase 4: Push more commits from local clone ---"
+
+cd "$LOCAL_CLONE"
+
+# Commit 4: larger changes
+for i in $(seq 21 50); do
+    printf "file-%03d: %s\n" "$i" "$(head -c 128 /dev/urandom | base64)" > "data/file-$(printf '%03d' $i).txt"
+done
+echo 'package main; import "fmt"; func main() { fmt.Println("v3") }' > src/main.go
+git add -A && git commit -m "expand data and update to v3" >/dev/null 2>&1
+COMMIT4=$(git rev-parse HEAD)
+
+# Commit 5: rename and delete
+git mv docs/guide.md docs/manual.md
+git rm data/file-001.txt >/dev/null 2>&1
+git commit -m "rename guide, remove file-001" >/dev/null 2>&1
+COMMIT5=$(git rev-parse HEAD)
+
+git push origin "$BRANCH" >/dev/null 2>&1
+pass "2 more commits pushed (5 total)"
+
+# --- Phase 5: Checkout older revision in on-mount clone -------------------
+
+echo "--- Phase 5: Checkout older revision in on-mount clone ---"
+
+cd "$MOUNT_CLONE"
+git checkout "$COMMIT2" >/dev/null 2>&1
+
+# Verify we're at commit 2 state
+DETACHED_HEAD=$(git rev-parse HEAD)
+assert_eq "$DETACHED_HEAD" "$COMMIT2" "on-mount clone at commit 2 (detached)"
+assert_file_not_exists "$MOUNT_CLONE/docs/guide.md" "docs/guide.md not in commit 2"
+assert_file_contains "$MOUNT_CLONE/src/main.go" 'v1' "src/main.go has v1 at commit 2"
+
+# --- Phase 6: Return to branch and pull -----------------------------------
+
+echo "--- Phase 6: Return to branch and pull with real changes ---"
+
+cd "$MOUNT_CLONE"
+git checkout "$BRANCH" >/dev/null 2>&1
+# At this point the on-mount clone is at commit 3, remote is at commit 5
+OLD_HEAD=$(git rev-parse HEAD)
+assert_eq "$OLD_HEAD" "$COMMIT3" "on-mount clone at commit 3 before pull"
+
+git pull >/dev/null 2>&1
+NEW_HEAD=$(git rev-parse HEAD)
+assert_eq "$NEW_HEAD" "$COMMIT5" "HEAD matches commit 5 after pull"
+
+# Verify commit 5 state
+assert_file_contains "$MOUNT_CLONE/src/main.go" 'v3' "src/main.go has v3 after pull"
+assert_file_exists "$MOUNT_CLONE/docs/manual.md" "docs/manual.md exists (renamed)"
+assert_file_not_exists "$MOUNT_CLONE/docs/guide.md" "docs/guide.md gone (renamed)"
+assert_file_not_exists "$MOUNT_CLONE/data/file-001.txt" "data/file-001.txt removed"
+assert_file_exists "$MOUNT_CLONE/data/file-050.txt" "data/file-050.txt exists"
+
+FINAL_COUNT=$(find "$MOUNT_CLONE/data" -name '*.txt' | wc -l | tr -d ' ')
+assert_eq "$FINAL_COUNT" "49" "data/ has 49 files (50 added, 1 removed)"
+
+# --- Phase 7: Verify git log integrity -----------------------------------
+
+echo "--- Phase 7: Verify git log integrity ---"
+
+cd "$MOUNT_CLONE"
+GIT_LOG=$(git log --format=%s)
+LOG_COUNT=$(echo "$GIT_LOG" | wc -l | tr -d ' ')
+assert_eq "$LOG_COUNT" "5" "git log shows 5 commits"
+
+# Verify commit messages
+echo "$GIT_LOG" | grep -qF "initial commit" && pass "commit 1 message in log" || fail "commit 1 message missing"
+echo "$GIT_LOG" | grep -qF "expand data" && pass "commit 4 message in log" || fail "commit 4 message missing"
+echo "$GIT_LOG" | grep -qF "rename guide" && pass "commit 5 message in log" || fail "commit 5 message missing"
+
+# --- Phase 8: Verify git status is clean ----------------------------------
+
+echo "--- Phase 8: Verify git status is clean ---"
+
+cd "$MOUNT_CLONE"
+STATUS=$(git status --porcelain)
+if [[ -z "$STATUS" ]]; then
+    pass "git status is clean"
+else
+    fail "git status has changes: $STATUS"
+fi
+
+# --- Results ---------------------------------------------------------------
+
+echo ""
+echo "========================================"
+echo "  Results: $PASS passed, $FAIL failed"
+echo "========================================"
+
+if [[ "$FAIL" -gt 0 ]]; then
+    exit 1
+fi

docker/entrypoint.sh

@@ -72,6 +72,20 @@ case "$1" in
   	exec /usr/bin/weed -logtostderr=true volume $ARGS $@
 	;;
 
+  'volume-rust')
+  	ARGS="-dir /data -max 0"
+  	if isArgPassed "-max" "$@"; then
+  	  ARGS="-dir /data"
+  	fi
+  	shift
+  	if [ ! -s /usr/bin/weed-volume ]; then
+  	  echo "Error: Rust volume server is not available on this platform ($(uname -m))." >&2
+  	  echo "Use 'volume' for the Go volume server instead." >&2
+  	  exit 1
+  	fi
+  	exec /usr/bin/weed-volume $ARGS $@
+	;;
+
   'server')
   	ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024"
   	if isArgPassed "-volume.max" "$@"; then

go.mod

@@ -46,8 +46,8 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
-	github.com/klauspost/compress v1.18.4
-	github.com/klauspost/reedsolomon v1.13.0
+	github.com/klauspost/compress v1.18.5
+	github.com/klauspost/reedsolomon v1.13.3
 	github.com/kurin/blazer v0.5.3
 	github.com/linxGnu/grocksdb v1.10.7
 	github.com/mailru/easyjson v0.9.1 // indirect
@@ -83,22 +83,22 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/viant/ptrie v1.0.1
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/scram v1.2.0
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	go.etcd.io/etcd/client/v3 v3.6.7
+	go.etcd.io/etcd/client/v3 v3.6.9
 	go.mongodb.org/mongo-driver v1.17.9
 	go.opencensus.io v0.24.0 // indirect
 	gocloud.dev v0.45.0
 	gocloud.dev/pubsub/natspubsub v0.45.0
-	gocloud.dev/pubsub/rabbitpubsub v0.44.0
-	golang.org/x/crypto v0.48.0
+	gocloud.dev/pubsub/rabbitpubsub v0.45.0
+	golang.org/x/crypto v0.49.0
 	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
-	golang.org/x/image v0.36.0
-	golang.org/x/net v0.50.0
+	golang.org/x/image v0.38.0
+	golang.org/x/net v0.51.0
 	golang.org/x/oauth2 v0.36.0
 	golang.org/x/sys v0.42.0
-	golang.org/x/text v0.34.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
 	golang.org/x/tools v0.42.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.267.0
@@ -114,7 +114,7 @@ require (
 )
 
 require (
-	cloud.google.com/go/kms v1.25.0
+	cloud.google.com/go/kms v1.26.0
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
 	github.com/Jille/raft-grpc-transport v1.6.1
 	github.com/ThreeDotsLabs/watermill v1.5.1
@@ -124,14 +124,14 @@ require (
 	github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
 	github.com/arangodb/go-driver v1.6.9
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.41.4
-	github.com/aws/aws-sdk-go-v2/config v1.32.9
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.12
+	github.com/aws/aws-sdk-go-v2 v1.41.5
+	github.com/aws/aws-sdk-go-v2/config v1.32.13
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.13
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
-	github.com/getsentry/sentry-go v0.43.0
-	github.com/go-ldap/ldap/v3 v3.4.12
+	github.com/getsentry/sentry-go v0.44.1
+	github.com/go-ldap/ldap/v3 v3.4.13
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
 	github.com/hashicorp/raft v1.7.3
@@ -149,7 +149,7 @@ require (
 	github.com/rdleal/intervalst v1.5.0
 	github.com/redis/go-redis/v9 v9.18.0
 	github.com/schollz/progressbar/v3 v3.19.0
-	github.com/seaweedfs/go-fuse/v2 v2.9.1
+	github.com/seaweedfs/go-fuse/v2 v2.9.2
 	github.com/shirou/gopsutil/v4 v4.26.2
 	github.com/tarantool/go-tarantool/v2 v2.4.2
 	github.com/testcontainers/testcontainers-go v0.40.0
@@ -157,9 +157,9 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
-	go.etcd.io/etcd/client/pkg/v3 v3.6.7
+	go.etcd.io/etcd/client/pkg/v3 v3.6.9
 	go.uber.org/atomic v1.11.0
-	golang.org/x/sync v0.19.0
+	golang.org/x/sync v0.20.0
 	golang.org/x/tools/godoc v0.1.0-deprecated
 	google.golang.org/grpc/security/advancedtls v1.0.0
 )
@@ -181,7 +181,7 @@ require (
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
 	github.com/apache/thrift v0.22.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/bazelbuild/rules_go v0.46.0 // indirect
 	github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
@@ -324,21 +324,21 @@ require (
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.9
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
 	github.com/aws/smithy-go v1.24.2
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
@@ -371,7 +371,7 @@ require (
 	github.com/geoffgarside/ber v1.2.0 // indirect
 	github.com/go-chi/chi/v5 v5.2.5 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -486,7 +486,7 @@ require (
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
 	go.etcd.io/bbolt v1.4.3 // indirect
-	go.etcd.io/etcd/api/v3 v3.6.7 // indirect
+	go.etcd.io/etcd/api/v3 v3.6.9 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
@@ -498,7 +498,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.40.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
-	golang.org/x/term v0.40.0 // indirect
+	golang.org/x/term v0.41.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect

go.sum

@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
 cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
 cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
 cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
-cloud.google.com/go/kms v1.25.0 h1:gVqvGGUmz0nYCmtoxWmdc1wli2L1apgP8U4fghPGSbQ=
-cloud.google.com/go/kms v1.25.0/go.mod h1:XIdHkzfj0bUO3E+LvwPg+oc7s58/Ns8Nd8Sdtljihbk=
+cloud.google.com/go/kms v1.26.0 h1:cK9mN2cf+9V63D3H1f6koxTatWy39aTI/hCjz1I+adU=
+cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58=
 cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
 cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
 cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
@@ -714,48 +714,48 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
-github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
+github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A=
-github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
+github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
+github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
 github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
 github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
@@ -1051,8 +1051,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
 github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
 github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
 github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
-github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
-github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
+github.com/getsentry/sentry-go v0.44.1 h1:/cPtrA5qB7uMRrhgSn9TYtcEF36auGP3Y6+ThvD/yaI=
+github.com/getsentry/sentry-go v0.44.1/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
 github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
@@ -1076,15 +1076,15 @@ github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
-github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
+github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U=
 github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk=
-github.com/go-ldap/ldap/v3 v3.4.12 h1:1b81mv7MagXZ7+1r7cLTWmyuTqVqdwbtJSjC0DAp9s4=
-github.com/go-ldap/ldap/v3 v3.4.12/go.mod h1:+SPAGcTtOfmGsCb3h1RFiq4xpp4N636G75OEace8lNo=
+github.com/go-ldap/ldap/v3 v3.4.13 h1:+x1nG9h+MZN7h/lUi5Q3UZ0fJ1GyDQYbPvbuH38baDQ=
+github.com/go-ldap/ldap/v3 v3.4.13/go.mod h1:LxsGZV6vbaK0sIvYfsv47rfh4ca0JXokCoKjZxsszv0=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
@@ -1475,15 +1475,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
-github.com/klauspost/reedsolomon v1.13.0 h1:E0Cmgf2kMuhZTj6eefnvpKC4/Q4jhCi9YIjcZjK4arc=
-github.com/klauspost/reedsolomon v1.13.0/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA=
+github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
+github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=
@@ -1838,8 +1838,8 @@ github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1Ivohy
 github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
 github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 h1:TgxPraf1NmF6XTcUG53ULpLQrKvhtUJxQ3hyekxSDNQ=
 github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
-github.com/seaweedfs/go-fuse/v2 v2.9.1 h1:gnKmfrKreCRGJmekGz5WMnNZqXEf9s9+V2hdWQdvx88=
-github.com/seaweedfs/go-fuse/v2 v2.9.1/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
+github.com/seaweedfs/go-fuse/v2 v2.9.2 h1:IfP/yFjLGO4rALcJY2Gb39PlebHxLnj7dkIiQAjFres=
+github.com/seaweedfs/go-fuse/v2 v2.9.2/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
 github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
 github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
 github.com/seaweedfs/raft v1.1.7 h1:3mVJZ2p4rdvBtbbrHROPjYKtH+q5qjMBc56G6VRu1kA=
@@ -2036,8 +2036,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
-github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
+github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -2101,12 +2101,12 @@ go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
 go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
 go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
-go.etcd.io/etcd/api/v3 v3.6.7 h1:7BNJ2gQmc3DNM+9cRkv7KkGQDayElg8x3X+tFDYS+E0=
-go.etcd.io/etcd/api/v3 v3.6.7/go.mod h1:xJ81TLj9hxrYYEDmXTeKURMeY3qEDN24hqe+q7KhbnI=
-go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/AhBVs=
-go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
-go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
-go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
+go.etcd.io/etcd/api/v3 v3.6.9 h1:UA7iKfEW1AzgihcBSGXci2kDGQiokSq41F9HMCI/RTI=
+go.etcd.io/etcd/api/v3 v3.6.9/go.mod h1:csEk/qTfxKL36NqJdU15Tgtl65A8dyEY2BYo7PRsIwk=
+go.etcd.io/etcd/client/pkg/v3 v3.6.9 h1:T8nuk8Lz64C+Hzb0coBFLMSlVSQZBpAtFk46swdM1DA=
+go.etcd.io/etcd/client/pkg/v3 v3.6.9/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
+go.etcd.io/etcd/client/v3 v3.6.9 h1:3X555hQXmhRr27O37wls53g68CpUiPOiHXrZfz2Al+o=
+go.etcd.io/etcd/client/v3 v3.6.9/go.mod h1:KO7H1HLYh1qaljuVZJQwBFk1lRce6pJzt+C81GEnrlM=
 go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
 go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -2184,8 +2184,8 @@ gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY=
 gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI=
 gocloud.dev/pubsub/natspubsub v0.45.0 h1:kfCupVejeynIQcS1GaIvkY3Nj/acLlM5yPWyxZN8wJ8=
 gocloud.dev/pubsub/natspubsub v0.45.0/go.mod h1:WU5SMDWuF7CCr2U8UpbrEONYYFOmvigCAvTpOOpUvYE=
-gocloud.dev/pubsub/rabbitpubsub v0.44.0 h1:MpRIO6XJ/JTqrlUWt3CxwDe1LvaiXUVu4sS5cv4f/AM=
-gocloud.dev/pubsub/rabbitpubsub v0.44.0/go.mod h1:BB9+qT3r6g4M5+4asiXaEeqw4QAOzsWusO5krYaqkdA=
+gocloud.dev/pubsub/rabbitpubsub v0.45.0 h1:hlejKjBFYQBUAj7ZsHKgW0JOER/3XUqrMFFxBOFUIOU=
+gocloud.dev/pubsub/rabbitpubsub v0.45.0/go.mod h1:Kqq6gppGY/RTrRQkKSHZq+/qbCufPEoQ0E1V3X389zM=
 golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
 golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -2208,8 +2208,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
-golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
+golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2240,8 +2240,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
 golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
-golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
-golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
+golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
+golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -2344,8 +2344,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
-golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
+golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
+golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2397,8 +2397,8 @@ golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -2523,8 +2523,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
-golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
-golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
+golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
+golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2545,8 +2545,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

install.sh

@@ -0,0 +1,275 @@
+#!/bin/bash
+#
+# SeaweedFS Installer
+# Downloads Go and/or Rust binaries from GitHub releases.
+#
+# Usage:
+#   curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
+#   curl -fsSL ... | bash -s -- --component volume-rust --large-disk
+#   curl -fsSL ... | bash -s -- --version v3.93 --dir /usr/local/bin
+#
+# Options:
+#   --component COMP   Which binary to install: weed, volume-rust, all (default: weed)
+#   --version VER      Release version tag (default: latest)
+#   --large-disk       Use large disk variant (5-byte offset, 8TB max volume)
+#   --dir DIR          Installation directory (default: /usr/local/bin)
+#   --help             Show this help message
+
+set -euo pipefail
+
+REPO="seaweedfs/seaweedfs"
+COMPONENT="weed"
+VERSION=""
+LARGE_DISK=false
+INSTALL_DIR="/usr/local/bin"
+
+# Colors (if terminal supports them)
+if [ -t 1 ]; then
+    RED='\033[0;31m'
+    GREEN='\033[0;32m'
+    YELLOW='\033[0;33m'
+    BLUE='\033[0;34m'
+    NC='\033[0m'
+else
+    RED='' GREEN='' YELLOW='' BLUE='' NC=''
+fi
+
+info()  { echo -e "${BLUE}[info]${NC} $*"; }
+ok()    { echo -e "${GREEN}[ok]${NC} $*"; }
+warn()  { echo -e "${YELLOW}[warn]${NC} $*"; }
+error() { echo -e "${RED}[error]${NC} $*" >&2; exit 1; }
+
+usage() {
+    sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//'
+    exit 0
+}
+
+# Parse arguments
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --component)  COMPONENT="$2"; shift 2 ;;
+        --version)    VERSION="$2"; shift 2 ;;
+        --large-disk) LARGE_DISK=true; shift ;;
+        --dir)        INSTALL_DIR="$2"; shift 2 ;;
+        --help|-h)    usage ;;
+        *)            error "Unknown option: $1. Use --help for usage." ;;
+    esac
+done
+
+# Detect OS and architecture
+detect_platform() {
+    local os arch
+
+    case "$(uname -s)" in
+        Linux*)   os="linux" ;;
+        Darwin*)  os="darwin" ;;
+        MINGW*|MSYS*|CYGWIN*) os="windows" ;;
+        FreeBSD*) os="freebsd" ;;
+        *)        error "Unsupported OS: $(uname -s)" ;;
+    esac
+
+    case "$(uname -m)" in
+        x86_64|amd64)   arch="amd64" ;;
+        aarch64|arm64)  arch="arm64" ;;
+        armv7l|armv6l)  arch="arm" ;;
+        *)              error "Unsupported architecture: $(uname -m)" ;;
+    esac
+
+    echo "${os}" "${arch}"
+}
+
+# Get latest release tag from GitHub API
+get_latest_version() {
+    local url="https://api.github.com/repos/${REPO}/releases/latest"
+    if command -v curl &>/dev/null; then
+        curl -fsSL "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
+    elif command -v wget &>/dev/null; then
+        wget -qO- "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
+    else
+        error "Neither curl nor wget found. Please install one."
+    fi
+}
+
+# Download a file
+download() {
+    local url="$1" dest="$2"
+    info "Downloading ${url}"
+    if command -v curl &>/dev/null; then
+        curl -fsSL -o "$dest" "$url"
+    elif command -v wget &>/dev/null; then
+        wget -qO "$dest" "$url"
+    fi
+}
+
+# Build Go weed binary asset name
+go_asset_name() {
+    local os="$1" arch="$2"
+    local suffix="${os}_${arch}"
+    if [ "$LARGE_DISK" = true ]; then
+        suffix="${suffix}_large_disk"
+    fi
+    echo "${suffix}.tar.gz"
+}
+
+# Build Rust volume server asset name
+rust_asset_name() {
+    local os="$1" arch="$2"
+    local prefix="weed-volume"
+    if [ "$LARGE_DISK" = true ]; then
+        prefix="weed-volume_large_disk"
+    else
+        prefix="weed-volume"
+    fi
+    local suffix="${os}_${arch}"
+    if [ "$os" = "windows" ]; then
+        echo "${prefix}_${suffix}.zip"
+    else
+        echo "${prefix}_${suffix}.tar.gz"
+    fi
+}
+
+# Install a single component
+install_component() {
+    local component="$1" os="$2" arch="$3"
+    local asset_name download_url tmpdir
+
+    tmpdir="$(mktemp -d)"
+    trap "rm -rf '$tmpdir'" EXIT
+
+    case "$component" in
+        weed)
+            asset_name="$(go_asset_name "$os" "$arch")"
+            download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
+            download "$download_url" "${tmpdir}/${asset_name}"
+
+            info "Extracting ${asset_name}..."
+            tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
+
+            # The Go release action puts the binary inside a directory
+            local weed_bin
+            weed_bin="$(find "$tmpdir" -name 'weed' -type f | head -1)"
+            if [ -z "$weed_bin" ]; then
+                weed_bin="$(find "$tmpdir" -name 'weed.exe' -type f | head -1)"
+            fi
+            if [ -z "$weed_bin" ]; then
+                error "Could not find weed binary in archive"
+            fi
+
+            chmod +x "$weed_bin"
+            install_binary "$weed_bin" "weed"
+            ok "Installed weed to ${INSTALL_DIR}/weed"
+            ;;
+
+        volume-rust)
+            # Check platform support for Rust volume server
+            case "$os" in
+                linux|darwin|windows) ;;
+                *) error "Rust volume server is not available for ${os}. Supported: linux, darwin, windows" ;;
+            esac
+            case "$arch" in
+                amd64|arm64) ;;
+                *) error "Rust volume server is not available for ${arch}. Supported: amd64, arm64" ;;
+            esac
+
+            asset_name="$(rust_asset_name "$os" "$arch")"
+            download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
+            download "$download_url" "${tmpdir}/${asset_name}"
+
+            info "Extracting ${asset_name}..."
+            if [ "$os" = "windows" ]; then
+                unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir"
+            else
+                tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
+            fi
+
+            local rust_bin
+            if [ "$LARGE_DISK" = true ]; then
+                rust_bin="$(find "$tmpdir" -name 'weed-volume-large-disk*' -type f | head -1)"
+            else
+                rust_bin="$(find "$tmpdir" -name 'weed-volume-normal*' -type f | head -1)"
+            fi
+            if [ -z "$rust_bin" ]; then
+                rust_bin="$(find "$tmpdir" -name 'weed-volume*' -type f | head -1)"
+            fi
+            if [ -z "$rust_bin" ]; then
+                error "Could not find weed-volume binary in archive"
+            fi
+
+            chmod +x "$rust_bin"
+            local dest_name="weed-volume"
+            if [ "$os" = "windows" ]; then
+                dest_name="weed-volume.exe"
+            fi
+            install_binary "$rust_bin" "$dest_name"
+            ok "Installed weed-volume to ${INSTALL_DIR}/${dest_name}"
+            ;;
+
+        *)
+            error "Unknown component: ${component}. Use: weed, volume-rust, all"
+            ;;
+    esac
+}
+
+# Copy binary to install dir, using sudo if needed
+install_binary() {
+    local src="$1" name="$2"
+    local dest="${INSTALL_DIR}/${name}"
+
+    mkdir -p "$INSTALL_DIR" 2>/dev/null || true
+
+    if [ -w "$INSTALL_DIR" ]; then
+        cp "$src" "$dest"
+    else
+        info "Need elevated permissions to write to ${INSTALL_DIR}"
+        sudo cp "$src" "$dest"
+    fi
+    chmod +x "$dest" 2>/dev/null || sudo chmod +x "$dest"
+}
+
+main() {
+    info "SeaweedFS Installer"
+
+    read -r os arch <<< "$(detect_platform)"
+    info "Detected platform: ${os}/${arch}"
+
+    if [ -z "$VERSION" ]; then
+        info "Resolving latest release..."
+        VERSION="$(get_latest_version)"
+        if [ -z "$VERSION" ]; then
+            error "Could not determine latest version. Specify with --version"
+        fi
+    fi
+    info "Version: ${VERSION}"
+
+    if [ "$LARGE_DISK" = true ]; then
+        info "Variant: large disk (8TB max volume)"
+    else
+        info "Variant: normal (32GB max volume)"
+    fi
+
+    case "$COMPONENT" in
+        all)
+            install_component "weed" "$os" "$arch"
+            install_component "volume-rust" "$os" "$arch"
+            ;;
+        *)
+            install_component "$COMPONENT" "$os" "$arch"
+            ;;
+    esac
+
+    echo ""
+    ok "Installation complete!"
+    if [ "$COMPONENT" = "weed" ] || [ "$COMPONENT" = "all" ]; then
+        info "  weed:            ${INSTALL_DIR}/weed"
+    fi
+    if [ "$COMPONENT" = "volume-rust" ] || [ "$COMPONENT" = "all" ]; then
+        info "  weed-volume:  ${INSTALL_DIR}/weed-volume"
+    fi
+    echo ""
+    info "Quick start:"
+    info "  weed master                          # Start master server"
+    info "  weed volume -mserver=localhost:9333   # Start Go volume server"
+    info "  weed-volume -mserver localhost:9333 # Start Rust volume server"
+}
+
+main

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.17"
+appVersion: "4.18"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.17.0
+version: 4.18.0

k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json

@@ -1168,6 +1168,108 @@
       "title": "Filer QPS",
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 28
+      },
+      "id": 91,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "width": 250
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "sum by (code) (rate(SeaweedFS_upload_error_total{namespace=\"$NAMESPACE\"}[$__rate_interval]))",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "{{code}}",
+          "refId": "A",
+          "step": 30
+        }
+      ],
+      "title": "Upload Errors",
+      "type": "timeseries"
+    },
     {
       "collapsed": false,
       "datasource": {
@@ -1178,7 +1280,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 28
+        "y": 35
       },
       "id": 61,
       "panels": [],
@@ -1251,7 +1353,7 @@
         "h": 7,
         "w": 8,
         "x": 0,
-        "y": 29
+        "y": 36
       },
       "id": 65,
       "links": [],
@@ -1357,7 +1459,7 @@
         "h": 7,
         "w": 8,
         "x": 8,
-        "y": 29
+        "y": 36
       },
       "id": 56,
       "links": [],
@@ -1463,7 +1565,7 @@
         "h": 7,
         "w": 8,
         "x": 16,
-        "y": 29
+        "y": 36
       },
       "id": 58,
       "links": [],
@@ -1520,7 +1622,7 @@
         "h": 7,
         "w": 12,
         "x": 0,
-        "y": 36
+        "y": 43
       },
       "id": 84,
       "links": [],
@@ -1565,7 +1667,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 36
+        "y": 43
       },
       "id": 85,
       "links": [],
@@ -1661,7 +1763,7 @@
         "h": 7,
         "w": 12,
         "x": 0,
-        "y": 41
+        "y": 48
       },
       "id": 86,
       "options": {
@@ -1756,7 +1858,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 41
+        "y": 48
       },
       "id": 72,
       "links": [],
@@ -1874,7 +1976,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 50
+        "y": 57
       },
       "id": 73,
       "links": [],
@@ -2030,7 +2132,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 57
+        "y": 64
       },
       "id": 55,
       "links": [],
@@ -2187,7 +2289,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 64
+        "y": 71
       },
       "hideTimeOverride": false,
       "id": 59,
@@ -2259,7 +2361,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 71
+        "y": 78
       },
       "id": 62,
       "panels": [],
@@ -2331,7 +2433,7 @@
         "h": 7,
         "w": 12,
         "x": 0,
-        "y": 72
+        "y": 79
       },
       "id": 47,
       "links": [],
@@ -2474,7 +2576,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 72
+        "y": 79
       },
       "id": 40,
       "links": [],
@@ -2571,7 +2673,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 79
+        "y": 86
       },
       "id": 48,
       "links": [],
@@ -2681,7 +2783,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 86
+        "y": 93
       },
       "id": 50,
       "links": [],
@@ -2783,7 +2885,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 93
+        "y": 100
       },
       "id": 51,
       "links": [],
@@ -2823,7 +2925,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 93
+        "y": 100
       },
       "id": 63,
       "panels": [],
@@ -2896,7 +2998,7 @@
         "h": 7,
         "w": 12,
         "x": 0,
-        "y": 101
+        "y": 108
       },
       "id": 12,
       "links": [],
@@ -2991,7 +3093,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 101
+        "y": 108
       },
       "id": 14,
       "links": [],
@@ -3033,7 +3135,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 108
+        "y": 115
       },
       "id": 64,
       "panels": [],
@@ -3106,7 +3208,7 @@
         "h": 7,
         "w": 12,
         "x": 0,
-        "y": 109
+        "y": 116
       },
       "id": 52,
       "links": [],
@@ -3234,7 +3336,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 109
+        "y": 116
       },
       "id": 54,
       "links": [],
@@ -3331,7 +3433,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 116
+        "y": 123
       },
       "id": 53,
       "links": [],
@@ -3426,7 +3528,7 @@
         "h": 7,
         "w": 12,
         "x": 0,
-        "y": 48
+        "y": 55
       },
       "id": 89,
       "options": {
@@ -3533,7 +3635,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 48
+        "y": 55
       },
       "id": 90,
       "options": {
@@ -3654,4 +3756,4 @@
   "uid": "a24009d7-cbda-4443-a132-1cc1c4677304",
   "version": 1,
   "weekStart": ""
-}
+}

k8s/charts/seaweedfs/templates/s3/s3-service.yaml

@@ -15,6 +15,7 @@ metadata:
     {{- toYaml .Values.s3.annotations | nindent 4 }}
 {{- end }}
 spec:
+  type: {{ .Values.s3.service.type }}
   internalTrafficPolicy: {{ .Values.s3.internalTrafficPolicy | default "Cluster" }}
   {{- $td := .Values.s3.trafficDistribution | default .Values.filer.s3.trafficDistribution }}
   {{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) $td }}

k8s/charts/seaweedfs/templates/sftp/sftp-service.yaml

@@ -15,6 +15,7 @@ metadata:
     {{- toYaml .Values.sftp.annotations | nindent 4 }}
 {{- end }}
 spec:
+  type: {{ .Values.sftp.service.type }}
   internalTrafficPolicy: {{ .Values.sftp.internalTrafficPolicy | default "Cluster" }}
   ports:
   - name: "swfs-sftp"
@@ -31,4 +32,4 @@ spec:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/component: sftp
-{{- end }}
+{{- end }}

k8s/charts/seaweedfs/values.yaml

@@ -1104,6 +1104,10 @@ s3:
     annotations: {}
     tls: []
 
+  # Service settings
+  service:
+    type: ClusterIP
+
   icebergIngress:
     enabled: false
     className: ""
@@ -1185,6 +1189,10 @@ sftp:
     failureThreshold: 100
     timeoutSeconds: 10
 
+  # Service settings
+  service:
+    type: ClusterIP
+
 admin:
   enabled: false
   imageOverride: null

other/java/client/src/main/proto/filer.proto

@@ -33,6 +33,9 @@ service SeaweedFiler {
     rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) {
     }
 
+    rpc StreamMutateEntry (stream StreamMutateEntryRequest) returns (stream StreamMutateEntryResponse) {
+    }
+
     rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
     }
 
@@ -81,6 +84,8 @@ service SeaweedFiler {
     // distributed lock management internal use only
     rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
     }
+    rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
+    }
 }
 
 //////////////////////////////////////////////////
@@ -203,9 +208,21 @@ message CreateEntryRequest {
     bool skip_check_parent_directory = 6;
 }
 
+// Structured error codes for filer entry operations.
+// Values are stable — do not reorder or reuse numbers.
+enum FilerError {
+    OK                    = 0;
+    ENTRY_NAME_TOO_LONG   = 1; // name exceeds max_file_name_length
+    PARENT_IS_FILE        = 2; // parent path component is a file, not a directory
+    EXISTING_IS_DIRECTORY = 3; // cannot overwrite directory with file
+    EXISTING_IS_FILE      = 4; // cannot overwrite file with directory
+    ENTRY_ALREADY_EXISTS  = 5; // O_EXCL and entry already exists
+}
+
 message CreateEntryResponse {
-    string error = 1;
+    string error = 1;                          // kept for human readability + backward compat
     SubscribeMetadataResponse metadata_event = 2;
+    FilerError error_code = 3;                 // machine-readable error code
 }
 
 message UpdateEntryRequest {
@@ -375,11 +392,23 @@ message SubscribeMetadataRequest {
     int64 until_ns = 8;
     int32 client_epoch = 9;
     repeated string directories = 10;  // exact directory to watch
+    bool client_supports_batching = 11; // client can unpack SubscribeMetadataResponse.events
+    bool client_supports_metadata_chunks = 12; // client can read log file chunks from volume servers
 }
 message SubscribeMetadataResponse {
     string directory = 1;
     EventNotification event_notification = 2;
     int64 ts_ns = 3;
+    repeated SubscribeMetadataResponse events = 4; // batch of additional events (backlog catch-up)
+    repeated LogFileChunkRef log_file_refs = 5;    // log file chunk refs for client direct-read
+}
+// A persisted log file that the client can read directly from volume servers.
+// The file format is: [4-byte size | protobuf LogEntry] repeated.
+// Each LogEntry.Data contains a marshaled SubscribeMetadataResponse.
+message LogFileChunkRef {
+    repeated FileChunk chunks = 1; // chunk references (fids) to read from volume servers
+    int64 file_ts_ns = 2;         // minute-level timestamp of the log file
+    string filer_id = 3;          // filer signature suffix from log filename
 }
 
 message TraverseBfsMetadataRequest {
@@ -495,6 +524,7 @@ message LockResponse {
     string lock_owner = 2;
     string lock_host_moved_to = 3;
     string error = 4;
+    int64 generation = 5;
 }
 message UnlockRequest {
     string name = 1;
@@ -517,9 +547,52 @@ message Lock {
     string renew_token = 2;
     int64 expired_at_ns = 3;
     string owner = 4;
+    int64 generation = 5;
+    bool is_backup = 6;
+    int64 seq = 7;
 }
 message TransferLocksRequest {
     repeated Lock locks = 1;
 }
 message TransferLocksResponse {
 }
+message ReplicateLockRequest {
+    string name = 1;
+    string renew_token = 2;
+    int64 expired_at_ns = 3;
+    string owner = 4;
+    int64 generation = 5;
+    bool is_unlock = 6;
+    int64 seq = 7;
+}
+message ReplicateLockResponse {
+}
+
+//////////////////////////////////////////////////
+// StreamMutateEntry: ordered bidirectional streaming for all filer mutations.
+// All create/update/delete/rename operations from a single mount go through
+// one stream, preserving mutation ordering and eliminating per-request
+// connection overhead.
+
+message StreamMutateEntryRequest {
+    uint64 request_id = 1;
+    oneof request {
+        CreateEntryRequest create_request = 2;
+        UpdateEntryRequest update_request = 3;
+        DeleteEntryRequest delete_request = 4;
+        StreamRenameEntryRequest rename_request = 5;
+    }
+}
+
+message StreamMutateEntryResponse {
+    uint64 request_id = 1;
+    bool is_last = 2; // always true except for rename, which sends multiple events
+    oneof response {
+        CreateEntryResponse create_response = 3;
+        UpdateEntryResponse update_response = 4;
+        DeleteEntryResponse delete_response = 5;
+        StreamRenameEntryResponse rename_response = 6;
+    }
+    string error = 7; // human-readable error message when the operation failed
+    int32 errno = 8;  // POSIX errno (e.g. ENOENT=2, ENOTEMPTY=66) for direct FUSE status mapping
+}

seaweed-volume/Cargo.toml

@@ -0,0 +1,139 @@
+[package]
+name = "weed-volume"
+version = "0.1.0"
+edition = "2021"
+description = "SeaweedFS Volume Server — Rust implementation"
+
+[lib]
+name = "seaweed_volume"
+
+[[bin]]
+name = "weed-volume"
+path = "src/main.rs"
+
+[features]
+# Default: 5-byte offsets (8TB max volume size), matching production Go builds (-tags 5BytesOffset).
+# Disable with --no-default-features for 4-byte offsets (32GB max volume size).
+default = ["5bytes"]
+5bytes = []
+
+[dependencies]
+# Async runtime
+tokio = { version = "1", features = ["full"] }
+tokio-stream = { version = "0.1", features = ["net"] }
+tokio-io-timeout = "1"
+
+# gRPC + protobuf
+tonic = { version = "0.12", features = ["tls"] }
+tonic-reflection = "0.12"
+prost = "0.13"
+prost-types = "0.13"
+
+# HTTP server
+axum = { version = "0.7", features = ["multipart"] }
+http-body = "1"
+hyper = { version = "1", features = ["full"] }
+hyper-util = { version = "0.1", features = ["tokio", "service", "server-auto", "http1", "http2"] }
+tower = "0.4"
+tower-http = { version = "0.5", features = ["cors", "trace"] }
+
+# CLI
+clap = { version = "4", features = ["derive"] }
+
+# Metrics
+prometheus = { version = "0.13", default-features = false, features = ["process"] }
+lazy_static = "1"
+
+# JWT
+jsonwebtoken = { version = "10", features = ["rust_crypto"] }
+
+# TLS
+rustls = "0.23"
+tokio-rustls = "0.26"
+rustls-pemfile = "2"
+
+# LevelDB (via RocksDB for better Rust support)
+# Using rusty-leveldb for pure Rust LevelDB
+rusty-leveldb = "3"
+
+# Disk-backed needle map (alternative to in-memory HashMap)
+redb = "3"
+
+# Reed-Solomon erasure coding
+reed-solomon-erasure = "6"
+
+# Logging
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+# Config
+toml = "0.8"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+serde_urlencoded = "0.7"
+
+# CRC32 — using Castagnoli polynomial (CRC32-C), matching Go's crc32.Castagnoli
+crc32c = "0.6"
+crc32fast = "1"
+
+# Memory-mapped files
+memmap2 = "0.9"
+
+# UUID
+uuid = { version = "1", features = ["v4"] }
+
+# HTTP client (for proxying, remote fetch)
+reqwest = { version = "0.12", features = ["rustls-tls", "stream", "multipart", "json"] }
+
+# Content hashing
+md-5 = "0.10"
+base64 = "0.22"
+
+# Compression
+flate2 = "1"
+
+# Image processing
+image = { version = "0.25", default-features = false, features = ["png", "jpeg", "gif", "webp"] }
+kamadak-exif = "0.5"
+
+# Multipart form-data parsing
+multer = "3"
+
+# MIME type guessing from file extensions
+mime_guess = "2"
+
+# Misc
+bytes = "1"
+rand = "0.8"
+chrono = "0.4"
+hex = "0.4"
+parking_lot = "0.12"
+dashmap = "6"
+thiserror = "1"
+anyhow = "1"
+async-trait = "0.1"
+futures = "0.3"
+async-stream = "0.3"
+x509-parser = "0.16"
+
+# Disk space checking
+sysinfo = "0.31"
+libc = "0.2"
+
+# AWS S3 SDK (for remote storage backends)
+aws-config = { version = "1", features = ["behavior-version-latest"] }
+aws-sdk-s3 = { version = "1.125.0", default-features = false, features = ["sigv4a", "http-1x", "default-https-client", "rt-tokio"] }
+aws-credential-types = "1"
+aws-types = "1"
+
+# pprof is Unix-only (requires libc/nix APIs not available on Windows)
+[target.'cfg(unix)'.dependencies]
+pprof = { version = "0.15", features = ["prost-codec"] }
+
+[dev-dependencies]
+tempfile = "3"
+
+[build-dependencies]
+tonic-build = "0.12"
+
+[patch.crates-io]
+reed-solomon-erasure = { path = "vendor/reed-solomon-erasure" }

seaweed-volume/DEV_PLAN.md

@@ -0,0 +1,105 @@
+# Rust Volume Server — Dev Plan
+
+## Current Status (2026-03-07)
+
+**HTTP tests**: 53/53 pass (100%)
+**gRPC tests**: 56/56 pass (100%) — includes TestVolumeMoveHandlesInFlightWrites with Rust multi-volume cluster
+**Rust integration tests**: 8/8 pass
+**S3 remote storage tests**: 3/3 pass
+**Total**: 117/117 (100%) + 8 Rust + 3 S3 tests
+**Rust unit tests**: 137 lib + 7 integration = 144
+
+## Completed Features
+
+All phases from the original plan are complete:
+
+- **Phase 1** — HTTP Core: CORS, OPTIONS, unsupported methods, static assets, path routing,
+  cookie validation, conditional headers, range requests, dedup 204, content-encoding,
+  readDeleted, chunk manifests, multipart validation, MD5 check, file size limit,
+  upload/download throttling, image resize/crop, download disposition
+- **Phase 2** — JWT/Security: signing keys from security.toml, token source precedence
+  (query > header > cookie), file_id claims, leeway=0
+- **Phase 3** — gRPC: maintenance mode, error message parity, ping routing, batch delete,
+  VolumeServerStatus (with real disk stats, data_center, rack), ReadVolumeFileStatus
+  (with timestamps)
+- **Phase 4** — Streaming gRPC: VolumeIncrementalCopy, CopyFile, ReceiveFile, ReadAllNeedles,
+  VolumeTailSender, VolumeCopy, VolumeTailReceiver, VacuumVolumeCheck
+- **Phase 5** — EC Shards: mount/unmount, delete, read, blob delete, rebuild, shards-to-volume,
+  copy, info
+- **Phase 6** — Advanced gRPC: ScrubVolume, ScrubEcVolume, Query, FetchAndWriteNeedle,
+  VolumeTierMoveDat (error paths)
+- **Phase 7** — Remote Storage: S3-compatible backend via aws-sdk-s3,
+  FetchAndWriteNeedle reads from S3/MinIO/SeaweedFS S3 and writes locally.
+  Supports all S3-compatible providers (AWS, Wasabi, Backblaze, Aliyun, etc.)
+- **Master Heartbeat** — Bidirectional streaming SendHeartbeat RPC, volume/EC registration,
+  leader changes, shutdown deregistration. Tested end-to-end with Go master.
+- **Production Sprint 1** — Quick wins:
+  - VolumeMarkReadonly master notification (triggers immediate heartbeat)
+  - Compaction throttling (`maybe_throttle_compaction()`)
+  - File size limit enforcement on upload
+  - `ts` query param for custom timestamps (upload + delete)
+  - TTL expiration check (was already implemented)
+  - Health check heartbeat status (returns 503 if disconnected from master)
+  - preStopSeconds graceful drain before shutdown
+  - S3 response passthrough headers (content-encoding, expires, content-language, content-disposition)
+  - .vif persistence for readonly state across restarts
+  - Webp image support for resize
+- **Production Sprint 2** — Compatibility:
+  - MIME type extraction from Content-Type header
+  - Stats endpoints (/stats/counter, /stats/memory, /stats/disk)
+  - JSON pretty print (?pretty=y) and JSONP (?callback=fn)
+  - Request ID generation (UUID if x-amz-request-id missing)
+  - Advanced Prometheus metrics (INFLIGHT_REQUESTS, VOLUME_FILE_COUNT gauges)
+- **Production Sprint 3** — Streaming & Multi-node:
+  - Streaming reads for large files (>1MB) via http_body::Body trait with spawn_blocking
+  - Meta-only needle reads (NeedleStreamInfo) to avoid loading full body for streaming
+  - Multi-volume Rust cluster support (RustMultiVolumeCluster test framework)
+  - TestVolumeMoveHandlesInFlightWrites now uses Rust volume servers
+  - CI skip list cleaned up (all tests pass with Rust)
+
+- **Production Sprint 4** — Advanced Features:
+  - BatchDelete EC shard support (ecx index lookup + ecj journal deletion)
+  - JPEG EXIF orientation auto-fix on upload (kamadak-exif + image crate)
+  - Async batched write processing (mpsc queue, up to 128 entries per batch)
+  - VolumeTierMoveDatToRemote/FromRemote (S3 multipart upload/download)
+  - S3TierRegistry for managing remote storage backends
+  - VolumeInfo (.vif) persistence for remote file references
+- **Production Sprint 5** — Upload Compatibility:
+  - TTL query parameter extraction during upload (`ttl=3m`)
+  - Auto-compression for compressible file types (text/*, .js, .css, .json, .svg, etc.)
+  - Seaweed-* custom metadata headers stored as needle pairs (JSON, max 64KB)
+  - Filename extraction from URL path stored in needle name field
+  - Upload response includes filename
+
+- **Production Sprint 6** — Storage & Networking:
+  - Redb disk-backed needle maps (pure Rust, no C deps) via `NeedleMap` enum
+  - Binary search for `VolumeIncrementalCopy` with `since_ns > 0`
+  - Proxy/redirect read modes for non-local volumes (master lookup, HTTP proxy, 301 redirect)
+
+## Remaining Work (Production Readiness)
+
+No major remaining items. All phases and production sprints are complete.
+
+## Test Commands
+
+```bash
+# Build
+cd seaweed-volume && cargo build --release
+
+# Run all Go integration tests with Rust volume server
+VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 1200s ./test/volume_server/grpc/... ./test/volume_server/http/...
+
+# Run S3 remote storage tests
+VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 180s -run "TestFetchAndWriteNeedle(FromS3|S3NotFound)" ./test/volume_server/grpc/...
+
+# Run specific test
+VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 60s -run "TestName" ./test/volume_server/http/...
+
+# Run Rust unit tests
+cd seaweed-volume && cargo test
+
+# Test heartbeat with Go master
+weed master -port=9333 &
+seaweed-volume --port 8080 --master localhost:9333 --dir /tmp/vol1 --max 7
+curl http://localhost:9333/dir/status  # should show Rust volume server registered
+```

seaweed-volume/MISSING_FEATURES.md

@@ -0,0 +1,288 @@
+# Rust Volume Server — Missing Features Audit
+
+Comprehensive line-by-line comparison of Go vs Rust volume server.
+Generated 2026-03-07 from 4 parallel audits covering HTTP, gRPC, storage, and infrastructure.
+
+## Executive Summary
+
+| Area | Total Features | Implemented | Partial | Missing |
+|------|---------------|-------------|---------|---------|
+| gRPC RPCs | 48 | 43 (90%) | 2 (4%) | 3 (6%) |
+| HTTP Handlers | 31 | 12 (39%) | 10 (32%) | 9 (29%) |
+| Storage Layer | 22 | 6 (27%) | 7 (32%) | 9 (41%) |
+| Infrastructure | 14 | 5 (36%) | 4 (29%) | 5 (36%) |
+
+---
+
+## Priority 1 — Critical for Production
+
+### P1.1 Streaming / Meta-Only Reads
+- **Go**: `ReadNeedleMeta()`, `ReadNeedleData()`, `ReadPagedData()` — reads only metadata or pages of large files
+- **Go**: `streamWriteResponseContent()` streams needle data in chunks
+- **Go**: `AttemptMetaOnly` / `MustMetaOnly` flags in `ReadOption`
+- **Rust**: Reads entire needle into memory always
+- **Impact**: OOM on large files; 8MB file = 8MB heap per request
+- **Files**: `weed/storage/needle/needle_read.go`, `weed/server/volume_server_handlers_read.go`
+- **Effort**: Medium
+
+### P1.2 Download Proxy/Redirect Fallback (ReadMode)
+- **Go**: `ReadMode` config: "local" | "proxy" | "redirect"
+- **Go**: `tryProxyToReplica()` probes replicas, `proxyReqToTargetServer()` streams response
+- **Rust**: Always returns 404 for non-local volumes
+- **Impact**: Clients must handle volume placement themselves; breaks transparent replication
+- **Files**: `weed/server/volume_server_handlers_read.go:138-250`
+- **Effort**: Medium
+
+### P1.3 TLS/HTTPS Support
+- **Go**: `LoadServerTLS()`, `LoadClientTLS()`, cert/key loading from security.toml
+- **Go**: Applied to both HTTP and gRPC servers
+- **Rust**: No TLS at all — plain TCP only
+- **Impact**: Cannot deploy in secure clusters
+- **Files**: `weed/security/tls.go`, `weed/command/volume.go`
+- **Effort**: Medium (rustls + tokio-rustls already in Cargo.toml)
+
+### P1.4 VolumeMarkReadonly/Writable Master Notification
+- **Go**: `notifyMasterVolumeReadonly()` updates master with readonly state
+- **Rust**: Only sets local in-memory flag
+- **Impact**: Master keeps directing writes to readonly volume
+- **Files**: `weed/server/volume_grpc_admin.go`
+- **Effort**: Low
+
+### P1.5 Compaction/Maintenance Throttling
+- **Go**: `WriteThrottler` with `MaybeSlowdown()` for MB/s rate limiting
+- **Rust**: Flags parsed but no throttle implementation
+- **Impact**: Compaction/copy operations can saturate disk IO
+- **Files**: `weed/util/throttler.go`
+- **Effort**: Low
+
+### P1.6 File Size Limit Enforcement
+- **Go**: `fileSizeLimitBytes` checked on upload, returns 400
+- **Rust**: No enforcement — accepts any size
+- **Impact**: Can write files larger than volume size limit
+- **Files**: `weed/server/volume_server_handlers_write.go`
+- **Effort**: Low
+
+---
+
+## Priority 2 — Important for Compatibility
+
+### P2.1 `ts` Query Param (Custom Timestamps)
+- **Go**: Upload and delete accept `ts` query param for custom Last-Modified time
+- **Rust**: Always uses current time
+- **Impact**: Replication timestamp fidelity; sync from external sources
+- **Files**: `weed/server/volume_server_handlers_write.go`, `volume_server_handlers_admin.go`
+- **Effort**: Low
+
+### P2.2 Multipart Form Upload Parsing
+- **Go**: `needle.CreateNeedleFromRequest()` parses multipart forms, extracts MIME type, custom headers/pairs
+- **Rust**: Reads raw body bytes only — no multipart form parsing for metadata
+- **Impact**: MIME type not stored; custom needle pairs not supported
+- **Files**: `weed/storage/needle/needle.go:CreateNeedleFromRequest`
+- **Effort**: Medium
+
+### P2.3 JPEG Orientation Auto-Fix
+- **Go**: `images.FixJpgOrientation()` on upload when enabled
+- **Rust**: Not implemented (flag exists but unused)
+- **Impact**: Mobile uploads may display rotated
+- **Files**: `weed/images/orientation.go`
+- **Effort**: Low (exif crate)
+
+### P2.4 TTL Expiration Enforcement
+- **Go**: Checks `HasTtl()` + `AppendAtNs` against current time on read path
+- **Rust**: TTL struct exists but no expiration checking
+- **Impact**: Expired needles still served
+- **Files**: `weed/storage/needle/volume_ttl.go`, `weed/storage/volume_read.go`
+- **Effort**: Low
+
+### P2.5 Health Check — Master Heartbeat Status
+- **Go**: Returns 503 if not heartbeating (can't reach master)
+- **Rust**: Only checks `is_stopping` flag
+- **Impact**: Load balancers won't detect disconnected volume servers
+- **Files**: `weed/server/volume_server.go`
+- **Effort**: Low
+
+### P2.6 Stats Endpoints
+- **Go**: `/stats/counter`, `/stats/memory`, `/stats/disk` (whitelist-guarded)
+- **Rust**: Not implemented
+- **Impact**: No operational visibility
+- **Files**: `weed/server/volume_server.go`
+- **Effort**: Low
+
+### P2.7 Webp Image Support
+- **Go**: `.webp` included in resize-eligible extensions
+- **Rust**: Only `.png`, `.jpg`, `.jpeg`, `.gif`
+- **Impact**: Webp images can't be resized on read
+- **Files**: `weed/server/volume_server_handlers_read.go`
+- **Effort**: Low (add webp feature to image crate)
+
+### P2.8 preStopSeconds Graceful Drain
+- **Go**: Stops heartbeat, waits N seconds, then shuts down servers
+- **Rust**: Immediate shutdown on signal
+- **Impact**: In-flight requests dropped; Kubernetes readiness race
+- **Files**: `weed/command/volume.go`
+- **Effort**: Low
+
+### P2.9 S3 Response Passthrough Headers
+- **Go**: `response-content-encoding`, `response-expires`, `response-content-language` query params
+- **Rust**: Only handles `response-content-type`, `response-cache-control`, `dl`
+- **Impact**: S3-compatible GET requests missing some override headers
+- **Files**: `weed/server/volume_server_handlers_read.go`
+- **Effort**: Low
+
+---
+
+## Priority 3 — Storage Layer Gaps
+
+### P3.1 LevelDB Needle Maps
+- **Go**: 5 needle map variants: memory, LevelDB, LevelDB-medium, LevelDB-large, sorted-file
+- **Rust**: Memory-only needle map
+- **Impact**: Large volumes (millions of needles) require too much RAM
+- **Files**: `weed/storage/needle_map_leveldb.go`
+- **Effort**: High (need LevelDB binding or alternative)
+
+### P3.2 Async Request Processing
+- **Go**: `asyncRequestsChan` with 128-entry queue, worker goroutine for batched writes
+- **Rust**: All writes synchronous
+- **Impact**: Write throughput limited by fsync latency
+- **Files**: `weed/storage/needle/async_request.go`
+- **Effort**: Medium
+
+### P3.3 Volume Scrubbing (Data Integrity)
+- **Go**: `ScrubIndex()`, `scrubVolumeData()` — full data + index verification
+- **Rust**: Stub only in gRPC (returns OK without actual scrubbing)
+- **Impact**: No way to verify data integrity
+- **Files**: `weed/storage/volume_checking.go`, `weed/storage/idx/check.go`
+- **Effort**: Medium
+
+### P3.4 Volume Backup / Sync
+- **Go**: Streaming backup, binary search for last modification, index generation scanner
+- **Rust**: Not implemented
+- **Impact**: No backup/restore capability
+- **Files**: `weed/storage/volume_backup.go`
+- **Effort**: Medium
+
+### P3.5 Volume Info (.vif) Persistence
+- **Go**: `.vif` files store tier/remote metadata, readonly state persists across restarts
+- **Rust**: No `.vif` support; readonly is in-memory only
+- **Impact**: Readonly state lost on restart; no tier metadata
+- **Files**: `weed/storage/volume_info/volume_info.go`
+- **Effort**: Low
+
+### P3.6 Disk Location Features
+- **Go**: Directory UUID tracking, disk space monitoring, min-free-space enforcement, tag-based grouping
+- **Rust**: Basic directory only
+- **Impact**: No disk-full protection
+- **Files**: `weed/storage/disk_location.go`
+- **Effort**: Medium
+
+### P3.7 Compact Map (Memory-Efficient Needle Map)
+- **Go**: `CompactMap` with overflow handling for memory optimization
+- **Rust**: Uses standard HashMap
+- **Impact**: Higher memory usage for index
+- **Files**: `weed/storage/needle_map/compact_map.go`
+- **Effort**: Medium
+
+---
+
+## Priority 4 — Nice to Have
+
+### P4.1 gRPC: VolumeTierMoveDatToRemote / FromRemote
+- **Go**: Full streaming implementation for tiering volumes to/from S3
+- **Rust**: Stub returning error
+- **Files**: `weed/server/volume_grpc_tier_upload.go`, `volume_grpc_tier_download.go`
+- **Effort**: High
+
+### P4.2 gRPC: Query (S3 Select)
+- **Go**: JSON/CSV query over needle data (S3 Select compatible)
+- **Rust**: Stub returning error
+- **Files**: `weed/server/volume_grpc_query.go`
+- **Effort**: High
+
+### P4.3 FetchAndWriteNeedle — Already Implemented
+- **Note**: The gRPC audit incorrectly flagged this as missing. It was implemented in a prior session with full S3 remote storage support.
+
+### P4.4 JSON Pretty Print + JSONP
+- **Go**: `?pretty` query param for indented JSON; `?callback=fn` for JSONP
+- **Rust**: Neither supported
+- **Effort**: Low
+
+### P4.5 Request ID Generation
+- **Go**: Generates UUID if `x-amz-request-id` header missing, propagates to gRPC context
+- **Rust**: Only echoes existing header
+- **Effort**: Low
+
+### P4.6 UI Status Page
+- **Go**: Full HTML template with volumes, disks, stats, uptime
+- **Rust**: Stub HTML
+- **Effort**: Medium
+
+### P4.7 Advanced Prometheus Metrics
+- **Go**: InFlightRequestsGauge, ConcurrentUploadLimit/DownloadLimit gauges, metrics push gateway
+- **Rust**: Basic request counter and histogram only
+- **Effort**: Low
+
+### P4.8 Profiling (pprof)
+- **Go**: CPU/memory profiling, /debug/pprof endpoints
+- **Rust**: Flags parsed but not wired
+- **Effort**: Medium (tokio-console or pprof-rs)
+
+### P4.9 EC Distribution / Rebalancing
+- **Go**: 17 files for EC operations including placement strategies, recovery, scrubbing
+- **Rust**: 6 files with basic encoder/decoder
+- **Effort**: High
+
+### P4.10 Cookie Mismatch Status Code
+- **Go**: Returns 406 Not Acceptable
+- **Rust**: Returns 400 Bad Request
+- **Effort**: Trivial
+
+---
+
+## Implementation Order Recommendation
+
+### Sprint 1 — Quick Wins (Low effort, high impact) ✅ DONE
+1. ✅ P1.4 VolumeMarkReadonly master notification — triggers immediate heartbeat
+2. ✅ P1.5 Compaction throttling — `maybe_throttle_compaction()` method added
+3. ✅ P1.6 File size limit enforcement — checks `file_size_limit_bytes` on upload
+4. ✅ P2.1 `ts` query param — custom timestamps for upload and delete
+5. ✅ P2.4 TTL expiration check — was already implemented
+6. ✅ P2.5 Health check heartbeat status — returns 503 if not heartbeating
+7. ✅ P2.8 preStopSeconds — graceful drain delay before shutdown
+8. ✅ P2.9 S3 passthrough headers — content-encoding, expires, content-language, content-disposition
+9. ✅ P3.5 .vif persistence — readonly state persists across restarts
+10. ✅ P2.7 Webp support — added to image resize-eligible extensions
+11. ~~P4.10 Cookie 406~~ — Go actually uses 404 for HTTP cookie mismatch (406 is gRPC batch delete only)
+
+### Sprint 2 — Core Read Path (Medium effort) — Partially Done
+1. P1.1 Streaming / meta-only reads — TODO (medium effort, no test coverage yet)
+2. ✅ P1.2 ReadMode proxy/redirect — was already implemented and tested
+3. ✅ P2.2 Multipart form parsing — MIME type extraction from Content-Type header
+4. P2.3 JPEG orientation fix — TODO (low effort, needs exif crate)
+5. ✅ P2.6 Stats endpoints — /stats/counter, /stats/memory, /stats/disk
+6. ✅ P2.7 Webp support — done in Sprint 1
+7. ✅ P4.4 JSON pretty print + JSONP — ?pretty=y and ?callback=fn
+8. ✅ P4.5 Request ID generation — generates UUID if x-amz-request-id missing
+9. ✅ P4.7 Advanced Prometheus metrics — INFLIGHT_REQUESTS gauge, VOLUME_FILE_COUNT gauge
+
+### Sprint 3 — Infrastructure (Medium effort) — Partially Done
+1. ✅ P1.3 TLS/HTTPS — rustls + tokio-rustls for HTTP, tonic ServerTlsConfig for gRPC
+2. P3.2 Async request processing — TODO (medium effort)
+3. ✅ P3.3 Volume scrubbing — CRC checksum verification of all needles
+4. ✅ P3.6 Disk location features — MinFreeSpace enforcement, background disk monitor
+
+### Sprint 4 — Storage Advanced (High effort) — Deferred
+No integration test coverage for these items. All existing tests pass.
+1. P3.1 LevelDB needle maps — needed only for volumes with millions of needles
+2. P3.4 Volume backup/sync — streaming backup, binary search
+3. P3.7 Compact map — memory optimization for needle index
+4. P4.1 VolumeTierMoveDat — full S3 tiering (currently error stub)
+5. P4.9 EC distribution — advanced EC placement/rebalancing
+
+### Sprint 5 — Polish — Deferred
+No integration test coverage for these items.
+1. P4.2 Query (S3 Select) — JSON/CSV query over needle data
+2. ✅ P4.4 JSON pretty/JSONP — done in Sprint 2
+3. ✅ P4.5 Request ID generation — done in Sprint 2
+4. P4.6 UI status page — HTML template with volume/disk/stats info
+5. ✅ P4.7 Advanced metrics — done in Sprint 2
+6. P4.8 Profiling — pprof-rs or tokio-console

seaweed-volume/PARITY_PLAN.md

@@ -0,0 +1,230 @@
+# Rust Volume Server Parity Plan
+
+Generated: 2026-03-16
+
+## Goal
+
+Make `seaweed-volume` a drop-in replacement for the Go volume server by:
+
+- comparing every Go volume-server code path against the Rust implementation,
+- recording file-level ownership and verification status,
+- closing verified behavior gaps one logic change per commit,
+- extending tests so regressions are caught by Go parity suites and Rust unit/integration tests.
+
+## Ground Truth
+
+Primary Go sources:
+
+- `weed/server/volume_server.go`
+- `weed/server/volume_server_handlers*.go`
+- `weed/server/volume_grpc_*.go`
+- `weed/server/constants/volume.go`
+- `weed/storage/store*.go`
+- `weed/storage/disk_location*.go`
+- `weed/storage/volume*.go`
+- `weed/storage/needle/*.go`
+- `weed/storage/idx/*.go`
+- `weed/storage/needle_map*.go`
+- `weed/storage/needle_map/*.go`
+- `weed/storage/super_block/*.go`
+- `weed/storage/erasure_coding/*.go`
+
+Supporting Go dependencies that affect drop-in behavior:
+
+- `weed/command/volume.go`
+- `weed/security/*.go`
+- `weed/images/*.go`
+- `weed/stats/*.go`
+
+Primary Rust sources:
+
+- `seaweed-volume/src/main.rs`
+- `seaweed-volume/src/config.rs`
+- `seaweed-volume/src/security.rs`
+- `seaweed-volume/src/images.rs`
+- `seaweed-volume/src/server/*.rs`
+- `seaweed-volume/src/storage/*.rs`
+- `seaweed-volume/src/storage/needle/*.rs`
+- `seaweed-volume/src/storage/idx/*.rs`
+- `seaweed-volume/src/storage/erasure_coding/*.rs`
+- `seaweed-volume/src/remote_storage/*.rs`
+
+## Audit Method
+
+For each Go file:
+
+1. Map it to the Rust file or files that should own the same behavior.
+2. Compare exported entry points, helper functions, state transitions, wire fields, and persistence side effects.
+3. Mark each file `implemented`, `partial`, `missing`, or `needs verification`.
+4. Link each behavior to an existing test or add a missing test.
+5. Only treat a gap as closed after code review plus local verification.
+
+## Acceptance Criteria
+
+The Rust server is a drop-in replacement only when all of these hold:
+
+- HTTP routes, status codes, headers, and body semantics match Go.
+- gRPC RPCs match Go request validation, response fields, streaming behavior, and maintenance/read-only semantics.
+- Master heartbeat and topology metadata match Go closely enough that the Go master treats Rust and Go volume servers the same.
+- On-disk volume behavior matches Go for normal volumes, EC shards, tiering metadata, and readonly persistence.
+- Startup flags and operational endpoints that affect production deployment behave equivalently or are explicitly documented as unsupported.
+- Existing Go integration suites pass with `VOLUME_SERVER_IMPL=rust`.
+
+## File Matrix
+
+### HTTP server surface
+
+| Go file | Rust counterpart | Status | Comparison focus |
+| --- | --- | --- | --- |
+| `weed/server/volume_server.go` | `seaweed-volume/src/main.rs`, `seaweed-volume/src/server/volume_server.rs`, `seaweed-volume/src/server/heartbeat.rs` | partial | startup wiring, routers, heartbeat, shutdown, metrics/debug listeners |
+| `weed/server/volume_server_handlers.go` | `seaweed-volume/src/server/volume_server.rs`, `seaweed-volume/src/server/handlers.rs` | needs verification | method dispatch, OPTIONS behavior, public/admin split |
+| `weed/server/volume_server_handlers_admin.go` | `seaweed-volume/src/server/handlers.rs` | implemented | `/status`, `/healthz`, stats, server headers |
+| `weed/server/volume_server_handlers_helper.go` | `seaweed-volume/src/server/handlers.rs` | needs verification | JSON encoding, request parsing, helper parity |
+| `weed/server/volume_server_handlers_read.go` | `seaweed-volume/src/server/handlers.rs` | needs verification | JWT, conditional reads, range reads, proxy/redirect, chunk manifests, image transforms |
+| `weed/server/volume_server_handlers_ui.go` | `seaweed-volume/src/server/handlers.rs`, embedded assets | partial | UI payload and HTML parity |
+| `weed/server/volume_server_handlers_write.go` | `seaweed-volume/src/server/handlers.rs`, `seaweed-volume/src/images.rs` | needs verification | multipart parsing, metadata, compression, ts, delete semantics |
+| `weed/server/constants/volume.go` | `seaweed-volume/src/server/heartbeat.rs`, config defaults | needs verification | heartbeat timing, constants parity |
+
+### gRPC server surface
+
+| Go file | Rust counterpart | Status | Comparison focus |
+| --- | --- | --- | --- |
+| `weed/server/volume_grpc_admin.go` | `seaweed-volume/src/server/grpc_server.rs` | needs verification | readonly/writable, allocate/delete/configure/mount/unmount |
+| `weed/server/volume_grpc_batch_delete.go` | `seaweed-volume/src/server/grpc_server.rs` | implemented | batch delete, EC delete path |
+| `weed/server/volume_grpc_client_to_master.go` | `seaweed-volume/src/server/heartbeat.rs` | partial | heartbeat fields, leader changes, metrics settings from master |
+| `weed/server/volume_grpc_copy.go` | `seaweed-volume/src/server/grpc_server.rs` | needs verification | full copy streams |
+| `weed/server/volume_grpc_copy_incremental.go` | `seaweed-volume/src/server/grpc_server.rs` | needs verification | incremental copy binary search, timestamps |
+| `weed/server/volume_grpc_erasure_coding.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/storage/erasure_coding/*.rs` | needs verification | shard read/write/delete/mount/unmount/rebuild |
+| `weed/server/volume_grpc_query.go` | `seaweed-volume/src/server/grpc_server.rs` | needs verification | query validation and error parity |
+| `weed/server/volume_grpc_read_all.go` | `seaweed-volume/src/server/grpc_server.rs` | needs verification | read-all ordering and tail semantics |
+| `weed/server/volume_grpc_read_write.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/storage/*.rs` | needs verification | blob/meta/page reads, write blob semantics |
+| `weed/server/volume_grpc_remote.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/remote_storage/*.rs` | needs verification | remote fetch/write and tier metadata |
+| `weed/server/volume_grpc_scrub.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/storage/*.rs` | needs verification | scrub result semantics |
+| `weed/server/volume_grpc_state.go` | `seaweed-volume/src/server/grpc_server.rs` | implemented | GetState/SetState/Status |
+| `weed/server/volume_grpc_tail.go` | `seaweed-volume/src/server/grpc_server.rs` | needs verification | tail streaming and idle timeout |
+| `weed/server/volume_grpc_tier_download.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/remote_storage/*.rs` | needs verification | tier download stream/error paths |
+| `weed/server/volume_grpc_tier_upload.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/remote_storage/*.rs` | needs verification | tier upload stream/error paths |
+| `weed/server/volume_grpc_vacuum.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/storage/*.rs` | needs verification | compact/commit/cleanup progress and readonly transitions |
+
+### Storage and persistence surface
+
+| Go file group | Rust counterpart | Status | Comparison focus |
+| --- | --- | --- | --- |
+| `weed/storage/store.go`, `store_state.go` | `seaweed-volume/src/storage/store.rs`, `seaweed-volume/src/server/heartbeat.rs` | partial | topology metadata, disk tags, server id, state persistence |
+| `weed/storage/store_vacuum.go` | `seaweed-volume/src/storage/store.rs`, `seaweed-volume/src/storage/volume.rs` | needs verification | vacuum sequencing |
+| `weed/storage/store_ec.go`, `store_ec_delete.go`, `store_ec_scrub.go` | `seaweed-volume/src/storage/store.rs`, `seaweed-volume/src/storage/erasure_coding/*.rs` | needs verification | EC lifecycle and scrub behavior |
+| `weed/storage/disk_location.go`, `disk_location_ec.go` | `seaweed-volume/src/storage/disk_location.rs`, `seaweed-volume/src/storage/store.rs` | partial | directory UUIDs, tags, load rules, disk space checks |
+| `weed/storage/volume.go`, `volume_loading.go` | `seaweed-volume/src/storage/volume.rs` | needs verification | load/reload/readonly/remote metadata |
+| `weed/storage/volume_super_block.go` | `seaweed-volume/src/storage/super_block.rs`, `seaweed-volume/src/storage/volume.rs` | implemented | super block parity |
+| `weed/storage/volume_read.go`, `volume_read_all.go` | `seaweed-volume/src/storage/volume.rs`, `seaweed-volume/src/server/handlers.rs` | needs verification | full/meta/page reads, TTL, streaming |
+| `weed/storage/volume_write.go` | `seaweed-volume/src/storage/volume.rs`, `seaweed-volume/src/server/write_queue.rs` | needs verification | dedup, sync/async writes, metadata flags |
+| `weed/storage/volume_vacuum.go` | `seaweed-volume/src/storage/volume.rs` | needs verification | compact and commit parity |
+| `weed/storage/volume_backup.go` | `seaweed-volume/src/storage/volume.rs`, `seaweed-volume/src/server/grpc_server.rs` | needs verification | backup/search logic |
+| `weed/storage/volume_checking.go` | `seaweed-volume/src/storage/volume.rs`, `seaweed-volume/src/storage/idx/mod.rs`, `seaweed-volume/src/server/grpc_server.rs` | needs verification | scrub and integrity checks |
+| `weed/storage/volume_info.go`, `volume_info/volume_info.go`, `volume_tier.go` | `seaweed-volume/src/storage/volume.rs`, `seaweed-volume/src/remote_storage/*.rs` | needs verification | `.vif` format and tiered file metadata |
+| `weed/storage/needle/*.go` | `seaweed-volume/src/storage/needle/*.rs` | needs verification | needle parsing, CRC, TTL, multipart metadata |
+| `weed/storage/idx/*.go` | `seaweed-volume/src/storage/idx/*.rs` | needs verification | index walking and binary search |
+| `weed/storage/needle_map*.go`, `needle_map/*.go` | `seaweed-volume/src/storage/needle_map.rs` | needs verification | map kind parity, persistence, memory behavior |
+| `weed/storage/super_block/*.go` | `seaweed-volume/src/storage/super_block.rs` | implemented | replica placement and TTL metadata |
+| `weed/storage/erasure_coding/*.go` | `seaweed-volume/src/storage/erasure_coding/*.rs` | needs verification | EC shard placement, encode/decode, journal deletes |
+
+### Supporting runtime surface
+
+| Go file | Rust counterpart | Status | Comparison focus |
+| --- | --- | --- | --- |
+| `weed/command/volume.go` | `seaweed-volume/src/config.rs`, `seaweed-volume/src/main.rs` | partial | flags, metrics/debug listeners, startup behavior |
+| `weed/security/*.go` | `seaweed-volume/src/security.rs`, `seaweed-volume/src/main.rs` | implemented | JWT and TLS loading |
+| `weed/images/*.go` | `seaweed-volume/src/images.rs`, `seaweed-volume/src/server/handlers.rs` | implemented | JPEG orientation and transforms |
+| `weed/stats/*.go` | `seaweed-volume/src/metrics.rs`, `seaweed-volume/src/server/handlers.rs` | partial | metrics endpoints, push-gateway integration |
+
+## Verified Gaps As Of 2026-03-08
+
+The startup/runtime gaps that were verified in the initial audit are now closed:
+
+1. Heartbeat metadata parity
+   Closed by `8ade1c51d` and retained in current HEAD.
+
+2. Dedicated metrics/debug listener parity
+   Closed by `fbe0e5829`.
+
+3. Master-provided metrics push settings
+   Closed by `fbe0e5829`.
+
+4. Slow-read tuning parity
+   Closed by `66e3900dc`.
+
+There are no remaining verified gaps from the initial startup/runtime audit. The broader line-by-line comparison batches below are still required to either confirm parity or surface new gaps.
+
+## Execution Status As Of 2026-03-16
+
+The file-by-file comparison and verification work executed in this round was:
+
+1. Startup and harness alignment
+   Compared `weed/command/volume.go`, `test/volume_server/framework/cluster*.go`, `seaweed-volume/src/config.rs`, and `seaweed-volume/src/main.rs` to ensure the Rust server is invoked with Go-compatible flags and is rebuilt from the current source during parity runs.
+
+2. HTTP admin surface
+   Compared `weed/server/volume_server_handlers_admin.go` against `seaweed-volume/src/server/handlers.rs` with emphasis on `/status` payload shape, disk-status fields, and volume ordering.
+
+3. gRPC admin surface
+   Compared `weed/server/volume_grpc_admin.go` against `seaweed-volume/src/server/grpc_server.rs` with emphasis on `Ping`, `VolumeConfigure`, readonly/writable flows, and error wrapping.
+
+4. Storage/index layout
+   Compared Go index-entry defaults in `weed/storage/types` and `weed/storage/idx/*.go` against the Rust default feature set in `seaweed-volume/Cargo.toml` and the Rust index reader/writer paths to confirm default binaries use the same offset width.
+
+5. End-to-end parity verification
+   Re-ran the Go HTTP and gRPC integration suites with `VOLUME_SERVER_IMPL=rust` after each fix to confirm wire-level compatibility.
+
+### Verified mismatches closed in this round
+
+- Rust parity runs could reuse a stale `weed-volume` binary across test invocations, hiding source and feature changes from the Go harness.
+- Rust defaulted to 5-byte index offsets, while the default Go `go build` path uses 4-byte offsets unless built with `-tags 5BytesOffset`.
+- Rust `/status` omitted Go fields in both `Volumes` and `DiskStatuses`, and did not sort volumes by `Id`.
+- Rust `Ping` treated an empty target as a self-ping and only performed a raw gRPC connect for filer targets; Go returns `remote_time_ns=0` for the empty request and performs a real filer `Ping` RPC.
+- Rust `VolumeNeedleStatus` dropped stored TTL metadata and reported `data_size` instead of Go’s `Size` field.
+- Rust multipart uploads ignored form fields such as `ts`, `ttl`, and `cm`, and also ignored part-level `Content-Encoding` and `Content-MD5`.
+- Rust only treated `dl=true` and `dl=1` as truthy, while Go accepts the full `strconv.ParseBool` set such as `dl=t` and `dl=True`.
+
+### Verification commands
+
+- `VOLUME_SERVER_IMPL=rust go test -count=1 -timeout 1200s ./test/volume_server/http/...`
+- `VOLUME_SERVER_IMPL=rust go test -count=1 -timeout 1200s ./test/volume_server/grpc/...`
+
+## Execution Plan
+
+### Batch 1: startup and heartbeat
+
+- Compare `weed/command/volume.go`, `weed/server/volume_server.go`, `weed/server/volume_grpc_client_to_master.go`, `weed/storage/store.go`, and `weed/storage/disk_location.go`.
+- Close metadata and startup parity gaps that affect master registration and deployment compatibility.
+- Add Rust unit tests for heartbeat payloads and config wiring.
+
+### Batch 2: HTTP read path
+
+- Compare `volume_server_handlers_read.go`, `volume_server_handlers_helper.go`, and related storage read functions line by line.
+- Verify JWT, path parsing, proxy/redirect, ranges, streaming, chunk manifests, image transforms, and response-header overrides.
+- Extend `test/volume_server/http/...` and Rust handler tests where parity is not covered.
+
+### Batch 3: HTTP write/delete path
+
+- Compare `volume_server_handlers_write.go` and write-related storage functions.
+- Verify multipart behavior, metadata, md5, compression, unchanged writes, delete edge cases, and timestamp handling.
+
+### Batch 4: gRPC admin and lifecycle
+
+- Compare `volume_grpc_admin.go`, `volume_grpc_state.go`, and `volume_grpc_vacuum.go`.
+- Verify readonly/writable flows, maintenance mode, status payloads, mount/unmount/delete/configure, and vacuum transitions.
+
+### Batch 5: gRPC data movement
+
+- Compare `volume_grpc_read_write.go`, `copy*.go`, `read_all.go`, `tail.go`, `remote.go`, and `query.go`.
+- Verify stream framing, binary search, idle timeout, and remote-storage semantics.
+
+### Batch 6: storage internals
+
+- Compare all `weed/storage` volume, needle, idx, needle map, and EC files line by line.
+- Focus on persistence rules, readonly semantics, TTL, recovery/scrub, backup, and memory/disk map behavior.
+
+## Commit Strategy
+
+- One commit for the audit/plan document if the document itself changes.
+- One commit per logic fix.
+- Every logic commit must include the smallest test addition that proves the new parity claim.

seaweed-volume/README.md

@@ -0,0 +1,140 @@
+# SeaweedFS Volume Server (Rust)
+
+A drop-in replacement for the [SeaweedFS](https://github.com/seaweedfs/seaweedfs) Go volume server, rewritten in Rust. It uses binary-compatible storage formats (`.dat`, `.idx`, `.vif`) and speaks the same HTTP and gRPC protocols, so it works with an unmodified Go master server.
+
+## Building
+
+Requires Rust 1.75+ (2021 edition).
+
+```bash
+cd seaweed-volume
+cargo build --release
+```
+
+The binary is produced at `target/release/seaweed-volume`.
+
+## Running
+
+Start a Go master server first, then point the Rust volume server at it:
+
+```bash
+# Minimal
+seaweed-volume --port 8080 --master localhost:9333 --dir /data/vol1 --max 7
+
+# Multiple data directories
+seaweed-volume --port 8080 --master localhost:9333 \
+  --dir /mnt/ssd1,/mnt/ssd2 --max 100,100 --disk ssd
+
+# With datacenter/rack topology
+seaweed-volume --port 8080 --master localhost:9333 --dir /data/vol1 --max 7 \
+  --dataCenter dc1 --rack rack1
+
+# With JWT authentication
+seaweed-volume --port 8080 --master localhost:9333 --dir /data/vol1 --max 7 \
+  --securityFile /etc/seaweedfs/security.toml
+
+# With TLS (configured in security.toml via [https.volume] and [grpc.volume] sections)
+seaweed-volume --port 8080 --master localhost:9333 --dir /data/vol1 --max 7 \
+  --securityFile /etc/seaweedfs/security.toml
+```
+
+### Common flags
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--port` | `8080` | HTTP listen port |
+| `--port.grpc` | `port+10000` | gRPC listen port |
+| `--master` | `localhost:9333` | Comma-separated master server addresses |
+| `--dir` | `/tmp` | Comma-separated data directories |
+| `--max` | `8` | Max volumes per directory (comma-separated) |
+| `--ip` | auto-detect | Server IP / identifier |
+| `--ip.bind` | same as `--ip` | Bind address |
+| `--dataCenter` | | Datacenter name |
+| `--rack` | | Rack name |
+| `--disk` | | Disk type tag: `hdd`, `ssd`, or custom |
+| `--index` | `memory` | Needle map type: `memory`, `leveldb`, `leveldbMedium`, `leveldbLarge` |
+| `--readMode` | `proxy` | Non-local read mode: `local`, `proxy`, `redirect` |
+| `--fileSizeLimitMB` | `256` | Max upload file size |
+| `--minFreeSpace` | `1` (percent) | Min free disk space before marking volumes read-only |
+| `--securityFile` | | Path to `security.toml` for JWT keys and TLS certs |
+| `--metricsPort` | `0` (disabled) | Prometheus metrics endpoint port |
+| `--whiteList` | | Comma-separated IPs with write permission |
+| `--preStopSeconds` | `10` | Graceful drain period before shutdown |
+| `--compactionMBps` | `0` (unlimited) | Compaction I/O rate limit |
+| `--pprof` | `false` | Enable pprof HTTP handlers |
+
+Set `RUST_LOG=debug` (or `trace`, `info`, `warn`) for log level control.
+Set `SEAWEED_WRITE_QUEUE=1` to enable batched async write processing.
+
+## Features
+
+- **Binary compatible** -- reads and writes the same `.dat`/`.idx`/`.vif` files as the Go server; seamless migration with no data conversion.
+- **HTTP + gRPC** -- full implementation of the volume server HTTP API and all gRPC RPCs including streaming operations (copy, tail, incremental copy, vacuum).
+- **Master heartbeat** -- bidirectional streaming heartbeat with the Go master server; volume and EC shard registration, leader failover, graceful shutdown deregistration.
+- **JWT authentication** -- signing key configuration via `security.toml` with token source precedence (query > header > cookie), file_id claims validation, and separate read/write keys.
+- **TLS** -- HTTPS for the HTTP API and mTLS for gRPC, configured through `security.toml`.
+- **Erasure coding** -- Reed-Solomon EC shard management: mount/unmount, read, rebuild, copy, delete, and shard-to-volume reconstruction.
+- **S3 remote storage** -- `FetchAndWriteNeedle` reads from any S3-compatible backend (AWS, MinIO, Wasabi, Backblaze, etc.) and writes locally. Supports `VolumeTierMoveDatToRemote`/`FromRemote` for tiered storage.
+- **Needle map backends** -- in-memory HashMap, LevelDB (via `rusty-leveldb`), or redb (pure Rust disk-backed) needle maps.
+- **Image processing** -- on-the-fly resize/crop, JPEG EXIF orientation auto-fix, WebP support.
+- **Streaming reads** -- large files (>1MB) are streamed via `spawn_blocking` to avoid blocking the async runtime.
+- **Auto-compression** -- compressible file types (text, JSON, CSS, JS, SVG, etc.) are gzip-compressed on upload.
+- **Prometheus metrics** -- counters, histograms, and gauges exported at a dedicated metrics port; optional push gateway support.
+- **Graceful shutdown** -- SIGINT/SIGTERM handling with configurable `preStopSeconds` drain period.
+
+## Testing
+
+### Rust unit tests
+
+```bash
+cd seaweed-volume
+cargo test
+```
+
+### Go integration tests
+
+The Go test suite can target either the Go or Rust volume server via the `VOLUME_SERVER_IMPL` environment variable:
+
+```bash
+# Run all HTTP + gRPC integration tests against the Rust server
+VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 1200s \
+  ./test/volume_server/grpc/... ./test/volume_server/http/...
+
+# Run a single test
+VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 60s \
+  -run "TestName" ./test/volume_server/http/...
+
+# Run S3 remote storage tests
+VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 180s \
+  -run "TestFetchAndWriteNeedle" ./test/volume_server/grpc/...
+```
+
+## Load testing
+
+A load test harness is available at `test/volume_server/loadtest/`. See that directory for usage instructions and scenarios.
+
+## Architecture
+
+The server runs three listeners concurrently:
+
+- **HTTP** (Axum 0.7) -- admin and public routers for file upload/download, status, and stats endpoints.
+- **gRPC** (Tonic 0.12) -- all `VolumeServer` RPCs from the SeaweedFS protobuf definition.
+- **Metrics** (optional) -- Prometheus scrape endpoint on a separate port.
+
+Key source modules:
+
+| Path | Description |
+|------|-------------|
+| `src/main.rs` | Entry point, server startup, signal handling |
+| `src/config.rs` | CLI parsing and configuration resolution |
+| `src/server/volume_server.rs` | HTTP router setup and middleware |
+| `src/server/handlers.rs` | HTTP request handlers (read, write, delete, status) |
+| `src/server/grpc_server.rs` | gRPC service implementation |
+| `src/server/heartbeat.rs` | Master heartbeat loop |
+| `src/storage/volume.rs` | Volume read/write/delete logic |
+| `src/storage/needle.rs` | Needle (file entry) serialization |
+| `src/storage/store.rs` | Multi-volume store management |
+| `src/security.rs` | JWT validation and IP whitelist guard |
+| `src/remote_storage/` | S3 remote storage backend |
+
+See [DEV_PLAN.md](DEV_PLAN.md) for the full development history and feature checklist.

seaweed-volume/build.rs

@@ -0,0 +1,17 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR")?);
+    tonic_build::configure()
+        .build_server(true)
+        .build_client(true)
+        .file_descriptor_set_path(out_dir.join("seaweed_descriptor.bin"))
+        .compile_protos(
+            &[
+                "proto/volume_server.proto",
+                "proto/master.proto",
+                "proto/remote.proto",
+                "../weed/pb/filer.proto",
+            ],
+            &["proto/", "../weed/pb/"],
+        )?;
+    Ok(())
+}

seaweed-volume/proto/master.proto

@@ -0,0 +1,474 @@
+syntax = "proto3";
+
+package master_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/master_pb";
+
+import "volume_server.proto";
+
+//////////////////////////////////////////////////
+
+service Seaweed {
+  rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) {
+  }
+  rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) {
+  }
+  rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
+  }
+  rpc Assign (AssignRequest) returns (AssignResponse) {
+  }
+  rpc StreamAssign (stream AssignRequest) returns (stream AssignResponse) {
+  }
+  rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
+  }
+  rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
+  }
+  rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) {
+  }
+  rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) {
+  }
+  rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) {
+  }
+  rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) {
+  }
+  rpc DisableVacuum (DisableVacuumRequest) returns (DisableVacuumResponse) {
+  }
+  rpc EnableVacuum (EnableVacuumRequest) returns (EnableVacuumResponse) {
+  }
+  rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
+  }
+  rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) {
+  }
+  rpc ListClusterNodes (ListClusterNodesRequest) returns (ListClusterNodesResponse) {
+  }
+  rpc LeaseAdminToken (LeaseAdminTokenRequest) returns (LeaseAdminTokenResponse) {
+  }
+  rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) {
+  }
+  rpc Ping (PingRequest) returns (PingResponse) {
+  }
+  rpc RaftListClusterServers (RaftListClusterServersRequest) returns (RaftListClusterServersResponse) {
+  }
+  rpc RaftAddServer (RaftAddServerRequest) returns (RaftAddServerResponse) {
+  }
+  rpc RaftRemoveServer (RaftRemoveServerRequest) returns (RaftRemoveServerResponse) {
+  }
+  rpc RaftLeadershipTransfer (RaftLeadershipTransferRequest) returns (RaftLeadershipTransferResponse) {
+  }
+  rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) {
+  }
+}
+
+//////////////////////////////////////////////////
+
+message DiskTag {
+  uint32 disk_id = 1;
+  repeated string tags = 2;
+}
+
+message Heartbeat {
+  string ip = 1;
+  uint32 port = 2;
+  string public_url = 3;
+  uint64 max_file_key = 5;
+  string data_center = 6;
+  string rack = 7;
+  uint32 admin_port = 8;
+  repeated VolumeInformationMessage volumes = 9;
+  // delta volumes
+  repeated VolumeShortInformationMessage new_volumes = 10;
+  repeated VolumeShortInformationMessage deleted_volumes = 11;
+  bool has_no_volumes = 12;
+
+  // erasure coding
+  repeated VolumeEcShardInformationMessage ec_shards = 16;
+  // delta erasure coding shards
+  repeated VolumeEcShardInformationMessage new_ec_shards = 17;
+  repeated VolumeEcShardInformationMessage deleted_ec_shards = 18;
+  bool has_no_ec_shards = 19;
+
+  map<string, uint32> max_volume_counts = 4;
+  uint32 grpc_port = 20;
+  repeated string location_uuids = 21;
+  string id = 22; // volume server id, independent of ip:port for stable identification
+
+  // state flags
+  volume_server_pb.VolumeServerState state = 23;
+
+  repeated DiskTag disk_tags = 24;
+}
+
+message HeartbeatResponse {
+  uint64 volume_size_limit = 1;
+  string leader = 2;
+  string metrics_address = 3;
+  uint32 metrics_interval_seconds = 4;
+  repeated StorageBackend storage_backends = 5;
+  repeated string duplicated_uuids = 6;
+  bool preallocate = 7;
+}
+
+message VolumeInformationMessage {
+  uint32 id = 1;
+  uint64 size = 2;
+  string collection = 3;
+  uint64 file_count = 4;
+  uint64 delete_count = 5;
+  uint64 deleted_byte_count = 6;
+  bool read_only = 7;
+  uint32 replica_placement = 8;
+  uint32 version = 9;
+  uint32 ttl = 10;
+  uint32 compact_revision = 11;
+  int64 modified_at_second = 12;
+  string remote_storage_name = 13;
+  string remote_storage_key = 14;
+  string disk_type = 15;
+  uint32 disk_id = 16;
+}
+
+message VolumeShortInformationMessage {
+  uint32 id = 1;
+  string collection = 3;
+  uint32 replica_placement = 8;
+  uint32 version = 9;
+  uint32 ttl = 10;
+  string disk_type = 15;
+  uint32 disk_id = 16;
+}
+
+message VolumeEcShardInformationMessage {
+  uint32 id = 1;
+  string collection = 2;
+  uint32 ec_index_bits = 3;
+  string disk_type = 4;
+  uint64 expire_at_sec = 5; // used to record the destruction time of ec volume
+  uint32 disk_id = 6;
+  repeated int64 shard_sizes = 7; // optimized: sizes for shards in order of set bits in ec_index_bits
+}
+
+message StorageBackend {
+  string type = 1;
+  string id = 2;
+  map<string, string> properties = 3;
+}
+
+message Empty {
+}
+
+message SuperBlockExtra {
+  message ErasureCoding {
+    uint32 data = 1;
+    uint32 parity = 2;
+    repeated uint32 volume_ids = 3;
+  }
+  ErasureCoding erasure_coding = 1;
+}
+
+message KeepConnectedRequest {
+  string client_type = 1;
+  string client_address = 3;
+  string version = 4;
+  string filer_group = 5;
+  string data_center = 6;
+  string rack = 7;
+}
+
+message VolumeLocation {
+  string url = 1;
+  string public_url = 2;
+  repeated uint32 new_vids = 3;
+  repeated uint32 deleted_vids = 4;
+  string leader = 5; // optional when leader is not itself
+  string data_center = 6; // optional when DataCenter is in use
+  uint32 grpc_port = 7;
+  repeated uint32 new_ec_vids = 8;
+  repeated uint32 deleted_ec_vids = 9;
+}
+
+message ClusterNodeUpdate {
+  string node_type = 1;
+  string address = 2;
+  bool is_add = 4;
+  string filer_group = 5;
+  int64 created_at_ns = 6;
+}
+
+message KeepConnectedResponse {
+  VolumeLocation volume_location = 1;
+  ClusterNodeUpdate cluster_node_update = 2;
+}
+
+message LookupVolumeRequest {
+  repeated string volume_or_file_ids = 1;
+  string collection = 2; // optional, a bit faster if provided.
+}
+message LookupVolumeResponse {
+  message VolumeIdLocation {
+    string volume_or_file_id = 1;
+    repeated Location locations = 2;
+    string error = 3;
+    string auth = 4;
+  }
+  repeated VolumeIdLocation volume_id_locations = 1;
+}
+
+message Location {
+  string url = 1;
+  string public_url = 2;
+  uint32 grpc_port = 3;
+  string data_center = 4;
+}
+
+message AssignRequest {
+  uint64 count = 1;
+  string replication = 2;
+  string collection = 3;
+  string ttl = 4;
+  string data_center = 5;
+  string rack = 6;
+  string data_node = 7;
+  uint32 memory_map_max_size_mb = 8;
+  uint32 writable_volume_count = 9;
+  string disk_type = 10;
+}
+
+message VolumeGrowRequest {
+  uint32 writable_volume_count = 1;
+  string replication = 2;
+  string collection = 3;
+  string ttl = 4;
+  string data_center = 5;
+  string rack = 6;
+  string data_node = 7;
+  uint32 memory_map_max_size_mb = 8;
+  string disk_type = 9;
+}
+
+message AssignResponse {
+  string fid = 1;
+  uint64 count = 4;
+  string error = 5;
+  string auth = 6;
+  repeated Location replicas = 7;
+  Location location = 8;
+}
+
+message StatisticsRequest {
+  string replication = 1;
+  string collection = 2;
+  string ttl = 3;
+  string disk_type = 4;
+}
+message StatisticsResponse {
+  uint64 total_size = 4;
+  uint64 used_size = 5;
+  uint64 file_count = 6;
+}
+
+//
+// collection related
+//
+message Collection {
+  string name = 1;
+}
+message CollectionListRequest {
+  bool include_normal_volumes = 1;
+  bool include_ec_volumes = 2;
+}
+message CollectionListResponse {
+  repeated Collection collections = 1;
+}
+
+message CollectionDeleteRequest {
+  string name = 1;
+}
+message CollectionDeleteResponse {
+}
+
+//
+// volume related
+//
+message DiskInfo {
+  string type = 1;
+  int64 volume_count = 2;
+  int64 max_volume_count = 3;
+  int64 free_volume_count = 4;
+  int64 active_volume_count = 5;
+  repeated VolumeInformationMessage volume_infos = 6;
+  repeated VolumeEcShardInformationMessage ec_shard_infos = 7;
+  int64 remote_volume_count = 8;
+  uint32 disk_id = 9;
+  repeated string tags = 10;
+}
+message DataNodeInfo {
+  string id = 1;
+  map<string, DiskInfo> diskInfos = 2;
+  uint32 grpc_port = 3;
+  string address = 4; // ip:port for connecting to the volume server
+}
+message RackInfo {
+  string id = 1;
+  repeated DataNodeInfo data_node_infos = 2;
+  map<string, DiskInfo> diskInfos = 3;
+}
+message DataCenterInfo {
+  string id = 1;
+  repeated RackInfo rack_infos = 2;
+  map<string, DiskInfo> diskInfos = 3;
+}
+message TopologyInfo {
+  string id = 1;
+  repeated DataCenterInfo data_center_infos = 2;
+  map<string, DiskInfo> diskInfos = 3;
+}
+message VolumeListRequest {
+}
+message VolumeListResponse {
+  TopologyInfo topology_info = 1;
+  uint64 volume_size_limit_mb = 2;
+}
+
+message LookupEcVolumeRequest {
+  uint32 volume_id = 1;
+}
+message LookupEcVolumeResponse {
+  uint32 volume_id = 1;
+  message EcShardIdLocation {
+    uint32 shard_id = 1;
+    repeated Location locations = 2;
+  }
+  repeated EcShardIdLocation shard_id_locations = 2;
+}
+
+message VacuumVolumeRequest {
+  float garbage_threshold = 1;
+  uint32 volume_id = 2;
+  string collection = 3;
+}
+message VacuumVolumeResponse {
+}
+
+message DisableVacuumRequest {
+}
+message DisableVacuumResponse {
+}
+
+message EnableVacuumRequest {
+}
+message EnableVacuumResponse {
+}
+
+message VolumeMarkReadonlyRequest {
+  string ip = 1;
+  uint32 port = 2;
+  uint32 volume_id = 4;
+  string collection = 5;
+  uint32 replica_placement = 6;
+  uint32 version = 7;
+  uint32 ttl = 8;
+  string disk_type = 9;
+  bool is_readonly = 10;
+}
+message VolumeMarkReadonlyResponse {
+}
+
+message GetMasterConfigurationRequest {
+}
+message GetMasterConfigurationResponse {
+  string metrics_address = 1;
+  uint32 metrics_interval_seconds = 2;
+  repeated StorageBackend storage_backends = 3;
+  string default_replication = 4;
+  string leader = 5;
+  uint32 volume_size_limit_m_b = 6;
+  bool volume_preallocate = 7;
+  // MIGRATION: fields 8-9 help migrate master.toml [master.maintenance] to admin script plugin. Remove after March 2027.
+  string maintenance_scripts = 8;
+  uint32 maintenance_sleep_minutes = 9;
+}
+
+message ListClusterNodesRequest {
+  string client_type = 1;
+  string filer_group = 2;
+  int32 limit = 4;
+}
+message ListClusterNodesResponse {
+  message ClusterNode {
+    string address = 1;
+    string version = 2;
+    int64 created_at_ns = 4;
+    string data_center = 5;
+    string rack = 6;
+  }
+  repeated ClusterNode cluster_nodes = 1;
+}
+
+message LeaseAdminTokenRequest {
+  int64 previous_token = 1;
+  int64 previous_lock_time = 2;
+  string lock_name = 3;
+  string client_name = 4;
+  string message = 5;
+}
+message LeaseAdminTokenResponse {
+  int64 token = 1;
+  int64 lock_ts_ns = 2;
+}
+
+message ReleaseAdminTokenRequest {
+  int64 previous_token = 1;
+  int64 previous_lock_time = 2;
+  string lock_name = 3;
+}
+message ReleaseAdminTokenResponse {
+}
+
+message PingRequest {
+  string target = 1; // default to ping itself
+  string target_type = 2;
+}
+message PingResponse {
+  int64 start_time_ns = 1;
+  int64 remote_time_ns = 2;
+  int64 stop_time_ns = 3;
+}
+
+message RaftAddServerRequest {
+  string id = 1;
+  string address = 2;
+  bool voter = 3;
+}
+message RaftAddServerResponse {
+}
+
+message RaftRemoveServerRequest {
+  string id = 1;
+  bool force = 2;
+}
+message RaftRemoveServerResponse {
+}
+
+message RaftListClusterServersRequest {
+}
+message RaftListClusterServersResponse {
+  message ClusterServers {
+    string id = 1;
+    string address = 2;
+    string suffrage = 3;
+    bool   isLeader = 4;
+  }
+  repeated ClusterServers cluster_servers = 1;
+}
+
+message RaftLeadershipTransferRequest {
+  string target_id = 1;       // Optional: target server ID. If empty, transfers to any eligible follower
+  string target_address = 2;  // Optional: target server address. Required if target_id is specified
+}
+message RaftLeadershipTransferResponse {
+  string previous_leader = 1;
+  string new_leader = 2;
+}
+
+message VolumeGrowResponse {
+}

seaweed-volume/proto/remote.proto

@@ -0,0 +1,76 @@
+syntax = "proto3";
+
+package remote_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb";
+option java_package = "seaweedfs.client";
+option java_outer_classname = "FilerProto";
+
+/////////////////////////
+// Remote Storage related
+/////////////////////////
+message RemoteConf {
+  string type = 1;
+  string name = 2;
+  string s3_access_key = 4;
+  string s3_secret_key = 5;
+  string s3_region = 6;
+  string s3_endpoint = 7;
+  string s3_storage_class = 8;
+  bool s3_force_path_style = 9;
+  bool s3_support_tagging = 13;
+  bool s3_v4_signature = 11;
+
+  string gcs_google_application_credentials = 10;
+  string gcs_project_id = 12;
+
+  string azure_account_name = 15;
+  string azure_account_key = 16;
+
+  string backblaze_key_id = 20;
+  string backblaze_application_key = 21;
+  string backblaze_endpoint = 22;
+  string backblaze_region = 23;
+
+  string aliyun_access_key = 25;
+  string aliyun_secret_key = 26;
+  string aliyun_endpoint = 27;
+  string aliyun_region = 28;
+
+  string tencent_secret_id = 30;
+  string tencent_secret_key = 31;
+  string tencent_endpoint = 32;
+
+  string baidu_access_key = 35;
+  string baidu_secret_key = 36;
+  string baidu_endpoint = 37;
+  string baidu_region = 38;
+
+  string wasabi_access_key = 40;
+  string wasabi_secret_key = 41;
+  string wasabi_endpoint = 42;
+  string wasabi_region = 43;
+
+  string filebase_access_key = 60;
+  string filebase_secret_key = 61;
+  string filebase_endpoint = 62;
+
+  string storj_access_key = 65;
+  string storj_secret_key = 66;
+  string storj_endpoint = 67;
+
+  string contabo_access_key = 68;
+  string contabo_secret_key = 69;
+  string contabo_endpoint = 70;
+  string contabo_region = 71;
+}
+
+message RemoteStorageMapping {
+  map<string,RemoteStorageLocation> mappings = 1;
+  string primary_bucket_storage_name = 2;
+}
+message RemoteStorageLocation {
+  string name = 1;
+  string bucket = 2;
+  string path = 3;
+}

seaweed-volume/proto/volume_server.proto

@@ -0,0 +1,759 @@
+syntax = "proto3";
+
+package volume_server_pb;
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb";
+
+import "remote.proto";
+
+//////////////////////////////////////////////////
+
+// Persistent state for volume servers.
+message VolumeServerState {
+    // whether the server is in maintenance (i.e. read-only) mode.
+    bool maintenance = 1;
+    // incremental version counter
+    uint32 version = 2;
+}
+
+//////////////////////////////////////////////////
+
+service VolumeServer {
+    //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
+    rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
+    }
+
+    rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
+    }
+    rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
+    }
+    rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
+    }
+    rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
+    }
+
+    rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
+    }
+    rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
+    }
+
+    rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
+    }
+    rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
+    }
+
+    rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
+    }
+    rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
+    }
+    rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
+    }
+    rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
+    }
+    rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
+    }
+    rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
+    }
+    rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
+    }
+
+    rpc GetState (GetStateRequest) returns (GetStateResponse) {
+    }
+    rpc SetState (SetStateRequest) returns (SetStateResponse) {
+    }
+
+    // copy the .idx .dat files, and mount this volume
+    rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
+    }
+    rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
+    }
+    rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
+    }
+    rpc ReceiveFile (stream ReceiveFileRequest) returns (ReceiveFileResponse) {
+    }
+
+    rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
+    }
+    rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) {
+    }
+    rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
+    }
+    rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
+    }
+
+    rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
+    }
+    rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
+    }
+
+    // erasure coding
+    rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
+    }
+    rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
+    }
+    rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
+    }
+    rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
+    }
+    rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
+    }
+    rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
+    }
+    rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
+    }
+    rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
+    }
+    rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
+    }
+    rpc VolumeEcShardsInfo (VolumeEcShardsInfoRequest) returns (VolumeEcShardsInfoResponse) {
+    }
+
+    // tiered storage
+    rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
+    }
+    rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
+    }
+
+    rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
+    }
+    rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
+    }
+
+    // remote storage
+    rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
+    }
+
+    // scrubbing
+    rpc ScrubVolume (ScrubVolumeRequest) returns (ScrubVolumeResponse) {
+    }
+    rpc ScrubEcVolume (ScrubEcVolumeRequest) returns (ScrubEcVolumeResponse) {
+    }
+
+    // <experimental> query
+    rpc Query (QueryRequest) returns (stream QueriedStripe) {
+    }
+
+    rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
+    }
+
+    rpc Ping (PingRequest) returns (PingResponse) {
+    }
+
+}
+
+//////////////////////////////////////////////////
+
+message BatchDeleteRequest {
+    repeated string file_ids = 1;
+    bool skip_cookie_check = 2;
+}
+
+message BatchDeleteResponse {
+    repeated DeleteResult results = 1;
+}
+message DeleteResult {
+    string file_id = 1;
+    int32 status = 2;
+    string error = 3;
+    uint32 size = 4;
+    uint32 version = 5;
+}
+
+message Empty {
+}
+
+message VacuumVolumeCheckRequest {
+    uint32 volume_id = 1;
+}
+message VacuumVolumeCheckResponse {
+    double garbage_ratio = 1;
+}
+
+message VacuumVolumeCompactRequest {
+    uint32 volume_id = 1;
+    int64 preallocate = 2;
+}
+message VacuumVolumeCompactResponse {
+    int64 processed_bytes = 1;
+    float load_avg_1m = 2;
+}
+
+message VacuumVolumeCommitRequest {
+    uint32 volume_id = 1;
+}
+message VacuumVolumeCommitResponse {
+    bool is_read_only = 1;
+    uint64 volume_size = 2;
+}
+
+message VacuumVolumeCleanupRequest {
+    uint32 volume_id = 1;
+}
+message VacuumVolumeCleanupResponse {
+}
+
+message DeleteCollectionRequest {
+    string collection = 1;
+}
+message DeleteCollectionResponse {
+}
+
+message AllocateVolumeRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    int64 preallocate = 3;
+    string replication = 4;
+    string ttl = 5;
+    uint32 memory_map_max_size_mb = 6;
+    string disk_type = 7;
+    uint32 version = 8;
+}
+message AllocateVolumeResponse {
+}
+
+message VolumeSyncStatusRequest {
+    uint32 volume_id = 1;
+}
+message VolumeSyncStatusResponse {
+    uint32 volume_id = 1;
+    string collection = 2;
+    string replication = 4;
+    string ttl = 5;
+    uint64 tail_offset = 6;
+    uint32 compact_revision = 7;
+    uint64 idx_file_size = 8;
+    uint32 version = 9;
+}
+
+message VolumeIncrementalCopyRequest {
+    uint32 volume_id = 1;
+    uint64 since_ns = 2;
+}
+message VolumeIncrementalCopyResponse {
+    bytes file_content = 1;
+}
+
+message VolumeMountRequest {
+    uint32 volume_id = 1;
+}
+message VolumeMountResponse {
+}
+
+message VolumeUnmountRequest {
+    uint32 volume_id = 1;
+}
+message VolumeUnmountResponse {
+}
+
+message VolumeDeleteRequest {
+    uint32 volume_id = 1;
+    bool only_empty = 2;
+}
+message VolumeDeleteResponse {
+}
+
+message VolumeMarkReadonlyRequest {
+    uint32 volume_id = 1;
+    bool persist = 2;
+}
+message VolumeMarkReadonlyResponse {
+}
+
+message VolumeMarkWritableRequest {
+    uint32 volume_id = 1;
+}
+message VolumeMarkWritableResponse {
+}
+
+message VolumeConfigureRequest {
+    uint32 volume_id = 1;
+    string replication = 2;
+}
+message VolumeConfigureResponse {
+    string error = 1;
+}
+
+message VolumeStatusRequest {
+    uint32 volume_id = 1;
+}
+message VolumeStatusResponse {
+    bool is_read_only = 1;
+    uint64 volume_size = 2;
+    uint64 file_count = 3;
+    uint64 file_deleted_count = 4;
+}
+
+message GetStateRequest {
+}
+message GetStateResponse {
+    VolumeServerState state = 1;
+}
+
+message SetStateRequest {
+    // SetState updates *all* volume server flags at once. Retrieve state with GetState(),
+    // modify individual flags as required, then call this RPC to update.
+    VolumeServerState state = 1;
+}
+message SetStateResponse {
+    VolumeServerState state = 1;
+}
+
+message VolumeCopyRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    string replication = 3;
+    string ttl = 4;
+    string source_data_node = 5;
+    string disk_type = 6;
+    int64 io_byte_per_second = 7;
+}
+message VolumeCopyResponse {
+    uint64 last_append_at_ns = 1;
+    int64 processed_bytes = 2;
+}
+
+message CopyFileRequest {
+    uint32 volume_id = 1;
+    string ext = 2;
+    uint32 compaction_revision = 3;
+    uint64 stop_offset = 4;
+    string collection = 5;
+    bool is_ec_volume = 6;
+    bool ignore_source_file_not_found = 7;
+}
+message CopyFileResponse {
+    bytes file_content = 1;
+    int64 modified_ts_ns = 2;
+}
+
+message ReceiveFileRequest {
+    oneof data {
+        ReceiveFileInfo info = 1;
+        bytes file_content = 2;
+    }
+}
+
+message ReceiveFileInfo {
+    uint32 volume_id = 1;
+    string ext = 2;
+    string collection = 3;
+    bool is_ec_volume = 4;
+    uint32 shard_id = 5;
+    uint64 file_size = 6;
+}
+
+message ReceiveFileResponse {
+    uint64 bytes_written = 1;
+    string error = 2;
+}
+
+message ReadNeedleBlobRequest {
+    uint32 volume_id = 1;
+    int64 offset = 3; // actual offset
+    int32 size = 4;
+}
+message ReadNeedleBlobResponse {
+    bytes needle_blob = 1;
+}
+
+message ReadNeedleMetaRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    int64 offset = 3; // actual offset
+    int32 size = 4;
+}
+message ReadNeedleMetaResponse {
+    uint32 cookie = 1;
+    uint64 last_modified = 2;
+    uint32 crc = 3;
+    string ttl = 4;
+    uint64 append_at_ns = 5;
+}
+
+message WriteNeedleBlobRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    int32 size = 3;
+    bytes needle_blob = 4;
+}
+message WriteNeedleBlobResponse {
+}
+
+message ReadAllNeedlesRequest {
+    repeated uint32 volume_ids = 1;
+}
+message ReadAllNeedlesResponse {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    uint32 cookie = 3;
+    bytes needle_blob = 5;
+    bool needle_blob_compressed = 6;
+    uint64 last_modified = 7;
+    uint32 crc = 8;
+    bytes name = 9;
+    bytes mime = 10;
+}
+
+message VolumeTailSenderRequest {
+    uint32 volume_id = 1;
+    uint64 since_ns = 2;
+    uint32 idle_timeout_seconds = 3;
+}
+message VolumeTailSenderResponse {
+    bytes needle_header = 1;
+    bytes needle_body = 2;
+    bool is_last_chunk = 3;
+    uint32 version = 4;
+}
+
+message VolumeTailReceiverRequest {
+    uint32 volume_id = 1;
+    uint64 since_ns = 2;
+    uint32 idle_timeout_seconds = 3;
+    string source_volume_server = 4;
+}
+message VolumeTailReceiverResponse {
+}
+
+message VolumeEcShardsGenerateRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+}
+message VolumeEcShardsGenerateResponse {
+}
+
+message VolumeEcShardsRebuildRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+}
+message VolumeEcShardsRebuildResponse {
+    repeated uint32 rebuilt_shard_ids = 1;
+}
+
+message VolumeEcShardsCopyRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    repeated uint32 shard_ids = 3;
+    bool copy_ecx_file = 4;
+    string source_data_node = 5;
+    bool copy_ecj_file = 6;
+    bool copy_vif_file = 7;
+    uint32 disk_id = 8;  // Target disk ID for storing EC shards
+}
+message VolumeEcShardsCopyResponse {
+}
+
+message VolumeEcShardsDeleteRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    repeated uint32 shard_ids = 3;
+}
+message VolumeEcShardsDeleteResponse {
+}
+
+message VolumeEcShardsMountRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    repeated uint32 shard_ids = 3;
+}
+message VolumeEcShardsMountResponse {
+}
+
+message VolumeEcShardsUnmountRequest {
+    uint32 volume_id = 1;
+    repeated uint32 shard_ids = 3;
+}
+message VolumeEcShardsUnmountResponse {
+}
+
+message VolumeEcShardReadRequest {
+    uint32 volume_id = 1;
+    uint32 shard_id = 2;
+    int64 offset = 3;
+    int64 size = 4;
+    uint64 file_key = 5;
+}
+message VolumeEcShardReadResponse {
+    bytes data = 1;
+    bool is_deleted = 2;
+}
+
+message VolumeEcBlobDeleteRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    uint64 file_key = 3;
+    uint32 version = 4;
+}
+message VolumeEcBlobDeleteResponse {
+}
+
+message VolumeEcShardsToVolumeRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+}
+message VolumeEcShardsToVolumeResponse {
+}
+
+message VolumeEcShardsInfoRequest {
+    uint32 volume_id = 1;
+}
+message VolumeEcShardsInfoResponse {
+    repeated EcShardInfo ec_shard_infos = 1;
+    uint64 volume_size = 2;
+    uint64 file_count = 3;
+    uint64 file_deleted_count = 4;
+}
+
+message EcShardInfo {
+    uint32 shard_id = 1;
+    int64 size = 2;
+    string collection = 3;
+    uint32 volume_id = 4;
+}
+
+message ReadVolumeFileStatusRequest {
+    uint32 volume_id = 1;
+}
+message ReadVolumeFileStatusResponse {
+    uint32 volume_id = 1;
+    uint64 idx_file_timestamp_seconds = 2;
+    uint64 idx_file_size = 3;
+    uint64 dat_file_timestamp_seconds = 4;
+    uint64 dat_file_size = 5;
+    uint64 file_count = 6;
+    uint32 compaction_revision = 7;
+    string collection = 8;
+    string disk_type = 9;
+    VolumeInfo volume_info = 10;
+    uint32 version = 11;
+}
+
+message DiskStatus {
+    string dir = 1;
+    uint64 all = 2;
+    uint64 used = 3;
+    uint64 free = 4;
+    float percent_free = 5;
+    float percent_used = 6;
+    string disk_type = 7;
+}
+
+message MemStatus {
+    int32 goroutines = 1;
+    uint64 all = 2;
+    uint64 used = 3;
+    uint64 free = 4;
+    uint64 self = 5;
+    uint64 heap = 6;
+    uint64 stack = 7;
+}
+
+// tired storage on volume servers
+message RemoteFile {
+    string backend_type = 1;
+    string backend_id = 2;
+    string key = 3;
+    uint64 offset = 4;
+    uint64 file_size = 5;
+    uint64 modified_time = 6;
+    string extension = 7;
+}
+message VolumeInfo {
+    repeated RemoteFile files = 1;
+    uint32 version = 2;
+    string replication = 3;
+    uint32 bytes_offset = 4;
+    int64 dat_file_size = 5; // store the original dat file size
+    uint64 expire_at_sec = 6; // expiration time of ec volume
+    bool read_only = 7;
+    EcShardConfig ec_shard_config = 8; // EC shard configuration (optional, null = use default 10+4)
+}
+
+// EcShardConfig specifies erasure coding shard configuration
+message EcShardConfig {
+    uint32 data_shards = 1;   // Number of data shards (e.g., 10)
+    uint32 parity_shards = 2; // Number of parity shards (e.g., 4)
+}
+message OldVersionVolumeInfo {
+    repeated RemoteFile files = 1;
+    uint32 version = 2;
+    string replication = 3;
+    uint32 BytesOffset = 4;
+    int64 dat_file_size = 5; // store the original dat file size
+    uint64 DestroyTime = 6; // expiration time of ec volume
+    bool read_only = 7;
+}
+
+// tiered storage
+message VolumeTierMoveDatToRemoteRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    string destination_backend_name = 3;
+    bool keep_local_dat_file = 4;
+}
+message VolumeTierMoveDatToRemoteResponse {
+    int64 processed = 1;
+    float processedPercentage = 2;
+}
+
+message VolumeTierMoveDatFromRemoteRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    bool keep_remote_dat_file = 3;
+}
+message VolumeTierMoveDatFromRemoteResponse {
+    int64 processed = 1;
+    float processedPercentage = 2;
+}
+
+message VolumeServerStatusRequest {
+
+}
+message VolumeServerStatusResponse {
+    repeated DiskStatus disk_statuses = 1;
+    MemStatus memory_status = 2;
+    string version = 3;
+    string data_center = 4;
+    string rack = 5;
+    VolumeServerState state = 6;
+}
+
+message VolumeServerLeaveRequest {
+}
+message VolumeServerLeaveResponse {
+}
+
+// remote storage
+message FetchAndWriteNeedleRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    uint32 cookie = 3;
+    int64 offset = 4;
+    int64 size = 5;
+    message Replica {
+        string url = 1;
+        string public_url = 2;
+        int32 grpc_port = 3;
+    }
+    repeated Replica replicas = 6;
+    string auth = 7;
+    // remote conf
+    remote_pb.RemoteConf remote_conf = 15;
+    remote_pb.RemoteStorageLocation remote_location = 16;
+}
+message FetchAndWriteNeedleResponse {
+    string e_tag = 1;
+}
+
+enum VolumeScrubMode {
+    UNKNOWN = 0;
+    INDEX = 1;
+    FULL = 2;
+    LOCAL = 3;
+}
+
+message ScrubVolumeRequest {
+    VolumeScrubMode mode = 1;
+    // optional list of volume IDs to scrub. if empty, all volumes for the server are scrubbed.
+    repeated uint32 volume_ids = 2;
+    bool mark_broken_volumes_readonly = 3;
+}
+message ScrubVolumeResponse {
+    uint64 total_volumes = 1;
+    uint64 total_files = 2;
+    repeated uint32 broken_volume_ids = 3;
+    repeated string details = 4;
+}
+
+message ScrubEcVolumeRequest {
+    VolumeScrubMode mode = 1;
+    // optional list of volume IDs to scrub. if empty, all EC volumes for the server are scrubbed.
+    repeated uint32 volume_ids = 2;
+}
+message ScrubEcVolumeResponse {
+    uint64 total_volumes = 1;
+    uint64 total_files = 2;
+    repeated uint32 broken_volume_ids = 3;
+    repeated EcShardInfo broken_shard_infos = 4;
+    repeated string details = 5;
+}
+
+// select on volume servers
+message QueryRequest {
+    repeated string selections = 1;
+    repeated string from_file_ids = 2;
+    message Filter {
+        string field = 1;
+        string operand = 2;
+        string value = 3;
+    }
+    Filter filter = 3;
+
+    message InputSerialization {
+        // NONE | GZIP | BZIP2
+        string compression_type = 1;
+        message CSVInput {
+            string file_header_info = 1; // Valid values: NONE | USE | IGNORE
+            string record_delimiter = 2; // Default: \n
+            string field_delimiter = 3; // Default: ,
+            string quote_character = 4; // Default: "
+            string quote_escape_character = 5; // Default: "
+            string comments = 6; // Default: #
+            // If true, records might contain record delimiters within quote characters
+            bool allow_quoted_record_delimiter = 7; // default False.
+        }
+        message JSONInput {
+            string type = 1; // Valid values: DOCUMENT | LINES
+        }
+        message ParquetInput {
+        }
+
+        CSVInput csv_input = 2;
+        JSONInput json_input = 3;
+        ParquetInput parquet_input = 4;
+    }
+    InputSerialization input_serialization = 4;
+
+    message OutputSerialization {
+        message CSVOutput {
+            string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
+            string record_delimiter = 2; // Default: \n
+            string field_delimiter = 3; // Default: ,
+            string quote_character = 4; // Default: "
+            string quote_escape_character = 5; // Default: "
+        }
+        message JSONOutput {
+            string record_delimiter = 1;
+        }
+
+        CSVOutput csv_output = 2;
+        JSONOutput json_output = 3;
+    }
+
+    OutputSerialization output_serialization = 5;
+}
+message QueriedStripe {
+    bytes records = 1;
+}
+
+message VolumeNeedleStatusRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+}
+message VolumeNeedleStatusResponse {
+    uint64 needle_id = 1;
+    uint32 cookie = 2;
+    uint32 size = 3;
+    uint64 last_modified = 4;
+    uint32 crc = 5;
+    string ttl = 6;
+}
+
+message PingRequest {
+    string target = 1; // default to ping itself
+    string target_type = 2;
+}
+message PingResponse {
+    int64 start_time_ns = 1;
+    int64 remote_time_ns = 2;
+    int64 stop_time_ns = 3;
+}

seaweed-volume/src/images.rs

@@ -0,0 +1,275 @@
+//! JPEG EXIF orientation auto-fix, matching Go's `FixJpgOrientation`.
+//!
+//! Reads the EXIF orientation tag from JPEG data and rotates/flips the image
+//! to normalize it to orientation 1 (top-left). If EXIF parsing fails or
+//! orientation is already normal, returns the original data unchanged.
+
+use std::io::Cursor;
+
+use image::{DynamicImage, GenericImageView, ImageFormat, RgbaImage};
+
+/// EXIF orientation tag values.
+/// See: <http://sylvana.net/jpegcrop/exif_orientation.html>
+const TOP_LEFT_SIDE: u32 = 1;
+const TOP_RIGHT_SIDE: u32 = 2;
+const BOTTOM_RIGHT_SIDE: u32 = 3;
+const BOTTOM_LEFT_SIDE: u32 = 4;
+const LEFT_SIDE_TOP: u32 = 5;
+const RIGHT_SIDE_TOP: u32 = 6;
+const RIGHT_SIDE_BOTTOM: u32 = 7;
+const LEFT_SIDE_BOTTOM: u32 = 8;
+
+/// Fix JPEG orientation based on EXIF data.
+///
+/// Reads the EXIF orientation tag and applies the appropriate rotation/flip
+/// to normalize the image to orientation 1 (top-left). Re-encodes as JPEG.
+///
+/// Returns the original data unchanged if:
+/// - EXIF data cannot be parsed
+/// - No orientation tag is present
+/// - Orientation is already 1 (normal)
+/// - Image decoding or re-encoding fails
+pub fn fix_jpg_orientation(data: &[u8]) -> Vec<u8> {
+    // Parse EXIF data
+    let orientation = match read_exif_orientation(data) {
+        Some(o) => o,
+        None => return data.to_vec(),
+    };
+
+    // Orientation 1 means normal — no transformation needed
+    if orientation == TOP_LEFT_SIDE {
+        return data.to_vec();
+    }
+
+    // Determine rotation angle and flip mode
+    let (angle, flip_horizontal) = match orientation {
+        TOP_RIGHT_SIDE => (0, true),
+        BOTTOM_RIGHT_SIDE => (180, false),
+        BOTTOM_LEFT_SIDE => (180, true),
+        LEFT_SIDE_TOP => (-90, true),
+        RIGHT_SIDE_TOP => (-90, false),
+        RIGHT_SIDE_BOTTOM => (90, true),
+        LEFT_SIDE_BOTTOM => (90, false),
+        _ => return data.to_vec(),
+    };
+
+    // Decode the image
+    let src_image = match image::load_from_memory_with_format(data, ImageFormat::Jpeg) {
+        Ok(img) => img,
+        Err(_) => return data.to_vec(),
+    };
+
+    // Apply rotation then flip (matching Go's flip(rotate(img, angle), flipMode))
+    let transformed = flip_horizontal_if(rotate(src_image, angle), flip_horizontal);
+
+    // Re-encode as JPEG
+    let mut buf = Cursor::new(Vec::new());
+    match transformed.write_to(&mut buf, ImageFormat::Jpeg) {
+        Ok(_) => buf.into_inner(),
+        Err(_) => data.to_vec(),
+    }
+}
+
+/// Read the EXIF orientation tag from JPEG data.
+/// Returns None if EXIF cannot be parsed or orientation tag is not present.
+fn read_exif_orientation(data: &[u8]) -> Option<u32> {
+    let exif_reader = exif::Reader::new();
+    let mut cursor = Cursor::new(data);
+    let exif_data = exif_reader.read_from_container(&mut cursor).ok()?;
+
+    let orientation_field = exif_data.get_field(exif::Tag::Orientation, exif::In::PRIMARY)?;
+    match orientation_field.value {
+        exif::Value::Short(ref v) if !v.is_empty() => Some(v[0] as u32),
+        _ => orientation_field.value.get_uint(0),
+    }
+}
+
+/// Rotate an image by the given angle (counter-clockwise, in degrees).
+/// Matches Go's rotate function.
+fn rotate(img: DynamicImage, angle: i32) -> DynamicImage {
+    let (width, height) = img.dimensions();
+
+    match angle {
+        90 => {
+            // 90 degrees counter-clockwise
+            let new_w = height;
+            let new_h = width;
+            let mut out = RgbaImage::new(new_w, new_h);
+            for y in 0..new_h {
+                for x in 0..new_w {
+                    out.put_pixel(x, y, img.get_pixel(new_h - 1 - y, x));
+                }
+            }
+            DynamicImage::ImageRgba8(out)
+        }
+        -90 => {
+            // 90 degrees clockwise (or 270 counter-clockwise)
+            let new_w = height;
+            let new_h = width;
+            let mut out = RgbaImage::new(new_w, new_h);
+            for y in 0..new_h {
+                for x in 0..new_w {
+                    out.put_pixel(x, y, img.get_pixel(y, new_w - 1 - x));
+                }
+            }
+            DynamicImage::ImageRgba8(out)
+        }
+        180 | -180 => {
+            let mut out = RgbaImage::new(width, height);
+            for y in 0..height {
+                for x in 0..width {
+                    out.put_pixel(x, y, img.get_pixel(width - 1 - x, height - 1 - y));
+                }
+            }
+            DynamicImage::ImageRgba8(out)
+        }
+        _ => img,
+    }
+}
+
+/// Flip the image horizontally if requested.
+/// In Go, flipMode 2 == FlipHorizontal. We simplify since only horizontal flip is used.
+fn flip_horizontal_if(img: DynamicImage, do_flip: bool) -> DynamicImage {
+    if !do_flip {
+        return img;
+    }
+    let (width, height) = img.dimensions();
+    let mut out = RgbaImage::new(width, height);
+    for y in 0..height {
+        for x in 0..width {
+            out.put_pixel(x, y, img.get_pixel(width - 1 - x, y));
+        }
+    }
+    DynamicImage::ImageRgba8(out)
+}
+
+/// Returns true if the given MIME type or file path extension indicates a JPEG file.
+pub fn is_jpeg(mime_type: &str, path: &str) -> bool {
+    if mime_type == "image/jpeg" {
+        return true;
+    }
+    let lower = path.to_lowercase();
+    lower.ends_with(".jpg") || lower.ends_with(".jpeg")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_non_jpeg_data_returned_unchanged() {
+        let data = b"not a jpeg file at all";
+        let result = fix_jpg_orientation(data);
+        assert_eq!(result, data);
+    }
+
+    #[test]
+    fn test_jpeg_without_exif_returned_unchanged() {
+        // Create a minimal JPEG without EXIF data
+        let img = DynamicImage::ImageRgba8(RgbaImage::new(2, 2));
+        let mut buf = Cursor::new(Vec::new());
+        img.write_to(&mut buf, ImageFormat::Jpeg).unwrap();
+        let jpeg_data = buf.into_inner();
+
+        let result = fix_jpg_orientation(&jpeg_data);
+        // Should return data unchanged (no EXIF orientation tag)
+        // Just verify it's still valid JPEG
+        assert!(!result.is_empty());
+        assert_eq!(&result[0..2], &[0xFF, 0xD8]); // JPEG magic bytes
+    }
+
+    #[test]
+    fn test_is_jpeg() {
+        assert!(is_jpeg("image/jpeg", ""));
+        assert!(is_jpeg("", "/3,abc.jpg"));
+        assert!(is_jpeg("", "/3,abc.JPEG"));
+        assert!(is_jpeg("application/octet-stream", "/3,abc.JPG"));
+        assert!(!is_jpeg("image/png", "/3,abc.png"));
+        assert!(!is_jpeg("", "/3,abc.png"));
+    }
+
+    #[test]
+    fn test_rotate_180() {
+        // Create a 2x2 image with distinct pixel colors
+        let mut img = RgbaImage::new(2, 2);
+        img.put_pixel(0, 0, image::Rgba([255, 0, 0, 255])); // red top-left
+        img.put_pixel(1, 0, image::Rgba([0, 255, 0, 255])); // green top-right
+        img.put_pixel(0, 1, image::Rgba([0, 0, 255, 255])); // blue bottom-left
+        img.put_pixel(1, 1, image::Rgba([255, 255, 0, 255])); // yellow bottom-right
+        let dynamic = DynamicImage::ImageRgba8(img);
+
+        let rotated = rotate(dynamic, 180);
+        let (w, h) = rotated.dimensions();
+        assert_eq!((w, h), (2, 2));
+        // After 180 rotation: top-left should be yellow, top-right should be blue
+        assert_eq!(rotated.get_pixel(0, 0), image::Rgba([255, 255, 0, 255]));
+        assert_eq!(rotated.get_pixel(1, 0), image::Rgba([0, 0, 255, 255]));
+        assert_eq!(rotated.get_pixel(0, 1), image::Rgba([0, 255, 0, 255]));
+        assert_eq!(rotated.get_pixel(1, 1), image::Rgba([255, 0, 0, 255]));
+    }
+
+    #[test]
+    fn test_rotate_90_ccw() {
+        // Create 3x2 image (width=3, height=2)
+        let mut img = RgbaImage::new(3, 2);
+        img.put_pixel(0, 0, image::Rgba([1, 0, 0, 255]));
+        img.put_pixel(1, 0, image::Rgba([2, 0, 0, 255]));
+        img.put_pixel(2, 0, image::Rgba([3, 0, 0, 255]));
+        img.put_pixel(0, 1, image::Rgba([4, 0, 0, 255]));
+        img.put_pixel(1, 1, image::Rgba([5, 0, 0, 255]));
+        img.put_pixel(2, 1, image::Rgba([6, 0, 0, 255]));
+        let dynamic = DynamicImage::ImageRgba8(img);
+
+        let rotated = rotate(dynamic, 90);
+        let (w, h) = rotated.dimensions();
+        // 90 CCW: width=3,height=2 -> new_w=2, new_h=3
+        assert_eq!((w, h), (2, 3));
+        // Top-right (2,0) should move to top-left (0,0) in CCW 90
+        assert_eq!(rotated.get_pixel(0, 0)[0], 3);
+        assert_eq!(rotated.get_pixel(1, 0)[0], 6);
+    }
+
+    #[test]
+    fn test_rotate_neg90_cw() {
+        // Create 3x2 image
+        let mut img = RgbaImage::new(3, 2);
+        img.put_pixel(0, 0, image::Rgba([1, 0, 0, 255]));
+        img.put_pixel(1, 0, image::Rgba([2, 0, 0, 255]));
+        img.put_pixel(2, 0, image::Rgba([3, 0, 0, 255]));
+        img.put_pixel(0, 1, image::Rgba([4, 0, 0, 255]));
+        img.put_pixel(1, 1, image::Rgba([5, 0, 0, 255]));
+        img.put_pixel(2, 1, image::Rgba([6, 0, 0, 255]));
+        let dynamic = DynamicImage::ImageRgba8(img);
+
+        let rotated = rotate(dynamic, -90);
+        let (w, h) = rotated.dimensions();
+        assert_eq!((w, h), (2, 3));
+        // -90 (CW 90): top-left (0,0) should go to top-right
+        assert_eq!(rotated.get_pixel(0, 0)[0], 4);
+        assert_eq!(rotated.get_pixel(1, 0)[0], 1);
+    }
+
+    #[test]
+    fn test_flip_horizontal() {
+        let mut img = RgbaImage::new(2, 1);
+        img.put_pixel(0, 0, image::Rgba([10, 0, 0, 255]));
+        img.put_pixel(1, 0, image::Rgba([20, 0, 0, 255]));
+        let dynamic = DynamicImage::ImageRgba8(img);
+
+        let flipped = flip_horizontal_if(dynamic, true);
+        assert_eq!(flipped.get_pixel(0, 0)[0], 20);
+        assert_eq!(flipped.get_pixel(1, 0)[0], 10);
+    }
+
+    #[test]
+    fn test_flip_horizontal_noop() {
+        let mut img = RgbaImage::new(2, 1);
+        img.put_pixel(0, 0, image::Rgba([10, 0, 0, 255]));
+        img.put_pixel(1, 0, image::Rgba([20, 0, 0, 255]));
+        let dynamic = DynamicImage::ImageRgba8(img);
+
+        let not_flipped = flip_horizontal_if(dynamic, false);
+        assert_eq!(not_flipped.get_pixel(0, 0)[0], 10);
+        assert_eq!(not_flipped.get_pixel(1, 0)[0], 20);
+    }
+}

seaweed-volume/src/lib.rs

@@ -0,0 +1,27 @@
+pub mod config;
+pub mod images;
+pub mod metrics;
+pub mod remote_storage;
+pub mod security;
+pub mod server;
+pub mod storage;
+pub mod version;
+
+/// Generated protobuf modules.
+pub mod pb {
+    pub const FILE_DESCRIPTOR_SET: &[u8] =
+        tonic::include_file_descriptor_set!("seaweed_descriptor");
+
+    pub mod remote_pb {
+        tonic::include_proto!("remote_pb");
+    }
+    pub mod volume_server_pb {
+        tonic::include_proto!("volume_server_pb");
+    }
+    pub mod master_pb {
+        tonic::include_proto!("master_pb");
+    }
+    pub mod filer_pb {
+        tonic::include_proto!("filer_pb");
+    }
+}

seaweed-volume/src/metrics.rs

@@ -0,0 +1,456 @@
+//! Prometheus metrics for the volume server.
+//!
+//! Mirrors the Go SeaweedFS volume server metrics.
+
+use prometheus::{
+    self, Encoder, GaugeVec, HistogramOpts, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec,
+    Opts, Registry, TextEncoder,
+};
+use std::sync::Once;
+
+use crate::version;
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct PushGatewayConfig {
+    pub address: String,
+    pub interval_seconds: u32,
+}
+
+lazy_static::lazy_static! {
+    pub static ref REGISTRY: Registry = Registry::new();
+
+    // ---- Request metrics (Go: VolumeServerRequestCounter, VolumeServerRequestHistogram) ----
+
+    /// Request counter with labels `type` (HTTP method) and `code` (HTTP status).
+    pub static ref REQUEST_COUNTER: IntCounterVec = IntCounterVec::new(
+        Opts::new("SeaweedFS_volumeServer_request_total", "Volume server requests"),
+        &["type", "code"],
+    ).expect("metric can be created");
+
+    /// Request duration histogram with label `type` (HTTP method).
+    pub static ref REQUEST_DURATION: HistogramVec = HistogramVec::new(
+        HistogramOpts::new(
+            "SeaweedFS_volumeServer_request_seconds",
+            "Volume server request duration in seconds",
+        ).buckets(exponential_buckets(0.0001, 2.0, 24)),
+        &["type"],
+    ).expect("metric can be created");
+
+    // ---- Handler counters (Go: VolumeServerHandlerCounter) ----
+
+    /// Handler-level operation counter with label `type`.
+    pub static ref HANDLER_COUNTER: IntCounterVec = IntCounterVec::new(
+        Opts::new("SeaweedFS_volumeServer_handler_total", "Volume server handler counters"),
+        &["type"],
+    ).expect("metric can be created");
+
+    // ---- Vacuuming metrics (Go: VolumeServerVacuuming*) ----
+
+    /// Vacuuming compact counter with label `success` (true/false).
+    pub static ref VACUUMING_COMPACT_COUNTER: IntCounterVec = IntCounterVec::new(
+        Opts::new("SeaweedFS_volumeServer_vacuuming_compact_count", "Counter of volume vacuuming Compact counter"),
+        &["success"],
+    ).expect("metric can be created");
+
+    /// Vacuuming commit counter with label `success` (true/false).
+    pub static ref VACUUMING_COMMIT_COUNTER: IntCounterVec = IntCounterVec::new(
+        Opts::new("SeaweedFS_volumeServer_vacuuming_commit_count", "Counter of volume vacuuming commit counter"),
+        &["success"],
+    ).expect("metric can be created");
+
+    /// Vacuuming duration histogram with label `type` (compact/commit).
+    pub static ref VACUUMING_HISTOGRAM: HistogramVec = HistogramVec::new(
+        HistogramOpts::new(
+            "SeaweedFS_volumeServer_vacuuming_seconds",
+            "Volume vacuuming duration in seconds",
+        ).buckets(exponential_buckets(0.0001, 2.0, 24)),
+        &["type"],
+    ).expect("metric can be created");
+
+    // ---- Volume gauges (Go: VolumeServerVolumeGauge, VolumeServerReadOnlyVolumeGauge) ----
+
+    /// Volumes per collection and type (volume/ec_shards).
+    pub static ref VOLUME_GAUGE: GaugeVec = GaugeVec::new(
+        Opts::new("SeaweedFS_volumeServer_volumes", "Number of volumes"),
+        &["collection", "type"],
+    ).expect("metric can be created");
+
+    /// Read-only volumes per collection and type.
+    pub static ref READ_ONLY_VOLUME_GAUGE: GaugeVec = GaugeVec::new(
+        Opts::new("SeaweedFS_volumeServer_read_only_volumes", "Number of read-only volumes."),
+        &["collection", "type"],
+    ).expect("metric can be created");
+
+    /// Maximum number of volumes this server can hold.
+    pub static ref MAX_VOLUMES: IntGauge = IntGauge::new(
+        "SeaweedFS_volumeServer_max_volumes",
+        "Maximum number of volumes",
+    ).expect("metric can be created");
+
+    // ---- Disk size gauges (Go: VolumeServerDiskSizeGauge) ----
+
+    /// Actual disk size used by volumes per collection and type (normal/deleted_bytes/ec).
+    pub static ref DISK_SIZE_GAUGE: GaugeVec = GaugeVec::new(
+        Opts::new("SeaweedFS_volumeServer_total_disk_size", "Actual disk size used by volumes"),
+        &["collection", "type"],
+    ).expect("metric can be created");
+
+    // ---- Resource gauges (Go: VolumeServerResourceGauge) ----
+
+    /// Disk resource usage per directory and type (all/used/free/avail).
+    pub static ref RESOURCE_GAUGE: GaugeVec = GaugeVec::new(
+        Opts::new("SeaweedFS_volumeServer_resource", "Server resource usage"),
+        &["name", "type"],
+    ).expect("metric can be created");
+
+    // ---- In-flight gauges (Go: VolumeServerInFlightRequestsGauge, InFlightDownload/UploadSize) ----
+
+    /// In-flight requests per HTTP method.
+    pub static ref INFLIGHT_REQUESTS_GAUGE: IntGaugeVec = IntGaugeVec::new(
+        Opts::new("SeaweedFS_volumeServer_in_flight_requests", "Current number of in-flight requests being handled by volume server."),
+        &["type"],
+    ).expect("metric can be created");
+
+    /// Concurrent download limit in bytes.
+    pub static ref CONCURRENT_DOWNLOAD_LIMIT: IntGauge = IntGauge::new(
+        "SeaweedFS_volumeServer_concurrent_download_limit",
+        "Limit for total concurrent download size in bytes",
+    ).expect("metric can be created");
+
+    /// Concurrent upload limit in bytes.
+    pub static ref CONCURRENT_UPLOAD_LIMIT: IntGauge = IntGauge::new(
+        "SeaweedFS_volumeServer_concurrent_upload_limit",
+        "Limit for total concurrent upload size in bytes",
+    ).expect("metric can be created");
+
+    /// Current in-flight download bytes.
+    pub static ref INFLIGHT_DOWNLOAD_SIZE: IntGauge = IntGauge::new(
+        "SeaweedFS_volumeServer_in_flight_download_size",
+        "In flight total download size.",
+    ).expect("metric can be created");
+
+    /// Current in-flight upload bytes.
+    pub static ref INFLIGHT_UPLOAD_SIZE: IntGauge = IntGauge::new(
+        "SeaweedFS_volumeServer_in_flight_upload_size",
+        "In flight total upload size.",
+    ).expect("metric can be created");
+
+    /// Upload error counter by HTTP status code. Code "0" = transport error (no response).
+    pub static ref UPLOAD_ERROR_COUNTER: IntCounterVec = IntCounterVec::new(
+        Opts::new("SeaweedFS_upload_error_total",
+            "Counter of upload errors by HTTP status code. Code 0 means transport error (no response received)."),
+        &["code"],
+    ).expect("metric can be created");
+
+    // ---- Legacy aliases for backward compat with existing code ----
+
+    /// Total number of volumes on this server (flat gauge).
+    pub static ref VOLUMES_TOTAL: IntGauge = IntGauge::new(
+        "volume_server_volumes_total",
+        "Total number of volumes",
+    ).expect("metric can be created");
+
+    /// Disk size in bytes per directory.
+    pub static ref DISK_SIZE_BYTES: IntGaugeVec = IntGaugeVec::new(
+        Opts::new("volume_server_disk_size_bytes", "Disk size in bytes"),
+        &["dir"],
+    ).expect("metric can be created");
+
+    /// Disk free bytes per directory.
+    pub static ref DISK_FREE_BYTES: IntGaugeVec = IntGaugeVec::new(
+        Opts::new("volume_server_disk_free_bytes", "Disk free space in bytes"),
+        &["dir"],
+    ).expect("metric can be created");
+
+    /// Current number of in-flight requests (flat gauge).
+    pub static ref INFLIGHT_REQUESTS: IntGauge = IntGauge::new(
+        "volume_server_inflight_requests",
+        "Current number of in-flight requests",
+    ).expect("metric can be created");
+
+    /// Total number of files stored across all volumes.
+    pub static ref VOLUME_FILE_COUNT: IntGauge = IntGauge::new(
+        "volume_server_volume_file_count",
+        "Total number of files stored across all volumes",
+    ).expect("metric can be created");
+
+    // ---- Build info (Go: BuildInfo) ----
+
+    /// Build information gauge, always set to 1. Matches Go:
+    /// Namespace="SeaweedFS", Subsystem="build", Name="info",
+    /// labels: version, commit, sizelimit, goos, goarch.
+    pub static ref BUILD_INFO: GaugeVec = GaugeVec::new(
+        Opts::new("SeaweedFS_build_info", "A metric with a constant '1' value labeled by version, commit, sizelimit, goos, and goarch from which SeaweedFS was built."),
+        &["version", "commit", "sizelimit", "goos", "goarch"],
+    ).expect("metric can be created");
+}
+
+/// Generate exponential bucket boundaries for histograms.
+fn exponential_buckets(start: f64, factor: f64, count: usize) -> Vec<f64> {
+    let mut buckets = Vec::with_capacity(count);
+    let mut val = start;
+    for _ in 0..count {
+        buckets.push(val);
+        val *= factor;
+    }
+    buckets
+}
+
+// Handler counter type constants (matches Go's metrics_names.go).
+pub const WRITE_TO_LOCAL_DISK: &str = "writeToLocalDisk";
+pub const WRITE_TO_REPLICAS: &str = "writeToReplicas";
+pub const DOWNLOAD_LIMIT_COND: &str = "downloadLimitCondition";
+pub const UPLOAD_LIMIT_COND: &str = "uploadLimitCondition";
+pub const READ_PROXY_REQ: &str = "readProxyRequest";
+pub const READ_REDIRECT_REQ: &str = "readRedirectRequest";
+pub const EMPTY_READ_PROXY_LOC: &str = "emptyReadProxyLocaction";
+pub const FAILED_READ_PROXY_REQ: &str = "failedReadProxyRequest";
+
+// Error metric name constants.
+pub const ERROR_SIZE_MISMATCH_OFFSET_SIZE: &str = "errorSizeMismatchOffsetSize";
+pub const ERROR_SIZE_MISMATCH: &str = "errorSizeMismatch";
+pub const ERROR_CRC: &str = "errorCRC";
+pub const ERROR_INDEX_OUT_OF_RANGE: &str = "errorIndexOutOfRange";
+pub const ERROR_GET_NOT_FOUND: &str = "errorGetNotFound";
+pub const ERROR_GET_INTERNAL: &str = "errorGetInternal";
+pub const ERROR_WRITE_TO_LOCAL_DISK: &str = "errorWriteToLocalDisk";
+pub const ERROR_UNMARSHAL_PAIRS: &str = "errorUnmarshalPairs";
+pub const ERROR_WRITE_TO_REPLICAS: &str = "errorWriteToReplicas";
+
+// Go volume heartbeat metric label values.
+pub const READ_ONLY_LABEL_IS_READ_ONLY: &str = "IsReadOnly";
+pub const READ_ONLY_LABEL_NO_WRITE_OR_DELETE: &str = "noWriteOrDelete";
+pub const READ_ONLY_LABEL_NO_WRITE_CAN_DELETE: &str = "noWriteCanDelete";
+pub const READ_ONLY_LABEL_IS_DISK_SPACE_LOW: &str = "isDiskSpaceLow";
+pub const DISK_SIZE_LABEL_NORMAL: &str = "normal";
+pub const DISK_SIZE_LABEL_DELETED_BYTES: &str = "deleted_bytes";
+pub const DISK_SIZE_LABEL_EC: &str = "ec";
+
+static REGISTER_METRICS: Once = Once::new();
+
+/// Register all metrics with the custom registry.
+/// Call this once at startup.
+pub fn register_metrics() {
+    REGISTER_METRICS.call_once(|| {
+        let metrics: Vec<Box<dyn prometheus::core::Collector>> = vec![
+            // New Go-compatible metrics
+            Box::new(REQUEST_COUNTER.clone()),
+            Box::new(REQUEST_DURATION.clone()),
+            Box::new(HANDLER_COUNTER.clone()),
+            Box::new(VACUUMING_COMPACT_COUNTER.clone()),
+            Box::new(VACUUMING_COMMIT_COUNTER.clone()),
+            Box::new(VACUUMING_HISTOGRAM.clone()),
+            Box::new(VOLUME_GAUGE.clone()),
+            Box::new(READ_ONLY_VOLUME_GAUGE.clone()),
+            Box::new(MAX_VOLUMES.clone()),
+            Box::new(DISK_SIZE_GAUGE.clone()),
+            Box::new(RESOURCE_GAUGE.clone()),
+            Box::new(INFLIGHT_REQUESTS_GAUGE.clone()),
+            Box::new(CONCURRENT_DOWNLOAD_LIMIT.clone()),
+            Box::new(CONCURRENT_UPLOAD_LIMIT.clone()),
+            Box::new(INFLIGHT_DOWNLOAD_SIZE.clone()),
+            Box::new(INFLIGHT_UPLOAD_SIZE.clone()),
+            Box::new(UPLOAD_ERROR_COUNTER.clone()),
+            // Legacy metrics
+            Box::new(VOLUMES_TOTAL.clone()),
+            Box::new(DISK_SIZE_BYTES.clone()),
+            Box::new(DISK_FREE_BYTES.clone()),
+            Box::new(INFLIGHT_REQUESTS.clone()),
+            Box::new(VOLUME_FILE_COUNT.clone()),
+            // Build info
+            Box::new(BUILD_INFO.clone()),
+        ];
+        for m in metrics {
+            REGISTRY.register(m).expect("metric registered");
+        }
+
+        // Set build info gauge to 1 with version/commit/sizelimit/os/arch labels (matches Go).
+        BUILD_INFO
+            .with_label_values(&[
+                version::version(),
+                version::commit(),
+                version::size_limit(),
+                std::env::consts::OS,
+                std::env::consts::ARCH,
+            ])
+            .set(1.0);
+    });
+}
+
+/// Gather all metrics and encode them in Prometheus text exposition format.
+pub fn gather_metrics() -> String {
+    let encoder = TextEncoder::new();
+    let metric_families = REGISTRY.gather();
+    let mut buffer = Vec::new();
+    encoder
+        .encode(&metric_families, &mut buffer)
+        .expect("encoding metrics");
+    String::from_utf8(buffer).expect("metrics are valid UTF-8")
+}
+
+pub fn delete_collection_metrics(collection: &str) {
+    // Mirrors Go's DeletePartialMatch(prometheus.Labels{"collection": collection})
+    // which removes ALL metric entries matching the collection label, regardless
+    // of other label values (like "type"). We gather the metric families to discover
+    // all type values dynamically, matching Go's partial-match behavior.
+    delete_partial_match_collection(&VOLUME_GAUGE, collection);
+    delete_partial_match_collection(&READ_ONLY_VOLUME_GAUGE, collection);
+    delete_partial_match_collection(&DISK_SIZE_GAUGE, collection);
+}
+
+/// Remove all metric entries from a GaugeVec where the "collection" label matches.
+/// This emulates Go's `DeletePartialMatch(prometheus.Labels{"collection": collection})`.
+fn delete_partial_match_collection(gauge: &GaugeVec, collection: &str) {
+    use prometheus::core::Collector;
+    let families = gauge.collect();
+    for family in &families {
+        for metric in family.get_metric() {
+            let labels = metric.get_label();
+            let mut matches_collection = false;
+            let mut type_value = None;
+            for label in labels {
+                if label.get_name() == "collection" && label.get_value() == collection {
+                    matches_collection = true;
+                }
+                if label.get_name() == "type" {
+                    type_value = Some(label.get_value().to_string());
+                }
+            }
+            if matches_collection {
+                if let Some(ref tv) = type_value {
+                    let _ = gauge.remove_label_values(&[collection, tv]);
+                }
+            }
+        }
+    }
+}
+
+pub fn build_pushgateway_url(address: &str, job: &str, instance: &str) -> String {
+    let base = if address.starts_with("http://") || address.starts_with("https://") {
+        address.to_string()
+    } else {
+        format!("http://{}", address)
+    };
+    let base = base.trim_end_matches('/');
+    format!("{}/metrics/job/{}/instance/{}", base, job, instance)
+}
+
+pub async fn push_metrics_once(
+    client: &reqwest::Client,
+    address: &str,
+    job: &str,
+    instance: &str,
+) -> Result<(), String> {
+    let url = build_pushgateway_url(address, job, instance);
+    let response = client
+        .put(&url)
+        .header(
+            reqwest::header::CONTENT_TYPE,
+            "text/plain; version=0.0.4; charset=utf-8",
+        )
+        .body(gather_metrics())
+        .send()
+        .await
+        .map_err(|e| format!("push metrics request failed: {}", e))?;
+
+    if response.status().is_success() {
+        Ok(())
+    } else {
+        Err(format!(
+            "push metrics failed with status {}",
+            response.status()
+        ))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use axum::{routing::put, Router};
+    use std::sync::{Arc, Mutex};
+
+    #[test]
+    fn test_gather_metrics_returns_text() {
+        register_metrics();
+        REQUEST_COUNTER.with_label_values(&["GET", "200"]).inc();
+        let output = gather_metrics();
+        assert!(output.contains("SeaweedFS_volumeServer_request_total"));
+    }
+
+    #[test]
+    fn test_build_pushgateway_url() {
+        assert_eq!(
+            build_pushgateway_url("localhost:9091", "volumeServer", "test-instance"),
+            "http://localhost:9091/metrics/job/volumeServer/instance/test-instance"
+        );
+        assert_eq!(
+            build_pushgateway_url("https://push.example", "volumeServer", "node-a"),
+            "https://push.example/metrics/job/volumeServer/instance/node-a"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_push_metrics_once() {
+        register_metrics();
+
+        let captured = Arc::new(Mutex::new(None::<String>));
+        let captured_clone = captured.clone();
+
+        let app = Router::new().route(
+            "/metrics/job/volumeServer/instance/test-instance",
+            put(move |body: String| {
+                let captured = captured_clone.clone();
+                async move {
+                    *captured.lock().unwrap() = Some(body);
+                    "ok"
+                }
+            }),
+        );
+
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = listener.local_addr().unwrap();
+        let server = tokio::spawn(async move {
+            axum::serve(listener, app).await.unwrap();
+        });
+
+        let client = reqwest::Client::new();
+        push_metrics_once(
+            &client,
+            &format!("127.0.0.1:{}", addr.port()),
+            "volumeServer",
+            "test-instance",
+        )
+        .await
+        .unwrap();
+
+        let body = captured.lock().unwrap().clone().unwrap();
+        assert!(body.contains("SeaweedFS_volumeServer_request_total"));
+
+        server.abort();
+    }
+
+    #[test]
+    fn test_delete_collection_metrics_removes_collection_labelsets() {
+        register_metrics();
+
+        VOLUME_GAUGE.with_label_values(&["pics", "volume"]).set(2.0);
+        VOLUME_GAUGE.with_label_values(&["pics", "ec_shards"]).set(3.0);
+        READ_ONLY_VOLUME_GAUGE
+            .with_label_values(&["pics", "volume"])
+            .set(1.0);
+        DISK_SIZE_GAUGE
+            .with_label_values(&["pics", "normal"])
+            .set(10.0);
+        DISK_SIZE_GAUGE
+            .with_label_values(&["pics", "deleted_bytes"])
+            .set(4.0);
+
+        delete_collection_metrics("pics");
+
+        let output = gather_metrics();
+        assert!(!output.contains("collection=\"pics\",type=\"volume\""));
+        assert!(!output.contains("collection=\"pics\",type=\"ec_shards\""));
+        assert!(!output.contains("collection=\"pics\",type=\"normal\""));
+        assert!(!output.contains("collection=\"pics\",type=\"deleted_bytes\""));
+    }
+}

seaweed-volume/src/remote_storage/mod.rs

@@ -0,0 +1,157 @@
+//! Remote storage backends for tiered storage support.
+//!
+//! Provides a trait-based abstraction over cloud storage providers (S3, GCS, Azure, etc.)
+//! and a registry to create clients from protobuf RemoteConf messages.
+
+pub mod s3;
+pub mod s3_tier;
+
+use crate::pb::remote_pb::{RemoteConf, RemoteStorageLocation};
+
+/// Error type for remote storage operations.
+#[derive(Debug, thiserror::Error)]
+pub enum RemoteStorageError {
+    #[error("remote storage type {0} not found")]
+    TypeNotFound(String),
+    #[error("remote object not found: {0}")]
+    ObjectNotFound(String),
+    #[error("remote storage error: {0}")]
+    Other(String),
+    #[error("io error: {0}")]
+    Io(#[from] std::io::Error),
+}
+
+/// Metadata about a remote file entry.
+#[derive(Debug, Clone)]
+pub struct RemoteEntry {
+    pub size: i64,
+    pub last_modified_at: i64, // Unix seconds
+    pub e_tag: String,
+    pub storage_name: String,
+}
+
+/// Trait for remote storage clients. Matches Go's RemoteStorageClient interface.
+#[async_trait::async_trait]
+pub trait RemoteStorageClient: Send + Sync {
+    /// Read (part of) a file from remote storage.
+    async fn read_file(
+        &self,
+        loc: &RemoteStorageLocation,
+        offset: i64,
+        size: i64,
+    ) -> Result<Vec<u8>, RemoteStorageError>;
+
+    /// Write a file to remote storage.
+    async fn write_file(
+        &self,
+        loc: &RemoteStorageLocation,
+        data: &[u8],
+    ) -> Result<RemoteEntry, RemoteStorageError>;
+
+    /// Get metadata for a file in remote storage.
+    async fn stat_file(
+        &self,
+        loc: &RemoteStorageLocation,
+    ) -> Result<RemoteEntry, RemoteStorageError>;
+
+    /// Delete a file from remote storage.
+    async fn delete_file(&self, loc: &RemoteStorageLocation) -> Result<(), RemoteStorageError>;
+
+    /// List all buckets.
+    async fn list_buckets(&self) -> Result<Vec<String>, RemoteStorageError>;
+
+    /// The RemoteConf used to create this client.
+    fn remote_conf(&self) -> &RemoteConf;
+}
+
+/// Create a new remote storage client from a RemoteConf.
+pub fn make_remote_storage_client(
+    conf: &RemoteConf,
+) -> Result<Box<dyn RemoteStorageClient>, RemoteStorageError> {
+    match conf.r#type.as_str() {
+        // All S3-compatible backends use the same client with different credentials
+        "s3" | "wasabi" | "backblaze" | "aliyun" | "tencent" | "baidu" | "filebase" | "storj"
+        | "contabo" => {
+            let (access_key, secret_key, endpoint, region) = extract_s3_credentials(conf);
+            Ok(Box::new(s3::S3RemoteStorageClient::new(
+                conf.clone(),
+                &access_key,
+                &secret_key,
+                &region,
+                &endpoint,
+                conf.s3_force_path_style,
+            )))
+        }
+        other => Err(RemoteStorageError::TypeNotFound(other.to_string())),
+    }
+}
+
+/// Extract S3-compatible credentials from a RemoteConf based on its type.
+fn extract_s3_credentials(conf: &RemoteConf) -> (String, String, String, String) {
+    match conf.r#type.as_str() {
+        "s3" => (
+            conf.s3_access_key.clone(),
+            conf.s3_secret_key.clone(),
+            conf.s3_endpoint.clone(),
+            if conf.s3_region.is_empty() {
+                "us-east-1".to_string()
+            } else {
+                conf.s3_region.clone()
+            },
+        ),
+        "wasabi" => (
+            conf.wasabi_access_key.clone(),
+            conf.wasabi_secret_key.clone(),
+            conf.wasabi_endpoint.clone(),
+            conf.wasabi_region.clone(),
+        ),
+        "backblaze" => (
+            conf.backblaze_key_id.clone(),
+            conf.backblaze_application_key.clone(),
+            conf.backblaze_endpoint.clone(),
+            conf.backblaze_region.clone(),
+        ),
+        "aliyun" => (
+            conf.aliyun_access_key.clone(),
+            conf.aliyun_secret_key.clone(),
+            conf.aliyun_endpoint.clone(),
+            conf.aliyun_region.clone(),
+        ),
+        "tencent" => (
+            conf.tencent_secret_id.clone(),
+            conf.tencent_secret_key.clone(),
+            conf.tencent_endpoint.clone(),
+            String::new(),
+        ),
+        "baidu" => (
+            conf.baidu_access_key.clone(),
+            conf.baidu_secret_key.clone(),
+            conf.baidu_endpoint.clone(),
+            conf.baidu_region.clone(),
+        ),
+        "filebase" => (
+            conf.filebase_access_key.clone(),
+            conf.filebase_secret_key.clone(),
+            conf.filebase_endpoint.clone(),
+            String::new(),
+        ),
+        "storj" => (
+            conf.storj_access_key.clone(),
+            conf.storj_secret_key.clone(),
+            conf.storj_endpoint.clone(),
+            String::new(),
+        ),
+        "contabo" => (
+            conf.contabo_access_key.clone(),
+            conf.contabo_secret_key.clone(),
+            conf.contabo_endpoint.clone(),
+            conf.contabo_region.clone(),
+        ),
+        _ => (
+            conf.s3_access_key.clone(),
+            conf.s3_secret_key.clone(),
+            conf.s3_endpoint.clone(),
+            conf.s3_region.clone(),
+        ),
+    }
+}

seaweed-volume/src/remote_storage/s3.rs

@@ -0,0 +1,186 @@
+//! S3-compatible remote storage client.
+//!
+//! Works with AWS S3, MinIO, SeaweedFS S3, and all S3-compatible providers.
+
+use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::Client;
+
+use super::{RemoteEntry, RemoteStorageClient, RemoteStorageError};
+use crate::pb::remote_pb::{RemoteConf, RemoteStorageLocation};
+
+/// S3-compatible remote storage client.
+pub struct S3RemoteStorageClient {
+    client: Client,
+    conf: RemoteConf,
+}
+
+impl S3RemoteStorageClient {
+    /// Create a new S3 client from credentials and endpoint configuration.
+    pub fn new(
+        conf: RemoteConf,
+        access_key: &str,
+        secret_key: &str,
+        region: &str,
+        endpoint: &str,
+        force_path_style: bool,
+    ) -> Self {
+        let region = if region.is_empty() {
+            "us-east-1"
+        } else {
+            region
+        };
+
+        let credentials = Credentials::new(
+            access_key,
+            secret_key,
+            None, // session token
+            None, // expiry
+            "seaweedfs-volume",
+        );
+
+        let mut s3_config = aws_sdk_s3::Config::builder()
+            .behavior_version(BehaviorVersion::latest())
+            .region(Region::new(region.to_string()))
+            .credentials_provider(credentials)
+            .force_path_style(force_path_style);
+
+        if !endpoint.is_empty() {
+            s3_config = s3_config.endpoint_url(endpoint);
+        }
+
+        let client = Client::from_conf(s3_config.build());
+
+        S3RemoteStorageClient { client, conf }
+    }
+}
+
+#[async_trait::async_trait]
+impl RemoteStorageClient for S3RemoteStorageClient {
+    async fn read_file(
+        &self,
+        loc: &RemoteStorageLocation,
+        offset: i64,
+        size: i64,
+    ) -> Result<Vec<u8>, RemoteStorageError> {
+        let key = loc.path.trim_start_matches('/');
+
+        let mut req = self.client.get_object().bucket(&loc.bucket).key(key);
+
+        // Set byte range if specified
+        if size > 0 {
+            let end = offset + size - 1;
+            req = req.range(format!("bytes={}-{}", offset, end));
+        } else if offset > 0 {
+            req = req.range(format!("bytes={}-", offset));
+        }
+
+        let resp = req.send().await.map_err(|e| {
+            let msg = format!("{}", e);
+            if msg.contains("NoSuchKey") || msg.contains("404") {
+                RemoteStorageError::ObjectNotFound(format!("{}/{}", loc.bucket, key))
+            } else {
+                RemoteStorageError::Other(format!("s3 get object: {}", e))
+            }
+        })?;
+
+        let data = resp
+            .body
+            .collect()
+            .await
+            .map_err(|e| RemoteStorageError::Other(format!("s3 read body: {}", e)))?;
+
+        Ok(data.into_bytes().to_vec())
+    }
+
+    async fn write_file(
+        &self,
+        loc: &RemoteStorageLocation,
+        data: &[u8],
+    ) -> Result<RemoteEntry, RemoteStorageError> {
+        let key = loc.path.trim_start_matches('/');
+
+        let resp = self
+            .client
+            .put_object()
+            .bucket(&loc.bucket)
+            .key(key)
+            .body(ByteStream::from(data.to_vec()))
+            .send()
+            .await
+            .map_err(|e| RemoteStorageError::Other(format!("s3 put object: {}", e)))?;
+
+        Ok(RemoteEntry {
+            size: data.len() as i64,
+            last_modified_at: std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap_or_default()
+                .as_secs() as i64,
+            e_tag: resp.e_tag().unwrap_or_default().to_string(),
+            storage_name: loc.name.clone(),
+        })
+    }
+
+    async fn stat_file(
+        &self,
+        loc: &RemoteStorageLocation,
+    ) -> Result<RemoteEntry, RemoteStorageError> {
+        let key = loc.path.trim_start_matches('/');
+
+        let resp = self
+            .client
+            .head_object()
+            .bucket(&loc.bucket)
+            .key(key)
+            .send()
+            .await
+            .map_err(|e| {
+                let msg = format!("{}", e);
+                if msg.contains("404") || msg.contains("NotFound") {
+                    RemoteStorageError::ObjectNotFound(format!("{}/{}", loc.bucket, key))
+                } else {
+                    RemoteStorageError::Other(format!("s3 head object: {}", e))
+                }
+            })?;
+
+        Ok(RemoteEntry {
+            size: resp.content_length().unwrap_or(0),
+            last_modified_at: resp.last_modified().map(|t| t.secs()).unwrap_or(0),
+            e_tag: resp.e_tag().unwrap_or_default().to_string(),
+            storage_name: loc.name.clone(),
+        })
+    }
+
+    async fn delete_file(&self, loc: &RemoteStorageLocation) -> Result<(), RemoteStorageError> {
+        let key = loc.path.trim_start_matches('/');
+
+        self.client
+            .delete_object()
+            .bucket(&loc.bucket)
+            .key(key)
+            .send()
+            .await
+            .map_err(|e| RemoteStorageError::Other(format!("s3 delete object: {}", e)))?;
+
+        Ok(())
+    }
+
+    async fn list_buckets(&self) -> Result<Vec<String>, RemoteStorageError> {
+        let resp = self
+            .client
+            .list_buckets()
+            .send()
+            .await
+            .map_err(|e| RemoteStorageError::Other(format!("s3 list buckets: {}", e)))?;
+
+        Ok(resp
+            .buckets()
+            .iter()
+            .filter_map(|b| b.name().map(String::from))
+            .collect())
+    }
+
+    fn remote_conf(&self) -> &RemoteConf {
+        &self.conf
+    }
+}

seaweed-volume/src/remote_storage/s3_tier.rs

@@ -0,0 +1,514 @@
+//! S3-compatible tiered storage backend for volume .dat file upload/download.
+//!
+//! Provides multipart upload and concurrent download with progress callbacks,
+//! matching the Go SeaweedFS S3 backend behavior.
+
+use std::collections::HashMap;
+use std::future::Future;
+use std::sync::{Arc, OnceLock, RwLock};
+
+use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+use aws_sdk_s3::Client;
+use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
+use tokio::sync::Semaphore;
+
+/// Concurrency limit for multipart upload/download (matches Go's s3manager).
+const CONCURRENCY: usize = 5;
+
+/// Configuration for an S3 tier backend.
+#[derive(Debug, Clone)]
+pub struct S3TierConfig {
+    pub access_key: String,
+    pub secret_key: String,
+    pub region: String,
+    pub bucket: String,
+    pub endpoint: String,
+    pub storage_class: String,
+    pub force_path_style: bool,
+}
+
+/// S3 tier backend for uploading/downloading volume .dat files.
+pub struct S3TierBackend {
+    client: Client,
+    pub bucket: String,
+    pub storage_class: String,
+}
+
+impl S3TierBackend {
+    /// Create a new S3 tier backend from configuration.
+    pub fn new(config: &S3TierConfig) -> Self {
+        let region = if config.region.is_empty() {
+            "us-east-1"
+        } else {
+            &config.region
+        };
+
+        let credentials = Credentials::new(
+            &config.access_key,
+            &config.secret_key,
+            None,
+            None,
+            "seaweedfs-volume-tier",
+        );
+
+        let mut s3_config = aws_sdk_s3::Config::builder()
+            .behavior_version(BehaviorVersion::latest())
+            .region(Region::new(region.to_string()))
+            .credentials_provider(credentials)
+            .force_path_style(config.force_path_style);
+
+        if !config.endpoint.is_empty() {
+            s3_config = s3_config.endpoint_url(&config.endpoint);
+        }
+
+        let client = Client::from_conf(s3_config.build());
+
+        S3TierBackend {
+            client,
+            bucket: config.bucket.clone(),
+            storage_class: if config.storage_class.is_empty() {
+                "STANDARD_IA".to_string()
+            } else {
+                config.storage_class.clone()
+            },
+        }
+    }
+
+    /// Upload a local file to S3 using multipart upload with concurrent parts
+    /// and progress reporting.
+    ///
+    /// Returns (s3_key, file_size) on success.
+    /// The progress callback receives (bytes_uploaded, percentage).
+    /// Uses 64MB part size and 5 concurrent uploads (matches Go s3manager).
+    pub async fn upload_file<F>(
+        &self,
+        file_path: &str,
+        progress_fn: F,
+    ) -> Result<(String, u64), String>
+    where
+        F: FnMut(i64, f32) + Send + Sync + 'static,
+    {
+        let key = uuid::Uuid::new_v4().to_string();
+
+        let metadata = tokio::fs::metadata(file_path)
+            .await
+            .map_err(|e| format!("failed to stat file {}: {}", file_path, e))?;
+        let file_size = metadata.len();
+
+        // Calculate part size: start at 64MB, scale up for very large files (matches Go)
+        let mut part_size: u64 = 64 * 1024 * 1024;
+        while part_size * 1000 < file_size {
+            part_size *= 4;
+        }
+
+        // Initiate multipart upload
+        let create_resp = self
+            .client
+            .create_multipart_upload()
+            .bucket(&self.bucket)
+            .key(&key)
+            .storage_class(
+                self.storage_class
+                    .parse()
+                    .unwrap_or(aws_sdk_s3::types::StorageClass::StandardIa),
+            )
+            .send()
+            .await
+            .map_err(|e| format!("failed to create multipart upload: {}", e))?;
+
+        let upload_id = create_resp
+            .upload_id()
+            .ok_or_else(|| "no upload_id in multipart upload response".to_string())?
+            .to_string();
+
+        // Build list of (part_number, offset, size) for all parts
+        let mut parts_plan: Vec<(i32, u64, usize)> = Vec::new();
+        let mut offset: u64 = 0;
+        let mut part_number: i32 = 1;
+        while offset < file_size {
+            let remaining = file_size - offset;
+            let this_part_size = std::cmp::min(part_size, remaining) as usize;
+            parts_plan.push((part_number, offset, this_part_size));
+            offset += this_part_size as u64;
+            part_number += 1;
+        }
+
+        // Upload parts concurrently with a semaphore limiting to CONCURRENCY
+        let semaphore = Arc::new(Semaphore::new(CONCURRENCY));
+        let client = &self.client;
+        let bucket = &self.bucket;
+        let file_path_owned = file_path.to_string();
+        let progress = Arc::new(std::sync::Mutex::new((0u64, progress_fn)));
+
+        let mut handles = Vec::with_capacity(parts_plan.len());
+        for (pn, off, size) in parts_plan {
+            let sem = semaphore.clone();
+            let client = client.clone();
+            let bucket = bucket.clone();
+            let key = key.clone();
+            let upload_id = upload_id.clone();
+            let fp = file_path_owned.clone();
+            let progress = progress.clone();
+
+            handles.push(tokio::spawn(async move {
+                let _permit = sem
+                    .acquire()
+                    .await
+                    .map_err(|e| format!("semaphore error: {}", e))?;
+
+                // Read this part's data from the file at the correct offset
+                let mut file = tokio::fs::File::open(&fp)
+                    .await
+                    .map_err(|e| format!("failed to open file {}: {}", fp, e))?;
+                file.seek(std::io::SeekFrom::Start(off))
+                    .await
+                    .map_err(|e| format!("failed to seek to offset {}: {}", off, e))?;
+                let mut buf = vec![0u8; size];
+                file.read_exact(&mut buf)
+                    .await
+                    .map_err(|e| format!("failed to read file at offset {}: {}", off, e))?;
+
+                let upload_part_resp = client
+                    .upload_part()
+                    .bucket(&bucket)
+                    .key(&key)
+                    .upload_id(&upload_id)
+                    .part_number(pn)
+                    .body(buf.into())
+                    .send()
+                    .await
+                    .map_err(|e| {
+                        format!("failed to upload part {} at offset {}: {}", pn, off, e)
+                    })?;
+
+                let e_tag = upload_part_resp.e_tag().unwrap_or_default().to_string();
+
+                // Report progress
+                {
+                    let mut guard = progress.lock().unwrap();
+                    guard.0 += size as u64;
+                    let uploaded = guard.0;
+                    let pct = if file_size > 0 {
+                        (uploaded as f32 * 100.0) / file_size as f32
+                    } else {
+                        100.0
+                    };
+                    (guard.1)(uploaded as i64, pct);
+                }
+
+                Ok::<_, String>(
+                    CompletedPart::builder()
+                        .e_tag(e_tag)
+                        .part_number(pn)
+                        .build(),
+                )
+            }));
+        }
+
+        // Collect results, preserving part order
+        let mut completed_parts = Vec::with_capacity(handles.len());
+        for handle in handles {
+            let part = handle
+                .await
+                .map_err(|e| format!("upload task panicked: {}", e))??;
+            completed_parts.push(part);
+        }
+
+        // Complete multipart upload
+        let completed_upload = CompletedMultipartUpload::builder()
+            .set_parts(Some(completed_parts))
+            .build();
+
+        self.client
+            .complete_multipart_upload()
+            .bucket(&self.bucket)
+            .key(&key)
+            .upload_id(&upload_id)
+            .multipart_upload(completed_upload)
+            .send()
+            .await
+            .map_err(|e| format!("failed to complete multipart upload: {}", e))?;
+
+        Ok((key, file_size))
+    }
+
+    /// Download a file from S3 to a local path with concurrent range requests
+    /// and progress reporting.
+    ///
+    /// Returns the file size on success.
+    /// Uses 64MB part size and 5 concurrent downloads (matches Go s3manager).
+    pub async fn download_file<F>(
+        &self,
+        dest_path: &str,
+        key: &str,
+        progress_fn: F,
+    ) -> Result<u64, String>
+    where
+        F: FnMut(i64, f32) + Send + Sync + 'static,
+    {
+        // Get file size first
+        let head_resp = self
+            .client
+            .head_object()
+            .bucket(&self.bucket)
+            .key(key)
+            .send()
+            .await
+            .map_err(|e| format!("failed to head object {}: {}", key, e))?;
+
+        let file_size = head_resp.content_length().unwrap_or(0) as u64;
+
+        // Pre-allocate file to full size so concurrent WriteAt-style writes work
+        {
+            let file = tokio::fs::OpenOptions::new()
+                .write(true)
+                .create(true)
+                .truncate(true)
+                .open(dest_path)
+                .await
+                .map_err(|e| format!("failed to open dest file {}: {}", dest_path, e))?;
+            file.set_len(file_size)
+                .await
+                .map_err(|e| format!("failed to set file length: {}", e))?;
+        }
+
+        let part_size: u64 = 64 * 1024 * 1024;
+
+        // Build list of (offset, size) for all parts
+        let mut parts_plan: Vec<(u64, u64)> = Vec::new();
+        let mut offset: u64 = 0;
+        while offset < file_size {
+            let remaining = file_size - offset;
+            let this_part_size = std::cmp::min(part_size, remaining);
+            parts_plan.push((offset, this_part_size));
+            offset += this_part_size;
+        }
+
+        // Download parts concurrently with a semaphore limiting to CONCURRENCY
+        let semaphore = Arc::new(Semaphore::new(CONCURRENCY));
+        let client = &self.client;
+        let bucket = &self.bucket;
+        let dest_path_owned = dest_path.to_string();
+        let key_owned = key.to_string();
+        let progress = Arc::new(std::sync::Mutex::new((0u64, progress_fn)));
+
+        let mut handles = Vec::with_capacity(parts_plan.len());
+        for (off, size) in parts_plan {
+            let sem = semaphore.clone();
+            let client = client.clone();
+            let bucket = bucket.clone();
+            let key = key_owned.clone();
+            let dp = dest_path_owned.clone();
+            let progress = progress.clone();
+
+            handles.push(tokio::spawn(async move {
+                let _permit = sem
+                    .acquire()
+                    .await
+                    .map_err(|e| format!("semaphore error: {}", e))?;
+
+                let end = off + size - 1;
+                let range = format!("bytes={}-{}", off, end);
+
+                let get_resp = client
+                    .get_object()
+                    .bucket(&bucket)
+                    .key(&key)
+                    .range(&range)
+                    .send()
+                    .await
+                    .map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
+
+                let body = get_resp
+                    .body
+                    .collect()
+                    .await
+                    .map_err(|e| format!("failed to read body: {}", e))?;
+                let bytes = body.into_bytes();
+
+                // Write at the correct offset (like Go's WriteAt)
+                let mut file = tokio::fs::OpenOptions::new()
+                    .write(true)
+                    .open(&dp)
+                    .await
+                    .map_err(|e| format!("failed to open dest file {}: {}", dp, e))?;
+                file.seek(std::io::SeekFrom::Start(off))
+                    .await
+                    .map_err(|e| format!("failed to seek to offset {}: {}", off, e))?;
+                file.write_all(&bytes)
+                    .await
+                    .map_err(|e| format!("failed to write to {}: {}", dp, e))?;
+
+                // Report progress
+                {
+                    let mut guard = progress.lock().unwrap();
+                    guard.0 += bytes.len() as u64;
+                    let downloaded = guard.0;
+                    let pct = if file_size > 0 {
+                        (downloaded as f32 * 100.0) / file_size as f32
+                    } else {
+                        100.0
+                    };
+                    (guard.1)(downloaded as i64, pct);
+                }
+
+                Ok::<_, String>(())
+            }));
+        }
+
+        // Wait for all download tasks
+        for handle in handles {
+            handle
+                .await
+                .map_err(|e| format!("download task panicked: {}", e))??;
+        }
+
+        Ok(file_size)
+    }
+
+    pub async fn read_range(&self, key: &str, offset: u64, size: usize) -> Result<Vec<u8>, String> {
+        let end = offset + (size as u64).saturating_sub(1);
+        let range = format!("bytes={}-{}", offset, end);
+        let resp = self
+            .client
+            .get_object()
+            .bucket(&self.bucket)
+            .key(key)
+            .range(&range)
+            .send()
+            .await
+            .map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
+
+        let body = resp
+            .body
+            .collect()
+            .await
+            .map_err(|e| format!("failed to read object {} body: {}", key, e))?;
+        Ok(body.into_bytes().to_vec())
+    }
+
+    /// Delete a file from S3.
+    pub async fn delete_file(&self, key: &str) -> Result<(), String> {
+        self.client
+            .delete_object()
+            .bucket(&self.bucket)
+            .key(key)
+            .send()
+            .await
+            .map_err(|e| format!("failed to delete object {}: {}", key, e))?;
+        Ok(())
+    }
+
+    pub fn delete_file_blocking(&self, key: &str) -> Result<(), String> {
+        let client = self.client.clone();
+        let bucket = self.bucket.clone();
+        let key = key.to_string();
+        block_on_tier_future(async move {
+            client
+                .delete_object()
+                .bucket(&bucket)
+                .key(&key)
+                .send()
+                .await
+                .map_err(|e| format!("failed to delete object {}: {}", key, e))?;
+            Ok(())
+        })
+    }
+
+    pub fn read_range_blocking(
+        &self,
+        key: &str,
+        offset: u64,
+        size: usize,
+    ) -> Result<Vec<u8>, String> {
+        let client = self.client.clone();
+        let bucket = self.bucket.clone();
+        let key = key.to_string();
+        block_on_tier_future(async move {
+            let end = offset + (size as u64).saturating_sub(1);
+            let range = format!("bytes={}-{}", offset, end);
+            let resp = client
+                .get_object()
+                .bucket(&bucket)
+                .key(&key)
+                .range(&range)
+                .send()
+                .await
+                .map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
+
+            let body = resp
+                .body
+                .collect()
+                .await
+                .map_err(|e| format!("failed to read object {} body: {}", key, e))?;
+            Ok(body.into_bytes().to_vec())
+        })
+    }
+}
+
+/// Parse a backend name like "s3" or "s3.default" into (backend_type, backend_id).
+/// Matches Go's `BackendNameToTypeId`.
+pub fn backend_name_to_type_id(backend_name: &str) -> (String, String) {
+    let parts: Vec<&str> = backend_name.split('.').collect();
+    match parts.len() {
+        1 => (backend_name.to_string(), "default".to_string()),
+        2 => (parts[0].to_string(), parts[1].to_string()),
+        _ => (String::new(), String::new()),
+    }
+}
+
+/// A registry of configured S3 tier backends, keyed by backend name (e.g., "s3.default").
+#[derive(Default)]
+pub struct S3TierRegistry {
+    backends: HashMap<String, Arc<S3TierBackend>>,
+}
+
+impl S3TierRegistry {
+    pub fn new() -> Self {
+        Self {
+            backends: HashMap::new(),
+        }
+    }
+
+    /// Register a backend with the given name.
+    pub fn register(&mut self, name: String, backend: S3TierBackend) {
+        self.backends.insert(name, Arc::new(backend));
+    }
+
+    /// Look up a backend by name.
+    pub fn get(&self, name: &str) -> Option<Arc<S3TierBackend>> {
+        self.backends.get(name).cloned()
+    }
+
+    /// List all registered backend names.
+    pub fn names(&self) -> Vec<String> {
+        self.backends.keys().cloned().collect()
+    }
+
+    pub fn clear(&mut self) {
+        self.backends.clear();
+    }
+}
+
+static GLOBAL_S3_TIER_REGISTRY: OnceLock<RwLock<S3TierRegistry>> = OnceLock::new();
+
+pub fn global_s3_tier_registry() -> &'static RwLock<S3TierRegistry> {
+    GLOBAL_S3_TIER_REGISTRY.get_or_init(|| RwLock::new(S3TierRegistry::new()))
+}
+
+fn block_on_tier_future<F, T>(future: F) -> Result<T, String>
+where
+    F: Future<Output = Result<T, String>> + Send + 'static,
+    T: Send + 'static,
+{
+    std::thread::spawn(move || {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .map_err(|e| format!("failed to build tokio runtime: {}", e))?;
+        runtime.block_on(future)
+    })
+    .join()
+    .map_err(|_| "tier runtime thread panicked".to_string())?
+}

seaweed-volume/src/security.rs

@@ -0,0 +1,481 @@
+//! Security: JWT validation and IP whitelist checking.
+//!
+//! Matches Go's security/guard.go and security/jwt.go.
+//! - Guard: combines whitelist IP checking with JWT token validation
+//! - JWT: HS256 HMAC signing with file-id claims
+
+pub mod tls;
+
+use std::collections::HashSet;
+use std::net::IpAddr;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use jsonwebtoken::{decode, encode, Algorithm, DecodingKey, EncodingKey, Header, Validation};
+use serde::{Deserialize, Serialize};
+
+// ============================================================================
+// JWT Claims
+// ============================================================================
+
+/// Claims for volume server file access tokens.
+/// Matches Go's `SeaweedFileIdClaims`.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct FileIdClaims {
+    /// File ID this token grants access to (e.g., "3,01637037d6").
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub fid: Option<String>,
+
+    /// Expiration time (Unix timestamp).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub exp: Option<u64>,
+
+    /// Not before (Unix timestamp).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub nbf: Option<u64>,
+}
+
+/// Signing key wrapper (empty = security disabled).
+#[derive(Clone)]
+pub struct SigningKey(pub Vec<u8>);
+
+impl SigningKey {
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    pub fn from_string(s: &str) -> Self {
+        SigningKey(s.as_bytes().to_vec())
+    }
+}
+
+/// Generate a JWT token for file access.
+pub fn gen_jwt(
+    signing_key: &SigningKey,
+    expires_after_sec: i64,
+    file_id: &str,
+) -> Result<String, JwtError> {
+    if signing_key.is_empty() {
+        return Err(JwtError::NoSigningKey);
+    }
+
+    let now = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs();
+
+    let claims = FileIdClaims {
+        fid: Some(file_id.to_string()),
+        exp: if expires_after_sec > 0 {
+            Some(now + expires_after_sec as u64)
+        } else {
+            None
+        },
+        nbf: None,
+    };
+
+    let token = encode(
+        &Header::new(Algorithm::HS256),
+        &claims,
+        &EncodingKey::from_secret(&signing_key.0),
+    )?;
+
+    Ok(token)
+}
+
+/// Decode and validate a JWT token.
+pub fn decode_jwt(signing_key: &SigningKey, token: &str) -> Result<FileIdClaims, JwtError> {
+    if signing_key.is_empty() {
+        return Err(JwtError::NoSigningKey);
+    }
+
+    let mut validation = Validation::new(Algorithm::HS256);
+    // Match Go behavior: tokens without exp are accepted (Go's jwt-go does not require exp)
+    // But if exp IS present, it must be valid (not expired).
+    validation.required_spec_claims.clear();
+    validation.validate_exp = true;
+    // Go's jwt-go/v5 validates nbf when present
+    validation.validate_nbf = true;
+    validation.leeway = 0;
+
+    let data = decode::<FileIdClaims>(
+        token,
+        &DecodingKey::from_secret(&signing_key.0),
+        &validation,
+    )?;
+
+    Ok(data.claims)
+}
+
+// ============================================================================
+// Guard
+// ============================================================================
+
+/// Security guard: IP whitelist + JWT token validation.
+pub struct Guard {
+    whitelist_ips: HashSet<String>,
+    whitelist_cidrs: Vec<(IpAddr, u8)>, // (network, prefix_len)
+    pub signing_key: SigningKey,
+    pub expires_after_sec: i64,
+    pub read_signing_key: SigningKey,
+    pub read_expires_after_sec: i64,
+    /// Combined flag: true when whitelist is non-empty OR signing key is present.
+    /// Matches Go's `isWriteActive = !isEmptyWhiteList || len(SigningKey) != 0`.
+    is_write_active: bool,
+}
+
+impl Guard {
+    pub fn new(
+        whitelist: &[String],
+        signing_key: SigningKey,
+        expires_after_sec: i64,
+        read_signing_key: SigningKey,
+        read_expires_after_sec: i64,
+    ) -> Self {
+        let mut guard = Guard {
+            whitelist_ips: HashSet::new(),
+            whitelist_cidrs: Vec::new(),
+            signing_key,
+            expires_after_sec,
+            read_signing_key,
+            read_expires_after_sec,
+            is_write_active: false,
+        };
+        guard.update_whitelist(whitelist);
+        guard
+    }
+
+    /// Update the IP whitelist.
+    pub fn update_whitelist(&mut self, entries: &[String]) {
+        self.whitelist_ips.clear();
+        self.whitelist_cidrs.clear();
+
+        for entry in entries {
+            let entry = entry.trim();
+            if entry.is_empty() {
+                continue;
+            }
+            if entry.contains('/') {
+                // CIDR range
+                if let Some((ip, prefix)) = parse_cidr(entry) {
+                    self.whitelist_cidrs.push((ip, prefix));
+                } else {
+                    tracing::error!("Parse CIDR {} in whitelist failed", entry);
+                }
+            } else {
+                // Exact IP/hostname
+                self.whitelist_ips.insert(entry.to_string());
+            }
+        }
+
+        // Match Go: isWriteActive = !isEmptyWhiteList || len(SigningKey) != 0
+        let is_empty_whitelist = self.whitelist_ips.is_empty() && self.whitelist_cidrs.is_empty();
+        self.is_write_active = !is_empty_whitelist || !self.signing_key.is_empty();
+    }
+
+    /// Check if a remote IP is in the whitelist.
+    /// Returns true if write security is inactive (no whitelist and no signing key),
+    /// if the whitelist is empty, or if the IP matches.
+    pub fn check_whitelist(&self, remote_addr: &str) -> bool {
+        if !self.is_write_active {
+            return true;
+        }
+        if self.whitelist_ips.is_empty() && self.whitelist_cidrs.is_empty() {
+            return true;
+        }
+
+        let host = extract_host(remote_addr);
+
+        // Check exact match
+        if self.whitelist_ips.contains(&host) {
+            return true;
+        }
+
+        // Check CIDR ranges
+        if let Ok(ip) = host.parse::<IpAddr>() {
+            for &(ref network, prefix_len) in &self.whitelist_cidrs {
+                if ip_in_cidr(&ip, network, prefix_len) {
+                    return true;
+                }
+            }
+        }
+
+        false
+    }
+
+    /// Check if a read signing key is configured.
+    pub fn has_read_signing_key(&self) -> bool {
+        !self.read_signing_key.is_empty()
+    }
+
+    /// Validate a request's JWT token.
+    /// `is_write` determines which signing key to use.
+    /// Returns Ok(()) if valid, or if security is disabled.
+    pub fn check_jwt(&self, token: Option<&str>, is_write: bool) -> Result<(), JwtError> {
+        let key = if is_write {
+            &self.signing_key
+        } else {
+            &self.read_signing_key
+        };
+
+        if key.is_empty() {
+            return Ok(()); // Security disabled for this operation type
+        }
+
+        let token = token.ok_or(JwtError::MissingToken)?;
+        decode_jwt(key, token)?;
+        Ok(())
+    }
+
+    /// Check JWT and validate the file ID claim matches.
+    pub fn check_jwt_for_file(
+        &self,
+        token: Option<&str>,
+        expected_fid: &str,
+        is_write: bool,
+    ) -> Result<(), JwtError> {
+        let key = if is_write {
+            &self.signing_key
+        } else {
+            &self.read_signing_key
+        };
+
+        if key.is_empty() {
+            return Ok(());
+        }
+
+        let token = token.ok_or(JwtError::MissingToken)?;
+        let claims = decode_jwt(key, token)?;
+
+        match claims.fid {
+            None => {
+                return Err(JwtError::MissingFileIdClaim);
+            }
+            Some(ref fid) if fid != expected_fid => {
+                return Err(JwtError::FileIdMismatch {
+                    expected: expected_fid.to_string(),
+                    got: fid.to_string(),
+                });
+            }
+            _ => {}
+        }
+
+        Ok(())
+    }
+}
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+/// Extract host from "host:port" or "[::1]:port" format.
+fn extract_host(addr: &str) -> String {
+    // Handle IPv6 with brackets
+    if addr.starts_with('[') {
+        if let Some(end) = addr.find(']') {
+            return addr[1..end].to_string();
+        }
+    }
+    // Handle host:port
+    if let Some(pos) = addr.rfind(':') {
+        return addr[..pos].to_string();
+    }
+    addr.to_string()
+}
+
+/// Parse CIDR notation "192.168.1.0/24" into (IpAddr, prefix_len).
+fn parse_cidr(cidr: &str) -> Option<(IpAddr, u8)> {
+    let parts: Vec<&str> = cidr.split('/').collect();
+    if parts.len() != 2 {
+        return None;
+    }
+    let ip: IpAddr = parts[0].parse().ok()?;
+    let prefix: u8 = parts[1].parse().ok()?;
+    Some((ip, prefix))
+}
+
+/// Check if an IP is within a CIDR range.
+fn ip_in_cidr(ip: &IpAddr, network: &IpAddr, prefix_len: u8) -> bool {
+    match (ip, network) {
+        (IpAddr::V4(ip), IpAddr::V4(net)) => {
+            let ip_bits = u32::from(*ip);
+            let net_bits = u32::from(*net);
+            let mask = if prefix_len == 0 {
+                0
+            } else if prefix_len >= 32 {
+                u32::MAX
+            } else {
+                u32::MAX << (32 - prefix_len)
+            };
+            (ip_bits & mask) == (net_bits & mask)
+        }
+        (IpAddr::V6(ip), IpAddr::V6(net)) => {
+            let ip_bits = u128::from(*ip);
+            let net_bits = u128::from(*net);
+            let mask = if prefix_len == 0 {
+                0
+            } else if prefix_len >= 128 {
+                u128::MAX
+            } else {
+                u128::MAX << (128 - prefix_len)
+            };
+            (ip_bits & mask) == (net_bits & mask)
+        }
+        _ => false, // V4/V6 mismatch
+    }
+}
+
+// ============================================================================
+// Errors
+// ============================================================================
+
+#[derive(Debug, thiserror::Error)]
+pub enum JwtError {
+    #[error("no signing key configured")]
+    NoSigningKey,
+
+    #[error("missing JWT token")]
+    MissingToken,
+
+    #[error("JWT error: {0}")]
+    Jwt(#[from] jsonwebtoken::errors::Error),
+
+    #[error("JWT token missing required fid claim")]
+    MissingFileIdClaim,
+
+    #[error("file ID mismatch: expected {expected}, got {got}")]
+    FileIdMismatch { expected: String, got: String },
+}
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_jwt_round_trip() {
+        let key = SigningKey::from_string("test-secret-key");
+        let token = gen_jwt(&key, 3600, "3,01637037d6").unwrap();
+        let claims = decode_jwt(&key, &token).unwrap();
+        assert_eq!(claims.fid, Some("3,01637037d6".to_string()));
+    }
+
+    #[test]
+    fn test_jwt_no_signing_key() {
+        let key = SigningKey(vec![]);
+        assert!(gen_jwt(&key, 3600, "1,abc").is_err());
+    }
+
+    #[test]
+    fn test_jwt_invalid_token() {
+        let key = SigningKey::from_string("secret");
+        let result = decode_jwt(&key, "invalid.token.here");
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_jwt_wrong_key() {
+        let key1 = SigningKey::from_string("secret1");
+        let key2 = SigningKey::from_string("secret2");
+        let token = gen_jwt(&key1, 3600, "1,abc").unwrap();
+        assert!(decode_jwt(&key2, &token).is_err());
+    }
+
+    #[test]
+    fn test_guard_empty_whitelist() {
+        let guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
+        assert!(guard.check_whitelist("192.168.1.1:8080"));
+    }
+
+    #[test]
+    fn test_guard_whitelist_exact() {
+        let guard = Guard::new(
+            &["192.168.1.1".to_string(), "10.0.0.1".to_string()],
+            SigningKey(vec![]),
+            0,
+            SigningKey(vec![]),
+            0,
+        );
+        assert!(guard.check_whitelist("192.168.1.1:8080"));
+        assert!(guard.check_whitelist("10.0.0.1:1234"));
+        assert!(!guard.check_whitelist("172.16.0.1:8080"));
+    }
+
+    #[test]
+    fn test_guard_whitelist_cidr() {
+        let guard = Guard::new(
+            &["10.0.0.0/8".to_string()],
+            SigningKey(vec![]),
+            0,
+            SigningKey(vec![]),
+            0,
+        );
+        assert!(guard.check_whitelist("10.1.2.3:8080"));
+        assert!(guard.check_whitelist("10.255.255.255:80"));
+        assert!(!guard.check_whitelist("11.0.0.1:80"));
+    }
+
+    #[test]
+    fn test_guard_check_jwt_disabled() {
+        let guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
+        // No signing key = security disabled
+        assert!(guard.check_jwt(None, true).is_ok());
+        assert!(guard.check_jwt(None, false).is_ok());
+    }
+
+    #[test]
+    fn test_guard_check_jwt_enabled() {
+        let key = SigningKey::from_string("write-secret");
+        let read_key = SigningKey::from_string("read-secret");
+        let guard = Guard::new(&[], key.clone(), 3600, read_key.clone(), 3600);
+
+        // Missing token
+        assert!(guard.check_jwt(None, true).is_err());
+
+        // Valid write token
+        let token = gen_jwt(&key, 3600, "1,abc").unwrap();
+        assert!(guard.check_jwt(Some(&token), true).is_ok());
+
+        // Write token for read should fail (different key)
+        assert!(guard.check_jwt(Some(&token), false).is_err());
+
+        // Valid read token
+        let read_token = gen_jwt(&read_key, 3600, "1,abc").unwrap();
+        assert!(guard.check_jwt(Some(&read_token), false).is_ok());
+    }
+
+    #[test]
+    fn test_guard_check_jwt_file_id() {
+        let key = SigningKey::from_string("secret");
+        let guard = Guard::new(&[], key.clone(), 3600, SigningKey(vec![]), 0);
+
+        let token = gen_jwt(&key, 3600, "3,01637037d6").unwrap();
+
+        // Correct file ID
+        assert!(guard
+            .check_jwt_for_file(Some(&token), "3,01637037d6", true)
+            .is_ok());
+
+        // Wrong file ID
+        let err = guard.check_jwt_for_file(Some(&token), "4,deadbeef", true);
+        assert!(matches!(err, Err(JwtError::FileIdMismatch { .. })));
+    }
+
+    #[test]
+    fn test_extract_host() {
+        assert_eq!(extract_host("192.168.1.1:8080"), "192.168.1.1");
+        assert_eq!(extract_host("[::1]:8080"), "::1");
+        assert_eq!(extract_host("localhost"), "localhost");
+    }
+
+    #[test]
+    fn test_ip_in_cidr() {
+        let net: IpAddr = "10.0.0.0".parse().unwrap();
+        let ip1: IpAddr = "10.1.2.3".parse().unwrap();
+        let ip2: IpAddr = "11.0.0.1".parse().unwrap();
+        assert!(ip_in_cidr(&ip1, &net, 8));
+        assert!(!ip_in_cidr(&ip2, &net, 8));
+    }
+}

seaweed-volume/src/security/tls.rs

@@ -0,0 +1,437 @@
+use std::collections::HashSet;
+use std::fmt;
+use std::sync::Arc;
+
+use rustls::client::danger::HandshakeSignatureValid;
+use rustls::crypto::aws_lc_rs;
+use rustls::crypto::CryptoProvider;
+use rustls::pki_types::UnixTime;
+use rustls::pki_types::{CertificateDer, PrivateKeyDer};
+use rustls::server::danger::{ClientCertVerified, ClientCertVerifier};
+use rustls::server::WebPkiClientVerifier;
+use rustls::{
+    CipherSuite, DigitallySignedStruct, DistinguishedName, RootCertStore, ServerConfig,
+    SignatureScheme, SupportedCipherSuite, SupportedProtocolVersion,
+};
+use x509_parser::prelude::{FromDer, X509Certificate};
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct TlsPolicy {
+    pub min_version: String,
+    pub max_version: String,
+    pub cipher_suites: String,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct GrpcClientAuthPolicy {
+    pub allowed_common_names: Vec<String>,
+    pub allowed_wildcard_domain: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TlsPolicyError(String);
+
+impl fmt::Display for TlsPolicyError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(&self.0)
+    }
+}
+
+impl std::error::Error for TlsPolicyError {}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+enum GoTlsVersion {
+    Ssl3,
+    Tls10,
+    Tls11,
+    Tls12,
+    Tls13,
+}
+
+#[derive(Debug)]
+struct CommonNameVerifier {
+    inner: Arc<dyn ClientCertVerifier>,
+    allowed_common_names: HashSet<String>,
+    allowed_wildcard_domain: String,
+}
+
+impl ClientCertVerifier for CommonNameVerifier {
+    fn offer_client_auth(&self) -> bool {
+        self.inner.offer_client_auth()
+    }
+
+    fn client_auth_mandatory(&self) -> bool {
+        self.inner.client_auth_mandatory()
+    }
+
+    fn root_hint_subjects(&self) -> &[DistinguishedName] {
+        self.inner.root_hint_subjects()
+    }
+
+    fn verify_client_cert(
+        &self,
+        end_entity: &CertificateDer<'_>,
+        intermediates: &[CertificateDer<'_>],
+        now: UnixTime,
+    ) -> Result<ClientCertVerified, rustls::Error> {
+        self.inner
+            .verify_client_cert(end_entity, intermediates, now)?;
+        let common_name = parse_common_name(end_entity).map_err(|e| {
+            rustls::Error::General(format!(
+                "parse client certificate common name failed: {}",
+                e
+            ))
+        })?;
+        if common_name_is_allowed(
+            &common_name,
+            &self.allowed_common_names,
+            &self.allowed_wildcard_domain,
+        ) {
+            return Ok(ClientCertVerified::assertion());
+        }
+        Err(rustls::Error::General(format!(
+            "Authenticate: invalid subject client common name: {}",
+            common_name
+        )))
+    }
+
+    fn verify_tls12_signature(
+        &self,
+        message: &[u8],
+        cert: &CertificateDer<'_>,
+        dss: &DigitallySignedStruct,
+    ) -> Result<HandshakeSignatureValid, rustls::Error> {
+        self.inner.verify_tls12_signature(message, cert, dss)
+    }
+
+    fn verify_tls13_signature(
+        &self,
+        message: &[u8],
+        cert: &CertificateDer<'_>,
+        dss: &DigitallySignedStruct,
+    ) -> Result<HandshakeSignatureValid, rustls::Error> {
+        self.inner.verify_tls13_signature(message, cert, dss)
+    }
+
+    fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
+        self.inner.supported_verify_schemes()
+    }
+}
+
+pub fn build_rustls_server_config(
+    cert_path: &str,
+    key_path: &str,
+    ca_path: &str,
+    policy: &TlsPolicy,
+) -> Result<ServerConfig, TlsPolicyError> {
+    build_rustls_server_config_with_client_auth(cert_path, key_path, ca_path, policy, None)
+}
+
+pub fn build_rustls_server_config_with_grpc_client_auth(
+    cert_path: &str,
+    key_path: &str,
+    ca_path: &str,
+    policy: &TlsPolicy,
+    client_auth_policy: &GrpcClientAuthPolicy,
+) -> Result<ServerConfig, TlsPolicyError> {
+    build_rustls_server_config_with_client_auth(
+        cert_path,
+        key_path,
+        ca_path,
+        policy,
+        Some(client_auth_policy),
+    )
+}
+
+fn build_rustls_server_config_with_client_auth(
+    cert_path: &str,
+    key_path: &str,
+    ca_path: &str,
+    policy: &TlsPolicy,
+    client_auth_policy: Option<&GrpcClientAuthPolicy>,
+) -> Result<ServerConfig, TlsPolicyError> {
+    let cert_chain = read_cert_chain(cert_path)?;
+    let private_key = read_private_key(key_path)?;
+    let provider = build_crypto_provider(policy)?;
+    let versions = build_supported_versions(policy)?;
+
+    let builder = ServerConfig::builder_with_provider(provider.clone())
+        .with_protocol_versions(&versions)
+        .map_err(|e| TlsPolicyError(format!("invalid TLS version policy: {}", e)))?;
+
+    let builder = if ca_path.is_empty() {
+        builder.with_no_client_auth()
+    } else {
+        let roots = read_root_store(ca_path)?;
+        let verifier =
+            WebPkiClientVerifier::builder_with_provider(Arc::new(roots), provider.clone())
+                .build()
+                .map_err(|e| TlsPolicyError(format!("build client verifier failed: {}", e)))?;
+        let verifier: Arc<dyn ClientCertVerifier> = if let Some(client_auth_policy) =
+            client_auth_policy.filter(|policy| {
+                !policy.allowed_common_names.is_empty()
+                    || !policy.allowed_wildcard_domain.is_empty()
+            }) {
+            Arc::new(CommonNameVerifier {
+                inner: verifier,
+                allowed_common_names: client_auth_policy
+                    .allowed_common_names
+                    .iter()
+                    .cloned()
+                    .collect(),
+                allowed_wildcard_domain: client_auth_policy.allowed_wildcard_domain.clone(),
+            })
+        } else {
+            verifier
+        };
+        builder.with_client_cert_verifier(verifier)
+    };
+
+    builder
+        .with_single_cert(cert_chain, private_key)
+        .map_err(|e| TlsPolicyError(format!("build rustls server config failed: {}", e)))
+}
+
+fn read_cert_chain(cert_path: &str) -> Result<Vec<CertificateDer<'static>>, TlsPolicyError> {
+    let cert_pem = std::fs::read(cert_path).map_err(|e| {
+        TlsPolicyError(format!(
+            "Failed to read TLS cert file '{}': {}",
+            cert_path, e
+        ))
+    })?;
+    rustls_pemfile::certs(&mut &cert_pem[..])
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|e| {
+            TlsPolicyError(format!(
+                "Failed to parse TLS cert PEM '{}': {}",
+                cert_path, e
+            ))
+        })
+}
+
+fn read_private_key(key_path: &str) -> Result<PrivateKeyDer<'static>, TlsPolicyError> {
+    let key_pem = std::fs::read(key_path).map_err(|e| {
+        TlsPolicyError(format!("Failed to read TLS key file '{}': {}", key_path, e))
+    })?;
+    rustls_pemfile::private_key(&mut &key_pem[..])
+        .map_err(|e| TlsPolicyError(format!("Failed to parse TLS key PEM '{}': {}", key_path, e)))?
+        .ok_or_else(|| TlsPolicyError(format!("No private key found in '{}'", key_path)))
+}
+
+fn read_root_store(ca_path: &str) -> Result<RootCertStore, TlsPolicyError> {
+    let ca_pem = std::fs::read(ca_path)
+        .map_err(|e| TlsPolicyError(format!("Failed to read TLS CA file '{}': {}", ca_path, e)))?;
+    let ca_certs = rustls_pemfile::certs(&mut &ca_pem[..])
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|e| TlsPolicyError(format!("Failed to parse TLS CA PEM '{}': {}", ca_path, e)))?;
+    let mut roots = RootCertStore::empty();
+    for cert in ca_certs {
+        roots
+            .add(cert)
+            .map_err(|e| TlsPolicyError(format!("Failed to add CA cert '{}': {}", ca_path, e)))?;
+    }
+    Ok(roots)
+}
+
+fn build_crypto_provider(policy: &TlsPolicy) -> Result<Arc<CryptoProvider>, TlsPolicyError> {
+    let mut provider = aws_lc_rs::default_provider();
+    let cipher_suites = parse_cipher_suites(&provider.cipher_suites, &policy.cipher_suites)?;
+    if !cipher_suites.is_empty() {
+        provider.cipher_suites = cipher_suites;
+    }
+    Ok(Arc::new(provider))
+}
+
+pub fn build_supported_versions(
+    policy: &TlsPolicy,
+) -> Result<Vec<&'static SupportedProtocolVersion>, TlsPolicyError> {
+    let min_version = parse_go_tls_version(&policy.min_version)?;
+    let max_version = parse_go_tls_version(&policy.max_version)?;
+    let versions = [&rustls::version::TLS13, &rustls::version::TLS12]
+        .into_iter()
+        .filter(|version| {
+            let current = go_tls_version_for_supported(version);
+            min_version.map(|min| current >= min).unwrap_or(true)
+                && max_version.map(|max| current <= max).unwrap_or(true)
+        })
+        .collect::<Vec<_>>();
+
+    if versions.is_empty() {
+        return Err(TlsPolicyError(format!(
+            "TLS version range min='{}' max='{}' is unsupported by rustls",
+            policy.min_version, policy.max_version
+        )));
+    }
+
+    Ok(versions)
+}
+
+fn parse_go_tls_version(value: &str) -> Result<Option<GoTlsVersion>, TlsPolicyError> {
+    match value.trim() {
+        "" => Ok(None),
+        "SSLv3" => Ok(Some(GoTlsVersion::Ssl3)),
+        "TLS 1.0" => Ok(Some(GoTlsVersion::Tls10)),
+        "TLS 1.1" => Ok(Some(GoTlsVersion::Tls11)),
+        "TLS 1.2" => Ok(Some(GoTlsVersion::Tls12)),
+        "TLS 1.3" => Ok(Some(GoTlsVersion::Tls13)),
+        other => Err(TlsPolicyError(format!("invalid TLS version {}", other))),
+    }
+}
+
+fn parse_cipher_suites(
+    available: &[SupportedCipherSuite],
+    value: &str,
+) -> Result<Vec<SupportedCipherSuite>, TlsPolicyError> {
+    let trimmed = value.trim();
+    if trimmed.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    trimmed
+        .split(',')
+        .map(|name| {
+            let suite = parse_cipher_suite_name(name.trim())?;
+            available
+                .iter()
+                .copied()
+                .find(|candidate| candidate.suite() == suite)
+                .ok_or_else(|| {
+                    TlsPolicyError(format!(
+                        "TLS cipher suite '{}' is unsupported by the Rust implementation",
+                        name.trim()
+                    ))
+                })
+        })
+        .collect()
+}
+
+fn parse_cipher_suite_name(value: &str) -> Result<CipherSuite, TlsPolicyError> {
+    match value {
+        "TLS_AES_128_GCM_SHA256" => Ok(CipherSuite::TLS13_AES_128_GCM_SHA256),
+        "TLS_AES_256_GCM_SHA384" => Ok(CipherSuite::TLS13_AES_256_GCM_SHA384),
+        "TLS_CHACHA20_POLY1305_SHA256" => Ok(CipherSuite::TLS13_CHACHA20_POLY1305_SHA256),
+        "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256" => {
+            Ok(CipherSuite::TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256)
+        }
+        "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" => {
+            Ok(CipherSuite::TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384)
+        }
+        "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256" => {
+            Ok(CipherSuite::TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256)
+        }
+        "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" => {
+            Ok(CipherSuite::TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256)
+        }
+        "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" => {
+            Ok(CipherSuite::TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384)
+        }
+        "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256" => {
+            Ok(CipherSuite::TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256)
+        }
+        other => Err(TlsPolicyError(format!(
+            "TLS cipher suite '{}' is unsupported by the Rust implementation",
+            other
+        ))),
+    }
+}
+
+fn parse_common_name(cert: &CertificateDer<'_>) -> Result<String, TlsPolicyError> {
+    let (_, certificate) = X509Certificate::from_der(cert.as_ref())
+        .map_err(|e| TlsPolicyError(format!("parse X.509 certificate failed: {}", e)))?;
+    let common_name = certificate
+        .subject()
+        .iter_common_name()
+        .next()
+        .and_then(|common_name| common_name.as_str().ok())
+        .map(str::to_string);
+    match common_name {
+        Some(common_name) => Ok(common_name),
+        None => Ok(String::new()),
+    }
+}
+
+fn common_name_is_allowed(
+    common_name: &str,
+    allowed_common_names: &HashSet<String>,
+    allowed_wildcard_domain: &str,
+) -> bool {
+    (!allowed_wildcard_domain.is_empty() && common_name.ends_with(allowed_wildcard_domain))
+        || allowed_common_names.contains(common_name)
+}
+
+fn go_tls_version_for_supported(version: &SupportedProtocolVersion) -> GoTlsVersion {
+    match version.version {
+        rustls::ProtocolVersion::TLSv1_2 => GoTlsVersion::Tls12,
+        rustls::ProtocolVersion::TLSv1_3 => GoTlsVersion::Tls13,
+        _ => unreachable!("rustls only exposes TLS 1.2 and 1.3"),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{build_supported_versions, common_name_is_allowed, parse_cipher_suites, TlsPolicy};
+    use rustls::crypto::aws_lc_rs;
+    use std::collections::HashSet;
+
+    #[test]
+    fn test_build_supported_versions_defaults_to_tls12_and_tls13() {
+        let versions = build_supported_versions(&TlsPolicy::default()).unwrap();
+        assert_eq!(
+            versions,
+            vec![&rustls::version::TLS13, &rustls::version::TLS12]
+        );
+    }
+
+    #[test]
+    fn test_build_supported_versions_filters_to_tls13() {
+        let versions = build_supported_versions(&TlsPolicy {
+            min_version: "TLS 1.3".to_string(),
+            max_version: "TLS 1.3".to_string(),
+            cipher_suites: String::new(),
+        })
+        .unwrap();
+        assert_eq!(versions, vec![&rustls::version::TLS13]);
+    }
+
+    #[test]
+    fn test_build_supported_versions_rejects_unsupported_legacy_range() {
+        let err = build_supported_versions(&TlsPolicy {
+            min_version: "TLS 1.0".to_string(),
+            max_version: "TLS 1.1".to_string(),
+            cipher_suites: String::new(),
+        })
+        .unwrap_err();
+        assert!(err.to_string().contains("unsupported by rustls"));
+    }
+
+    #[test]
+    fn test_parse_cipher_suites_accepts_go_names() {
+        let cipher_suites = parse_cipher_suites(
+            &aws_lc_rs::default_provider().cipher_suites,
+            "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_AES_128_GCM_SHA256",
+        )
+        .unwrap();
+        assert_eq!(cipher_suites.len(), 2);
+    }
+
+    #[test]
+    fn test_common_name_is_allowed_matches_exact_and_wildcard() {
+        let allowed_common_names =
+            HashSet::from([String::from("volume-a.internal"), String::from("worker-7")]);
+        assert!(common_name_is_allowed(
+            "volume-a.internal",
+            &allowed_common_names,
+            "",
+        ));
+        assert!(common_name_is_allowed(
+            "node.prod.example.com",
+            &allowed_common_names,
+            ".example.com",
+        ));
+        assert!(!common_name_is_allowed(
+            "node.prod.other.net",
+            &allowed_common_names,
+            ".example.com",
+        ));
+    }
+}

seaweed-volume/src/server/debug.rs

@@ -0,0 +1,159 @@
+use axum::body::Body;
+use axum::extract::Query;
+use axum::http::{header, StatusCode};
+use axum::response::{IntoResponse, Response};
+use axum::routing::{any, get};
+use axum::Router;
+use pprof::protos::Message;
+use serde::Deserialize;
+
+#[derive(Deserialize, Default)]
+struct ProfileQuery {
+    seconds: Option<u64>,
+}
+
+pub fn build_debug_router() -> Router {
+    Router::new()
+        .route("/debug/pprof/", get(pprof_index_handler))
+        .route("/debug/pprof/cmdline", get(pprof_cmdline_handler))
+        .route("/debug/pprof/profile", get(pprof_profile_handler))
+        .route("/debug/pprof/symbol", any(pprof_symbol_handler))
+        .route("/debug/pprof/trace", get(pprof_trace_handler))
+}
+
+async fn pprof_index_handler() -> Response {
+    let body = concat!(
+        "<html><head><title>/debug/pprof/</title></head><body>",
+        "<a href=\"cmdline\">cmdline</a><br>",
+        "<a href=\"profile\">profile</a><br>",
+        "<a href=\"symbol\">symbol</a><br>",
+        "<a href=\"trace\">trace</a><br>",
+        "</body></html>",
+    );
+    (
+        StatusCode::OK,
+        [(header::CONTENT_TYPE, "text/html; charset=utf-8")],
+        body,
+    )
+        .into_response()
+}
+
+async fn pprof_cmdline_handler() -> Response {
+    let body = std::env::args().collect::<Vec<_>>().join("\0");
+    (
+        StatusCode::OK,
+        [(header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+        body,
+    )
+        .into_response()
+}
+
+async fn pprof_profile_handler(Query(query): Query<ProfileQuery>) -> Response {
+    let seconds = query.seconds.unwrap_or(30).clamp(1, 300);
+    let guard = match pprof::ProfilerGuard::new(100) {
+        Ok(guard) => guard,
+        Err(e) => {
+            return (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                format!("failed to start profiler: {}", e),
+            )
+                .into_response();
+        }
+    };
+
+    tokio::time::sleep(std::time::Duration::from_secs(seconds)).await;
+
+    let report = match guard.report().build() {
+        Ok(report) => report,
+        Err(e) => {
+            return (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                format!("failed to build profile report: {}", e),
+            )
+                .into_response();
+        }
+    };
+
+    let profile = match report.pprof() {
+        Ok(profile) => profile,
+        Err(e) => {
+            return (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                format!("failed to encode profile: {}", e),
+            )
+                .into_response();
+        }
+    };
+
+    let mut bytes = Vec::new();
+    if let Err(e) = profile.encode(&mut bytes) {
+        return (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            format!("failed to serialize profile: {}", e),
+        )
+            .into_response();
+    }
+
+    (
+        StatusCode::OK,
+        [(header::CONTENT_TYPE, "application/octet-stream")],
+        bytes,
+    )
+        .into_response()
+}
+
+async fn pprof_symbol_handler() -> Response {
+    (
+        StatusCode::OK,
+        [(header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+        "num_symbols: 0\n",
+    )
+        .into_response()
+}
+
+async fn pprof_trace_handler(Query(query): Query<ProfileQuery>) -> Response {
+    let seconds = query.seconds.unwrap_or(1).clamp(1, 30);
+    tokio::time::sleep(std::time::Duration::from_secs(seconds)).await;
+    Response::builder()
+        .status(StatusCode::OK)
+        .header(header::CONTENT_TYPE, "application/octet-stream")
+        .body(Body::from(Vec::<u8>::new()))
+        .unwrap()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use axum::http::Request;
+    use tower::ServiceExt;
+
+    #[tokio::test]
+    async fn test_debug_index_route() {
+        let app = build_debug_router();
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .uri("/debug/pprof/")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .await
+            .unwrap();
+        assert_eq!(response.status(), StatusCode::OK);
+    }
+
+    #[tokio::test]
+    async fn test_debug_cmdline_route() {
+        let app = build_debug_router();
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .uri("/debug/pprof/cmdline")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .await
+            .unwrap();
+        assert_eq!(response.status(), StatusCode::OK);
+    }
+}

seaweed-volume/src/server/grpc_client.rs

@@ -0,0 +1,206 @@
+use std::error::Error;
+use std::fmt;
+use std::time::Duration;
+
+use hyper::http::Uri;
+use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity};
+
+use crate::config::VolumeServerConfig;
+
+pub const GRPC_MAX_MESSAGE_SIZE: usize = 1 << 30;
+const GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(60);
+const GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(20);
+const GRPC_INITIAL_WINDOW_SIZE: u32 = 16 * 1024 * 1024;
+
+#[derive(Clone, Debug)]
+pub struct OutgoingGrpcTlsConfig {
+    cert_pem: String,
+    key_pem: String,
+    ca_pem: String,
+}
+
+#[derive(Debug)]
+pub struct GrpcClientError(String);
+
+impl fmt::Display for GrpcClientError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(&self.0)
+    }
+}
+
+impl Error for GrpcClientError {}
+
+pub fn load_outgoing_grpc_tls(
+    config: &VolumeServerConfig,
+) -> Result<Option<OutgoingGrpcTlsConfig>, GrpcClientError> {
+    if config.grpc_cert_file.is_empty()
+        || config.grpc_key_file.is_empty()
+        || config.grpc_ca_file.is_empty()
+    {
+        return Ok(None);
+    }
+
+    let cert_pem = std::fs::read_to_string(&config.grpc_cert_file).map_err(|e| {
+        GrpcClientError(format!(
+            "Failed to read outgoing gRPC cert '{}': {}",
+            config.grpc_cert_file, e
+        ))
+    })?;
+    let key_pem = std::fs::read_to_string(&config.grpc_key_file).map_err(|e| {
+        GrpcClientError(format!(
+            "Failed to read outgoing gRPC key '{}': {}",
+            config.grpc_key_file, e
+        ))
+    })?;
+    let ca_pem = std::fs::read_to_string(&config.grpc_ca_file).map_err(|e| {
+        GrpcClientError(format!(
+            "Failed to read outgoing gRPC CA '{}': {}",
+            config.grpc_ca_file, e
+        ))
+    })?;
+
+    Ok(Some(OutgoingGrpcTlsConfig {
+        cert_pem,
+        key_pem,
+        ca_pem,
+    }))
+}
+
+pub fn grpc_endpoint_uri(grpc_host_port: &str, tls: Option<&OutgoingGrpcTlsConfig>) -> String {
+    let scheme = if tls.is_some() { "https" } else { "http" };
+    format!("{}://{}", scheme, grpc_host_port)
+}
+
+pub fn build_grpc_endpoint(
+    grpc_host_port: &str,
+    tls: Option<&OutgoingGrpcTlsConfig>,
+) -> Result<Endpoint, GrpcClientError> {
+    let uri = grpc_endpoint_uri(grpc_host_port, tls);
+    let mut endpoint = Channel::from_shared(uri.clone())
+        .map_err(|e| GrpcClientError(format!("invalid gRPC endpoint {}: {}", uri, e)))?
+        .http2_keep_alive_interval(GRPC_KEEPALIVE_INTERVAL)
+        .keep_alive_timeout(GRPC_KEEPALIVE_TIMEOUT)
+        .keep_alive_while_idle(false)
+        .initial_stream_window_size(Some(GRPC_INITIAL_WINDOW_SIZE))
+        .initial_connection_window_size(Some(GRPC_INITIAL_WINDOW_SIZE))
+        .http2_adaptive_window(false);
+
+    if let Some(tls) = tls {
+        let parsed = uri
+            .parse::<Uri>()
+            .map_err(|e| GrpcClientError(format!("invalid gRPC endpoint {}: {}", uri, e)))?;
+        let host = parsed
+            .host()
+            .ok_or_else(|| GrpcClientError(format!("missing host in gRPC endpoint {}", uri)))?;
+        let tls_config = ClientTlsConfig::new()
+            .identity(Identity::from_pem(
+                tls.cert_pem.clone(),
+                tls.key_pem.clone(),
+            ))
+            .ca_certificate(Certificate::from_pem(tls.ca_pem.clone()))
+            .domain_name(host.to_string());
+        endpoint = endpoint.tls_config(tls_config).map_err(|e| {
+            GrpcClientError(format!("configure gRPC TLS for {} failed: {}", uri, e))
+        })?;
+    }
+
+    Ok(endpoint)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{build_grpc_endpoint, grpc_endpoint_uri, load_outgoing_grpc_tls};
+    use crate::config::{NeedleMapKind, ReadMode, VolumeServerConfig};
+    use crate::security::tls::TlsPolicy;
+
+    fn sample_config() -> VolumeServerConfig {
+        VolumeServerConfig {
+            port: 8080,
+            grpc_port: 18080,
+            public_port: 8080,
+            ip: "127.0.0.1".to_string(),
+            bind_ip: String::new(),
+            public_url: "127.0.0.1:8080".to_string(),
+            id: String::new(),
+            masters: vec![],
+            pre_stop_seconds: 0,
+            idle_timeout: 0,
+            data_center: String::new(),
+            rack: String::new(),
+            index_type: NeedleMapKind::InMemory,
+            disk_type: String::new(),
+            folders: vec![],
+            folder_max_limits: vec![],
+            folder_tags: vec![],
+            min_free_spaces: vec![],
+            disk_types: vec![],
+            idx_folder: String::new(),
+            white_list: vec![],
+            fix_jpg_orientation: false,
+            read_mode: ReadMode::Local,
+            cpu_profile: String::new(),
+            mem_profile: String::new(),
+            compaction_byte_per_second: 0,
+            maintenance_byte_per_second: 0,
+            file_size_limit_bytes: 0,
+            concurrent_upload_limit: 0,
+            concurrent_download_limit: 0,
+            inflight_upload_data_timeout: std::time::Duration::from_secs(0),
+            inflight_download_data_timeout: std::time::Duration::from_secs(0),
+            has_slow_read: false,
+            read_buffer_size_mb: 0,
+            ldb_timeout: 0,
+            pprof: false,
+            metrics_port: 0,
+            metrics_ip: String::new(),
+            debug: false,
+            debug_port: 0,
+            ui_enabled: false,
+            jwt_signing_key: vec![],
+            jwt_signing_expires_seconds: 0,
+            jwt_read_signing_key: vec![],
+            jwt_read_signing_expires_seconds: 0,
+            https_cert_file: String::new(),
+            https_key_file: String::new(),
+            https_ca_file: String::new(),
+            https_client_enabled: false,
+            https_client_cert_file: String::new(),
+            https_client_key_file: String::new(),
+            https_client_ca_file: String::new(),
+            grpc_cert_file: String::new(),
+            grpc_key_file: String::new(),
+            grpc_ca_file: String::new(),
+            grpc_allowed_wildcard_domain: String::new(),
+            grpc_volume_allowed_common_names: vec![],
+            tls_policy: TlsPolicy::default(),
+            enable_write_queue: false,
+            security_file: String::new(),
+        }
+    }
+
+    #[test]
+    fn test_grpc_endpoint_uri_uses_https_when_tls_enabled() {
+        let tls = super::OutgoingGrpcTlsConfig {
+            cert_pem: "cert".to_string(),
+            key_pem: "key".to_string(),
+            ca_pem: "ca".to_string(),
+        };
+        assert_eq!(
+            grpc_endpoint_uri("master.example.com:19333", Some(&tls)),
+            "https://master.example.com:19333"
+        );
+    }
+
+    #[test]
+    fn test_load_outgoing_grpc_tls_requires_cert_key_and_ca() {
+        let mut config = sample_config();
+        config.grpc_cert_file = "/tmp/client.pem".to_string();
+        assert!(load_outgoing_grpc_tls(&config).unwrap().is_none());
+    }
+
+    #[test]
+    fn test_build_grpc_endpoint_without_tls_uses_http_scheme() {
+        let endpoint = build_grpc_endpoint("127.0.0.1:19333", None).unwrap();
+        assert_eq!(endpoint.uri().scheme_str(), Some("http"));
+    }
+}

seaweed-volume/src/server/memory_status.rs

@@ -0,0 +1,102 @@
+use crate::pb::volume_server_pb;
+
+pub fn collect_mem_status() -> volume_server_pb::MemStatus {
+    #[allow(unused_mut)]
+    let mut mem = volume_server_pb::MemStatus {
+        goroutines: 1,
+        ..Default::default()
+    };
+
+    #[cfg(target_os = "linux")]
+    {
+        if let Some((all, free)) = get_system_memory_linux() {
+            mem.all = all;
+            mem.free = free;
+            mem.used = all.saturating_sub(free);
+        }
+
+        if let Some(status) = read_process_status_linux() {
+            if status.threads > 0 {
+                mem.goroutines = status.threads as i32;
+            }
+            if let Some(rss) = status.rss {
+                mem.self_ = rss;
+            }
+            if let Some(heap) = status.data.or(status.rss) {
+                mem.heap = heap;
+            }
+            if let Some(stack) = status.stack {
+                mem.stack = stack;
+            }
+        }
+    }
+
+    mem
+}
+
+#[cfg(target_os = "linux")]
+fn get_system_memory_linux() -> Option<(u64, u64)> {
+    unsafe {
+        let mut info: libc::sysinfo = std::mem::zeroed();
+        if libc::sysinfo(&mut info) == 0 {
+            let unit = info.mem_unit as u64;
+            let total = info.totalram as u64 * unit;
+            let free = info.freeram as u64 * unit;
+            return Some((total, free));
+        }
+    }
+    None
+}
+
+#[cfg(target_os = "linux")]
+#[derive(Default)]
+struct ProcessStatus {
+    threads: u64,
+    rss: Option<u64>,
+    data: Option<u64>,
+    stack: Option<u64>,
+}
+
+#[cfg(target_os = "linux")]
+fn read_process_status_linux() -> Option<ProcessStatus> {
+    let status = std::fs::read_to_string("/proc/self/status").ok()?;
+    let mut out = ProcessStatus::default();
+
+    for line in status.lines() {
+        if let Some(value) = line.strip_prefix("Threads:") {
+            out.threads = value.trim().parse().ok()?;
+            continue;
+        }
+        if let Some(value) = parse_proc_status_kib_field(line, "VmRSS:") {
+            out.rss = Some(value);
+            continue;
+        }
+        if let Some(value) = parse_proc_status_kib_field(line, "VmData:") {
+            out.data = Some(value);
+            continue;
+        }
+        if let Some(value) = parse_proc_status_kib_field(line, "VmStk:") {
+            out.stack = Some(value);
+        }
+    }
+
+    Some(out)
+}
+
+#[cfg(target_os = "linux")]
+fn parse_proc_status_kib_field(line: &str, prefix: &str) -> Option<u64> {
+    let raw = line.strip_prefix(prefix)?.trim();
+    let value = raw.strip_suffix(" kB").unwrap_or(raw).trim();
+    value.parse::<u64>().ok().map(|kib| kib * 1024)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::collect_mem_status;
+
+    #[test]
+    fn test_collect_mem_status_reports_live_process_state() {
+        let mem = collect_mem_status();
+        assert!(mem.goroutines > 0);
+    }
+}

seaweed-volume/src/server/mod.rs

@@ -0,0 +1,14 @@
+#[cfg(unix)]
+pub mod debug;
+pub mod grpc_client;
+pub mod grpc_server;
+pub mod handlers;
+pub mod heartbeat;
+pub mod memory_status;
+#[cfg(unix)]
+pub mod profiling;
+pub mod request_id;
+pub mod server_stats;
+pub mod ui;
+pub mod volume_server;
+pub mod write_queue;

seaweed-volume/src/server/profiling.rs

@@ -0,0 +1,187 @@
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+
+use pprof::protos::Message;
+
+use crate::config::VolumeServerConfig;
+
+const GO_CPU_PROFILE_FREQUENCY: i32 = 100;
+const GO_PPROF_BLOCKLIST: [&str; 4] = ["libc", "libgcc", "pthread", "vdso"];
+
+pub struct CpuProfileSession {
+    output_path: PathBuf,
+    guard: pprof::ProfilerGuard<'static>,
+}
+
+impl CpuProfileSession {
+    pub fn start(config: &VolumeServerConfig) -> Result<Option<Self>, String> {
+        if config.cpu_profile.is_empty() {
+            if !config.mem_profile.is_empty() && !config.pprof {
+                tracing::warn!(
+                    "--memprofile is not yet supported in the Rust volume server; ignoring '{}'",
+                    config.mem_profile
+                );
+            }
+            return Ok(None);
+        }
+
+        if config.pprof {
+            tracing::info!(
+                "--pprof is enabled; ignoring --cpuprofile '{}' and --memprofile '{}'",
+                config.cpu_profile,
+                config.mem_profile
+            );
+            return Ok(None);
+        }
+
+        if !config.mem_profile.is_empty() {
+            tracing::warn!(
+                "--memprofile is not yet supported in the Rust volume server; only --cpuprofile '{}' will be written",
+                config.cpu_profile
+            );
+        }
+
+        let guard = pprof::ProfilerGuardBuilder::default()
+            .frequency(GO_CPU_PROFILE_FREQUENCY)
+            .blocklist(&GO_PPROF_BLOCKLIST)
+            .build()
+            .map_err(|e| {
+                format!(
+                    "Failed to start CPU profiler '{}': {}",
+                    config.cpu_profile, e
+                )
+            })?;
+
+        Ok(Some(Self {
+            output_path: PathBuf::from(&config.cpu_profile),
+            guard,
+        }))
+    }
+
+    pub fn finish(self) -> Result<(), String> {
+        let report = self
+            .guard
+            .report()
+            .build()
+            .map_err(|e| format!("Failed to build CPU profile report: {}", e))?;
+        let profile = report
+            .pprof()
+            .map_err(|e| format!("Failed to encode CPU profile report: {}", e))?;
+
+        let mut bytes = Vec::new();
+        profile
+            .encode(&mut bytes)
+            .map_err(|e| format!("Failed to serialize CPU profile report: {}", e))?;
+
+        let mut file = File::create(&self.output_path).map_err(|e| {
+            format!(
+                "Failed to create CPU profile '{}': {}",
+                self.output_path.display(),
+                e
+            )
+        })?;
+        file.write_all(&bytes).map_err(|e| {
+            format!(
+                "Failed to write CPU profile '{}': {}",
+                self.output_path.display(),
+                e
+            )
+        })?;
+        file.flush().map_err(|e| {
+            format!(
+                "Failed to flush CPU profile '{}': {}",
+                self.output_path.display(),
+                e
+            )
+        })?;
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::CpuProfileSession;
+    use crate::config::{NeedleMapKind, ReadMode, VolumeServerConfig};
+    use crate::security::tls::TlsPolicy;
+
+    fn sample_config() -> VolumeServerConfig {
+        VolumeServerConfig {
+            port: 8080,
+            grpc_port: 18080,
+            public_port: 8080,
+            ip: "127.0.0.1".to_string(),
+            bind_ip: "127.0.0.1".to_string(),
+            public_url: "127.0.0.1:8080".to_string(),
+            id: "127.0.0.1:8080".to_string(),
+            masters: vec![],
+            pre_stop_seconds: 0,
+            idle_timeout: 0,
+            data_center: String::new(),
+            rack: String::new(),
+            index_type: NeedleMapKind::InMemory,
+            disk_type: String::new(),
+            folders: vec!["/tmp".to_string()],
+            folder_max_limits: vec![8],
+            folder_tags: vec![vec![]],
+            min_free_spaces: vec![],
+            disk_types: vec![String::new()],
+            idx_folder: String::new(),
+            white_list: vec![],
+            fix_jpg_orientation: false,
+            read_mode: ReadMode::Local,
+            cpu_profile: String::new(),
+            mem_profile: String::new(),
+            compaction_byte_per_second: 0,
+            maintenance_byte_per_second: 0,
+            file_size_limit_bytes: 0,
+            concurrent_upload_limit: 0,
+            concurrent_download_limit: 0,
+            inflight_upload_data_timeout: std::time::Duration::from_secs(0),
+            inflight_download_data_timeout: std::time::Duration::from_secs(0),
+            has_slow_read: false,
+            read_buffer_size_mb: 4,
+            ldb_timeout: 0,
+            pprof: false,
+            metrics_port: 0,
+            metrics_ip: String::new(),
+            debug: false,
+            debug_port: 0,
+            ui_enabled: false,
+            jwt_signing_key: vec![],
+            jwt_signing_expires_seconds: 0,
+            jwt_read_signing_key: vec![],
+            jwt_read_signing_expires_seconds: 0,
+            https_cert_file: String::new(),
+            https_key_file: String::new(),
+            https_ca_file: String::new(),
+            https_client_enabled: false,
+            https_client_cert_file: String::new(),
+            https_client_key_file: String::new(),
+            https_client_ca_file: String::new(),
+            grpc_cert_file: String::new(),
+            grpc_key_file: String::new(),
+            grpc_ca_file: String::new(),
+            grpc_allowed_wildcard_domain: String::new(),
+            grpc_volume_allowed_common_names: vec![],
+            tls_policy: TlsPolicy::default(),
+            enable_write_queue: false,
+            security_file: String::new(),
+        }
+    }
+
+    #[test]
+    fn test_cpu_profile_session_skips_when_disabled() {
+        let config = sample_config();
+        assert!(CpuProfileSession::start(&config).unwrap().is_none());
+    }
+
+    #[test]
+    fn test_cpu_profile_session_skips_when_pprof_enabled() {
+        let mut config = sample_config();
+        config.cpu_profile = "/tmp/cpu.pb".to_string();
+        config.pprof = true;
+        assert!(CpuProfileSession::start(&config).unwrap().is_none());
+    }
+}

seaweed-volume/src/server/request_id.rs

@@ -0,0 +1,137 @@
+use std::future::Future;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use hyper::http::{self, HeaderValue};
+use tonic::metadata::MetadataValue;
+use tonic::{Request, Status};
+use tower::{Layer, Service};
+
+tokio::task_local! {
+    static CURRENT_REQUEST_ID: String;
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct GrpcRequestIdLayer;
+
+#[derive(Clone, Debug)]
+pub struct GrpcRequestIdService<S> {
+    inner: S,
+}
+
+impl<S> Layer<S> for GrpcRequestIdLayer {
+    type Service = GrpcRequestIdService<S>;
+
+    fn layer(&self, inner: S) -> Self::Service {
+        GrpcRequestIdService { inner }
+    }
+}
+
+impl<S, B> Service<http::Request<B>> for GrpcRequestIdService<S>
+where
+    S: Service<http::Request<B>, Response = http::Response<tonic::body::BoxBody>> + Send + 'static,
+    S::Future: Send + 'static,
+    B: Send + 'static,
+{
+    type Response = http::Response<tonic::body::BoxBody>;
+    type Error = S::Error;
+    type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        self.inner.poll_ready(cx)
+    }
+
+    fn call(&mut self, mut request: http::Request<B>) -> Self::Future {
+        let request_id = match request.headers().get("x-amz-request-id") {
+            Some(value) => match value.to_str() {
+                Ok(value) if !value.is_empty() => value.to_owned(),
+                _ => generate_grpc_request_id(),
+            },
+            None => generate_grpc_request_id(),
+        };
+
+        if let Ok(value) = HeaderValue::from_str(&request_id) {
+            request.headers_mut().insert("x-amz-request-id", value);
+        }
+
+        let future = self.inner.call(request);
+
+        Box::pin(async move {
+            let mut response: http::Response<tonic::body::BoxBody> =
+                scope_request_id(request_id.clone(), future).await?;
+            if let Ok(value) = HeaderValue::from_str(&request_id) {
+                response.headers_mut().insert("x-amz-request-id", value);
+            }
+            Ok(response)
+        })
+    }
+}
+
+pub async fn scope_request_id<F, T>(request_id: String, future: F) -> T
+where
+    F: Future<Output = T>,
+{
+    CURRENT_REQUEST_ID.scope(request_id, future).await
+}
+
+pub fn current_request_id() -> Option<String> {
+    CURRENT_REQUEST_ID.try_with(Clone::clone).ok()
+}
+
+pub fn outgoing_request_id_interceptor(mut request: Request<()>) -> Result<Request<()>, Status> {
+    if let Some(request_id) = current_request_id() {
+        let value = MetadataValue::try_from(request_id.as_str())
+            .map_err(|_| Status::internal("invalid scoped request id"))?;
+        request.metadata_mut().insert("x-amz-request-id", value);
+    }
+    Ok(request)
+}
+
+pub fn generate_http_request_id() -> String {
+    use rand::Rng;
+
+    let nanos = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_nanos() as u64;
+    let rand_val: u32 = rand::thread_rng().gen();
+    format!("{:X}{:08X}", nanos, rand_val)
+}
+
+fn generate_grpc_request_id() -> String {
+    uuid::Uuid::new_v4().to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{current_request_id, outgoing_request_id_interceptor, scope_request_id};
+    use tonic::Request;
+
+    #[tokio::test]
+    async fn test_scope_request_id_exposes_current_value() {
+        let request_id = "req-123".to_string();
+        let current = scope_request_id(
+            request_id.clone(),
+            async move { current_request_id().unwrap() },
+        )
+        .await;
+        assert_eq!(current, request_id);
+    }
+
+    #[tokio::test]
+    async fn test_outgoing_request_id_interceptor_propagates_scope() {
+        let request = scope_request_id("req-456".to_string(), async move {
+            outgoing_request_id_interceptor(Request::new(())).unwrap()
+        })
+        .await;
+        assert_eq!(
+            request
+                .metadata()
+                .get("x-amz-request-id")
+                .unwrap()
+                .to_str()
+                .unwrap(),
+            "req-456"
+        );
+    }
+}

seaweed-volume/src/server/server_stats.rs

@@ -0,0 +1,248 @@
+use chrono::{Datelike, Local, Timelike};
+use serde::Serialize;
+use std::sync::{LazyLock, Mutex};
+use std::time::Instant;
+
+static START_TIME: LazyLock<Instant> = LazyLock::new(Instant::now);
+static SERVER_STATS: LazyLock<ServerStats> = LazyLock::new(ServerStats::default);
+
+#[derive(Default)]
+pub struct ServerStats {
+    inner: Mutex<ServerStatsInner>,
+}
+
+#[derive(Default)]
+struct ServerStatsInner {
+    requests: DurationCounter,
+    connections: DurationCounter,
+    assign_requests: DurationCounter,
+    read_requests: DurationCounter,
+    write_requests: DurationCounter,
+    delete_requests: DurationCounter,
+    bytes_in: DurationCounter,
+    bytes_out: DurationCounter,
+}
+
+#[derive(Clone, Serialize)]
+#[serde(rename_all = "PascalCase")]
+pub struct ServerStatsSnapshot {
+    pub requests: DurationCounterSnapshot,
+    pub connections: DurationCounterSnapshot,
+    pub assign_requests: DurationCounterSnapshot,
+    pub read_requests: DurationCounterSnapshot,
+    pub write_requests: DurationCounterSnapshot,
+    pub delete_requests: DurationCounterSnapshot,
+    pub bytes_in: DurationCounterSnapshot,
+    pub bytes_out: DurationCounterSnapshot,
+}
+
+#[derive(Clone, Serialize)]
+#[serde(rename_all = "PascalCase")]
+pub struct DurationCounterSnapshot {
+    pub minute_counter: RoundRobinCounterSnapshot,
+    pub hour_counter: RoundRobinCounterSnapshot,
+    pub day_counter: RoundRobinCounterSnapshot,
+    pub week_counter: RoundRobinCounterSnapshot,
+}
+
+#[derive(Clone, Serialize)]
+#[serde(rename_all = "PascalCase")]
+pub struct RoundRobinCounterSnapshot {
+    pub last_index: i32,
+    pub values: Vec<i64>,
+    pub counts: Vec<i64>,
+}
+
+#[derive(Clone)]
+struct DurationCounter {
+    minute_counter: RoundRobinCounter,
+    hour_counter: RoundRobinCounter,
+    day_counter: RoundRobinCounter,
+    week_counter: RoundRobinCounter,
+}
+
+#[derive(Clone)]
+struct RoundRobinCounter {
+    last_index: i32,
+    values: Vec<i64>,
+    counts: Vec<i64>,
+}
+
+impl Default for DurationCounter {
+    fn default() -> Self {
+        Self {
+            minute_counter: RoundRobinCounter::new(60),
+            hour_counter: RoundRobinCounter::new(60),
+            day_counter: RoundRobinCounter::new(24),
+            week_counter: RoundRobinCounter::new(7),
+        }
+    }
+}
+
+impl RoundRobinCounter {
+    fn new(slots: usize) -> Self {
+        Self {
+            last_index: -1,
+            values: vec![0; slots],
+            counts: vec![0; slots],
+        }
+    }
+
+    fn add(&mut self, index: usize, val: i64) {
+        if index >= self.values.len() {
+            return;
+        }
+        while self.last_index != index as i32 {
+            self.last_index = (self.last_index + 1).rem_euclid(self.values.len() as i32);
+            self.values[self.last_index as usize] = 0;
+            self.counts[self.last_index as usize] = 0;
+        }
+        self.values[index] += val;
+        self.counts[index] += 1;
+    }
+
+    fn snapshot(&self) -> RoundRobinCounterSnapshot {
+        RoundRobinCounterSnapshot {
+            last_index: self.last_index,
+            values: self.values.clone(),
+            counts: self.counts.clone(),
+        }
+    }
+}
+
+impl DurationCounter {
+    fn add_now(&mut self, val: i64) {
+        let now = Local::now();
+        self.minute_counter.add(now.second() as usize, val);
+        self.hour_counter.add(now.minute() as usize, val);
+        self.day_counter.add(now.hour() as usize, val);
+        self.week_counter
+            .add(now.weekday().num_days_from_sunday() as usize, val);
+    }
+
+    fn snapshot(&self) -> DurationCounterSnapshot {
+        DurationCounterSnapshot {
+            minute_counter: self.minute_counter.snapshot(),
+            hour_counter: self.hour_counter.snapshot(),
+            day_counter: self.day_counter.snapshot(),
+            week_counter: self.week_counter.snapshot(),
+        }
+    }
+}
+
+impl ServerStatsInner {
+    fn snapshot(&self) -> ServerStatsSnapshot {
+        ServerStatsSnapshot {
+            requests: self.requests.snapshot(),
+            connections: self.connections.snapshot(),
+            assign_requests: self.assign_requests.snapshot(),
+            read_requests: self.read_requests.snapshot(),
+            write_requests: self.write_requests.snapshot(),
+            delete_requests: self.delete_requests.snapshot(),
+            bytes_in: self.bytes_in.snapshot(),
+            bytes_out: self.bytes_out.snapshot(),
+        }
+    }
+}
+
+impl ServerStats {
+    fn update<F>(&self, update: F)
+    where
+        F: FnOnce(&mut ServerStatsInner),
+    {
+        let mut inner = self.inner.lock().unwrap();
+        update(&mut inner);
+    }
+
+    fn snapshot(&self) -> ServerStatsSnapshot {
+        self.inner.lock().unwrap().snapshot()
+    }
+}
+
+impl RoundRobinCounterSnapshot {
+    pub fn to_list(&self) -> Vec<i64> {
+        if self.values.is_empty() {
+            return Vec::new();
+        }
+        let mut ret = Vec::with_capacity(self.values.len());
+        let mut index = self.last_index;
+        let mut step = self.values.len();
+        while step > 0 {
+            step -= 1;
+            index += 1;
+            if index >= self.values.len() as i32 {
+                index = 0;
+            }
+            ret.push(self.values[index as usize]);
+        }
+        ret
+    }
+}
+
+pub fn init_process_start() {
+    LazyLock::force(&START_TIME);
+    LazyLock::force(&SERVER_STATS);
+}
+
+pub fn uptime_string() -> String {
+    let secs = START_TIME.elapsed().as_secs();
+    let hours = secs / 3600;
+    let minutes = (secs % 3600) / 60;
+    let seconds = secs % 60;
+    let mut out = String::new();
+    if hours > 0 {
+        out.push_str(&format!("{}h", hours));
+    }
+    if hours > 0 || minutes > 0 {
+        out.push_str(&format!("{}m", minutes));
+    }
+    out.push_str(&format!("{}s", seconds));
+    out
+}
+
+pub fn snapshot() -> ServerStatsSnapshot {
+    SERVER_STATS.snapshot()
+}
+
+pub fn record_request_open() {
+    SERVER_STATS.update(|inner| inner.requests.add_now(1));
+}
+
+pub fn record_request_close() {
+    SERVER_STATS.update(|inner| inner.requests.add_now(-1));
+}
+
+pub fn record_connection_open() {
+    SERVER_STATS.update(|inner| inner.connections.add_now(1));
+}
+
+pub fn record_connection_close() {
+    SERVER_STATS.update(|inner| inner.connections.add_now(-1));
+}
+
+pub fn record_read_request() {
+    SERVER_STATS.update(|inner| inner.read_requests.add_now(1));
+}
+
+pub fn record_write_request() {
+    SERVER_STATS.update(|inner| inner.write_requests.add_now(1));
+}
+
+pub fn record_delete_request() {
+    SERVER_STATS.update(|inner| inner.delete_requests.add_now(1));
+}
+
+pub fn record_bytes_in(bytes: i64) {
+    SERVER_STATS.update(|inner| inner.bytes_in.add_now(bytes));
+}
+
+pub fn record_bytes_out(bytes: i64) {
+    SERVER_STATS.update(|inner| inner.bytes_out.add_now(bytes));
+}
+
+#[cfg(test)]
+pub fn reset_for_tests() {
+    LazyLock::force(&START_TIME);
+    let mut inner = SERVER_STATS.inner.lock().unwrap();
+    *inner = ServerStatsInner::default();
+}

seaweed-volume/src/server/ui.rs

@@ -0,0 +1,507 @@
+use std::fmt::Write as _;
+
+use crate::server::server_stats;
+use crate::server::volume_server::VolumeServerState;
+use crate::storage::store::Store;
+
+pub struct EmbeddedAsset {
+    pub content_type: &'static str,
+    pub bytes: &'static [u8],
+}
+
+struct UiDiskRow {
+    dir: String,
+    disk_type: String,
+    all: u64,
+    free: u64,
+    used: u64,
+}
+
+struct UiVolumeRow {
+    id: u32,
+    collection: String,
+    disk_type: String,
+    size: u64,
+    file_count: i64,
+    delete_count: i64,
+    deleted_byte_count: u64,
+    ttl: String,
+    read_only: bool,
+    version: u32,
+    remote_storage_name: String,
+    remote_storage_key: String,
+}
+
+struct UiEcShardRow {
+    shard_id: u8,
+    size: u64,
+}
+
+struct UiEcVolumeRow {
+    volume_id: u32,
+    collection: String,
+    size: u64,
+    shards: Vec<UiEcShardRow>,
+    created_at: String,
+}
+
+pub fn favicon_asset() -> EmbeddedAsset {
+    EmbeddedAsset {
+        content_type: "image/x-icon",
+        bytes: include_bytes!(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/../weed/static/favicon.ico"
+        )),
+    }
+}
+
+pub fn lookup_static_asset(path: &str) -> Option<EmbeddedAsset> {
+    let path = path.trim_start_matches('/');
+    let asset = match path {
+        "bootstrap/3.3.1/css/bootstrap.min.css" => EmbeddedAsset {
+            content_type: "text/css; charset=utf-8",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/bootstrap/3.3.1/css/bootstrap.min.css"
+            )),
+        },
+        "bootstrap/3.3.1/fonts/glyphicons-halflings-regular.eot" => EmbeddedAsset {
+            content_type: "application/vnd.ms-fontobject",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.eot"
+            )),
+        },
+        "bootstrap/3.3.1/fonts/glyphicons-halflings-regular.svg" => EmbeddedAsset {
+            content_type: "image/svg+xml",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.svg"
+            )),
+        },
+        "bootstrap/3.3.1/fonts/glyphicons-halflings-regular.ttf" => EmbeddedAsset {
+            content_type: "font/ttf",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.ttf"
+            )),
+        },
+        "bootstrap/3.3.1/fonts/glyphicons-halflings-regular.woff" => EmbeddedAsset {
+            content_type: "font/woff",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.woff"
+            )),
+        },
+        "images/folder.gif" => EmbeddedAsset {
+            content_type: "image/gif",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/images/folder.gif"
+            )),
+        },
+        "javascript/jquery-3.6.0.min.js" => EmbeddedAsset {
+            content_type: "application/javascript; charset=utf-8",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/javascript/jquery-3.6.0.min.js"
+            )),
+        },
+        "javascript/jquery-sparklines/2.1.2/jquery.sparkline.min.js" => EmbeddedAsset {
+            content_type: "application/javascript; charset=utf-8",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/javascript/jquery-sparklines/2.1.2/jquery.sparkline.min.js"
+            )),
+        },
+        "seaweed50x50.png" => EmbeddedAsset {
+            content_type: "image/png",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/seaweed50x50.png"
+            )),
+        },
+        _ => return None,
+    };
+    Some(asset)
+}
+
+pub fn render_volume_server_html(state: &VolumeServerState) -> String {
+    let counters = server_stats::snapshot();
+    let (disk_rows, volume_rows, remote_volume_rows, ec_volume_rows) = {
+        let store = state.store.read().unwrap();
+        collect_ui_data(&store)
+    };
+
+    let masters = if state.master_urls.is_empty() {
+        "[]".to_string()
+    } else {
+        format!("[{}]", state.master_urls.join(" "))
+    };
+    let uptime = server_stats::uptime_string();
+    let read_week = join_i64(&counters.read_requests.week_counter.to_list());
+    let read_day = join_i64(&counters.read_requests.day_counter.to_list());
+    let read_hour = join_i64(&counters.read_requests.hour_counter.to_list());
+    let read_minute = join_i64(&counters.read_requests.minute_counter.to_list());
+
+    let mut disk_rows_html = String::new();
+    for disk in &disk_rows {
+        let _ = write!(
+            disk_rows_html,
+            "<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{:.2}%</td></tr>",
+            escape_html(&disk.dir),
+            escape_html(&disk.disk_type),
+            bytes_to_human_readable(disk.all),
+            bytes_to_human_readable(disk.free),
+            percent_from(disk.all, disk.used),
+        );
+    }
+
+    let mut volume_rows_html = String::new();
+    for volume in &volume_rows {
+        let _ = write!(
+            volume_rows_html,
+            "<tr><td><code>{}</code></td><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{} / {}</td><td>{}</td><td>{}</td><td>{}</td></tr>",
+            volume.id,
+            escape_html(&volume.collection),
+            escape_html(&volume.disk_type),
+            bytes_to_human_readable(volume.size),
+            volume.file_count,
+            volume.delete_count,
+            bytes_to_human_readable(volume.deleted_byte_count),
+            escape_html(&volume.ttl),
+            volume.read_only,
+            volume.version,
+        );
+    }
+
+    let remote_section = if remote_volume_rows.is_empty() {
+        String::new()
+    } else {
+        let mut remote_rows_html = String::new();
+        for volume in &remote_volume_rows {
+            let _ = write!(
+                remote_rows_html,
+                "<tr><td><code>{}</code></td><td>{}</td><td>{}</td><td>{}</td><td>{} / {}</td><td>{}</td><td>{}</td></tr>",
+                volume.id,
+                escape_html(&volume.collection),
+                bytes_to_human_readable(volume.size),
+                volume.file_count,
+                volume.delete_count,
+                bytes_to_human_readable(volume.deleted_byte_count),
+                escape_html(&volume.remote_storage_name),
+                escape_html(&volume.remote_storage_key),
+            );
+        }
+        format!(
+            r#"<div class="row">
+            <h2>Remote Volumes</h2>
+            <table class="table table-striped">
+                <thead>
+                    <tr>
+                        <th>Id</th>
+                        <th>Collection</th>
+                        <th>Size</th>
+                        <th>Files</th>
+                        <th>Trash</th>
+                        <th>Remote</th>
+                        <th>Key</th>
+                    </tr>
+                </thead>
+                <tbody>{}</tbody>
+            </table>
+        </div>"#,
+            remote_rows_html
+        )
+    };
+
+    let ec_section = if ec_volume_rows.is_empty() {
+        String::new()
+    } else {
+        let mut ec_rows_html = String::new();
+        for ec in &ec_volume_rows {
+            let mut shard_labels = String::new();
+            for shard in &ec.shards {
+                let _ = write!(
+                    shard_labels,
+                    "<span class=\"label label-info\" style=\"margin-right: 5px;\">{}: {}</span>",
+                    shard.shard_id,
+                    bytes_to_human_readable(shard.size)
+                );
+            }
+            let _ = write!(
+                ec_rows_html,
+                "<tr><td><code>{}</code></td><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>",
+                ec.volume_id,
+                escape_html(&ec.collection),
+                bytes_to_human_readable(ec.size),
+                shard_labels,
+                escape_html(&ec.created_at),
+            );
+        }
+        format!(
+            r#"<div class="row">
+            <h2>Erasure Coding Shards</h2>
+            <table class="table table-striped">
+                <thead>
+                    <tr>
+                        <th>Id</th>
+                        <th>Collection</th>
+                        <th>Total Size</th>
+                        <th>Shard Details</th>
+                        <th>CreatedAt</th>
+                    </tr>
+                </thead>
+                <tbody>{}</tbody>
+            </table>
+        </div>"#,
+            ec_rows_html
+        )
+    };
+
+    format!(
+        r#"<!DOCTYPE html>
+<html>
+<head>
+    <title>SeaweedFS {version}</title>
+    <link rel="stylesheet" href="/seaweedfsstatic/bootstrap/3.3.1/css/bootstrap.min.css">
+    <script type="text/javascript" src="/seaweedfsstatic/javascript/jquery-3.6.0.min.js"></script>
+    <script type="text/javascript" src="/seaweedfsstatic/javascript/jquery-sparklines/2.1.2/jquery.sparkline.min.js"></script>
+    <script type="text/javascript">
+        $(function () {{
+            var periods = ['second', 'minute', 'hour', 'day'];
+            for (var i = 0; i < periods.length; i++) {{
+                var period = periods[i];
+                $('.inlinesparkline-' + period).sparkline('html', {{
+                    type: 'line',
+                    barColor: 'red',
+                    tooltipSuffix: ' request per ' + period
+                }});
+            }}
+        }});
+    </script>
+    <style>
+        #jqstooltip {{
+            height: 28px !important;
+            width: 150px !important;
+        }}
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="page-header">
+            <h1>
+                <a href="https://github.com/seaweedfs/seaweedfs"><img src="/seaweedfsstatic/seaweed50x50.png" alt="SeaweedFS"></a>
+                SeaweedFS <small>{version}</small>
+            </h1>
+        </div>
+
+        <div class="row">
+            <div class="col-sm-6">
+                <h2>Disk Stats</h2>
+                <table class="table table-striped">
+                    <thead>
+                        <tr>
+                            <th>Path</th>
+                            <th>Disk</th>
+                            <th>Total</th>
+                            <th>Free</th>
+                            <th>Usage</th>
+                        </tr>
+                    </thead>
+                    <tbody>{disk_rows_html}</tbody>
+                </table>
+            </div>
+
+            <div class="col-sm-6">
+                <h2>System Stats</h2>
+                <table class="table table-condensed table-striped">
+                    <tr><th>Masters</th><td>{masters}</td></tr>
+                    <tr><th>Weekly # ReadRequests</th><td><span class="inlinesparkline-day">{read_week}</span></td></tr>
+                    <tr><th>Daily # ReadRequests</th><td><span class="inlinesparkline-hour">{read_day}</span></td></tr>
+                    <tr><th>Hourly # ReadRequests</th><td><span class="inlinesparkline-minute">{read_hour}</span></td></tr>
+                    <tr><th>Last Minute # ReadRequests</th><td><span class="inlinesparkline-second">{read_minute}</span></td></tr>
+                    <tr><th>Up Time</th><td>{uptime}</td></tr>
+                </table>
+            </div>
+        </div>
+
+        <div class="row">
+            <h2>Volumes</h2>
+            <table class="table table-striped">
+                <thead>
+                    <tr>
+                        <th>Id</th>
+                        <th>Collection</th>
+                        <th>Disk</th>
+                        <th>Data Size</th>
+                        <th>Files</th>
+                        <th>Trash</th>
+                        <th>TTL</th>
+                        <th>ReadOnly</th>
+                        <th>Version</th>
+                    </tr>
+                </thead>
+                <tbody>{volume_rows_html}</tbody>
+            </table>
+        </div>
+
+        {remote_section}
+        {ec_section}
+    </div>
+</body>
+</html>"#,
+        version = escape_html(crate::version::version()),
+        disk_rows_html = disk_rows_html,
+        masters = escape_html(&masters),
+        read_week = read_week,
+        read_day = read_day,
+        read_hour = read_hour,
+        read_minute = read_minute,
+        uptime = escape_html(&uptime),
+        volume_rows_html = volume_rows_html,
+        remote_section = remote_section,
+        ec_section = ec_section,
+    )
+}
+
+fn collect_ui_data(
+    store: &Store,
+) -> (
+    Vec<UiDiskRow>,
+    Vec<UiVolumeRow>,
+    Vec<UiVolumeRow>,
+    Vec<UiEcVolumeRow>,
+) {
+    let mut disk_rows = Vec::new();
+    let mut volumes = Vec::new();
+    let mut remote_volumes = Vec::new();
+    let mut ec_volumes = Vec::new();
+
+    for loc in &store.locations {
+        let dir = absolute_display_path(&loc.directory);
+        let (all, free) = crate::storage::disk_location::get_disk_stats(&dir);
+        disk_rows.push(UiDiskRow {
+            dir,
+            disk_type: loc.disk_type.to_string(),
+            all,
+            free,
+            used: all.saturating_sub(free),
+        });
+
+        for (_, volume) in loc.volumes() {
+            let (remote_storage_name, remote_storage_key) = volume.remote_storage_name_key();
+            let row = UiVolumeRow {
+                id: volume.id.0,
+                collection: volume.collection.clone(),
+                disk_type: loc.disk_type.to_string(),
+                size: volume.content_size(),
+                file_count: volume.file_count(),
+                delete_count: volume.deleted_count(),
+                deleted_byte_count: volume.deleted_size(),
+                ttl: volume.super_block.ttl.to_string(),
+                read_only: volume.is_read_only(),
+                version: volume.version().0 as u32,
+                remote_storage_name,
+                remote_storage_key,
+            };
+            if row.remote_storage_name.is_empty() {
+                volumes.push(row);
+            } else {
+                remote_volumes.push(row);
+            }
+        }
+
+        for (_, ec_volume) in loc.ec_volumes() {
+            let mut shards = Vec::new();
+            let mut total_size = 0u64;
+            let mut created_at = String::from("-");
+            for shard in ec_volume.shards.iter().flatten() {
+                let shard_size = shard.file_size().max(0) as u64;
+                total_size = total_size.saturating_add(shard_size);
+                shards.push(UiEcShardRow {
+                    shard_id: shard.shard_id,
+                    size: shard_size,
+                });
+                if created_at == "-" {
+                    if let Ok(metadata) = std::fs::metadata(shard.file_name()) {
+                        if let Ok(modified) = metadata.modified() {
+                            let ts: chrono::DateTime<chrono::Local> = modified.into();
+                            created_at = ts.format("%Y-%m-%d %H:%M").to_string();
+                        }
+                    }
+                }
+            }
+            let preferred_size = ec_volume.dat_file_size.max(0) as u64;
+            ec_volumes.push(UiEcVolumeRow {
+                volume_id: ec_volume.volume_id.0,
+                collection: ec_volume.collection.clone(),
+                size: preferred_size.max(total_size),
+                shards,
+                created_at,
+            });
+        }
+    }
+
+    disk_rows.sort_by(|left, right| left.dir.cmp(&right.dir));
+    volumes.sort_by_key(|row| row.id);
+    remote_volumes.sort_by_key(|row| row.id);
+    ec_volumes.sort_by_key(|row| row.volume_id);
+
+    (disk_rows, volumes, remote_volumes, ec_volumes)
+}
+
+fn absolute_display_path(path: &str) -> String {
+    let p = std::path::Path::new(path);
+    if p.is_absolute() {
+        return path.to_string();
+    }
+    std::env::current_dir()
+        .map(|cwd| cwd.join(p).to_string_lossy().to_string())
+        .unwrap_or_else(|_| path.to_string())
+}
+
+fn join_i64(values: &[i64]) -> String {
+    values
+        .iter()
+        .map(std::string::ToString::to_string)
+        .collect::<Vec<_>>()
+        .join(",")
+}
+
+fn percent_from(total: u64, part: u64) -> f64 {
+    if total == 0 {
+        return 0.0;
+    }
+    (part as f64 / total as f64) * 100.0
+}
+
+fn bytes_to_human_readable(bytes: u64) -> String {
+    const UNIT: u64 = 1024;
+    if bytes < UNIT {
+        return format!("{} B", bytes);
+    }
+
+    let mut div = UNIT;
+    let mut exp = 0usize;
+    let mut n = bytes / UNIT;
+    while n >= UNIT {
+        div *= UNIT;
+        n /= UNIT;
+        exp += 1;
+    }
+
+    format!(
+        "{:.2} {}iB",
+        bytes as f64 / div as f64,
+        ["K", "M", "G", "T", "P", "E"][exp]
+    )
+}
+
+fn escape_html(input: &str) -> String {
+    input
+        .replace('&', "&amp;")
+        .replace('<', "&lt;")
+        .replace('>', "&gt;")
+        .replace('"', "&quot;")
+}

seaweed-volume/src/server/volume_server.rs

@@ -0,0 +1,394 @@
+//! VolumeServer: the main HTTP server for volume operations.
+//!
+//! Routes:
+//!   GET/HEAD /{vid},{fid}     — read a file
+//!   POST/PUT /{vid},{fid}     — write a file
+//!   DELETE   /{vid},{fid}     — delete a file
+//!   GET      /status          — server status
+//!   GET      /healthz         — health check
+//!
+//! Matches Go's server/volume_server.go.
+
+use std::net::SocketAddr;
+use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU32, Ordering};
+use std::sync::{Arc, RwLock};
+
+use axum::{
+    extract::{connect_info::ConnectInfo, Request, State},
+    http::{header, HeaderValue, Method, StatusCode},
+    middleware::{self, Next},
+    response::{IntoResponse, Response},
+    routing::{any, get},
+    Router,
+};
+
+use crate::config::ReadMode;
+use crate::security::Guard;
+use crate::storage::store::Store;
+
+use super::grpc_client::OutgoingGrpcTlsConfig;
+use super::handlers;
+use super::write_queue::WriteQueue;
+
+#[derive(Clone, Debug, Default)]
+pub struct RuntimeMetricsConfig {
+    pub push_gateway: crate::metrics::PushGatewayConfig,
+}
+
+/// Shared state for the volume server.
+pub struct VolumeServerState {
+    pub store: RwLock<Store>,
+    pub guard: RwLock<Guard>,
+    pub is_stopping: RwLock<bool>,
+    /// Maintenance mode flag.
+    pub maintenance: AtomicBool,
+    /// State version — incremented on each SetState call.
+    pub state_version: AtomicU32,
+    /// Throttling: concurrent upload/download limits (in bytes, 0 = disabled).
+    pub concurrent_upload_limit: i64,
+    pub concurrent_download_limit: i64,
+    pub inflight_upload_data_timeout: std::time::Duration,
+    pub inflight_download_data_timeout: std::time::Duration,
+    /// Current in-flight upload/download bytes.
+    pub inflight_upload_bytes: AtomicI64,
+    pub inflight_download_bytes: AtomicI64,
+    /// Notify waiters when inflight bytes decrease.
+    pub upload_notify: tokio::sync::Notify,
+    pub download_notify: tokio::sync::Notify,
+    /// Data center name from config.
+    pub data_center: String,
+    /// Rack name from config.
+    pub rack: String,
+    /// File size limit in bytes (0 = no limit).
+    pub file_size_limit_bytes: i64,
+    /// Default IO rate limit for maintenance copy/replication work.
+    pub maintenance_byte_per_second: i64,
+    /// Whether the server is connected to master (heartbeat active).
+    pub is_heartbeating: AtomicBool,
+    /// Whether master addresses are configured.
+    pub has_master: bool,
+    /// Seconds to wait before shutting down servers (graceful drain).
+    pub pre_stop_seconds: u32,
+    /// Notify heartbeat to send an immediate update when volume state changes.
+    pub volume_state_notify: tokio::sync::Notify,
+    /// Optional batched write queue for improved throughput under load.
+    pub write_queue: std::sync::OnceLock<WriteQueue>,
+    /// Registry of S3 tier backends for tiered storage operations.
+    pub s3_tier_registry: std::sync::RwLock<crate::remote_storage::s3_tier::S3TierRegistry>,
+    /// Read mode: local, proxy, or redirect for non-local volumes.
+    pub read_mode: ReadMode,
+    /// First master address for volume lookups (e.g., "localhost:9333").
+    pub master_url: String,
+    /// Seed master addresses for UI rendering.
+    pub master_urls: Vec<String>,
+    /// This server's own address (ip:port) for filtering self from lookup results.
+    pub self_url: String,
+    /// HTTP client for proxy requests and master lookups.
+    pub http_client: reqwest::Client,
+    /// Scheme used for outgoing master and peer HTTP requests ("http" or "https").
+    pub outgoing_http_scheme: String,
+    /// Optional client TLS material for outgoing gRPC connections.
+    pub outgoing_grpc_tls: Option<OutgoingGrpcTlsConfig>,
+    /// Metrics push settings learned from master heartbeat responses.
+    pub metrics_runtime: std::sync::RwLock<RuntimeMetricsConfig>,
+    pub metrics_notify: tokio::sync::Notify,
+    /// Whether JPEG uploads should be normalized using EXIF orientation.
+    pub fix_jpg_orientation: bool,
+    /// Read tuning flags for large-file streaming.
+    pub has_slow_read: bool,
+    pub read_buffer_size_bytes: usize,
+    /// Path to security.toml — stored for SIGHUP reload.
+    pub security_file: String,
+    /// Original CLI whitelist entries — stored for SIGHUP reload.
+    pub cli_white_list: Vec<String>,
+    /// Path to state.pb file for persisting VolumeServerState across restarts.
+    pub state_file_path: String,
+}
+
+impl VolumeServerState {
+    /// Check if the server is in maintenance mode; return gRPC error if so.
+    pub fn check_maintenance(&self) -> Result<(), tonic::Status> {
+        if self.maintenance.load(Ordering::Relaxed) {
+            let id = self.store.read().unwrap().id.clone();
+            return Err(tonic::Status::unavailable(format!(
+                "volume server {} is in maintenance mode",
+                id
+            )));
+        }
+        Ok(())
+    }
+}
+
+pub fn build_metrics_router() -> Router {
+    Router::new().route("/metrics", get(handlers::metrics_handler))
+}
+
+pub fn normalize_outgoing_http_url(scheme: &str, raw_target: &str) -> Result<String, String> {
+    if raw_target.starts_with("http://") || raw_target.starts_with("https://") {
+        let mut url = reqwest::Url::parse(raw_target)
+            .map_err(|e| format!("invalid url {}: {}", raw_target, e))?;
+        url.set_scheme(scheme)
+            .map_err(|_| format!("invalid scheme {}", scheme))?;
+        return Ok(url.to_string());
+    }
+    Ok(format!("{}://{}", scheme, raw_target))
+}
+
+fn request_remote_addr(request: &Request) -> Option<SocketAddr> {
+    request
+        .extensions()
+        .get::<ConnectInfo<SocketAddr>>()
+        .map(|info| info.0)
+}
+
+fn request_is_whitelisted(state: &VolumeServerState, request: &Request) -> bool {
+    request_remote_addr(request)
+        .map(|remote_addr| {
+            state
+                .guard
+                .read()
+                .unwrap()
+                .check_whitelist(&remote_addr.to_string())
+        })
+        .unwrap_or(true)
+}
+
+/// Middleware: set Server header, echo x-amz-request-id, set CORS if Origin present.
+async fn common_headers_middleware(request: Request, next: Next) -> Response {
+    let origin = request.headers().get("origin").cloned();
+    let request_id = super::request_id::generate_http_request_id();
+
+    let mut response =
+        super::request_id::scope_request_id(
+            request_id.clone(),
+            async move { next.run(request).await },
+        )
+        .await;
+
+    let headers = response.headers_mut();
+    if let Ok(val) = HeaderValue::from_str(crate::version::server_header()) {
+        headers.insert("Server", val);
+    }
+
+    if let Ok(val) = HeaderValue::from_str(&request_id) {
+        headers.insert("X-Request-Id", val.clone());
+        headers.insert("x-amz-request-id", val);
+    }
+
+    if origin.is_some() {
+        headers.insert("Access-Control-Allow-Origin", HeaderValue::from_static("*"));
+        headers.insert(
+            "Access-Control-Allow-Credentials",
+            HeaderValue::from_static("true"),
+        );
+    }
+
+    response
+}
+
+/// Admin store handler — dispatches based on HTTP method.
+/// Matches Go's privateStoreHandler: GET/HEAD → read, POST/PUT → write,
+/// DELETE → delete, OPTIONS → CORS headers, anything else → 400.
+async fn admin_store_handler(state: State<Arc<VolumeServerState>>, request: Request) -> Response {
+    let start = std::time::Instant::now();
+    let method = request.method().clone();
+    let mut method_str = method.as_str().to_string();
+    let request_bytes = request
+        .headers()
+        .get(header::CONTENT_LENGTH)
+        .and_then(|value| value.to_str().ok())
+        .and_then(|value| value.parse::<i64>().ok())
+        .filter(|value| *value > 0)
+        .unwrap_or(0);
+    super::server_stats::record_request_open();
+    crate::metrics::INFLIGHT_REQUESTS_GAUGE
+        .with_label_values(&[&method_str])
+        .inc();
+    let whitelist_rejected = matches!(method, Method::POST | Method::PUT | Method::DELETE)
+        && !request_is_whitelisted(&state, &request);
+    let response = match method.clone() {
+        _ if whitelist_rejected => StatusCode::UNAUTHORIZED.into_response(),
+        Method::GET | Method::HEAD => {
+            super::server_stats::record_read_request();
+            handlers::get_or_head_handler_from_request(state, request).await
+        }
+        Method::POST | Method::PUT => {
+            super::server_stats::record_write_request();
+            if request_bytes > 0 {
+                super::server_stats::record_bytes_in(request_bytes);
+            }
+            handlers::post_handler(state, request).await
+        }
+        Method::DELETE => {
+            super::server_stats::record_delete_request();
+            handlers::delete_handler(state, request).await
+        }
+        Method::OPTIONS => {
+            super::server_stats::record_read_request();
+            admin_options_response()
+        }
+        _ => {
+            let method_name = request.method().to_string();
+            let query = request.uri().query().map(|q| q.to_string());
+            method_str = "INVALID".to_string();
+            handlers::json_error_with_query(
+                StatusCode::BAD_REQUEST,
+                format!("unsupported method {}", method_name),
+                query.as_deref(),
+            )
+        }
+    };
+    if method == Method::GET {
+        if let Some(response_bytes) = response
+            .headers()
+            .get(header::CONTENT_LENGTH)
+            .and_then(|value| value.to_str().ok())
+            .and_then(|value| value.parse::<i64>().ok())
+            .filter(|value| *value > 0)
+        {
+            super::server_stats::record_bytes_out(response_bytes);
+        }
+    }
+    super::server_stats::record_request_close();
+    crate::metrics::INFLIGHT_REQUESTS_GAUGE
+        .with_label_values(&[&method_str])
+        .dec();
+    crate::metrics::REQUEST_COUNTER
+        .with_label_values(&[&method_str, response.status().as_str()])
+        .inc();
+    crate::metrics::REQUEST_DURATION
+        .with_label_values(&[&method_str])
+        .observe(start.elapsed().as_secs_f64());
+    response
+}
+
+/// Public store handler — dispatches based on HTTP method.
+/// Matches Go's publicReadOnlyHandler: GET/HEAD → read, OPTIONS → CORS,
+/// anything else → 200 (passthrough no-op).
+async fn public_store_handler(state: State<Arc<VolumeServerState>>, request: Request) -> Response {
+    let start = std::time::Instant::now();
+    let method = request.method().clone();
+    let method_str = method.as_str().to_string();
+    super::server_stats::record_request_open();
+    crate::metrics::INFLIGHT_REQUESTS_GAUGE
+        .with_label_values(&[&method_str])
+        .inc();
+    let response = match method.clone() {
+        Method::GET | Method::HEAD => {
+            super::server_stats::record_read_request();
+            handlers::get_or_head_handler_from_request(state, request).await
+        }
+        Method::OPTIONS => {
+            super::server_stats::record_read_request();
+            public_options_response()
+        }
+        _ => StatusCode::OK.into_response(),
+    };
+    if method == Method::GET {
+        if let Some(response_bytes) = response
+            .headers()
+            .get(header::CONTENT_LENGTH)
+            .and_then(|value| value.to_str().ok())
+            .and_then(|value| value.parse::<i64>().ok())
+            .filter(|value| *value > 0)
+        {
+            super::server_stats::record_bytes_out(response_bytes);
+        }
+    }
+    super::server_stats::record_request_close();
+    crate::metrics::INFLIGHT_REQUESTS_GAUGE
+        .with_label_values(&[&method_str])
+        .dec();
+    crate::metrics::REQUEST_COUNTER
+        .with_label_values(&[&method_str, response.status().as_str()])
+        .inc();
+    crate::metrics::REQUEST_DURATION
+        .with_label_values(&[&method_str])
+        .observe(start.elapsed().as_secs_f64());
+    response
+}
+
+/// Build OPTIONS response for admin port.
+fn admin_options_response() -> Response {
+    let mut response = StatusCode::OK.into_response();
+    let headers = response.headers_mut();
+    headers.insert(
+        "Access-Control-Allow-Methods",
+        HeaderValue::from_static("PUT, POST, GET, DELETE, OPTIONS"),
+    );
+    headers.insert(
+        "Access-Control-Allow-Headers",
+        HeaderValue::from_static("*"),
+    );
+    response
+}
+
+/// Build OPTIONS response for public port.
+fn public_options_response() -> Response {
+    let mut response = StatusCode::OK.into_response();
+    let headers = response.headers_mut();
+    headers.insert(
+        "Access-Control-Allow-Methods",
+        HeaderValue::from_static("GET, OPTIONS"),
+    );
+    headers.insert(
+        "Access-Control-Allow-Headers",
+        HeaderValue::from_static("*"),
+    );
+    response
+}
+
+/// Build the admin (private) HTTP router — supports all operations.
+/// UI route is only registered when no signing keys are configured,
+/// matching Go's `if signingKey == "" || enableUiAccess` check.
+pub fn build_admin_router(state: Arc<VolumeServerState>) -> Router {
+    let guard = state.guard.read().unwrap();
+    // This helper can only derive the default Go behavior from the guard state:
+    // UI stays enabled when the write signing key is empty. The explicit
+    // `access.ui` override is handled by `build_admin_router_with_ui(...)`.
+    let ui_enabled = guard.signing_key.0.is_empty();
+    drop(guard);
+    build_admin_router_with_ui(state, ui_enabled)
+}
+
+/// Build the admin router with an explicit UI exposure flag.
+pub fn build_admin_router_with_ui(state: Arc<VolumeServerState>, ui_enabled: bool) -> Router {
+    let mut router = Router::new()
+        .route("/status", get(handlers::status_handler))
+        .route("/healthz", get(handlers::healthz_handler))
+        .route("/favicon.ico", get(handlers::favicon_handler))
+        .route(
+            "/seaweedfsstatic/*path",
+            get(handlers::static_asset_handler),
+        )
+        .route("/", any(admin_store_handler))
+        .route("/:path", any(admin_store_handler))
+        .route("/:vid/:fid", any(admin_store_handler))
+        .route("/:vid/:fid/:filename", any(admin_store_handler))
+        .fallback(admin_store_handler);
+    if ui_enabled {
+        // Note: /stats/* endpoints are commented out in Go's volume_server.go (L130-134).
+        // Only the UI endpoint is registered when UI access is enabled.
+        router = router.route("/ui/index.html", get(handlers::ui_handler));
+    }
+    router
+        .layer(middleware::from_fn(common_headers_middleware))
+        .with_state(state)
+}
+
+/// Build the public (read-only) HTTP router — only GET/HEAD.
+pub fn build_public_router(state: Arc<VolumeServerState>) -> Router {
+    Router::new()
+        .route("/favicon.ico", get(handlers::favicon_handler))
+        .route(
+            "/seaweedfsstatic/*path",
+            get(handlers::static_asset_handler),
+        )
+        .route("/", any(public_store_handler))
+        .route("/:path", any(public_store_handler))
+        .route("/:vid/:fid", any(public_store_handler))
+        .route("/:vid/:fid/:filename", any(public_store_handler))
+        .fallback(public_store_handler)
+        .layer(middleware::from_fn(common_headers_middleware))
+        .with_state(state)
+}

seaweed-volume/src/server/write_queue.rs

@@ -0,0 +1,330 @@
+//! Async batched write processing for the volume server.
+//!
+//! Instead of each upload handler directly calling `write_needle` and syncing,
+//! writes are submitted to a queue. A background worker drains the queue in
+//! batches (up to 128 entries), groups them by volume ID, processes them
+//! together, and syncs once per volume for the entire batch.
+
+use std::sync::Arc;
+
+use tokio::sync::{mpsc, oneshot};
+use tracing::debug;
+
+use crate::storage::needle::needle::Needle;
+use crate::storage::types::{Size, VolumeId};
+use crate::storage::volume::VolumeError;
+
+use super::volume_server::VolumeServerState;
+
+/// Result of a single write operation: (offset, size, is_unchanged).
+pub type WriteResult = Result<(u64, Size, bool), VolumeError>;
+
+/// A request to write a needle, submitted to the write queue.
+pub struct WriteRequest {
+    pub volume_id: VolumeId,
+    pub needle: Needle,
+    pub response_tx: oneshot::Sender<WriteResult>,
+}
+
+/// Maximum number of write requests to batch together.
+const MAX_BATCH_SIZE: usize = 128;
+
+/// Maximum bytes to accumulate per batch before breaking (matches Go's 4MB limit).
+/// This prevents large writes from accumulating unbounded latency.
+const MAX_BATCH_BYTES: usize = 4 * 1024 * 1024;
+
+/// Handle for submitting write requests to the background worker.
+#[derive(Clone)]
+pub struct WriteQueue {
+    tx: mpsc::Sender<WriteRequest>,
+}
+
+impl WriteQueue {
+    /// Create a new write queue and spawn the background worker.
+    ///
+    /// `capacity` controls the channel buffer size (backpressure kicks in when full).
+    /// The worker holds a reference to `state` for accessing the store.
+    pub fn new(state: Arc<VolumeServerState>, capacity: usize) -> Self {
+        let (tx, rx) = mpsc::channel(capacity);
+        let worker = WriteQueueWorker { rx, state };
+        tokio::spawn(worker.run());
+        WriteQueue { tx }
+    }
+
+    /// Submit a write request and wait for the result.
+    ///
+    /// Returns `Err` if the worker has shut down or the response channel was dropped.
+    pub async fn submit(&self, volume_id: VolumeId, needle: Needle) -> WriteResult {
+        let (response_tx, response_rx) = oneshot::channel();
+        let request = WriteRequest {
+            volume_id,
+            needle,
+            response_tx,
+        };
+
+        // Send to queue; this awaits if the channel is full (backpressure).
+        if self.tx.send(request).await.is_err() {
+            return Err(VolumeError::Io(std::io::Error::new(
+                std::io::ErrorKind::BrokenPipe,
+                "write queue worker has shut down",
+            )));
+        }
+
+        // Wait for the worker to process our request.
+        match response_rx.await {
+            Ok(result) => result,
+            Err(_) => Err(VolumeError::Io(std::io::Error::new(
+                std::io::ErrorKind::BrokenPipe,
+                "write queue worker dropped response channel",
+            ))),
+        }
+    }
+}
+
+/// Background worker that drains write requests and processes them in batches.
+struct WriteQueueWorker {
+    rx: mpsc::Receiver<WriteRequest>,
+    state: Arc<VolumeServerState>,
+}
+
+impl WriteQueueWorker {
+    async fn run(mut self) {
+        debug!("write queue worker started");
+
+        loop {
+            // Wait for the first request (blocks until one arrives or channel closes).
+            let first = match self.rx.recv().await {
+                Some(req) => req,
+                None => {
+                    debug!("write queue channel closed, worker exiting");
+                    return;
+                }
+            };
+
+            // Drain as many additional requests as available, up to MAX_BATCH_SIZE
+            // or MAX_BATCH_BYTES (matches Go: 128 requests or 4MB, whichever comes first).
+            let mut batch = Vec::with_capacity(MAX_BATCH_SIZE);
+            let mut batch_bytes: usize = first.needle.data.len();
+            batch.push(first);
+
+            while batch.len() < MAX_BATCH_SIZE && batch_bytes < MAX_BATCH_BYTES {
+                match self.rx.try_recv() {
+                    Ok(req) => {
+                        batch_bytes += req.needle.data.len();
+                        batch.push(req);
+                    }
+                    Err(_) => break,
+                }
+            }
+
+            let batch_size = batch.len();
+            debug!("processing write batch of {} requests", batch_size);
+
+            // Process the batch in spawn_blocking since write_needle does file I/O.
+            let state = self.state.clone();
+            let _ = tokio::task::spawn_blocking(move || {
+                process_batch(state, batch);
+            })
+            .await;
+        }
+    }
+}
+
+/// Process a batch of write requests, grouped by volume ID.
+///
+/// Groups writes by volume to minimize the number of store lock acquisitions,
+/// then sends results back via each request's oneshot channel.
+fn process_batch(state: Arc<VolumeServerState>, batch: Vec<WriteRequest>) {
+    // Group requests by volume ID for efficient processing.
+    // We use a Vec of (VolumeId, Vec<(Needle, Sender)>) to preserve order
+    // and avoid requiring Hash on VolumeId.
+    let mut groups: Vec<(VolumeId, Vec<(Needle, oneshot::Sender<WriteResult>)>)> = Vec::new();
+
+    for req in batch {
+        let vid = req.volume_id;
+        if let Some(group) = groups.iter_mut().find(|(v, _)| *v == vid) {
+            group.1.push((req.needle, req.response_tx));
+        } else {
+            groups.push((vid, vec![(req.needle, req.response_tx)]));
+        }
+    }
+
+    // Process each volume group under a single store lock.
+    let mut store = state.store.write().unwrap();
+
+    for (vid, entries) in groups {
+        for (mut needle, response_tx) in entries {
+            let result = store.write_volume_needle(vid, &mut needle);
+            // Send result back; ignore error if receiver dropped.
+            let _ = response_tx.send(result);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::storage::types::VolumeId;
+
+    /// Helper to create a minimal VolumeServerState for testing.
+    fn make_test_state() -> Arc<VolumeServerState> {
+        use crate::security::{Guard, SigningKey};
+        use crate::server::volume_server::RuntimeMetricsConfig;
+        use crate::storage::needle_map::NeedleMapKind;
+        use crate::storage::store::Store;
+        use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU32};
+        use std::sync::RwLock;
+
+        let store = Store::new(NeedleMapKind::InMemory);
+        let guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
+
+        Arc::new(VolumeServerState {
+            store: RwLock::new(store),
+            guard: RwLock::new(guard),
+            is_stopping: RwLock::new(false),
+            maintenance: AtomicBool::new(false),
+            state_version: AtomicU32::new(0),
+            concurrent_upload_limit: 0,
+            concurrent_download_limit: 0,
+            inflight_upload_data_timeout: std::time::Duration::ZERO,
+            inflight_download_data_timeout: std::time::Duration::ZERO,
+            inflight_upload_bytes: AtomicI64::new(0),
+            inflight_download_bytes: AtomicI64::new(0),
+            upload_notify: tokio::sync::Notify::new(),
+            download_notify: tokio::sync::Notify::new(),
+            data_center: String::new(),
+            rack: String::new(),
+            file_size_limit_bytes: 0,
+            maintenance_byte_per_second: 0,
+            is_heartbeating: AtomicBool::new(false),
+            has_master: false,
+            pre_stop_seconds: 0,
+            volume_state_notify: tokio::sync::Notify::new(),
+            write_queue: std::sync::OnceLock::new(),
+            s3_tier_registry: std::sync::RwLock::new(
+                crate::remote_storage::s3_tier::S3TierRegistry::new(),
+            ),
+            read_mode: crate::config::ReadMode::Local,
+            master_url: String::new(),
+            master_urls: Vec::new(),
+            self_url: String::new(),
+            http_client: reqwest::Client::new(),
+            outgoing_http_scheme: "http".to_string(),
+            outgoing_grpc_tls: None,
+            metrics_runtime: std::sync::RwLock::new(RuntimeMetricsConfig::default()),
+            metrics_notify: tokio::sync::Notify::new(),
+            fix_jpg_orientation: false,
+            has_slow_read: true,
+            read_buffer_size_bytes: 4 * 1024 * 1024,
+            security_file: String::new(),
+            cli_white_list: vec![],
+            state_file_path: String::new(),
+        })
+    }
+
+    #[tokio::test]
+    async fn test_write_queue_submit_no_volume() {
+        // Submit a write to a non-existent volume -- should return VolumeError::NotFound.
+        let state = make_test_state();
+        let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
+
+        let needle = Needle {
+            id: 1.into(),
+            cookie: 0x12345678.into(),
+            data: vec![1, 2, 3],
+            data_size: 3,
+            ..Needle::default()
+        };
+
+        let result = queue.submit(VolumeId(999), needle).await;
+        assert!(result.is_err());
+        match result {
+            Err(VolumeError::NotFound) => {} // expected
+            other => panic!("expected NotFound, got {:?}", other),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_write_queue_concurrent_submissions() {
+        // Submit multiple concurrent writes -- all should complete (with errors since no volume).
+        let state = make_test_state();
+        let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
+
+        let mut handles = Vec::new();
+        for i in 0..10u64 {
+            let q = queue.clone();
+            handles.push(tokio::spawn(async move {
+                let needle = Needle {
+                    id: i.into(),
+                    cookie: 0xABCD.into(),
+                    data: vec![i as u8; 10],
+                    data_size: 10,
+                    ..Needle::default()
+                };
+                q.submit(VolumeId(1), needle).await
+            }));
+        }
+
+        for handle in handles {
+            let result = handle.await.unwrap();
+            // All should fail with NotFound since there's no volume 1
+            assert!(matches!(result, Err(VolumeError::NotFound)));
+        }
+    }
+
+    #[tokio::test]
+    async fn test_write_queue_batching() {
+        // Verify that many concurrent writes get processed (testing the batching path).
+        let state = make_test_state();
+        let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
+
+        // Submit MAX_BATCH_SIZE requests concurrently
+        let mut handles = Vec::new();
+        for i in 0..MAX_BATCH_SIZE as u64 {
+            let q = queue.clone();
+            handles.push(tokio::spawn(async move {
+                let needle = Needle {
+                    id: i.into(),
+                    cookie: 0x1111.into(),
+                    data: vec![0u8; 4],
+                    data_size: 4,
+                    ..Needle::default()
+                };
+                q.submit(VolumeId(42), needle).await
+            }));
+        }
+
+        let mut results = Vec::new();
+        for handle in handles {
+            results.push(handle.await.unwrap());
+        }
+
+        // All should complete (with NotFound errors since no volume exists)
+        assert_eq!(results.len(), MAX_BATCH_SIZE);
+        for r in results {
+            assert!(matches!(r, Err(VolumeError::NotFound)));
+        }
+    }
+
+    #[tokio::test]
+    async fn test_write_queue_dropped_sender() {
+        // When the queue is dropped, subsequent submits should fail gracefully.
+        let state = make_test_state();
+        let queue = WriteQueue::new(state, 1);
+
+        // Clone then drop the original -- the worker keeps running via its rx handle.
+        let queue2 = queue.clone();
+        drop(queue);
+
+        // This should still work since the worker is alive.
+        let needle = Needle {
+            id: 1.into(),
+            cookie: 0.into(),
+            data: vec![],
+            data_size: 0,
+            ..Needle::default()
+        };
+        let result = queue2.submit(VolumeId(1), needle).await;
+        assert!(result.is_err()); // NotFound is fine -- the point is it doesn't panic
+    }
+}

seaweed-volume/src/storage/disk_location.rs

@@ -0,0 +1,951 @@
+//! DiskLocation: manages volumes on a single disk/directory.
+//!
+//! Each DiskLocation represents one storage directory containing .dat + .idx files.
+//! A Store contains one or more DiskLocations (one per configured directory).
+//! Matches Go's storage/disk_location.go.
+
+use std::collections::{HashMap, HashSet};
+use std::fs;
+use std::io;
+use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
+use std::sync::Arc;
+
+use tracing::{info, warn};
+
+use crate::config::MinFreeSpace;
+use crate::storage::erasure_coding::ec_shard::{
+    EcVolumeShard, DATA_SHARDS_COUNT, ERASURE_CODING_LARGE_BLOCK_SIZE,
+    ERASURE_CODING_SMALL_BLOCK_SIZE,
+};
+use crate::storage::erasure_coding::ec_volume::EcVolume;
+use crate::storage::needle_map::NeedleMapKind;
+use crate::storage::super_block::ReplicaPlacement;
+use crate::storage::types::*;
+use crate::storage::volume::{remove_volume_files, volume_file_name, Volume, VolumeError};
+
+/// A single disk location managing volumes in one directory.
+pub struct DiskLocation {
+    pub directory: String,
+    pub idx_directory: String,
+    pub directory_uuid: String,
+    pub disk_type: DiskType,
+    pub tags: Vec<String>,
+    pub max_volume_count: AtomicI32,
+    pub original_max_volume_count: i32,
+    volumes: HashMap<VolumeId, Volume>,
+    ec_volumes: HashMap<VolumeId, EcVolume>,
+    pub is_disk_space_low: Arc<AtomicBool>,
+    pub available_space: AtomicU64,
+    pub min_free_space: MinFreeSpace,
+}
+
+impl DiskLocation {
+    const UUID_FILE_NAME: &'static str = "vol_dir.uuid";
+
+    pub fn new(
+        directory: &str,
+        idx_directory: &str,
+        max_volume_count: i32,
+        disk_type: DiskType,
+        min_free_space: MinFreeSpace,
+        tags: Vec<String>,
+    ) -> io::Result<Self> {
+        fs::create_dir_all(directory)?;
+
+        let idx_dir = if idx_directory.is_empty() {
+            directory.to_string()
+        } else {
+            fs::create_dir_all(idx_directory)?;
+            idx_directory.to_string()
+        };
+        let directory_uuid = Self::generate_directory_uuid(directory)?;
+
+        Ok(DiskLocation {
+            directory: directory.to_string(),
+            idx_directory: idx_dir,
+            directory_uuid,
+            disk_type,
+            tags,
+            max_volume_count: AtomicI32::new(max_volume_count),
+            original_max_volume_count: max_volume_count,
+            volumes: HashMap::new(),
+            ec_volumes: HashMap::new(),
+            is_disk_space_low: Arc::new(AtomicBool::new(false)),
+            available_space: AtomicU64::new(0),
+            min_free_space,
+        })
+    }
+
+    fn generate_directory_uuid(directory: &str) -> io::Result<String> {
+        let path = std::path::Path::new(directory).join(Self::UUID_FILE_NAME);
+        if path.exists() {
+            let existing = fs::read_to_string(&path)?;
+            if !existing.trim().is_empty() {
+                return Ok(existing);
+            }
+        }
+
+        let dir_uuid = uuid::Uuid::new_v4().to_string();
+        fs::write(path, &dir_uuid)?;
+        Ok(dir_uuid)
+    }
+
+    // ---- Volume management ----
+
+    /// Load existing volumes from the directory.
+    ///
+    /// Matches Go's `loadExistingVolume`: checks for incomplete volumes (.note file),
+    /// validates EC shards before skipping .dat loading, and cleans up stale
+    /// compaction temp files (.cpd/.cpx).
+    pub fn load_existing_volumes(&mut self, needle_map_kind: NeedleMapKind) -> io::Result<()> {
+        // Ensure directory exists
+        fs::create_dir_all(&self.directory)?;
+        if self.directory != self.idx_directory {
+            fs::create_dir_all(&self.idx_directory)?;
+        }
+
+        // Scan for .dat files
+        let entries = fs::read_dir(&self.directory)?;
+        let mut dat_files: Vec<(String, VolumeId)> = Vec::new();
+        let mut seen = HashSet::new();
+
+        for entry in entries {
+            let entry = entry?;
+            let name = entry.file_name().into_string().unwrap_or_default();
+            if let Some((collection, vid)) = parse_volume_filename(&name) {
+                if seen.insert((collection.clone(), vid)) {
+                    dat_files.push((collection, vid));
+                }
+            }
+        }
+
+        for (collection, vid) in dat_files {
+            let volume_name = volume_file_name(&self.directory, &collection, vid);
+            let idx_name = volume_file_name(&self.idx_directory, &collection, vid);
+
+            // Check for incomplete volume (.note file means a VolumeCopy was interrupted)
+            let note_path = format!("{}.note", volume_name);
+            if std::path::Path::new(&note_path).exists() {
+                let note = fs::read_to_string(&note_path).unwrap_or_default();
+                warn!(
+                    volume_id = vid.0,
+                    "volume was not completed: {}, removing files", note
+                );
+                remove_volume_files(&volume_name);
+                remove_volume_files(&idx_name);
+                continue;
+            }
+
+            // If valid EC shards exist (.ecx file present), skip loading .dat
+            let ecx_path = format!("{}.ecx", idx_name);
+            let ecx_exists = if std::path::Path::new(&ecx_path).exists() {
+                true
+            } else if self.idx_directory != self.directory {
+                // .ecx may have been created before -dir.idx was configured
+                let fallback = format!("{}.ecx", volume_name);
+                std::path::Path::new(&fallback).exists()
+            } else {
+                false
+            };
+            if ecx_exists {
+                if self.validate_ec_volume(&collection, vid) {
+                    // Valid EC volume — don't load .dat
+                    continue;
+                } else {
+                    warn!(
+                        volume_id = vid.0,
+                        "EC volume validation failed, removing incomplete EC files"
+                    );
+                    self.remove_ec_volume_files(&collection, vid);
+                    // Fall through to load .dat file
+                }
+            }
+
+            // Clean up stale compaction temp files
+            let cpd_path = format!("{}.cpd", volume_name);
+            let cpx_path = format!("{}.cpx", idx_name);
+            if std::path::Path::new(&cpd_path).exists() {
+                info!(volume_id = vid.0, "removing stale compaction file .cpd");
+                let _ = fs::remove_file(&cpd_path);
+            }
+            if std::path::Path::new(&cpx_path).exists() {
+                info!(volume_id = vid.0, "removing stale compaction file .cpx");
+                let _ = fs::remove_file(&cpx_path);
+            }
+
+            // Skip if already loaded (e.g., from a previous call)
+            if self.volumes.contains_key(&vid) {
+                continue;
+            }
+
+            match Volume::new(
+                &self.directory,
+                &self.idx_directory,
+                &collection,
+                vid,
+                needle_map_kind,
+                None, // replica placement read from superblock
+                None, // TTL read from superblock
+                0,    // no preallocate on load
+                Version::current(),
+            ) {
+                Ok(mut v) => {
+                    v.location_disk_space_low = self.is_disk_space_low.clone();
+                    crate::metrics::VOLUME_GAUGE
+                        .with_label_values(&[&collection, "volume"])
+                        .inc();
+                    self.volumes.insert(vid, v);
+                }
+                Err(e) => {
+                    warn!(volume_id = vid.0, error = %e, "failed to load volume");
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Validate EC volume shards: all shards must be same size, and if .dat exists,
+    /// need at least DATA_SHARDS_COUNT shards with size matching expected.
+    fn validate_ec_volume(&self, collection: &str, vid: VolumeId) -> bool {
+        let base = volume_file_name(&self.directory, collection, vid);
+        let dat_path = format!("{}.dat", base);
+
+        let mut expected_shard_size: Option<i64> = None;
+        let dat_exists = std::path::Path::new(&dat_path).exists();
+
+        if dat_exists {
+            if let Ok(meta) = fs::metadata(&dat_path) {
+                expected_shard_size = Some(calculate_expected_shard_size(meta.len() as i64));
+            } else {
+                return false;
+            }
+        }
+
+        let mut shard_count = 0usize;
+        let mut actual_shard_size: Option<i64> = None;
+        const MAX_SHARD_COUNT: usize = 32;
+
+        for i in 0..MAX_SHARD_COUNT {
+            let shard_path = format!("{}.ec{:02}", base, i);
+            match fs::metadata(&shard_path) {
+                Ok(meta) if meta.len() > 0 => {
+                    let size = meta.len() as i64;
+                    if let Some(prev) = actual_shard_size {
+                        if size != prev {
+                            warn!(
+                                volume_id = vid.0,
+                                shard = i,
+                                size,
+                                expected = prev,
+                                "EC shard size mismatch"
+                            );
+                            return false;
+                        }
+                    } else {
+                        actual_shard_size = Some(size);
+                    }
+                    shard_count += 1;
+                }
+                Err(e) if e.kind() != io::ErrorKind::NotFound => {
+                    warn!(
+                        volume_id = vid.0,
+                        shard = i,
+                        error = %e,
+                        "failed to stat EC shard"
+                    );
+                    return false;
+                }
+                _ => {} // not found or zero size — skip
+            }
+        }
+
+        // If .dat exists, validate shard size matches expected
+        if dat_exists {
+            if let (Some(actual), Some(expected)) = (actual_shard_size, expected_shard_size) {
+                if actual != expected {
+                    warn!(
+                        volume_id = vid.0,
+                        actual_shard_size = actual,
+                        expected_shard_size = expected,
+                        "EC shard size doesn't match .dat file"
+                    );
+                    return false;
+                }
+            }
+        }
+
+        // Distributed EC (no .dat): any shard count is valid
+        if !dat_exists {
+            return true;
+        }
+
+        // With .dat: need at least DATA_SHARDS_COUNT shards
+        if shard_count < DATA_SHARDS_COUNT {
+            warn!(
+                volume_id = vid.0,
+                shard_count,
+                required = DATA_SHARDS_COUNT,
+                "EC volume has .dat but too few shards"
+            );
+            return false;
+        }
+
+        true
+    }
+
+    /// Remove all EC-related files for a volume.
+    fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) {
+        let base = volume_file_name(&self.directory, collection, vid);
+        let idx_base = volume_file_name(&self.idx_directory, collection, vid);
+        const MAX_SHARD_COUNT: usize = 32;
+
+        // Remove index files from idx directory (.ecx, .ecj)
+        let _ = fs::remove_file(format!("{}.ecx", idx_base));
+        let _ = fs::remove_file(format!("{}.ecj", idx_base));
+        // Also try data directory in case .ecx/.ecj were created before -dir.idx was configured
+        if self.idx_directory != self.directory {
+            let _ = fs::remove_file(format!("{}.ecx", base));
+            let _ = fs::remove_file(format!("{}.ecj", base));
+        }
+
+        // Remove all EC shard files (.ec00 ~ .ec31)
+        for i in 0..MAX_SHARD_COUNT {
+            let _ = fs::remove_file(format!("{}.ec{:02}", base, i));
+        }
+    }
+
+    /// Find a volume by ID.
+    pub fn find_volume(&self, vid: VolumeId) -> Option<&Volume> {
+        self.volumes.get(&vid)
+    }
+
+    /// Find a volume by ID (mutable).
+    pub fn find_volume_mut(&mut self, vid: VolumeId) -> Option<&mut Volume> {
+        self.volumes.get_mut(&vid)
+    }
+
+    /// Add a volume to this location.
+    pub fn set_volume(&mut self, vid: VolumeId, volume: Volume) {
+        let collection = volume.collection.clone();
+        self.volumes.insert(vid, volume);
+        crate::metrics::VOLUME_GAUGE
+            .with_label_values(&[&collection, "volume"])
+            .inc();
+    }
+
+    /// Create a new volume in this location.
+    pub fn create_volume(
+        &mut self,
+        vid: VolumeId,
+        collection: &str,
+        needle_map_kind: NeedleMapKind,
+        replica_placement: Option<ReplicaPlacement>,
+        ttl: Option<crate::storage::needle::ttl::TTL>,
+        preallocate: u64,
+        version: Version,
+    ) -> Result<(), VolumeError> {
+        let mut v = Volume::new(
+            &self.directory,
+            &self.idx_directory,
+            collection,
+            vid,
+            needle_map_kind,
+            replica_placement,
+            ttl,
+            preallocate,
+            version,
+        )?;
+        v.location_disk_space_low = self.is_disk_space_low.clone();
+        crate::metrics::VOLUME_GAUGE
+            .with_label_values(&[collection, "volume"])
+            .inc();
+        self.volumes.insert(vid, v);
+        Ok(())
+    }
+
+    /// Remove and close a volume.
+    pub fn unload_volume(&mut self, vid: VolumeId) -> Option<Volume> {
+        if let Some(mut v) = self.volumes.remove(&vid) {
+            crate::metrics::VOLUME_GAUGE
+                .with_label_values(&[&v.collection, "volume"])
+                .dec();
+            v.close();
+            Some(v)
+        } else {
+            None
+        }
+    }
+
+    /// Remove, close, and delete all files for a volume.
+    pub fn delete_volume(&mut self, vid: VolumeId, only_empty: bool) -> Result<(), VolumeError> {
+        if let Some(mut v) = self.volumes.remove(&vid) {
+            crate::metrics::VOLUME_GAUGE
+                .with_label_values(&[&v.collection, "volume"])
+                .dec();
+            v.destroy(only_empty)?;
+            Ok(())
+        } else {
+            Err(VolumeError::NotFound)
+        }
+    }
+
+    /// Delete all volumes in a collection.
+    pub fn delete_collection(&mut self, collection: &str) -> Result<(), VolumeError> {
+        let vids: Vec<VolumeId> = self
+            .volumes
+            .iter()
+            .filter(|(_, v)| v.collection == collection && !v.is_compacting())
+            .map(|(vid, _)| *vid)
+            .collect();
+
+        for vid in vids {
+            if let Some(mut v) = self.volumes.remove(&vid) {
+                crate::metrics::VOLUME_GAUGE
+                    .with_label_values(&[&v.collection, "volume"])
+                    .dec();
+                if let Err(e) = v.destroy(false) {
+                    warn!(volume_id = vid.0, error = %e, "delete collection: failed to destroy volume");
+                }
+            }
+        }
+
+        let ec_vids: Vec<VolumeId> = self
+            .ec_volumes
+            .iter()
+            .filter(|(_, v)| v.collection == collection)
+            .map(|(vid, _)| *vid)
+            .collect();
+
+        for vid in ec_vids {
+            if let Some(mut ec_vol) = self.ec_volumes.remove(&vid) {
+                for _ in 0..ec_vol.shard_count() {
+                    crate::metrics::VOLUME_GAUGE
+                        .with_label_values(&[collection, "ec_shards"])
+                        .dec();
+                }
+                ec_vol.destroy();
+            }
+        }
+        Ok(())
+    }
+
+    // ---- Metrics ----
+
+    /// Number of volumes on this disk.
+    pub fn volumes_len(&self) -> usize {
+        self.volumes.len()
+    }
+
+    /// Get all volume IDs, sorted.
+    pub fn volume_ids(&self) -> Vec<VolumeId> {
+        let mut ids: Vec<VolumeId> = self.volumes.keys().copied().collect();
+        ids.sort();
+        ids
+    }
+
+    /// Iterate over all volumes.
+    pub fn iter_volumes(&self) -> impl Iterator<Item = (&VolumeId, &Volume)> {
+        self.volumes.iter()
+    }
+
+    /// Number of free volume slots.
+    /// Matches Go's FindFreeLocation formula:
+    ///   free = ((MaxVolumeCount - VolumesLen()) * DataShardsCount - EcShardCount()) / DataShardsCount
+    pub fn free_volume_count(&self) -> i32 {
+        use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
+        let max = self.max_volume_count.load(Ordering::Relaxed);
+        let free_count = (max as i64 - self.volumes.len() as i64)
+            * DATA_SHARDS_COUNT as i64
+            - self.ec_shard_count() as i64;
+        let effective_free = free_count / DATA_SHARDS_COUNT as i64;
+        if effective_free > 0 {
+            effective_free as i32
+        } else {
+            0
+        }
+    }
+
+    /// Iterate over all volumes.
+    pub fn volumes(&self) -> impl Iterator<Item = (&VolumeId, &Volume)> {
+        self.volumes.iter()
+    }
+
+    /// Iterate over all volumes (mutable).
+    pub fn volumes_mut(&mut self) -> impl Iterator<Item = (&VolumeId, &mut Volume)> {
+        self.volumes.iter_mut()
+    }
+
+    /// Sum of unused space in writable volumes (volumeSizeLimit - actual size per volume).
+    /// Used by auto-max-volume-count to estimate how many more volumes can fit.
+    pub fn unused_space(&self, volume_size_limit: u64) -> u64 {
+        let mut unused: u64 = 0;
+        for vol in self.volumes.values() {
+            if vol.is_read_only() {
+                continue;
+            }
+            let dat_size = vol.dat_file_size().unwrap_or(0);
+            let idx_size = vol.idx_file_size();
+            let used = dat_size + idx_size;
+            if volume_size_limit > used {
+                unused += volume_size_limit - used;
+            }
+        }
+        unused
+    }
+
+    /// Check disk space against min_free_space and update is_disk_space_low.
+    pub fn check_disk_space(&self) {
+        let (total, free) = get_disk_stats(&self.directory);
+        if total == 0 {
+            return;
+        }
+        let used = total.saturating_sub(free);
+        let is_low = match &self.min_free_space {
+            MinFreeSpace::Percent(pct) => {
+                let free_pct = (free as f64 / total as f64) * 100.0;
+                free_pct < *pct
+            }
+            MinFreeSpace::Bytes(min_bytes) => free < *min_bytes,
+        };
+        self.is_disk_space_low.store(is_low, Ordering::Relaxed);
+        self.available_space.store(free, Ordering::Relaxed);
+
+        // Update resource gauges
+        crate::metrics::RESOURCE_GAUGE
+            .with_label_values(&[&self.directory, "all"])
+            .set(total as f64);
+        crate::metrics::RESOURCE_GAUGE
+            .with_label_values(&[&self.directory, "used"])
+            .set(used as f64);
+        crate::metrics::RESOURCE_GAUGE
+            .with_label_values(&[&self.directory, "free"])
+            .set(free as f64);
+        // "avail" is same as "free" for us (Go subtracts reserved blocks but we use statvfs f_bavail)
+        crate::metrics::RESOURCE_GAUGE
+            .with_label_values(&[&self.directory, "avail"])
+            .set(free as f64);
+    }
+
+    // ---- EC volume operations ----
+
+    /// Find an EC volume by ID.
+    pub fn find_ec_volume(&self, vid: VolumeId) -> Option<&EcVolume> {
+        self.ec_volumes.get(&vid)
+    }
+
+    /// Find an EC volume by ID (mutable).
+    pub fn find_ec_volume_mut(&mut self, vid: VolumeId) -> Option<&mut EcVolume> {
+        self.ec_volumes.get_mut(&vid)
+    }
+
+    /// Check if this location has an EC volume.
+    pub fn has_ec_volume(&self, vid: VolumeId) -> bool {
+        self.ec_volumes.contains_key(&vid)
+    }
+
+    /// Remove an EC volume, returning it.
+    pub fn remove_ec_volume(&mut self, vid: VolumeId) -> Option<EcVolume> {
+        self.ec_volumes.remove(&vid)
+    }
+
+    /// Mount EC shards for a volume on this location.
+    pub fn mount_ec_shards(
+        &mut self,
+        vid: VolumeId,
+        collection: &str,
+        shard_ids: &[u32],
+    ) -> Result<(), VolumeError> {
+        let dir = self.directory.clone();
+        let idx_dir = self.idx_directory.clone();
+        let ec_vol = self
+            .ec_volumes
+            .entry(vid)
+            .or_insert_with(|| EcVolume::new(&dir, &idx_dir, collection, vid).unwrap());
+        ec_vol.disk_type = self.disk_type.clone();
+
+        for &shard_id in shard_ids {
+            let shard = EcVolumeShard::new(&dir, collection, vid, shard_id as u8);
+            ec_vol.add_shard(shard).map_err(VolumeError::Io)?;
+            crate::metrics::VOLUME_GAUGE
+                .with_label_values(&[collection, "ec_shards"])
+                .inc();
+        }
+        Ok(())
+    }
+
+    /// Unmount EC shards for a volume on this location.
+    pub fn unmount_ec_shards(&mut self, vid: VolumeId, shard_ids: &[u32]) {
+        if let Some(ec_vol) = self.ec_volumes.get_mut(&vid) {
+            let collection = ec_vol.collection.clone();
+            for &shard_id in shard_ids {
+                ec_vol.remove_shard(shard_id as u8);
+                crate::metrics::VOLUME_GAUGE
+                    .with_label_values(&[&collection, "ec_shards"])
+                    .dec();
+            }
+            if ec_vol.shard_count() == 0 {
+                let mut vol = self.ec_volumes.remove(&vid).unwrap();
+                vol.close();
+            }
+        }
+    }
+
+    /// Total number of EC shards on this location.
+    pub fn ec_shard_count(&self) -> usize {
+        self.ec_volumes
+            .values()
+            .map(|ecv| ecv.shards.iter().filter(|s| s.is_some()).count())
+            .sum()
+    }
+
+    /// Iterate over all EC volumes.
+    pub fn ec_volumes(&self) -> impl Iterator<Item = (&VolumeId, &EcVolume)> {
+        self.ec_volumes.iter()
+    }
+
+    /// Close all volumes.
+    pub fn close(&mut self) {
+        for (_, v) in self.volumes.iter_mut() {
+            v.close();
+        }
+        self.volumes.clear();
+        for (_, mut ec_vol) in self.ec_volumes.drain() {
+            ec_vol.close();
+        }
+    }
+}
+
+/// Get total and free disk space for a given path.
+/// Returns (total_bytes, free_bytes).
+pub fn get_disk_stats(path: &str) -> (u64, u64) {
+    #[cfg(unix)]
+    {
+        use std::ffi::CString;
+        let c_path = match CString::new(path) {
+            Ok(p) => p,
+            Err(_) => return (0, 0),
+        };
+        unsafe {
+            let mut stat: libc::statvfs = std::mem::zeroed();
+            if libc::statvfs(c_path.as_ptr(), &mut stat) == 0 {
+                let all = stat.f_blocks as u64 * stat.f_frsize as u64;
+                let free = stat.f_bavail as u64 * stat.f_frsize as u64;
+                return (all, free);
+            }
+        }
+        (0, 0)
+    }
+    #[cfg(not(unix))]
+    {
+        let _ = path;
+        (0, 0)
+    }
+}
+
+/// Calculate expected EC shard size from .dat file size.
+/// Matches Go's `calculateExpectedShardSize`: large blocks (1GB * data_shards) first,
+/// then small blocks (1MB * data_shards) for the remainder.
+fn calculate_expected_shard_size(dat_file_size: i64) -> i64 {
+    let large_batch_size = ERASURE_CODING_LARGE_BLOCK_SIZE as i64 * DATA_SHARDS_COUNT as i64;
+    let num_large_batches = dat_file_size / large_batch_size;
+    let mut shard_size = num_large_batches * ERASURE_CODING_LARGE_BLOCK_SIZE as i64;
+    let remaining = dat_file_size - (num_large_batches * large_batch_size);
+
+    if remaining > 0 {
+        let small_batch_size = ERASURE_CODING_SMALL_BLOCK_SIZE as i64 * DATA_SHARDS_COUNT as i64;
+        // Ceiling division
+        let num_small_batches = (remaining + small_batch_size - 1) / small_batch_size;
+        shard_size += num_small_batches * ERASURE_CODING_SMALL_BLOCK_SIZE as i64;
+    }
+
+    shard_size
+}
+
+/// Parse a volume filename like "collection_42.dat" or "42.dat" into (collection, VolumeId).
+fn parse_volume_filename(filename: &str) -> Option<(String, VolumeId)> {
+    let stem = filename
+        .strip_suffix(".dat")
+        .or_else(|| filename.strip_suffix(".vif"))
+        .or_else(|| filename.strip_suffix(".idx"))?;
+    if let Some(pos) = stem.rfind('_') {
+        let collection = &stem[..pos];
+        let id_str = &stem[pos + 1..];
+        let id: u32 = id_str.parse().ok()?;
+        Some((collection.to_string(), VolumeId(id)))
+    } else {
+        let id: u32 = stem.parse().ok()?;
+        Some((String::new(), VolumeId(id)))
+    }
+}
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    #[test]
+    fn test_parse_volume_filename() {
+        assert_eq!(
+            parse_volume_filename("42.dat"),
+            Some(("".to_string(), VolumeId(42)))
+        );
+        assert_eq!(
+            parse_volume_filename("pics_7.dat"),
+            Some(("pics".to_string(), VolumeId(7)))
+        );
+        assert_eq!(
+            parse_volume_filename("42.vif"),
+            Some(("".to_string(), VolumeId(42)))
+        );
+        assert_eq!(
+            parse_volume_filename("pics_7.idx"),
+            Some(("pics".to_string(), VolumeId(7)))
+        );
+        assert_eq!(parse_volume_filename("notadat.idx"), None);
+        assert_eq!(parse_volume_filename("bad.dat"), None);
+    }
+
+    #[test]
+    fn test_disk_location_create_volume() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let mut loc = DiskLocation::new(
+            dir,
+            dir,
+            10,
+            DiskType::HardDrive,
+            MinFreeSpace::Percent(1.0),
+            Vec::new(),
+        )
+        .unwrap();
+
+        loc.create_volume(
+            VolumeId(1),
+            "",
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+
+        assert_eq!(loc.volumes_len(), 1);
+        assert!(loc.find_volume(VolumeId(1)).is_some());
+        assert!(loc.find_volume(VolumeId(99)).is_none());
+        assert_eq!(loc.free_volume_count(), 9);
+    }
+
+    #[test]
+    fn test_disk_location_load_existing() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+
+        // Create volumes
+        {
+            let mut loc = DiskLocation::new(
+                dir,
+                dir,
+                10,
+                DiskType::HardDrive,
+                MinFreeSpace::Percent(1.0),
+                Vec::new(),
+            )
+            .unwrap();
+            loc.create_volume(
+                VolumeId(1),
+                "",
+                NeedleMapKind::InMemory,
+                None,
+                None,
+                0,
+                Version::current(),
+            )
+            .unwrap();
+            loc.create_volume(
+                VolumeId(2),
+                "test",
+                NeedleMapKind::InMemory,
+                None,
+                None,
+                0,
+                Version::current(),
+            )
+            .unwrap();
+            loc.close();
+        }
+
+        // Reload
+        let mut loc = DiskLocation::new(
+            dir,
+            dir,
+            10,
+            DiskType::HardDrive,
+            MinFreeSpace::Percent(1.0),
+            Vec::new(),
+        )
+        .unwrap();
+        loc.load_existing_volumes(NeedleMapKind::InMemory).unwrap();
+        assert_eq!(loc.volumes_len(), 2);
+
+        let ids = loc.volume_ids();
+        assert!(ids.contains(&VolumeId(1)));
+        assert!(ids.contains(&VolumeId(2)));
+    }
+
+    #[test]
+    fn test_disk_location_delete_volume() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let mut loc = DiskLocation::new(
+            dir,
+            dir,
+            10,
+            DiskType::HardDrive,
+            MinFreeSpace::Percent(1.0),
+            Vec::new(),
+        )
+        .unwrap();
+
+        loc.create_volume(
+            VolumeId(1),
+            "",
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+        loc.create_volume(
+            VolumeId(2),
+            "",
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+        assert_eq!(loc.volumes_len(), 2);
+
+        loc.delete_volume(VolumeId(1), false).unwrap();
+        assert_eq!(loc.volumes_len(), 1);
+        assert!(loc.find_volume(VolumeId(1)).is_none());
+    }
+
+    #[test]
+    fn test_disk_location_delete_collection() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let mut loc = DiskLocation::new(
+            dir,
+            dir,
+            10,
+            DiskType::HardDrive,
+            MinFreeSpace::Percent(1.0),
+            Vec::new(),
+        )
+        .unwrap();
+
+        loc.create_volume(
+            VolumeId(1),
+            "pics",
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+        loc.create_volume(
+            VolumeId(2),
+            "pics",
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+        loc.create_volume(
+            VolumeId(3),
+            "docs",
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+        assert_eq!(loc.volumes_len(), 3);
+
+        loc.delete_collection("pics").unwrap();
+        assert_eq!(loc.volumes_len(), 1);
+        assert!(loc.find_volume(VolumeId(3)).is_some());
+    }
+
+    #[test]
+    fn test_disk_location_delete_collection_removes_ec_volumes() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let mut loc = DiskLocation::new(
+            dir,
+            dir,
+            10,
+            DiskType::HardDrive,
+            MinFreeSpace::Percent(1.0),
+            Vec::new(),
+        )
+        .unwrap();
+
+        let shard_path = format!("{}/pics_7.ec00", dir);
+        std::fs::write(&shard_path, b"ec-shard").unwrap();
+
+        loc.mount_ec_shards(VolumeId(7), "pics", &[0]).unwrap();
+        assert!(loc.has_ec_volume(VolumeId(7)));
+        assert!(std::path::Path::new(&shard_path).exists());
+        assert!(std::path::Path::new(&format!("{}/pics_7.ecj", dir)).exists());
+
+        loc.delete_collection("pics").unwrap();
+
+        assert!(!loc.has_ec_volume(VolumeId(7)));
+        assert!(!std::path::Path::new(&shard_path).exists());
+        assert!(!std::path::Path::new(&format!("{}/pics_7.ecj", dir)).exists());
+    }
+
+    #[test]
+    fn test_disk_location_persists_directory_uuid_and_tags() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+
+        let loc = DiskLocation::new(
+            dir,
+            dir,
+            10,
+            DiskType::HardDrive,
+            MinFreeSpace::Percent(1.0),
+            vec!["fast".to_string(), "ssd".to_string()],
+        )
+        .unwrap();
+        let directory_uuid = loc.directory_uuid.clone();
+        assert_eq!(loc.tags, vec!["fast".to_string(), "ssd".to_string()]);
+        drop(loc);
+
+        let reloaded = DiskLocation::new(
+            dir,
+            dir,
+            10,
+            DiskType::HardDrive,
+            MinFreeSpace::Percent(1.0),
+            Vec::new(),
+        )
+        .unwrap();
+        assert_eq!(reloaded.directory_uuid, directory_uuid);
+    }
+}

seaweed-volume/src/storage/erasure_coding/ec_decoder.rs

@@ -0,0 +1,261 @@
+//! EC decoding: reconstruct a .dat file from EC shards.
+//!
+//! Rebuilds the original .dat + .idx files from data shards (.ec00-.ec09)
+//! and the sorted index (.ecx) + deletion journal (.ecj).
+
+use std::fs::File;
+use std::io::{self, Read, Write};
+
+use crate::storage::erasure_coding::ec_shard::*;
+use crate::storage::idx;
+use crate::storage::needle::needle::get_actual_size;
+use crate::storage::super_block::SUPER_BLOCK_SIZE;
+use crate::storage::types::*;
+use crate::storage::volume::volume_file_name;
+
+/// Calculate .dat file size from the max offset entry in .ecx.
+/// Reads the volume version from the first EC shard (.ec00) superblock,
+/// then scans .ecx entries to find the largest (offset + needle_actual_size).
+pub fn find_dat_file_size(dir: &str, collection: &str, volume_id: VolumeId) -> io::Result<i64> {
+    let base = volume_file_name(dir, collection, volume_id);
+
+    // Read volume version from .ec00 superblock
+    let ec00_path = format!("{}.ec00", base);
+    let mut ec00 = File::open(&ec00_path)?;
+    let mut sb_buf = [0u8; SUPER_BLOCK_SIZE];
+    ec00.read_exact(&mut sb_buf)?;
+    let version = Version(sb_buf[0]);
+
+    // Start with at least the superblock size
+    let mut dat_size: i64 = SUPER_BLOCK_SIZE as i64;
+
+    // Scan .ecx entries
+    let ecx_path = format!("{}.ecx", base);
+    let ecx_data = std::fs::read(&ecx_path)?;
+    let entry_count = ecx_data.len() / NEEDLE_MAP_ENTRY_SIZE;
+
+    for i in 0..entry_count {
+        let start = i * NEEDLE_MAP_ENTRY_SIZE;
+        let (_, offset, size) =
+            idx_entry_from_bytes(&ecx_data[start..start + NEEDLE_MAP_ENTRY_SIZE]);
+        if size.is_deleted() {
+            continue;
+        }
+        let entry_stop = offset.to_actual_offset() + get_actual_size(size, version);
+        if entry_stop > dat_size {
+            dat_size = entry_stop;
+        }
+    }
+
+    Ok(dat_size)
+}
+
+/// Reconstruct a .dat file from EC data shards.
+///
+/// Reads from .ec00-.ec09 and writes a new .dat file.
+pub fn write_dat_file_from_shards(
+    dir: &str,
+    collection: &str,
+    volume_id: VolumeId,
+    dat_file_size: i64,
+    data_shards: usize,
+) -> io::Result<()> {
+    let base = volume_file_name(dir, collection, volume_id);
+    let dat_path = format!("{}.dat", base);
+
+    // Open data shards
+    let mut shards: Vec<EcVolumeShard> = (0..data_shards as u8)
+        .map(|i| EcVolumeShard::new(dir, collection, volume_id, i))
+        .collect();
+
+    for shard in &mut shards {
+        shard.open()?;
+    }
+
+    let mut dat_file = File::create(&dat_path)?;
+    let mut remaining = dat_file_size;
+    let large_block_size = ERASURE_CODING_LARGE_BLOCK_SIZE;
+    let small_block_size = ERASURE_CODING_SMALL_BLOCK_SIZE;
+    let large_row_size = (large_block_size * data_shards) as i64;
+
+    let mut shard_offset: u64 = 0;
+
+    // Read large blocks
+    while remaining >= large_row_size {
+        for i in 0..data_shards {
+            let mut buf = vec![0u8; large_block_size];
+            shards[i].read_at(&mut buf, shard_offset)?;
+            let to_write = large_block_size.min(remaining as usize);
+            dat_file.write_all(&buf[..to_write])?;
+            remaining -= to_write as i64;
+            if remaining <= 0 {
+                break;
+            }
+        }
+        shard_offset += large_block_size as u64;
+    }
+
+    // Read small blocks
+    while remaining > 0 {
+        for i in 0..data_shards {
+            let mut buf = vec![0u8; small_block_size];
+            shards[i].read_at(&mut buf, shard_offset)?;
+            let to_write = small_block_size.min(remaining as usize);
+            dat_file.write_all(&buf[..to_write])?;
+            remaining -= to_write as i64;
+            if remaining <= 0 {
+                break;
+            }
+        }
+        shard_offset += small_block_size as u64;
+    }
+
+    for shard in &mut shards {
+        shard.close();
+    }
+
+    dat_file.sync_all()?;
+    Ok(())
+}
+
+/// Write .idx file from .ecx index + .ecj deletion journal.
+///
+/// Copies sorted .ecx entries to .idx, then appends tombstones for
+/// deleted needles from .ecj.
+pub fn write_idx_file_from_ec_index(
+    dir: &str,
+    collection: &str,
+    volume_id: VolumeId,
+) -> io::Result<()> {
+    let base = volume_file_name(dir, collection, volume_id);
+    let ecx_path = format!("{}.ecx", base);
+    let ecj_path = format!("{}.ecj", base);
+    let idx_path = format!("{}.idx", base);
+
+    // Copy .ecx to .idx
+    std::fs::copy(&ecx_path, &idx_path)?;
+
+    // Append deletions from .ecj as tombstones
+    if std::path::Path::new(&ecj_path).exists() {
+        let ecj_data = std::fs::read(&ecj_path)?;
+        if !ecj_data.is_empty() {
+            let mut idx_file = std::fs::OpenOptions::new()
+                .write(true)
+                .append(true)
+                .open(&idx_path)?;
+
+            let count = ecj_data.len() / NEEDLE_ID_SIZE;
+            for i in 0..count {
+                let start = i * NEEDLE_ID_SIZE;
+                let needle_id = NeedleId::from_bytes(&ecj_data[start..start + NEEDLE_ID_SIZE]);
+                idx::write_index_entry(
+                    &mut idx_file,
+                    needle_id,
+                    Offset::default(),
+                    TOMBSTONE_FILE_SIZE,
+                )?;
+            }
+        }
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::storage::erasure_coding::ec_encoder;
+    use crate::storage::needle::needle::Needle;
+    use crate::storage::needle_map::NeedleMapKind;
+    use crate::storage::volume::Volume;
+    use tempfile::TempDir;
+
+    #[test]
+    fn test_ec_full_round_trip() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+
+        // Create volume with data
+        let mut v = Volume::new(
+            dir,
+            dir,
+            "",
+            VolumeId(1),
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+
+        let test_data: Vec<(NeedleId, Vec<u8>)> = (1..=3)
+            .map(|i| {
+                let data = format!("EC round trip data for needle {}", i);
+                (NeedleId(i), data.into_bytes())
+            })
+            .collect();
+
+        for (id, data) in &test_data {
+            let mut n = Needle {
+                id: *id,
+                cookie: Cookie(id.0 as u32),
+                data: data.clone(),
+                data_size: data.len() as u32,
+                ..Needle::default()
+            };
+            v.write_needle(&mut n, true).unwrap();
+        }
+        v.sync_to_disk().unwrap();
+        let original_dat_size = v.dat_file_size().unwrap();
+        v.close();
+
+        // Read original .dat for comparison
+        let original_dat = std::fs::read(format!("{}/1.dat", dir)).unwrap();
+
+        // Encode to EC
+        let data_shards = 10;
+        let parity_shards = 4;
+        ec_encoder::write_ec_files(dir, dir, "", VolumeId(1), data_shards, parity_shards).unwrap();
+
+        // Delete original .dat and .idx
+        std::fs::remove_file(format!("{}/1.dat", dir)).unwrap();
+        std::fs::remove_file(format!("{}/1.idx", dir)).unwrap();
+
+        // Reconstruct from EC shards
+        write_dat_file_from_shards(dir, "", VolumeId(1), original_dat_size as i64, data_shards)
+            .unwrap();
+        write_idx_file_from_ec_index(dir, "", VolumeId(1)).unwrap();
+
+        // Verify reconstructed .dat matches original
+        let reconstructed_dat = std::fs::read(format!("{}/1.dat", dir)).unwrap();
+        assert_eq!(
+            original_dat[..original_dat_size as usize],
+            reconstructed_dat[..original_dat_size as usize],
+            "reconstructed .dat should match original"
+        );
+
+        // Verify we can load and read from reconstructed volume
+        let v2 = Volume::new(
+            dir,
+            dir,
+            "",
+            VolumeId(1),
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+
+        for (id, expected_data) in &test_data {
+            let mut n = Needle {
+                id: *id,
+                ..Needle::default()
+            };
+            v2.read_needle(&mut n).unwrap();
+            assert_eq!(&n.data, expected_data, "needle {} data should match", id);
+        }
+    }
+}

seaweed-volume/src/storage/erasure_coding/ec_encoder.rs

@@ -0,0 +1,824 @@
+//! EC encoding: convert a .dat file into 10 data + 4 parity shards.
+//!
+//! Uses Reed-Solomon erasure coding. The .dat file is split into blocks
+//! (1GB large, 1MB small) and encoded across 14 shard files.
+
+use std::fs::File;
+use std::io;
+#[cfg(not(unix))]
+use std::io::{Read, Seek, SeekFrom};
+
+use reed_solomon_erasure::galois_8::ReedSolomon;
+
+use crate::storage::erasure_coding::ec_shard::*;
+use crate::storage::idx;
+use crate::storage::types::*;
+use crate::storage::volume::volume_file_name;
+
+/// Encode a .dat file into EC shard files.
+///
+/// Creates .ec00-.ec13 files in the same directory.
+/// Also creates a sorted .ecx index from the .idx file.
+pub fn write_ec_files(
+    dir: &str,
+    idx_dir: &str,
+    collection: &str,
+    volume_id: VolumeId,
+    data_shards: usize,
+    parity_shards: usize,
+) -> io::Result<()> {
+    let base = volume_file_name(dir, collection, volume_id);
+    let dat_path = format!("{}.dat", base);
+    let idx_base = volume_file_name(idx_dir, collection, volume_id);
+    let idx_path = format!("{}.idx", idx_base);
+
+    // Create sorted .ecx from .idx
+    write_sorted_ecx_from_idx(&idx_path, &format!("{}.ecx", base))?;
+
+    // Encode .dat into shards
+    let dat_file = File::open(&dat_path)?;
+    let dat_size = dat_file.metadata()?.len() as i64;
+
+    let rs = ReedSolomon::new(data_shards, parity_shards)
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("reed-solomon init: {:?}", e)))?;
+
+    // Create shard files
+    let total_shards = data_shards + parity_shards;
+    let mut shards: Vec<EcVolumeShard> = (0..total_shards as u8)
+        .map(|i| EcVolumeShard::new(dir, collection, volume_id, i))
+        .collect();
+
+    for shard in &mut shards {
+        shard.create()?;
+    }
+
+    // Encode in large blocks, then small blocks
+    encode_dat_file(
+        &dat_file,
+        dat_size,
+        &rs,
+        &mut shards,
+        data_shards,
+        parity_shards,
+    )?;
+
+    // Close all shards
+    for shard in &mut shards {
+        shard.close();
+    }
+
+    Ok(())
+}
+
+/// Rebuild missing EC shard files from existing shards using Reed-Solomon reconstruct.
+///
+/// This does not require the `.dat` file, only the existing `.ecXX` shard files.
+pub fn rebuild_ec_files(
+    dir: &str,
+    collection: &str,
+    volume_id: VolumeId,
+    missing_shard_ids: &[u32],
+    data_shards: usize,
+    parity_shards: usize,
+) -> io::Result<()> {
+    if missing_shard_ids.is_empty() {
+        return Ok(());
+    }
+
+    let rs = ReedSolomon::new(data_shards, parity_shards)
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("reed-solomon init: {:?}", e)))?;
+
+    let total_shards = data_shards + parity_shards;
+    let mut shards: Vec<EcVolumeShard> = (0..total_shards as u8)
+        .map(|i| EcVolumeShard::new(dir, collection, volume_id, i))
+        .collect();
+
+    // Determine the exact shard size from the first available existing shard
+    let mut shard_size = 0;
+    for (i, shard) in shards.iter_mut().enumerate() {
+        if !missing_shard_ids.contains(&(i as u32)) {
+            if let Ok(_) = shard.open() {
+                let size = shard.file_size();
+                if size > shard_size {
+                    shard_size = size;
+                }
+            } else {
+                return Err(io::Error::new(
+                    io::ErrorKind::NotFound,
+                    format!("missing non-rebuild shard {}", i),
+                ));
+            }
+        }
+    }
+
+    if shard_size == 0 {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            "all existing shards are empty or cannot find an existing shard to determine size",
+        ));
+    }
+
+    // Create the missing shards for writing
+    for i in missing_shard_ids {
+        if let Some(shard) = shards.get_mut(*i as usize) {
+            shard.create()?;
+        }
+    }
+
+    let block_size = ERASURE_CODING_SMALL_BLOCK_SIZE;
+    let mut remaining = shard_size;
+    let mut offset: u64 = 0;
+
+    // Process all data in blocks
+    while remaining > 0 {
+        let to_process = remaining.min(block_size as i64) as usize;
+
+        // Allocate buffers for all shards. Option<Vec<u8>> is required by rs.reconstruct()
+        let mut buffers: Vec<Option<Vec<u8>>> = vec![None; total_shards];
+
+        // Read available shards
+        for (i, shard) in shards.iter().enumerate() {
+            if !missing_shard_ids.contains(&(i as u32)) {
+                let mut buf = vec![0u8; to_process];
+                shard.read_at(&mut buf, offset)?;
+                buffers[i] = Some(buf);
+            }
+        }
+
+        // Reconstruct missing shards
+        rs.reconstruct(&mut buffers).map_err(|e| {
+            io::Error::new(
+                io::ErrorKind::Other,
+                format!("reed-solomon reconstruct: {:?}", e),
+            )
+        })?;
+
+        // Write recovered data into the missing shards
+        for i in missing_shard_ids {
+            let idx = *i as usize;
+            if let Some(buf) = buffers[idx].take() {
+                shards[idx].write_all(&buf)?;
+            }
+        }
+
+        offset += to_process as u64;
+        remaining -= to_process as i64;
+    }
+
+    // Close all shards
+    for shard in &mut shards {
+        shard.close();
+    }
+
+    Ok(())
+}
+
+/// Verify EC shards by computing parity against the existing data and identifying corrupted shards.
+pub fn verify_ec_shards(
+    dir: &str,
+    collection: &str,
+    volume_id: VolumeId,
+    data_shards: usize,
+    parity_shards: usize,
+) -> io::Result<(Vec<u32>, Vec<String>)> {
+    let rs = ReedSolomon::new(data_shards, parity_shards)
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("reed-solomon init: {:?}", e)))?;
+
+    let total_shards = data_shards + parity_shards;
+    let mut shards: Vec<EcVolumeShard> = (0..total_shards as u8)
+        .map(|i| EcVolumeShard::new(dir, collection, volume_id, i))
+        .collect();
+
+    let mut shard_size = 0;
+    let mut broken_shards = std::collections::HashSet::new();
+    let mut details = Vec::new();
+
+    for (i, shard) in shards.iter_mut().enumerate() {
+        if let Ok(_) = shard.open() {
+            let size = shard.file_size();
+            if size > shard_size {
+                shard_size = size;
+            }
+        } else {
+            broken_shards.insert(i as u32);
+            details.push(format!("failed to open or missing shard {}", i));
+        }
+    }
+
+    if shard_size == 0 || broken_shards.len() >= parity_shards {
+        // Can't do much if we don't know the size or have too many missing
+        return Ok((broken_shards.into_iter().collect(), details));
+    }
+
+    let block_size = ERASURE_CODING_SMALL_BLOCK_SIZE;
+    let mut remaining = shard_size;
+    let mut offset: u64 = 0;
+
+    while remaining > 0 {
+        let to_process = remaining.min(block_size as i64) as usize;
+        let mut buffers = vec![vec![0u8; to_process]; total_shards];
+
+        let mut read_failed = false;
+        for i in 0..total_shards {
+            if !broken_shards.contains(&(i as u32)) {
+                if let Err(e) = shards[i].read_at(&mut buffers[i], offset) {
+                    broken_shards.insert(i as u32);
+                    details.push(format!("read error shard {}: {}", i, e));
+                    read_failed = true;
+                }
+            } else {
+                read_failed = true;
+            }
+        }
+
+        // Only do verification if all shards were readable
+        if !read_failed {
+            // Need to convert Vec<Vec<u8>> to &[&[u8]] for rs.verify
+            let slice_ptrs: Vec<&[u8]> = buffers.iter().map(|v| v.as_slice()).collect();
+            if let Ok(is_valid) = rs.verify(&slice_ptrs) {
+                if !is_valid {
+                    // Reed-Solomon verification failed. We cannot easily pinpoint which shard
+                    // is corrupted without recalculating parities or syndromes, so we just
+                    // log that this batch has corruption. Wait, we can test each parity shard!
+                    // Let's re-encode from the first `data_shards` and compare to the actual `parity_shards`.
+
+                    let mut verify_buffers = buffers.clone();
+                    // Clear the parity parts
+                    for i in data_shards..total_shards {
+                        verify_buffers[i].fill(0);
+                    }
+                    if rs.encode(&mut verify_buffers).is_ok() {
+                        for i in 0..total_shards {
+                            if buffers[i] != verify_buffers[i] {
+                                broken_shards.insert(i as u32);
+                                details.push(format!(
+                                    "parity mismatch on shard {} at offset {}",
+                                    i, offset
+                                ));
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        offset += to_process as u64;
+        remaining -= to_process as i64;
+    }
+
+    // Close all shards
+    for shard in &mut shards {
+        shard.close();
+    }
+
+    let mut broken_vec: Vec<u32> = broken_shards.into_iter().collect();
+    broken_vec.sort_unstable();
+
+    Ok((broken_vec, details))
+}
+
+/// Write sorted .ecx index from .idx file.
+fn write_sorted_ecx_from_idx(idx_path: &str, ecx_path: &str) -> io::Result<()> {
+    if !std::path::Path::new(idx_path).exists() {
+        return Err(io::Error::new(
+            io::ErrorKind::NotFound,
+            "idx file not found",
+        ));
+    }
+
+    // Read all idx entries
+    let mut idx_file = File::open(idx_path)?;
+    let mut entries: Vec<(NeedleId, Offset, Size)> = Vec::new();
+
+    idx::walk_index_file(&mut idx_file, 0, |key, offset, size| {
+        entries.push((key, offset, size));
+        Ok(())
+    })?;
+
+    // Sort by NeedleId, then by actual offset so later entries come last
+    entries.sort_by_key(|&(key, offset, _)| (key, offset.to_actual_offset()));
+
+    // Remove duplicates (keep last/latest entry for each key).
+    // dedup_by_key keeps the first in each run, so we reverse first,
+    // dedup, then reverse back.
+    entries.reverse();
+    entries.dedup_by_key(|entry| entry.0);
+    entries.reverse();
+
+    // Write sorted entries to .ecx
+    let mut ecx_file = File::create(ecx_path)?;
+    for &(key, offset, size) in &entries {
+        idx::write_index_entry(&mut ecx_file, key, offset, size)?;
+    }
+
+    Ok(())
+}
+
+/// Rebuild the .ecx index file by walking needles in the EC data shards.
+///
+/// This is the equivalent of Go's `RebuildEcxFile`. It reads the logical .dat
+/// content from the EC data shards, walks through needle headers to extract
+/// (needle_id, offset, size) entries, deduplicates them, and writes a sorted
+/// .ecx index file.
+pub fn rebuild_ecx_file(
+    dir: &str,
+    collection: &str,
+    volume_id: VolumeId,
+    data_shards: usize,
+) -> io::Result<()> {
+    use crate::storage::needle::needle::get_actual_size;
+    use crate::storage::super_block::SUPER_BLOCK_SIZE;
+
+    let base = volume_file_name(dir, collection, volume_id);
+    let ecx_path = format!("{}.ecx", base);
+
+    // Open data shards to read logical .dat content
+    let mut shards: Vec<EcVolumeShard> = (0..data_shards as u8)
+        .map(|i| EcVolumeShard::new(dir, collection, volume_id, i))
+        .collect();
+
+    for shard in &mut shards {
+        if let Err(_) = shard.open() {
+            // If a data shard is missing, we can't rebuild ecx
+            for s in &mut shards {
+                s.close();
+            }
+            return Err(io::Error::new(
+                io::ErrorKind::NotFound,
+                format!("cannot open data shard for ecx rebuild"),
+            ));
+        }
+    }
+
+    // Determine total logical data size from shard sizes
+    let shard_size = shards.iter().map(|s| s.file_size()).max().unwrap_or(0);
+    let total_data_size = shard_size as i64 * data_shards as i64;
+
+    // Read version from superblock (first byte of logical data)
+    let mut sb_buf = [0u8; SUPER_BLOCK_SIZE];
+    read_from_data_shards(&shards, &mut sb_buf, 0, data_shards)?;
+    let version = Version(sb_buf[0]);
+
+    // Walk needles starting after superblock
+    let mut offset = SUPER_BLOCK_SIZE as i64;
+    let header_size = NEEDLE_HEADER_SIZE;
+    let mut entries: Vec<(NeedleId, Offset, Size)> = Vec::new();
+
+    while offset + header_size as i64 <= total_data_size {
+        // Read needle header (cookie + needle_id + size = 16 bytes)
+        let mut header_buf = [0u8; NEEDLE_HEADER_SIZE];
+        if read_from_data_shards(&shards, &mut header_buf, offset as u64, data_shards).is_err() {
+            break;
+        }
+
+        let cookie = Cookie::from_bytes(&header_buf[..COOKIE_SIZE]);
+        let needle_id = NeedleId::from_bytes(&header_buf[COOKIE_SIZE..COOKIE_SIZE + NEEDLE_ID_SIZE]);
+        let size = Size::from_bytes(&header_buf[COOKIE_SIZE + NEEDLE_ID_SIZE..header_size]);
+
+        // Validate: stop if we hit zero cookie+id (end of data)
+        if cookie.0 == 0 && needle_id.0 == 0 {
+            break;
+        }
+
+        // Validate size is reasonable
+        if size.0 < 0 && !size.is_deleted() {
+            break;
+        }
+
+        let actual_size = get_actual_size(size, version);
+        if actual_size <= 0 || offset + actual_size > total_data_size {
+            break;
+        }
+
+        entries.push((needle_id, Offset::from_actual_offset(offset), size));
+
+        // Advance to next needle (aligned to NEEDLE_PADDING_SIZE)
+        offset += actual_size;
+        let padding_rem = offset % NEEDLE_PADDING_SIZE as i64;
+        if padding_rem != 0 {
+            offset += NEEDLE_PADDING_SIZE as i64 - padding_rem;
+        }
+    }
+
+    for shard in &mut shards {
+        shard.close();
+    }
+
+    // Sort by NeedleId, then by offset (later entries override earlier)
+    entries.sort_by_key(|&(key, offset, _)| (key, offset.to_actual_offset()));
+
+    // Deduplicate: keep latest entry per needle_id
+    entries.reverse();
+    entries.dedup_by_key(|entry| entry.0);
+    entries.reverse();
+
+    // Write sorted .ecx
+    let mut ecx_file = File::create(&ecx_path)?;
+    for &(key, offset, size) in &entries {
+        idx::write_index_entry(&mut ecx_file, key, offset, size)?;
+    }
+    ecx_file.sync_all()?;
+
+    Ok(())
+}
+
+/// Read bytes from EC data shards at a logical offset in the .dat file.
+fn read_from_data_shards(
+    shards: &[EcVolumeShard],
+    buf: &mut [u8],
+    logical_offset: u64,
+    data_shards: usize,
+) -> io::Result<()> {
+    let small_block = ERASURE_CODING_SMALL_BLOCK_SIZE as u64;
+    let data_shards_u64 = data_shards as u64;
+
+    let mut bytes_read = 0u64;
+    let mut remaining = buf.len() as u64;
+    let mut current_offset = logical_offset;
+
+    while remaining > 0 {
+        // Determine which shard and at what shard-offset this logical offset maps to.
+        // The data is interleaved: large blocks first, then small blocks.
+        // For simplicity, use the small block size for all calculations since
+        // large blocks are multiples of small blocks.
+        let row_size = small_block * data_shards_u64;
+        let row_index = current_offset / row_size;
+        let row_offset = current_offset % row_size;
+        let shard_index = (row_offset / small_block) as usize;
+        let shard_offset = row_index * small_block + (row_offset % small_block);
+
+        if shard_index >= data_shards {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "shard index out of range",
+            ));
+        }
+
+        // How many bytes can we read from this position in this shard block
+        let bytes_left_in_block = small_block - (row_offset % small_block);
+        let to_read = remaining.min(bytes_left_in_block) as usize;
+
+        let dest = &mut buf[bytes_read as usize..bytes_read as usize + to_read];
+        shards[shard_index].read_at(dest, shard_offset)?;
+
+        bytes_read += to_read as u64;
+        remaining -= to_read as u64;
+        current_offset += to_read as u64;
+    }
+
+    Ok(())
+}
+
+/// Encode the .dat file data into shard files.
+///
+/// Uses a two-phase approach matching Go's ec_encoder.go:
+/// 1. Process as many large blocks (1GB) as possible
+/// 2. Process remaining data with small blocks (1MB)
+fn encode_dat_file(
+    dat_file: &File,
+    dat_size: i64,
+    rs: &ReedSolomon,
+    shards: &mut [EcVolumeShard],
+    data_shards: usize,
+    parity_shards: usize,
+) -> io::Result<()> {
+    let mut remaining = dat_size;
+    let mut offset: u64 = 0;
+
+    // Phase 1: Process large blocks (1GB each) while enough data remains
+    let large_block_size = ERASURE_CODING_LARGE_BLOCK_SIZE;
+    let large_row_size = large_block_size * data_shards;
+
+    while remaining >= large_row_size as i64 {
+        encode_one_batch(
+            dat_file,
+            offset,
+            large_block_size,
+            rs,
+            shards,
+            data_shards,
+            parity_shards,
+        )?;
+        offset += large_row_size as u64;
+        remaining -= large_row_size as i64;
+    }
+
+    // Phase 2: Process remaining data with small blocks (1MB each)
+    let small_block_size = ERASURE_CODING_SMALL_BLOCK_SIZE;
+    let small_row_size = small_block_size * data_shards;
+
+    while remaining > 0 {
+        let to_process = remaining.min(small_row_size as i64);
+        encode_one_batch(
+            dat_file,
+            offset,
+            small_block_size,
+            rs,
+            shards,
+            data_shards,
+            parity_shards,
+        )?;
+        offset += to_process as u64;
+        remaining -= to_process;
+    }
+
+    Ok(())
+}
+
+/// Encode one batch (row) of data.
+fn encode_one_batch(
+    dat_file: &File,
+    offset: u64,
+    block_size: usize,
+    rs: &ReedSolomon,
+    shards: &mut [EcVolumeShard],
+    data_shards: usize,
+    parity_shards: usize,
+) -> io::Result<()> {
+    let total_shards = data_shards + parity_shards;
+    // Each batch allocates block_size * total_shards bytes.
+    // With large blocks (1 GiB) this is 14 GiB -- guard against OOM.
+    let total_alloc = block_size.checked_mul(total_shards).ok_or_else(|| {
+        io::Error::new(
+            io::ErrorKind::InvalidInput,
+            "block_size * shard count overflows usize",
+        )
+    })?;
+    // Large-block encoding uses 1 GiB * 14 shards = 14 GiB; allow up to 16 GiB.
+    const MAX_BATCH_ALLOC: usize = 16 * 1024 * 1024 * 1024; // 16 GiB safety limit
+    if total_alloc > MAX_BATCH_ALLOC {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidInput,
+            format!(
+                "batch allocation too large ({} bytes, limit {} bytes); block_size={} shards={}",
+                total_alloc, MAX_BATCH_ALLOC, block_size, total_shards,
+            ),
+        ));
+    }
+
+    // Allocate buffers for all shards
+    let mut buffers: Vec<Vec<u8>> = (0..total_shards).map(|_| vec![0u8; block_size]).collect();
+
+    // Read data shards from .dat file
+    for i in 0..data_shards {
+        let read_offset = offset + (i * block_size) as u64;
+
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::FileExt;
+            dat_file.read_at(&mut buffers[i], read_offset)?;
+        }
+
+        #[cfg(not(unix))]
+        {
+            let mut f = dat_file.try_clone()?;
+            f.seek(SeekFrom::Start(read_offset))?;
+            f.read(&mut buffers[i])?;
+        }
+    }
+
+    // Encode parity shards
+    rs.encode(&mut buffers).map_err(|e| {
+        io::Error::new(
+            io::ErrorKind::Other,
+            format!("reed-solomon encode: {:?}", e),
+        )
+    })?;
+
+    // Write all shard buffers to files
+    for (i, buf) in buffers.iter().enumerate() {
+        shards[i].write_all(buf)?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::storage::needle::needle::Needle;
+    use crate::storage::needle_map::NeedleMapKind;
+    use crate::storage::volume::Volume;
+    use tempfile::TempDir;
+
+    #[test]
+    fn test_ec_encode_decode_round_trip() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+
+        // Create a volume with some data
+        let mut v = Volume::new(
+            dir,
+            dir,
+            "",
+            VolumeId(1),
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+
+        for i in 1..=5 {
+            let data = format!("test data for needle {}", i);
+            let mut n = Needle {
+                id: NeedleId(i),
+                cookie: Cookie(i as u32),
+                data: data.as_bytes().to_vec(),
+                data_size: data.len() as u32,
+                ..Needle::default()
+            };
+            v.write_needle(&mut n, true).unwrap();
+        }
+        v.sync_to_disk().unwrap();
+        v.close();
+
+        // Encode to EC shards
+        let data_shards = 10;
+        let parity_shards = 4;
+        let total_shards = data_shards + parity_shards;
+        write_ec_files(dir, dir, "", VolumeId(1), data_shards, parity_shards).unwrap();
+
+        // Verify shard files exist
+        for i in 0..total_shards {
+            let path = format!("{}/{}.ec{:02}", dir, 1, i);
+            assert!(
+                std::path::Path::new(&path).exists(),
+                "shard file {} should exist",
+                path
+            );
+        }
+
+        // Verify .ecx exists
+        let ecx_path = format!("{}/1.ecx", dir);
+        assert!(std::path::Path::new(&ecx_path).exists());
+    }
+
+    #[test]
+    fn test_reed_solomon_basic() {
+        let data_shards = 10;
+        let parity_shards = 4;
+        let total_shards = data_shards + parity_shards;
+        let rs = ReedSolomon::new(data_shards, parity_shards).unwrap();
+        let block_size = 1024;
+        let mut shards: Vec<Vec<u8>> = (0..total_shards)
+            .map(|i| {
+                if i < data_shards {
+                    vec![(i as u8).wrapping_mul(7); block_size]
+                } else {
+                    vec![0u8; block_size]
+                }
+            })
+            .collect();
+
+        // Encode
+        rs.encode(&mut shards).unwrap();
+
+        // Verify parity is non-zero (at least some)
+        let parity_nonzero: bool = shards[data_shards..]
+            .iter()
+            .any(|s| s.iter().any(|&b| b != 0));
+        assert!(parity_nonzero);
+
+        // Simulate losing 4 shards and reconstructing
+        let original_0 = shards[0].clone();
+        let original_1 = shards[1].clone();
+
+        let mut shard_opts: Vec<Option<Vec<u8>>> = shards.into_iter().map(Some).collect();
+        shard_opts[0] = None;
+        shard_opts[1] = None;
+        shard_opts[2] = None;
+        shard_opts[3] = None;
+
+        rs.reconstruct(&mut shard_opts).unwrap();
+
+        assert_eq!(shard_opts[0].as_ref().unwrap(), &original_0);
+        assert_eq!(shard_opts[1].as_ref().unwrap(), &original_1);
+    }
+
+    /// EC encode must read .idx from a separate index directory when configured.
+    #[test]
+    fn test_ec_encode_with_separate_idx_dir() {
+        let dat_tmp = TempDir::new().unwrap();
+        let idx_tmp = TempDir::new().unwrap();
+        let dat_dir = dat_tmp.path().to_str().unwrap();
+        let idx_dir = idx_tmp.path().to_str().unwrap();
+
+        // Create a volume with separate data and index directories
+        let mut v = Volume::new(
+            dat_dir,
+            idx_dir,
+            "",
+            VolumeId(1),
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+
+        for i in 1..=5 {
+            let data = format!("needle {} payload", i);
+            let mut n = Needle {
+                id: NeedleId(i),
+                cookie: Cookie(i as u32),
+                data: data.as_bytes().to_vec(),
+                data_size: data.len() as u32,
+                ..Needle::default()
+            };
+            v.write_needle(&mut n, true).unwrap();
+        }
+        v.sync_to_disk().unwrap();
+        v.close();
+
+        // Verify .dat is in data dir, .idx is in idx dir
+        assert!(std::path::Path::new(&format!("{}/1.dat", dat_dir)).exists());
+        assert!(!std::path::Path::new(&format!("{}/1.idx", dat_dir)).exists());
+        assert!(std::path::Path::new(&format!("{}/1.idx", idx_dir)).exists());
+        assert!(!std::path::Path::new(&format!("{}/1.dat", idx_dir)).exists());
+
+        // EC encode with separate idx dir
+        let data_shards = 10;
+        let parity_shards = 4;
+        let total_shards = data_shards + parity_shards;
+        write_ec_files(
+            dat_dir,
+            idx_dir,
+            "",
+            VolumeId(1),
+            data_shards,
+            parity_shards,
+        )
+        .unwrap();
+
+        // Verify all 14 shard files in data dir
+        for i in 0..total_shards {
+            let path = format!("{}/1.ec{:02}", dat_dir, i);
+            assert!(
+                std::path::Path::new(&path).exists(),
+                "shard {} should exist in data dir",
+                path
+            );
+        }
+
+        // Verify .ecx in data dir (not idx dir)
+        assert!(std::path::Path::new(&format!("{}/1.ecx", dat_dir)).exists());
+        assert!(!std::path::Path::new(&format!("{}/1.ecx", idx_dir)).exists());
+
+        // Verify no shard files leaked into idx dir
+        for i in 0..total_shards {
+            let path = format!("{}/1.ec{:02}", idx_dir, i);
+            assert!(
+                !std::path::Path::new(&path).exists(),
+                "shard {} should NOT exist in idx dir",
+                path
+            );
+        }
+    }
+
+    /// EC encode should fail gracefully when .idx is only in the data dir
+    /// but we pass a wrong idx_dir. This guards against regressions where
+    /// write_ec_files ignores the idx_dir parameter.
+    #[test]
+    fn test_ec_encode_fails_with_wrong_idx_dir() {
+        let dat_tmp = TempDir::new().unwrap();
+        let idx_tmp = TempDir::new().unwrap();
+        let wrong_tmp = TempDir::new().unwrap();
+        let dat_dir = dat_tmp.path().to_str().unwrap();
+        let idx_dir = idx_tmp.path().to_str().unwrap();
+        let wrong_dir = wrong_tmp.path().to_str().unwrap();
+
+        let mut v = Volume::new(
+            dat_dir,
+            idx_dir,
+            "",
+            VolumeId(1),
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+
+        let mut n = Needle {
+            id: NeedleId(1),
+            cookie: Cookie(1),
+            data: b"hello".to_vec(),
+            data_size: 5,
+            ..Needle::default()
+        };
+        v.write_needle(&mut n, true).unwrap();
+        v.sync_to_disk().unwrap();
+        v.close();
+
+        // Should fail: .idx is in idx_dir, not wrong_dir
+        let result = write_ec_files(dat_dir, wrong_dir, "", VolumeId(1), 10, 4);
+        assert!(
+            result.is_err(),
+            "should fail when idx_dir doesn't contain .idx"
+        );
+    }
+}

seaweed-volume/src/storage/erasure_coding/ec_locate.rs

@@ -0,0 +1,223 @@
+//! EC data location: maps needle offset/size to shard intervals.
+//!
+//! Determines which shard(s) contain data for a given needle and at what
+//! offsets within those shards. Handles both large (1GB) and small (1MB)
+//! block sections.
+
+use crate::storage::erasure_coding::ec_shard::*;
+use crate::storage::types::*;
+
+/// An interval to read from EC shards.
+#[derive(Debug, Clone)]
+pub struct Interval {
+    pub block_index: usize,
+    pub inner_block_offset: i64,
+    pub size: i64,
+    pub is_large_block: bool,
+    pub large_block_rows_count: usize,
+}
+
+impl Interval {
+    pub fn to_shard_id_and_offset(&self, data_shards: u32) -> (ShardId, i64) {
+        let data_shards_usize = data_shards as usize;
+        let shard_id = (self.block_index % data_shards_usize) as ShardId;
+        let row_index = self.block_index / data_shards_usize;
+
+        let block_size = if self.is_large_block {
+            ERASURE_CODING_LARGE_BLOCK_SIZE as i64
+        } else {
+            ERASURE_CODING_SMALL_BLOCK_SIZE as i64
+        };
+
+        let mut offset = row_index as i64 * block_size + self.inner_block_offset;
+        if !self.is_large_block {
+            // Small blocks come after large blocks in the shard file
+            offset += self.large_block_rows_count as i64 * ERASURE_CODING_LARGE_BLOCK_SIZE as i64;
+        }
+
+        (shard_id, offset)
+    }
+}
+
+/// Locate the EC shard intervals needed to read data at the given offset and size.
+///
+/// `shard_size` is the size of a single shard file.
+pub fn locate_data(offset: i64, size: Size, shard_size: i64, data_shards: u32) -> Vec<Interval> {
+    let mut intervals = Vec::new();
+    let data_size = size.0 as i64;
+
+    if data_size <= 0 || shard_size <= 0 {
+        return intervals;
+    }
+
+    let large_block_size = ERASURE_CODING_LARGE_BLOCK_SIZE as i64;
+    let small_block_size = ERASURE_CODING_SMALL_BLOCK_SIZE as i64;
+    let large_row_size = large_block_size * data_shards as i64;
+    let small_row_size = small_block_size * data_shards as i64;
+
+    // Number of large block rows
+    let n_large_block_rows = if shard_size > 0 {
+        ((shard_size - 1) / large_block_size) as usize
+    } else {
+        0
+    };
+    let large_section_size = n_large_block_rows as i64 * large_row_size;
+
+    let mut remaining_offset = offset;
+    let mut remaining_size = data_size;
+
+    // In large block section?
+    if remaining_offset < large_section_size {
+        let available_in_large = large_section_size - remaining_offset;
+        let to_read = remaining_size.min(available_in_large);
+
+        add_intervals(
+            &mut intervals,
+            remaining_offset,
+            to_read,
+            large_block_size,
+            large_row_size,
+            true,
+            n_large_block_rows,
+        );
+
+        remaining_offset += to_read;
+        remaining_size -= to_read;
+    }
+
+    // In small block section?
+    if remaining_size > 0 {
+        let small_offset = remaining_offset - large_section_size;
+        add_intervals(
+            &mut intervals,
+            small_offset,
+            remaining_size,
+            small_block_size,
+            small_row_size,
+            false,
+            n_large_block_rows,
+        );
+    }
+
+    intervals
+}
+
+fn add_intervals(
+    intervals: &mut Vec<Interval>,
+    offset: i64,
+    size: i64,
+    block_size: i64,
+    _row_size: i64,
+    is_large_block: bool,
+    large_block_rows_count: usize,
+) {
+    let mut pos = offset;
+    let end = offset + size;
+
+    while pos < end {
+        let block_index = (pos / block_size) as usize;
+        let inner_offset = pos % block_size;
+        let remaining_in_block = block_size - inner_offset;
+        let interval_size = remaining_in_block.min(end - pos);
+
+        intervals.push(Interval {
+            block_index,
+            inner_block_offset: inner_offset,
+            size: interval_size,
+            is_large_block,
+            large_block_rows_count,
+        });
+
+        pos += interval_size;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_interval_to_shard_id() {
+        let data_shards = 10;
+        let large_block_size = ERASURE_CODING_LARGE_BLOCK_SIZE as i64;
+        let _shard_size = 1024 * 1024; // Example shard size
+
+        // Block index 0 → shard 0
+        let interval = Interval {
+            block_index: 0,
+            inner_block_offset: 100,
+            size: 50,
+            is_large_block: true,
+            large_block_rows_count: 1,
+        };
+        let (shard_id, offset) = interval.to_shard_id_and_offset(data_shards);
+        assert_eq!(shard_id, 0);
+        assert_eq!(offset, 100);
+
+        // Block index 5 → shard 5
+        let interval = Interval {
+            block_index: 5,
+            inner_block_offset: 0,
+            size: 1024,
+            is_large_block: true,
+            large_block_rows_count: 1,
+        };
+        let (shard_id, _offset) = interval.to_shard_id_and_offset(data_shards);
+        assert_eq!(shard_id, 5);
+
+        // Block index 12 (data_shards=10) → row_index 1, shard_id 2
+        let interval = Interval {
+            block_index: 12,
+            inner_block_offset: 200,
+            size: 50,
+            is_large_block: true,
+            large_block_rows_count: 5,
+        };
+        let (shard_id, offset) = interval.to_shard_id_and_offset(data_shards);
+        assert_eq!(shard_id, 2); // 12 % 10 = 2
+        assert_eq!(offset, large_block_size + 200); // row 1 offset + inner_block_offset
+
+        // Block index 10 → shard 0 (second row)
+        let interval = Interval {
+            block_index: 10,
+            inner_block_offset: 0,
+            size: 100,
+            is_large_block: true,
+            large_block_rows_count: 2,
+        };
+        let (shard_id, offset) = interval.to_shard_id_and_offset(data_shards);
+        assert_eq!(shard_id, 0);
+        assert_eq!(offset, ERASURE_CODING_LARGE_BLOCK_SIZE as i64); // row 1 offset
+    }
+
+    #[test]
+    fn test_locate_data_small_file() {
+        // Small file: 100 bytes at offset 50, shard size = 1MB
+        let intervals = locate_data(50, Size(100), 1024 * 1024, 10);
+        assert!(!intervals.is_empty());
+
+        // Should be a single small block interval (no large block rows for 1MB shard)
+        assert_eq!(intervals.len(), 1);
+        assert!(!intervals[0].is_large_block);
+    }
+
+    #[test]
+    fn test_locate_data_empty() {
+        let intervals = locate_data(0, Size(0), 1024 * 1024, 10);
+        assert!(intervals.is_empty());
+    }
+
+    #[test]
+    fn test_small_block_after_large() {
+        let interval = Interval {
+            block_index: 0,
+            inner_block_offset: 0,
+            size: 100,
+            is_large_block: false,
+            large_block_rows_count: 2,
+        };
+        let (_shard_id, offset) = interval.to_shard_id_and_offset(10);
+        // Should be after 2 large block rows
+        assert_eq!(offset, 2 * ERASURE_CODING_LARGE_BLOCK_SIZE as i64);
+    }
+}

seaweed-volume/src/storage/erasure_coding/ec_shard.rs

@@ -0,0 +1,225 @@
+//! EcVolumeShard: a single shard file (.ec00-.ec13) of an erasure-coded volume.
+
+use std::fs::{self, File, OpenOptions};
+use std::io::{self, Write};
+
+use crate::storage::types::*;
+
+pub const DATA_SHARDS_COUNT: usize = 10;
+pub const PARITY_SHARDS_COUNT: usize = 4;
+pub const TOTAL_SHARDS_COUNT: usize = DATA_SHARDS_COUNT + PARITY_SHARDS_COUNT;
+pub const MAX_SHARD_COUNT: usize = 32;
+pub const MIN_TOTAL_DISKS: usize = TOTAL_SHARDS_COUNT / PARITY_SHARDS_COUNT + 1;
+pub const ERASURE_CODING_LARGE_BLOCK_SIZE: usize = 1024 * 1024 * 1024; // 1GB
+pub const ERASURE_CODING_SMALL_BLOCK_SIZE: usize = 1024 * 1024; // 1MB
+
+pub type ShardId = u8;
+
+/// A single erasure-coded shard file.
+pub struct EcVolumeShard {
+    pub volume_id: VolumeId,
+    pub shard_id: ShardId,
+    pub collection: String,
+    pub dir: String,
+    pub disk_type: DiskType,
+    ecd_file: Option<File>,
+    ecd_file_size: i64,
+}
+
+impl EcVolumeShard {
+    /// Create a new shard reference (does not open the file).
+    pub fn new(dir: &str, collection: &str, volume_id: VolumeId, shard_id: ShardId) -> Self {
+        EcVolumeShard {
+            volume_id,
+            shard_id,
+            collection: collection.to_string(),
+            dir: dir.to_string(),
+            disk_type: DiskType::default(),
+            ecd_file: None,
+            ecd_file_size: 0,
+        }
+    }
+
+    /// Shard file name, e.g. "dir/collection_42.ec03"
+    pub fn file_name(&self) -> String {
+        let base =
+            crate::storage::volume::volume_file_name(&self.dir, &self.collection, self.volume_id);
+        format!("{}.ec{:02}", base, self.shard_id)
+    }
+
+    /// Open the shard file for reading.
+    pub fn open(&mut self) -> io::Result<()> {
+        let path = self.file_name();
+        let file = File::open(&path)?;
+        self.ecd_file_size = file.metadata()?.len() as i64;
+        self.ecd_file = Some(file);
+        Ok(())
+    }
+
+    /// Create the shard file for writing.
+    pub fn create(&mut self) -> io::Result<()> {
+        let path = self.file_name();
+        let file = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .truncate(true)
+            .open(&path)?;
+        self.ecd_file = Some(file);
+        self.ecd_file_size = 0;
+        Ok(())
+    }
+
+    /// Read data at a specific offset.
+    pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize> {
+        let file = self
+            .ecd_file
+            .as_ref()
+            .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "shard file not open"))?;
+
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::FileExt;
+            file.read_at(buf, offset)
+        }
+
+        #[cfg(not(unix))]
+        {
+            use std::io::{Read, Seek, SeekFrom};
+            // File::read_at is unix-only; fall back to seek + read.
+            // We need a mutable reference for seek/read, so clone the handle.
+            let mut f = file.try_clone()?;
+            f.seek(SeekFrom::Start(offset))?;
+            f.read(buf)
+        }
+    }
+
+    /// Write data to the shard file (appends).
+    pub fn write_all(&mut self, data: &[u8]) -> io::Result<()> {
+        let file = self
+            .ecd_file
+            .as_mut()
+            .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "shard file not open"))?;
+        file.write_all(data)?;
+        self.ecd_file_size += data.len() as i64;
+        Ok(())
+    }
+
+    pub fn file_size(&self) -> i64 {
+        self.ecd_file_size
+    }
+
+    /// Close the shard file.
+    pub fn close(&mut self) {
+        if let Some(ref file) = self.ecd_file {
+            let _ = file.sync_all();
+        }
+        self.ecd_file = None;
+    }
+
+    /// Delete the shard file from disk.
+    pub fn destroy(&mut self) {
+        self.close();
+        let _ = fs::remove_file(self.file_name());
+    }
+}
+
+/// ShardBits: bitmap tracking which shards are present.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct ShardBits(pub u32);
+
+impl ShardBits {
+    pub fn add_shard_id(&mut self, id: ShardId) {
+        assert!((id as usize) < 32, "shard id {} out of bounds (max 31)", id,);
+        self.0 |= 1 << id;
+    }
+
+    pub fn remove_shard_id(&mut self, id: ShardId) {
+        assert!((id as usize) < 32, "shard id {} out of bounds (max 31)", id,);
+        self.0 &= !(1 << id);
+    }
+
+    pub fn has_shard_id(&self, id: ShardId) -> bool {
+        if (id as usize) >= 32 {
+            return false;
+        }
+        self.0 & (1 << id) != 0
+    }
+
+    pub fn shard_id_count(&self) -> usize {
+        self.0.count_ones() as usize
+    }
+
+    /// Iterator over present shard IDs.
+    pub fn shard_ids(&self) -> Vec<ShardId> {
+        let mut ids = Vec::with_capacity(self.shard_id_count());
+        for i in 0..32 {
+            if self.has_shard_id(i) {
+                ids.push(i);
+            }
+        }
+        ids
+    }
+
+    pub fn minus(&self, other: ShardBits) -> ShardBits {
+        ShardBits(self.0 & !other.0)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_shard_bits() {
+        let mut bits = ShardBits::default();
+        assert_eq!(bits.shard_id_count(), 0);
+
+        bits.add_shard_id(0);
+        bits.add_shard_id(3);
+        bits.add_shard_id(13);
+        assert_eq!(bits.shard_id_count(), 3);
+        assert!(bits.has_shard_id(0));
+        assert!(bits.has_shard_id(3));
+        assert!(!bits.has_shard_id(1));
+
+        bits.remove_shard_id(3);
+        assert!(!bits.has_shard_id(3));
+        assert_eq!(bits.shard_id_count(), 2);
+    }
+
+    #[test]
+    fn test_shard_bits_ids() {
+        let mut bits = ShardBits::default();
+        bits.add_shard_id(1);
+        bits.add_shard_id(5);
+        bits.add_shard_id(9);
+        assert_eq!(bits.shard_ids(), vec![1, 5, 9]);
+    }
+
+    #[test]
+    fn test_shard_bits_minus() {
+        let mut a = ShardBits::default();
+        a.add_shard_id(0);
+        a.add_shard_id(1);
+        a.add_shard_id(2);
+
+        let mut b = ShardBits::default();
+        b.add_shard_id(1);
+
+        let c = a.minus(b);
+        assert_eq!(c.shard_ids(), vec![0, 2]);
+    }
+
+    #[test]
+    fn test_shard_file_name() {
+        let shard = EcVolumeShard::new("/data", "pics", VolumeId(42), 3);
+        assert_eq!(shard.file_name(), "/data/pics_42.ec03");
+    }
+
+    #[test]
+    fn test_shard_file_name_no_collection() {
+        let shard = EcVolumeShard::new("/data", "", VolumeId(7), 13);
+        assert_eq!(shard.file_name(), "/data/7.ec13");
+    }
+}

seaweed-volume/src/storage/erasure_coding/ec_volume.rs

@@ -0,0 +1,944 @@
+//! EcVolume: an erasure-coded volume with up to 14 shards.
+//!
+//! Each EcVolume has a sorted index (.ecx) and a deletion journal (.ecj).
+//! Shards (.ec00-.ec13) may be distributed across multiple servers.
+
+use std::collections::HashMap;
+use std::fs::{self, File, OpenOptions};
+use std::io::{self, Write};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use crate::pb::master_pb;
+use crate::storage::erasure_coding::ec_locate;
+use crate::storage::erasure_coding::ec_shard::*;
+use crate::storage::needle::needle::{get_actual_size, Needle};
+use crate::storage::types::*;
+
+/// An erasure-coded volume managing its local shards and index.
+pub struct EcVolume {
+    pub volume_id: VolumeId,
+    pub collection: String,
+    pub dir: String,
+    pub dir_idx: String,
+    pub version: Version,
+    pub shards: Vec<Option<EcVolumeShard>>, // indexed by ShardId (0..14)
+    pub dat_file_size: i64,
+    pub data_shards: u32,
+    pub parity_shards: u32,
+    ecx_file: Option<File>,
+    ecx_file_size: i64,
+    ecj_file: Option<File>,
+    pub disk_type: DiskType,
+    /// Directory where .ecx/.ecj were actually found (may differ from dir_idx after fallback).
+    ecx_actual_dir: String,
+    /// Maps shard ID -> list of server addresses where that shard exists.
+    /// Used for distributed EC reads across the cluster.
+    pub shard_locations: HashMap<ShardId, Vec<String>>,
+    /// EC volume expiration time (unix epoch seconds), set during EC encode from TTL.
+    pub expire_at_sec: u64,
+}
+
+pub fn read_ec_shard_config(dir: &str, collection: &str, volume_id: VolumeId) -> (u32, u32) {
+    let mut data_shards = crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT as u32;
+    let mut parity_shards = crate::storage::erasure_coding::ec_shard::PARITY_SHARDS_COUNT as u32;
+    let base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
+    let vif_path = format!("{}.vif", base);
+    if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
+        if let Ok(vif_info) =
+            serde_json::from_str::<crate::storage::volume::VifVolumeInfo>(&vif_content)
+        {
+            if let Some(ec) = vif_info.ec_shard_config {
+                if ec.data_shards > 0
+                    && ec.parity_shards > 0
+                    && (ec.data_shards + ec.parity_shards) <= TOTAL_SHARDS_COUNT as u32
+                {
+                    data_shards = ec.data_shards;
+                    parity_shards = ec.parity_shards;
+                }
+            }
+        }
+    }
+    (data_shards, parity_shards)
+}
+
+impl EcVolume {
+    /// Create a new EcVolume. Loads .ecx index and .ecj journal if present.
+    pub fn new(
+        dir: &str,
+        dir_idx: &str,
+        collection: &str,
+        volume_id: VolumeId,
+    ) -> io::Result<Self> {
+        let (data_shards, parity_shards) = read_ec_shard_config(dir, collection, volume_id);
+
+        let total_shards = (data_shards + parity_shards) as usize;
+        let mut shards = Vec::with_capacity(total_shards);
+        for _ in 0..total_shards {
+            shards.push(None);
+        }
+
+        // Read expire_at_sec and version from .vif if present (matches Go's MaybeLoadVolumeInfo)
+        let (expire_at_sec, vif_version) = {
+            let base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
+            let vif_path = format!("{}.vif", base);
+            if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
+                if let Ok(vif_info) =
+                    serde_json::from_str::<crate::storage::volume::VifVolumeInfo>(&vif_content)
+                {
+                    let ver = if vif_info.version > 0 {
+                        Version(vif_info.version as u8)
+                    } else {
+                        Version::current()
+                    };
+                    (vif_info.expire_at_sec, ver)
+                } else {
+                    (0, Version::current())
+                }
+            } else {
+                (0, Version::current())
+            }
+        };
+
+        let mut vol = EcVolume {
+            volume_id,
+            collection: collection.to_string(),
+            dir: dir.to_string(),
+            dir_idx: dir_idx.to_string(),
+            version: vif_version,
+            shards,
+            dat_file_size: 0,
+            data_shards,
+            parity_shards,
+            ecx_file: None,
+            ecx_file_size: 0,
+            ecj_file: None,
+            disk_type: DiskType::default(),
+            ecx_actual_dir: dir_idx.to_string(),
+            shard_locations: HashMap::new(),
+            expire_at_sec,
+        };
+
+        // Open .ecx file (sorted index) in read/write mode for in-place deletion marking.
+        // Matches Go which opens ecx for writing via MarkNeedleDeleted.
+        let ecx_path = vol.ecx_file_name();
+        if std::path::Path::new(&ecx_path).exists() {
+            let file = OpenOptions::new().read(true).write(true).open(&ecx_path)?;
+            vol.ecx_file_size = file.metadata()?.len() as i64;
+            vol.ecx_file = Some(file);
+        } else if dir_idx != dir {
+            // Fall back to data directory if .ecx was created before -dir.idx was configured
+            let data_base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
+            let fallback_ecx = format!("{}.ecx", data_base);
+            if std::path::Path::new(&fallback_ecx).exists() {
+                tracing::info!(
+                    volume_id = volume_id.0,
+                    "ecx file not found in idx dir, falling back to data dir"
+                );
+                let file = OpenOptions::new().read(true).write(true).open(&fallback_ecx)?;
+                vol.ecx_file_size = file.metadata()?.len() as i64;
+                vol.ecx_file = Some(file);
+                vol.ecx_actual_dir = dir.to_string();
+            }
+        }
+
+        // Replay .ecj journal into .ecx on startup (matches Go's RebuildEcxFile).
+        vol.rebuild_ecx_from_journal()?;
+
+        // Open .ecj file (deletion journal) — use ecx_actual_dir for consistency
+        let ecj_base =
+            crate::storage::volume::volume_file_name(&vol.ecx_actual_dir, collection, volume_id);
+        let ecj_path = format!("{}.ecj", ecj_base);
+        let ecj_file = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .append(true)
+            .open(&ecj_path)?;
+        vol.ecj_file = Some(ecj_file);
+
+        Ok(vol)
+    }
+
+    // ---- File names ----
+
+    #[allow(dead_code)]
+    fn base_name(&self) -> String {
+        crate::storage::volume::volume_file_name(&self.dir, &self.collection, self.volume_id)
+    }
+
+    fn idx_base_name(&self) -> String {
+        crate::storage::volume::volume_file_name(&self.dir_idx, &self.collection, self.volume_id)
+    }
+
+    pub fn ecx_file_name(&self) -> String {
+        format!("{}.ecx", self.idx_base_name())
+    }
+
+    pub fn ecj_file_name(&self) -> String {
+        format!("{}.ecj", self.idx_base_name())
+    }
+
+    /// Sync the EC volume's journal and index files to disk (matching Go's ecv.Sync()).
+    /// Go flushes both .ecj and .ecx to ensure in-place deletion marks are persisted.
+    pub fn sync_to_disk(&self) -> io::Result<()> {
+        if let Some(ref ecj_file) = self.ecj_file {
+            ecj_file.sync_all()?;
+        }
+        if let Some(ref ecx_file) = self.ecx_file {
+            ecx_file.sync_all()?;
+        }
+        Ok(())
+    }
+
+    // ---- Shard management ----
+
+    /// Add a shard to this volume.
+    pub fn add_shard(&mut self, mut shard: EcVolumeShard) -> io::Result<()> {
+        let id = shard.shard_id as usize;
+        let total_shards = (self.data_shards + self.parity_shards) as usize;
+        if id >= total_shards {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                format!("invalid shard id: {} (max {})", id, total_shards - 1),
+            ));
+        }
+        shard.open()?;
+        self.shards[id] = Some(shard);
+        Ok(())
+    }
+
+    /// Remove and close a shard.
+    pub fn remove_shard(&mut self, shard_id: ShardId) {
+        if let Some(ref mut shard) = self.shards[shard_id as usize] {
+            shard.close();
+        }
+        self.shards[shard_id as usize] = None;
+    }
+
+    /// Get a ShardBits bitmap of locally available shards.
+    pub fn shard_bits(&self) -> ShardBits {
+        let mut bits = ShardBits::default();
+        for (i, shard) in self.shards.iter().enumerate() {
+            if shard.is_some() {
+                bits.add_shard_id(i as ShardId);
+            }
+        }
+        bits
+    }
+
+    /// Count of locally available shards.
+    pub fn shard_count(&self) -> usize {
+        self.shards.iter().filter(|s| s.is_some()).count()
+    }
+
+    pub fn is_time_to_destroy(&self) -> bool {
+        self.expire_at_sec > 0
+            && SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap_or_default()
+                .as_secs()
+                > self.expire_at_sec
+    }
+
+    pub fn to_volume_ec_shard_information_messages(
+        &self,
+        disk_id: u32,
+    ) -> Vec<master_pb::VolumeEcShardInformationMessage> {
+        let mut ec_index_bits: u32 = 0;
+        let mut shard_sizes = Vec::new();
+        for shard in self.shards.iter().flatten() {
+            ec_index_bits |= 1u32 << shard.shard_id;
+            shard_sizes.push(shard.file_size());
+        }
+
+        if ec_index_bits == 0 {
+            return Vec::new();
+        }
+
+        vec![master_pb::VolumeEcShardInformationMessage {
+            id: self.volume_id.0,
+            collection: self.collection.clone(),
+            ec_index_bits,
+            shard_sizes,
+            disk_type: self.disk_type.to_string(),
+            expire_at_sec: self.expire_at_sec,
+            disk_id,
+            ..Default::default()
+        }]
+    }
+
+    // ---- Shard locations (distributed tracking) ----
+
+    /// Set the list of server addresses for a given shard ID.
+    pub fn set_shard_locations(&mut self, shard_id: ShardId, locations: Vec<String>) {
+        self.shard_locations.insert(shard_id, locations);
+    }
+
+    /// Get the list of server addresses for a given shard ID.
+    pub fn get_shard_locations(&self, shard_id: ShardId) -> &[String] {
+        self.shard_locations
+            .get(&shard_id)
+            .map(|v| v.as_slice())
+            .unwrap_or(&[])
+    }
+
+    // ---- Index operations ----
+
+    /// Find a needle's offset and size in the sorted .ecx index via binary search.
+    pub fn find_needle_from_ecx(&self, needle_id: NeedleId) -> io::Result<Option<(Offset, Size)>> {
+        let ecx_file = self
+            .ecx_file
+            .as_ref()
+            .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecx file not open"))?;
+
+        let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
+        if entry_count == 0 {
+            return Ok(None);
+        }
+
+        // Binary search
+        let mut lo: usize = 0;
+        let mut hi: usize = entry_count;
+        let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
+
+        while lo < hi {
+            let mid = lo + (hi - lo) / 2;
+            let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;
+
+            #[cfg(unix)]
+            {
+                use std::os::unix::fs::FileExt;
+                ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
+            }
+
+            let (key, offset, size) = idx_entry_from_bytes(&entry_buf);
+            if key == needle_id {
+                return Ok(Some((offset, size)));
+            } else if key < needle_id {
+                lo = mid + 1;
+            } else {
+                hi = mid;
+            }
+        }
+
+        Ok(None)
+    }
+
+    /// Locate the EC shard intervals needed to read a needle.
+    pub fn locate_needle(
+        &self,
+        needle_id: NeedleId,
+    ) -> io::Result<Option<(Offset, Size, Vec<ec_locate::Interval>)>> {
+        let (offset, size) = match self.find_needle_from_ecx(needle_id)? {
+            Some((o, s)) => (o, s),
+            None => return Ok(None),
+        };
+
+        if size.is_deleted() || offset.is_zero() {
+            return Ok(None);
+        }
+
+        // Match Go's LocateEcShardNeedleInterval: shardSize = shard.ecdFileSize - 1
+        // Shards are usually padded to ErasureCodingSmallBlockSize, so subtract 1
+        // to avoid off-by-one in large block row count calculation.
+        // If datFileSize is known, use datFileSize / DataShards instead.
+        let shard_size = if self.dat_file_size > 0 {
+            self.dat_file_size / self.data_shards as i64
+        } else {
+            self.shard_file_size() - 1
+        };
+        // Pass the actual on-disk size (header+body+checksum+timestamp+padding)
+        // to locate_data, matching Go: types.Size(needle.GetActualSize(size, version))
+        let actual = get_actual_size(size, self.version);
+        let intervals = ec_locate::locate_data(
+            offset.to_actual_offset(),
+            Size(actual as i32),
+            shard_size,
+            self.data_shards,
+        );
+
+        Ok(Some((offset, size, intervals)))
+    }
+
+    /// Read a full needle from locally available EC shards.
+    ///
+    /// Locates the needle in the .ecx index, determines which shard intervals
+    /// contain its data, reads from local shards, and parses the result into
+    /// a fully populated Needle (including last_modified, checksum, ttl).
+    ///
+    /// Returns `Ok(None)` if the needle is not found or is deleted.
+    /// Returns an error if a required shard is not available locally.
+    pub fn read_ec_shard_needle(&self, needle_id: NeedleId) -> io::Result<Option<Needle>> {
+        let (offset, size, intervals) = match self.locate_needle(needle_id)? {
+            Some(v) => v,
+            None => return Ok(None),
+        };
+
+        if intervals.is_empty() {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                "no intervals for needle",
+            ));
+        }
+
+        // Compute the total bytes we need to read (full needle on disk)
+        let actual_size = get_actual_size(size, self.version) as usize;
+        let mut bytes = Vec::with_capacity(actual_size);
+
+        for interval in &intervals {
+            let (shard_id, shard_offset) = interval.to_shard_id_and_offset(self.data_shards);
+            let shard = self
+                .shards
+                .get(shard_id as usize)
+                .and_then(|s| s.as_ref())
+                .ok_or_else(|| {
+                    io::Error::new(
+                        io::ErrorKind::NotFound,
+                        format!("ec shard {} not available locally", shard_id),
+                    )
+                })?;
+
+            let mut buf = vec![0u8; interval.size as usize];
+            shard.read_at(&mut buf, shard_offset as u64)?;
+            bytes.extend_from_slice(&buf);
+        }
+
+        // Truncate to exact actual_size (intervals may span more than needed)
+        bytes.truncate(actual_size);
+
+        if bytes.len() < actual_size {
+            return Err(io::Error::new(
+                io::ErrorKind::UnexpectedEof,
+                format!(
+                    "read {} bytes but need {} for needle {}",
+                    bytes.len(),
+                    actual_size,
+                    needle_id
+                ),
+            ));
+        }
+
+        let mut n = Needle::default();
+        n.read_bytes(&bytes, offset.to_actual_offset(), size, self.version)
+            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{}", e)))?;
+
+        Ok(Some(n))
+    }
+
+    /// Get the size of a single shard (all shards are the same size).
+    fn shard_file_size(&self) -> i64 {
+        for shard in &self.shards {
+            if let Some(s) = shard {
+                return s.file_size();
+            }
+        }
+        0
+    }
+
+    /// Walk the .ecx index and return (file_count, file_deleted_count, total_size).
+    /// total_size sums size.Raw() for all entries (including deleted), matching Go's WalkIndex.
+    pub fn walk_ecx_stats(&self) -> io::Result<(u64, u64, u64)> {
+        let ecx_file = match self.ecx_file.as_ref() {
+            Some(f) => f,
+            None => return Ok((0, 0, 0)),
+        };
+
+        let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
+        let mut files: u64 = 0;
+        let mut files_deleted: u64 = 0;
+        let mut total_size: u64 = 0;
+        let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
+
+        for i in 0..entry_count {
+            let file_offset = (i * NEEDLE_MAP_ENTRY_SIZE) as u64;
+            #[cfg(unix)]
+            {
+                use std::os::unix::fs::FileExt;
+                ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
+            }
+            let (_key, _offset, size) = idx_entry_from_bytes(&entry_buf);
+            // Match Go's Size.Raw(): tombstone (-1) returns 0, other negatives return abs
+            if !size.is_tombstone() {
+                total_size += size.0.unsigned_abs() as u64;
+            }
+            if size.is_deleted() {
+                files_deleted += 1;
+            } else {
+                files += 1;
+            }
+        }
+
+        Ok((files, files_deleted, total_size))
+    }
+
+    /// ScrubIndex verifies index integrity of an EC volume.
+    /// Matches Go's `(ev *EcVolume) ScrubIndex()` → `idx.CheckIndexFile()`.
+    /// Returns (entry_count, errors).
+    pub fn scrub_index(&self) -> (u64, Vec<String>) {
+        let ecx_file = match self.ecx_file.as_ref() {
+            Some(f) => f,
+            None => {
+                return (
+                    0,
+                    vec![format!(
+                        "no ECX file associated with EC volume {}",
+                        self.volume_id.0
+                    )],
+                )
+            }
+        };
+
+        if self.ecx_file_size == 0 {
+            return (
+                0,
+                vec![format!(
+                    "zero-size ECX file for EC volume {}",
+                    self.volume_id.0
+                )],
+            );
+        }
+
+        let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
+        let mut entries: Vec<(usize, NeedleId, i64, Size)> = Vec::with_capacity(entry_count);
+        let mut errs: Vec<String> = Vec::new();
+        let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
+
+        // Walk all entries
+        for i in 0..entry_count {
+            let file_offset = (i * NEEDLE_MAP_ENTRY_SIZE) as u64;
+            #[cfg(unix)]
+            {
+                use std::os::unix::fs::FileExt;
+                if let Err(e) = ecx_file.read_exact_at(&mut entry_buf, file_offset) {
+                    errs.push(format!("read ecx entry {}: {}", i, e));
+                    continue;
+                }
+            }
+            let (key, offset, size) = idx_entry_from_bytes(&entry_buf);
+            entries.push((i, key, offset.to_actual_offset(), size));
+        }
+
+        // Sort by offset, then size
+        entries.sort_by(|a, b| a.2.cmp(&b.2).then(a.3 .0.cmp(&b.3 .0)));
+
+        // Check for overlapping needles
+        for i in 1..entries.len() {
+            let (idx, id, offset, size) = entries[i];
+            let (_, last_id, last_offset, last_size) = entries[i - 1];
+
+            let actual_size =
+                crate::storage::needle::needle::get_actual_size(size, self.version);
+            let end = if actual_size != 0 {
+                offset + actual_size - 1
+            } else {
+                offset
+            };
+
+            let last_actual_size =
+                crate::storage::needle::needle::get_actual_size(last_size, self.version);
+            let last_end = if last_actual_size != 0 {
+                last_offset + last_actual_size - 1
+            } else {
+                last_offset
+            };
+
+            if offset <= last_end {
+                errs.push(format!(
+                    "needle {} (#{}) at [{}-{}] overlaps needle {} at [{}-{}]",
+                    id.0,
+                    idx + 1,
+                    offset,
+                    end,
+                    last_id.0,
+                    last_offset,
+                    last_end
+                ));
+            }
+        }
+
+        // Verify file size matches entry count
+        let expected_size = entry_count as i64 * NEEDLE_MAP_ENTRY_SIZE as i64;
+        if expected_size != self.ecx_file_size {
+            errs.push(format!(
+                "expected an index file of size {}, got {}",
+                expected_size, self.ecx_file_size
+            ));
+        }
+
+        (entries.len() as u64, errs)
+    }
+
+    // ---- Deletion ----
+
+    /// Mark a needle as deleted in the .ecx file in-place.
+    /// Matches Go's MarkNeedleDeleted: binary search the .ecx, then overwrite
+    /// the size field with TOMBSTONE_FILE_SIZE.
+    fn mark_needle_deleted_in_ecx(&self, needle_id: NeedleId) -> io::Result<bool> {
+        let ecx_file = match self.ecx_file.as_ref() {
+            Some(f) => f,
+            None => return Ok(false),
+        };
+
+        let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
+        if entry_count == 0 {
+            return Ok(false);
+        }
+
+        // Binary search for the needle
+        let mut lo: usize = 0;
+        let mut hi: usize = entry_count;
+        let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
+
+        while lo < hi {
+            let mid = lo + (hi - lo) / 2;
+            let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;
+
+            #[cfg(unix)]
+            {
+                use std::os::unix::fs::FileExt;
+                ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
+            }
+
+            let (key, _offset, _size) = idx_entry_from_bytes(&entry_buf);
+            if key == needle_id {
+                // Found — overwrite the size field with TOMBSTONE_FILE_SIZE
+                let size_offset = file_offset + NEEDLE_ID_SIZE as u64 + OFFSET_SIZE as u64;
+                let mut size_buf = [0u8; SIZE_SIZE];
+                TOMBSTONE_FILE_SIZE.to_bytes(&mut size_buf);
+                #[cfg(unix)]
+                {
+                    use std::os::unix::fs::FileExt;
+                    ecx_file.write_all_at(&size_buf, size_offset)?;
+                }
+                return Ok(true);
+            } else if key < needle_id {
+                lo = mid + 1;
+            } else {
+                hi = mid;
+            }
+        }
+
+        Ok(false) // not found
+    }
+
+    /// Replay .ecj journal entries into .ecx on startup.
+    /// Matches Go's RebuildEcxFile: for each needle ID in .ecj, marks it
+    /// deleted in .ecx, then removes the .ecj file.
+    fn rebuild_ecx_from_journal(&mut self) -> io::Result<()> {
+        let ecj_path = self.ecj_file_name();
+        if !std::path::Path::new(&ecj_path).exists() {
+            return Ok(());
+        }
+
+        let data = fs::read(&ecj_path)?;
+        if data.is_empty() {
+            return Ok(());
+        }
+
+        let count = data.len() / NEEDLE_ID_SIZE;
+        for i in 0..count {
+            let start = i * NEEDLE_ID_SIZE;
+            if start + NEEDLE_ID_SIZE > data.len() {
+                break;
+            }
+            let needle_id = NeedleId::from_bytes(&data[start..start + NEEDLE_ID_SIZE]);
+            // Errors for individual entries are non-fatal (needle may not exist in .ecx)
+            let _ = self.mark_needle_deleted_in_ecx(needle_id);
+        }
+
+        // Remove the .ecj file after replay (matches Go)
+        let _ = fs::remove_file(&ecj_path);
+
+        // Re-create .ecj for future deletions
+        let ecj_file = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .append(true)
+            .open(&ecj_path)?;
+        self.ecj_file = Some(ecj_file);
+
+        Ok(())
+    }
+
+    // ---- Deletion journal ----
+
+    /// Append a deleted needle ID to the .ecj journal and mark in .ecx.
+    /// Matches Go's DeleteNeedleFromEcx: marks in .ecx first, then journals.
+    pub fn journal_delete(&mut self, needle_id: NeedleId) -> io::Result<()> {
+        // Mark deleted in .ecx in-place (matches Go's MarkNeedleDeleted)
+        let _ = self.mark_needle_deleted_in_ecx(needle_id);
+        let ecj_file = self
+            .ecj_file
+            .as_mut()
+            .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecj file not open"))?;
+
+        let mut buf = [0u8; NEEDLE_ID_SIZE];
+        needle_id.to_bytes(&mut buf);
+        ecj_file.write_all(&buf)?;
+        ecj_file.sync_all()?;
+        Ok(())
+    }
+
+    /// Append a deleted needle ID to the .ecj journal, validating the cookie first.
+    /// Matches Go's DeleteEcShardNeedle which validates cookie before journaling.
+    /// A cookie of 0 means skip cookie check (e.g., orphan cleanup).
+    pub fn journal_delete_with_cookie(
+        &mut self,
+        needle_id: NeedleId,
+        cookie: crate::storage::types::Cookie,
+    ) -> io::Result<()> {
+        // cookie == 0 indicates SkipCookieCheck was requested
+        if cookie.0 != 0 {
+            // Try to read the needle's cookie from the EC shards to validate
+            // Look up the needle in ecx index to find its offset, then read header from shard
+            if let Ok(Some((offset, size))) = self.find_needle_from_ecx(needle_id) {
+                if !size.is_deleted() && !offset.is_zero() {
+                    let actual_offset = offset.to_actual_offset() as u64;
+                    // Determine which shard contains this offset and read the cookie
+                    let shard_size = self
+                        .shards
+                        .iter()
+                        .filter_map(|s| s.as_ref())
+                        .map(|s| s.file_size())
+                        .next()
+                        .unwrap_or(0) as u64;
+                    if shard_size > 0 {
+                        let shard_id = (actual_offset / shard_size) as usize;
+                        let shard_offset = actual_offset % shard_size;
+                        if let Some(Some(shard)) = self.shards.get(shard_id) {
+                            let mut header_buf = [0u8; 4]; // cookie is first 4 bytes of needle
+                            if shard.read_at(&mut header_buf, shard_offset).is_ok() {
+                                let needle_cookie =
+                                    crate::storage::types::Cookie(u32::from_be_bytes(header_buf));
+                                if needle_cookie != cookie {
+                                    return Err(io::Error::new(
+                                        io::ErrorKind::InvalidData,
+                                        format!("unexpected cookie {:x}", cookie.0),
+                                    ));
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        self.journal_delete(needle_id)
+    }
+
+    /// Read all deleted needle IDs from the .ecj journal.
+    pub fn read_deleted_needles(&self) -> io::Result<Vec<NeedleId>> {
+        let ecj_path = self.ecj_file_name();
+        if !std::path::Path::new(&ecj_path).exists() {
+            return Ok(Vec::new());
+        }
+
+        let data = fs::read(&ecj_path)?;
+        let count = data.len() / NEEDLE_ID_SIZE;
+        let mut needles = Vec::with_capacity(count);
+        for i in 0..count {
+            let start = i * NEEDLE_ID_SIZE;
+            let id = NeedleId::from_bytes(&data[start..start + NEEDLE_ID_SIZE]);
+            needles.push(id);
+        }
+        Ok(needles)
+    }
+
+    // ---- Lifecycle ----
+
+    pub fn close(&mut self) {
+        for shard in &mut self.shards {
+            if let Some(s) = shard {
+                s.close();
+            }
+            *shard = None;
+        }
+        // Sync .ecx before closing to flush in-place deletion marks (matches Go's ev.ecxFile.Sync())
+        if let Some(ref ecx_file) = self.ecx_file {
+            let _ = ecx_file.sync_all();
+        }
+        self.ecx_file = None;
+        self.ecj_file = None;
+    }
+
+    pub fn destroy(&mut self) {
+        for shard in &mut self.shards {
+            if let Some(s) = shard {
+                s.destroy();
+            }
+            *shard = None;
+        }
+        // Remove .ecx/.ecj/.vif from ecx_actual_dir (where they were found)
+        // Go's Destroy() removes .ecx, .ecj, and .vif files.
+        let actual_base = crate::storage::volume::volume_file_name(
+            &self.ecx_actual_dir,
+            &self.collection,
+            self.volume_id,
+        );
+        let _ = fs::remove_file(format!("{}.ecx", actual_base));
+        let _ = fs::remove_file(format!("{}.ecj", actual_base));
+        let _ = fs::remove_file(format!("{}.vif", actual_base));
+        // Also try the configured idx dir and data dir in case files exist in either
+        if self.ecx_actual_dir != self.dir_idx {
+            let _ = fs::remove_file(self.ecx_file_name());
+            let _ = fs::remove_file(self.ecj_file_name());
+            let idx_base = crate::storage::volume::volume_file_name(
+                &self.dir_idx,
+                &self.collection,
+                self.volume_id,
+            );
+            let _ = fs::remove_file(format!("{}.vif", idx_base));
+        }
+        if self.ecx_actual_dir != self.dir && self.dir_idx != self.dir {
+            let data_base = crate::storage::volume::volume_file_name(
+                &self.dir,
+                &self.collection,
+                self.volume_id,
+            );
+            let _ = fs::remove_file(format!("{}.ecx", data_base));
+            let _ = fs::remove_file(format!("{}.ecj", data_base));
+            let _ = fs::remove_file(format!("{}.vif", data_base));
+        }
+        self.ecx_file = None;
+        self.ecj_file = None;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    fn write_ecx_file(
+        dir: &str,
+        collection: &str,
+        vid: VolumeId,
+        entries: &[(NeedleId, Offset, Size)],
+    ) {
+        let base = crate::storage::volume::volume_file_name(dir, collection, vid);
+        let ecx_path = format!("{}.ecx", base);
+        let mut file = File::create(&ecx_path).unwrap();
+
+        // Write sorted entries
+        for &(key, offset, size) in entries {
+            let mut buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
+            idx_entry_to_bytes(&mut buf, key, offset, size);
+            file.write_all(&buf).unwrap();
+        }
+    }
+
+    #[test]
+    fn test_ec_volume_find_needle() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+
+        // Write sorted ecx entries
+        let entries = vec![
+            (NeedleId(1), Offset::from_actual_offset(8), Size(100)),
+            (NeedleId(5), Offset::from_actual_offset(200), Size(200)),
+            (NeedleId(10), Offset::from_actual_offset(500), Size(300)),
+        ];
+        write_ecx_file(dir, "", VolumeId(1), &entries);
+
+        let vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
+
+        // Found
+        let result = vol.find_needle_from_ecx(NeedleId(5)).unwrap();
+        assert!(result.is_some());
+        let (offset, size) = result.unwrap();
+        assert_eq!(offset.to_actual_offset(), 200);
+        assert_eq!(size, Size(200));
+
+        // Not found
+        let result = vol.find_needle_from_ecx(NeedleId(7)).unwrap();
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_ec_volume_journal() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+
+        // Need ecx file for EcVolume::new to succeed
+        write_ecx_file(dir, "", VolumeId(1), &[]);
+
+        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
+
+        vol.journal_delete(NeedleId(10)).unwrap();
+        vol.journal_delete(NeedleId(20)).unwrap();
+
+        let deleted = vol.read_deleted_needles().unwrap();
+        assert_eq!(deleted, vec![NeedleId(10), NeedleId(20)]);
+    }
+
+    #[test]
+    fn test_ec_volume_shard_bits() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        write_ecx_file(dir, "", VolumeId(1), &[]);
+
+        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
+        assert_eq!(vol.shard_count(), 0);
+
+        // Create a shard file so we can add it
+        let mut shard = EcVolumeShard::new(dir, "", VolumeId(1), 3);
+        shard.create().unwrap();
+        shard.write_all(&[0u8; 100]).unwrap();
+        shard.close();
+
+        vol.add_shard(EcVolumeShard::new(dir, "", VolumeId(1), 3))
+            .unwrap();
+        assert_eq!(vol.shard_count(), 1);
+        assert!(vol.shard_bits().has_shard_id(3));
+    }
+
+    #[test]
+    fn test_ec_volume_uses_collection_prefixed_vif_config() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        write_ecx_file(dir, "pics", VolumeId(1), &[]);
+
+        let vif = crate::storage::volume::VifVolumeInfo {
+            ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
+                data_shards: 6,
+                parity_shards: 3,
+            }),
+            ..Default::default()
+        };
+        let base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
+        std::fs::write(
+            format!("{}.vif", base),
+            serde_json::to_string_pretty(&vif).unwrap(),
+        )
+        .unwrap();
+
+        let vol = EcVolume::new(dir, dir, "pics", VolumeId(1)).unwrap();
+        assert_eq!(vol.data_shards, 6);
+        assert_eq!(vol.parity_shards, 3);
+    }
+
+    #[test]
+    fn test_ec_volume_invalid_vif_config_falls_back_to_defaults() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        write_ecx_file(dir, "pics", VolumeId(1), &[]);
+
+        let vif = crate::storage::volume::VifVolumeInfo {
+            ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
+                data_shards: 10,
+                parity_shards: 10,
+            }),
+            ..Default::default()
+        };
+        let base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
+        std::fs::write(
+            format!("{}.vif", base),
+            serde_json::to_string_pretty(&vif).unwrap(),
+        )
+        .unwrap();
+
+        let vol = EcVolume::new(dir, dir, "pics", VolumeId(1)).unwrap();
+        assert_eq!(vol.data_shards, DATA_SHARDS_COUNT as u32);
+        assert_eq!(vol.parity_shards, PARITY_SHARDS_COUNT as u32);
+    }
+}

seaweed-volume/src/storage/erasure_coding/mod.rs

@@ -0,0 +1,16 @@
+//! Erasure coding module for volume data protection.
+//!
+//! Encodes a volume's .dat file into 10 data + 4 parity shards using
+//! Reed-Solomon erasure coding. Can reconstruct from any 10 of 14 shards.
+
+pub mod ec_decoder;
+pub mod ec_encoder;
+pub mod ec_locate;
+pub mod ec_shard;
+pub mod ec_volume;
+
+pub use ec_shard::{
+    EcVolumeShard, ShardId, DATA_SHARDS_COUNT, MAX_SHARD_COUNT, MIN_TOTAL_DISKS,
+    PARITY_SHARDS_COUNT, TOTAL_SHARDS_COUNT,
+};
+pub use ec_volume::EcVolume;

seaweed-volume/src/storage/idx/mod.rs

@@ -0,0 +1,116 @@
+//! Index file (.idx) format: sequential 17-byte entries.
+//!
+//! Each entry: NeedleId(8) + Offset(5) + Size(4) = 17 bytes.
+
+use crate::storage::types::*;
+use std::io::{self, Read, Seek, SeekFrom};
+
+const ROWS_TO_READ: usize = 1024;
+
+/// Walk all entries in an .idx file, calling `f` for each.
+/// Mirrors Go's `WalkIndexFile()`.
+pub fn walk_index_file<R, F>(reader: &mut R, start_from: u64, mut f: F) -> io::Result<()>
+where
+    R: Read + Seek,
+    F: FnMut(NeedleId, Offset, Size) -> io::Result<()>,
+{
+    let reader_offset = start_from * NEEDLE_MAP_ENTRY_SIZE as u64;
+    reader.seek(SeekFrom::Start(reader_offset))?;
+
+    let mut buf = vec![0u8; NEEDLE_MAP_ENTRY_SIZE * ROWS_TO_READ];
+
+    loop {
+        let count = match reader.read(&mut buf) {
+            Ok(0) => return Ok(()),
+            Ok(n) => n,
+            Err(ref e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(()),
+            Err(e) => return Err(e),
+        };
+
+        let mut i = 0;
+        while i + NEEDLE_MAP_ENTRY_SIZE <= count {
+            let (key, offset, size) = idx_entry_from_bytes(&buf[i..i + NEEDLE_MAP_ENTRY_SIZE]);
+            f(key, offset, size)?;
+            i += NEEDLE_MAP_ENTRY_SIZE;
+        }
+    }
+}
+
+/// Write a single index entry to a writer.
+pub fn write_index_entry<W: io::Write>(
+    writer: &mut W,
+    key: NeedleId,
+    offset: Offset,
+    size: Size,
+) -> io::Result<()> {
+    let mut buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
+    idx_entry_to_bytes(&mut buf, key, offset, size);
+    writer.write_all(&buf)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Cursor;
+
+    #[test]
+    fn test_walk_index_file() {
+        // Create a small index with 3 entries
+        let mut data = Vec::new();
+        let entries = vec![
+            (NeedleId(1), Offset::from_actual_offset(0), Size(100)),
+            (NeedleId(2), Offset::from_actual_offset(128), Size(200)),
+            (NeedleId(3), Offset::from_actual_offset(384), Size(300)),
+        ];
+        for (key, offset, size) in &entries {
+            let mut buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
+            idx_entry_to_bytes(&mut buf, *key, *offset, *size);
+            data.extend_from_slice(&buf);
+        }
+
+        let mut cursor = Cursor::new(data);
+        let mut collected = Vec::new();
+        walk_index_file(&mut cursor, 0, |key, offset, size| {
+            collected.push((key, offset.to_actual_offset(), size));
+            Ok(())
+        })
+        .unwrap();
+
+        assert_eq!(collected.len(), 3);
+        assert_eq!(collected[0].0, NeedleId(1));
+        assert_eq!(collected[0].1, 0);
+        assert_eq!(collected[0].2, Size(100));
+        assert_eq!(collected[1].0, NeedleId(2));
+        assert_eq!(collected[2].0, NeedleId(3));
+    }
+
+    #[test]
+    fn test_walk_empty() {
+        let mut cursor = Cursor::new(Vec::new());
+        let mut count = 0;
+        walk_index_file(&mut cursor, 0, |_, _, _| {
+            count += 1;
+            Ok(())
+        })
+        .unwrap();
+        assert_eq!(count, 0);
+    }
+
+    #[test]
+    fn test_write_index_entry() {
+        let mut buf = Vec::new();
+        write_index_entry(
+            &mut buf,
+            NeedleId(42),
+            Offset::from_actual_offset(8 * 10),
+            Size(512),
+        )
+        .unwrap();
+        assert_eq!(buf.len(), NEEDLE_MAP_ENTRY_SIZE);
+
+        let (key, offset, size) = idx_entry_from_bytes(&buf);
+        assert_eq!(key, NeedleId(42));
+        assert_eq!(offset.to_actual_offset(), 80);
+        assert_eq!(size, Size(512));
+    }
+}

seaweed-volume/src/storage/mod.rs

@@ -0,0 +1,9 @@
+pub mod disk_location;
+pub mod erasure_coding;
+pub mod idx;
+pub mod needle;
+pub mod needle_map;
+pub mod store;
+pub mod super_block;
+pub mod types;
+pub mod volume;

seaweed-volume/src/storage/needle/crc.rs

@@ -0,0 +1,73 @@
+//! CRC32-Castagnoli checksum for needle data integrity.
+//!
+//! Matches Go's `crc32.MakeTable(crc32.Castagnoli)` exactly.
+//! The CRC is stored as raw u32 (not the `.Value()` legacy transform).
+
+/// CRC32-Castagnoli checksum wrapper.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct CRC(pub u32);
+
+impl CRC {
+    /// Compute CRC from a byte slice (starting from 0).
+    pub fn new(data: &[u8]) -> Self {
+        CRC(0).update(data)
+    }
+
+    /// Update the CRC with additional bytes.
+    pub fn update(self, data: &[u8]) -> Self {
+        CRC(crc32c::crc32c_append(self.0, data))
+    }
+
+    /// Legacy `.Value()` function — deprecated in Go but needed for backward compat check.
+    /// Formula: (crc >> 15 | crc << 17) + 0xa282ead8
+    pub fn legacy_value(&self) -> u32 {
+        (self.0 >> 15 | self.0 << 17).wrapping_add(0xa282ead8)
+    }
+}
+
+impl From<u32> for CRC {
+    fn from(v: u32) -> Self {
+        CRC(v)
+    }
+}
+
+impl From<CRC> for u32 {
+    fn from(c: CRC) -> Self {
+        c.0
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_crc_empty() {
+        let crc = CRC::new(&[]);
+        assert_eq!(crc.0, 0);
+    }
+
+    #[test]
+    fn test_crc_known_value() {
+        // CRC32-C of "hello" — verify it produces a non-zero deterministic value
+        let crc = CRC::new(b"hello");
+        assert_ne!(crc.0, 0);
+        // Same input produces same output
+        assert_eq!(crc, CRC::new(b"hello"));
+    }
+
+    #[test]
+    fn test_crc_incremental() {
+        let crc1 = CRC::new(b"hello world");
+        let crc2 = CRC::new(b"hello").update(b" world");
+        assert_eq!(crc1, crc2);
+    }
+
+    #[test]
+    fn test_crc_legacy_value() {
+        let crc = CRC(0x12345678);
+        let v = crc.legacy_value();
+        let expected = (0x12345678u32 >> 15 | 0x12345678u32 << 17).wrapping_add(0xa282ead8);
+        assert_eq!(v, expected);
+    }
+}

seaweed-volume/src/storage/needle/mod.rs

@@ -0,0 +1,7 @@
+pub mod crc;
+pub mod needle;
+pub mod ttl;
+
+pub use crc::CRC;
+pub use needle::Needle;
+pub use ttl::TTL;

seaweed-volume/src/storage/needle/needle.rs

@@ -0,0 +1,944 @@
+//! Needle: the individual file object stored in a volume.
+//!
+//! Binary format (Version 2/3):
+//!   Header (16 bytes): Cookie(4) + NeedleId(8) + Size(4)
+//!   Body (Size bytes):
+//!     DataSize(4) + Data(DataSize) + Flags(1)
+//!     [if HasName]:    NameSize(1) + Name(NameSize)
+//!     [if HasMime]:    MimeSize(1) + Mime(MimeSize)
+//!     [if HasLastMod]: LastModified(5)
+//!     [if HasTtl]:     TTL(2)
+//!     [if HasPairs]:   PairsSize(2) + Pairs(PairsSize)
+//!   Tail:
+//!     Checksum(4) + [if V3: AppendAtNs(8)] + Padding(0-7)
+
+use super::crc::CRC;
+use super::ttl::TTL;
+use crate::storage::types::*;
+
+// Flag bits (matching Go constants)
+pub const FLAG_IS_COMPRESSED: u8 = 0x01;
+pub const FLAG_HAS_NAME: u8 = 0x02;
+pub const FLAG_HAS_MIME: u8 = 0x04;
+pub const FLAG_HAS_LAST_MODIFIED_DATE: u8 = 0x08;
+pub const FLAG_HAS_TTL: u8 = 0x10;
+pub const FLAG_HAS_PAIRS: u8 = 0x20;
+pub const FLAG_IS_CHUNK_MANIFEST: u8 = 0x80;
+
+pub const LAST_MODIFIED_BYTES_LENGTH: usize = 5;
+pub const TTL_BYTES_LENGTH: usize = 2;
+
+#[derive(Debug, Clone, Default)]
+pub struct Needle {
+    pub cookie: Cookie,
+    pub id: NeedleId,
+    pub size: Size, // sum of body content fields
+
+    // Version 2+ fields
+    pub data_size: u32,
+    pub data: Vec<u8>,
+    pub flags: u8,
+    pub name_size: u8,
+    pub name: Vec<u8>, // max 255 bytes
+    pub mime_size: u8,
+    pub mime: Vec<u8>, // max 255 bytes
+    pub pairs_size: u16,
+    pub pairs: Vec<u8>,     // max 64KB, JSON
+    pub last_modified: u64, // stored as 5 bytes on disk
+    pub ttl: Option<TTL>,
+
+    // Tail fields
+    pub checksum: CRC,
+    pub append_at_ns: u64, // Version 3 only
+    pub padding: Vec<u8>,
+}
+
+impl Needle {
+    // ---- Flag accessors (matching Go) ----
+
+    pub fn is_compressed(&self) -> bool {
+        self.flags & FLAG_IS_COMPRESSED != 0
+    }
+    pub fn set_is_compressed(&mut self) {
+        self.flags |= FLAG_IS_COMPRESSED;
+    }
+
+    pub fn has_name(&self) -> bool {
+        self.flags & FLAG_HAS_NAME != 0
+    }
+    pub fn set_has_name(&mut self) {
+        self.flags |= FLAG_HAS_NAME;
+    }
+
+    pub fn has_mime(&self) -> bool {
+        self.flags & FLAG_HAS_MIME != 0
+    }
+    pub fn set_has_mime(&mut self) {
+        self.flags |= FLAG_HAS_MIME;
+    }
+
+    pub fn has_last_modified_date(&self) -> bool {
+        self.flags & FLAG_HAS_LAST_MODIFIED_DATE != 0
+    }
+    pub fn set_has_last_modified_date(&mut self) {
+        self.flags |= FLAG_HAS_LAST_MODIFIED_DATE;
+    }
+
+    pub fn has_ttl(&self) -> bool {
+        self.flags & FLAG_HAS_TTL != 0
+    }
+    pub fn set_has_ttl(&mut self) {
+        self.flags |= FLAG_HAS_TTL;
+    }
+
+    pub fn has_pairs(&self) -> bool {
+        self.flags & FLAG_HAS_PAIRS != 0
+    }
+    pub fn set_has_pairs(&mut self) {
+        self.flags |= FLAG_HAS_PAIRS;
+    }
+
+    pub fn is_chunk_manifest(&self) -> bool {
+        self.flags & FLAG_IS_CHUNK_MANIFEST != 0
+    }
+    pub fn set_is_chunk_manifest(&mut self) {
+        self.flags |= FLAG_IS_CHUNK_MANIFEST;
+    }
+
+    // ---- Header parsing ----
+
+    /// Parse the 16-byte needle header.
+    pub fn parse_header(bytes: &[u8]) -> (Cookie, NeedleId, Size) {
+        assert!(bytes.len() >= NEEDLE_HEADER_SIZE);
+        let cookie = Cookie::from_bytes(&bytes[0..COOKIE_SIZE]);
+        let id = NeedleId::from_bytes(&bytes[COOKIE_SIZE..COOKIE_SIZE + NEEDLE_ID_SIZE]);
+        let size = Size::from_bytes(&bytes[COOKIE_SIZE + NEEDLE_ID_SIZE..NEEDLE_HEADER_SIZE]);
+        (cookie, id, size)
+    }
+
+    /// Parse needle header into self.
+    pub fn read_header(&mut self, bytes: &[u8]) {
+        let (cookie, id, size) = Self::parse_header(bytes);
+        self.cookie = cookie;
+        self.id = id;
+        self.size = size;
+    }
+
+    // ---- Body reading (Version 2/3) ----
+
+    /// Read version 2/3 body metadata only — skips copying the data payload.
+    /// Sets `data_size` and all metadata fields but leaves `data` empty.
+    pub fn read_body_v2_meta_only(&mut self, bytes: &[u8]) -> Result<(), NeedleError> {
+        let len_bytes = bytes.len();
+        let mut index = 0;
+
+        // DataSize (4 bytes)
+        if index + 4 > len_bytes {
+            return Err(NeedleError::IndexOutOfRange(1));
+        }
+        self.data_size = u32::from_be_bytes([
+            bytes[index],
+            bytes[index + 1],
+            bytes[index + 2],
+            bytes[index + 3],
+        ]);
+        index += 4;
+
+        // Skip data bytes (do NOT copy them)
+        if index + self.data_size as usize > len_bytes {
+            return Err(NeedleError::IndexOutOfRange(1));
+        }
+        index += self.data_size as usize;
+
+        // Read non-data metadata
+        self.read_body_v2_non_data(&bytes[index..])?;
+        Ok(())
+    }
+
+    /// Read full needle from bytes but skip copying the data payload.
+    /// Sets all metadata fields, checksum, etc. but leaves `data` empty.
+    pub fn read_bytes_meta_only(
+        &mut self,
+        bytes: &[u8],
+        offset: i64,
+        expected_size: Size,
+        version: Version,
+    ) -> Result<(), NeedleError> {
+        self.read_header(bytes);
+
+        if self.size != expected_size {
+            return Err(NeedleError::SizeMismatch {
+                offset,
+                id: self.id,
+                found: self.size,
+                expected: expected_size,
+            });
+        }
+
+        let body_start = NEEDLE_HEADER_SIZE;
+        let body_end = body_start + self.size.0 as usize;
+
+        if version == VERSION_1 {
+            // V1 has no metadata — data is the entire body
+            self.data_size = self.size.0 as u32;
+        } else if self.size.0 == 0 {
+            // Tombstones have no DataSize/body section; metadata starts at the tail.
+            self.data_size = 0;
+        } else {
+            self.read_body_v2_meta_only(&bytes[body_start..body_end])?;
+        }
+
+        // Read tail but skip CRC validation (no data to check against)
+        self.read_tail_meta_only(&bytes[body_end..], version)?;
+        Ok(())
+    }
+
+    /// Paged meta-only parse: accepts the 20-byte header+DataSize prefix and the
+    /// meta tail bytes (everything after the data payload). This avoids reading
+    /// the data payload from disk at all, matching Go's `ReadNeedleMeta`.
+    pub fn read_paged_meta(
+        &mut self,
+        header_bytes: &[u8],   // first 20 bytes: NEEDLE_HEADER_SIZE + DATA_SIZE_SIZE
+        meta_bytes: &[u8],     // tail: non-data body metadata + checksum + timestamp + padding
+        offset: i64,
+        expected_size: Size,
+        version: Version,
+    ) -> Result<(), NeedleError> {
+        // Parse the 16-byte header
+        self.read_header(header_bytes);
+
+        if self.size != expected_size {
+            return Err(NeedleError::SizeMismatch {
+                offset,
+                id: self.id,
+                found: self.size,
+                expected: expected_size,
+            });
+        }
+
+        if version == VERSION_1 {
+            self.data_size = self.size.0 as u32;
+        } else if self.size.0 == 0 {
+            // Tombstone
+            self.data_size = 0;
+        } else {
+            // Extract DataSize from bytes 16..20
+            self.data_size = u32::from_be_bytes([
+                header_bytes[NEEDLE_HEADER_SIZE],
+                header_bytes[NEEDLE_HEADER_SIZE + 1],
+                header_bytes[NEEDLE_HEADER_SIZE + 2],
+                header_bytes[NEEDLE_HEADER_SIZE + 3],
+            ]);
+
+            // meta_bytes starts with the non-data body metadata (flags, name, mime, etc.)
+            // followed by the tail (checksum + timestamp + padding).
+            // readNeedleDataVersion2NonData returns the index where it stopped.
+            let index = self.read_body_v2_non_data(meta_bytes)?;
+            self.read_tail_meta_only(&meta_bytes[index..], version)?;
+            return Ok(());
+        }
+
+        // For VERSION_1 or tombstones, meta_bytes IS the tail
+        self.read_tail_meta_only(meta_bytes, version)?;
+        Ok(())
+    }
+
+    /// Read tail without CRC validation (used when data was not read).
+    fn read_tail_meta_only(
+        &mut self,
+        tail_bytes: &[u8],
+        version: Version,
+    ) -> Result<(), NeedleError> {
+        if tail_bytes.len() < NEEDLE_CHECKSUM_SIZE {
+            return Err(NeedleError::TailTooShort);
+        }
+
+        self.checksum = CRC(u32::from_be_bytes([
+            tail_bytes[0],
+            tail_bytes[1],
+            tail_bytes[2],
+            tail_bytes[3],
+        ]));
+
+        if version == VERSION_3 {
+            let ts_offset = NEEDLE_CHECKSUM_SIZE;
+            if tail_bytes.len() < ts_offset + TIMESTAMP_SIZE {
+                return Err(NeedleError::TailTooShort);
+            }
+            self.append_at_ns = u64::from_be_bytes([
+                tail_bytes[ts_offset],
+                tail_bytes[ts_offset + 1],
+                tail_bytes[ts_offset + 2],
+                tail_bytes[ts_offset + 3],
+                tail_bytes[ts_offset + 4],
+                tail_bytes[ts_offset + 5],
+                tail_bytes[ts_offset + 6],
+                tail_bytes[ts_offset + 7],
+            ]);
+        }
+
+        Ok(())
+    }
+
+    /// Read the version 2/3 body data from bytes (size bytes starting after header).
+    /// Returns IndexOutOfRange errors for truncated data (matching Go's readNeedleDataVersion2).
+    pub fn read_body_v2(&mut self, bytes: &[u8]) -> Result<(), NeedleError> {
+        let len_bytes = bytes.len();
+        let mut index = 0;
+
+        // DataSize (4 bytes)
+        if index + 4 > len_bytes {
+            return Ok(()); // tolerate EOF
+        }
+        self.data_size = u32::from_be_bytes([
+            bytes[index],
+            bytes[index + 1],
+            bytes[index + 2],
+            bytes[index + 3],
+        ]);
+        index += 4;
+
+        // Data
+        if index + self.data_size as usize > len_bytes {
+            return Err(NeedleError::IndexOutOfRange(1));
+        }
+        self.data = bytes[index..index + self.data_size as usize].to_vec();
+        index += self.data_size as usize;
+
+        // Read non-data metadata
+        self.read_body_v2_non_data(&bytes[index..])?;
+        Ok(())
+    }
+
+    /// Read version 2/3 metadata fields (everything after Data).
+    /// Returns IndexOutOfRange errors for truncated data (matching Go's readNeedleDataVersion2).
+    fn read_body_v2_non_data(&mut self, bytes: &[u8]) -> Result<usize, NeedleError> {
+        let len_bytes = bytes.len();
+        let mut index = 0;
+
+        // Flags (1 byte)
+        if index < len_bytes {
+            self.flags = bytes[index];
+            index += 1;
+        } else {
+            return Ok(index);
+        }
+
+        // Name
+        if index < len_bytes && self.has_name() {
+            self.name_size = bytes[index];
+            index += 1;
+            if index + self.name_size as usize > len_bytes {
+                return Err(NeedleError::IndexOutOfRange(2));
+            }
+            self.name = bytes[index..index + self.name_size as usize].to_vec();
+            index += self.name_size as usize;
+        }
+
+        // Mime
+        if index < len_bytes && self.has_mime() {
+            self.mime_size = bytes[index];
+            index += 1;
+            if index + self.mime_size as usize > len_bytes {
+                return Err(NeedleError::IndexOutOfRange(3));
+            }
+            self.mime = bytes[index..index + self.mime_size as usize].to_vec();
+            index += self.mime_size as usize;
+        }
+
+        // LastModified (5 bytes)
+        if index < len_bytes && self.has_last_modified_date() {
+            if index + LAST_MODIFIED_BYTES_LENGTH > len_bytes {
+                return Err(NeedleError::IndexOutOfRange(4));
+            }
+            self.last_modified = bytes_to_u64_5(&bytes[index..index + LAST_MODIFIED_BYTES_LENGTH]);
+            index += LAST_MODIFIED_BYTES_LENGTH;
+        }
+
+        // TTL (2 bytes)
+        if index < len_bytes && self.has_ttl() {
+            if index + TTL_BYTES_LENGTH > len_bytes {
+                return Err(NeedleError::IndexOutOfRange(5));
+            }
+            self.ttl = Some(TTL::from_bytes(&bytes[index..index + TTL_BYTES_LENGTH]));
+            index += TTL_BYTES_LENGTH;
+        }
+
+        // Pairs
+        if index < len_bytes && self.has_pairs() {
+            if index + 2 > len_bytes {
+                return Err(NeedleError::IndexOutOfRange(6));
+            }
+            self.pairs_size = u16::from_be_bytes([bytes[index], bytes[index + 1]]);
+            index += 2;
+            if index + self.pairs_size as usize > len_bytes {
+                return Err(NeedleError::IndexOutOfRange(7));
+            }
+            self.pairs = bytes[index..index + self.pairs_size as usize].to_vec();
+            index += self.pairs_size as usize;
+        }
+
+        Ok(index)
+    }
+
+    // ---- Tail reading ----
+
+    /// Read the needle tail (checksum + optional timestamp + padding).
+    pub fn read_tail(&mut self, tail_bytes: &[u8], version: Version) -> Result<(), NeedleError> {
+        if tail_bytes.len() < NEEDLE_CHECKSUM_SIZE {
+            return Err(NeedleError::TailTooShort);
+        }
+
+        let expected_checksum = CRC(u32::from_be_bytes([
+            tail_bytes[0],
+            tail_bytes[1],
+            tail_bytes[2],
+            tail_bytes[3],
+        ]));
+
+        if !self.data.is_empty() {
+            let data_checksum = CRC::new(&self.data);
+            // Go double-checks: n.Checksum != crc && uint32(n.Checksum) != crc.Value()
+            // The crc.Value() path is a deprecated legacy transform for backward compat
+            // with seaweed versions prior to commit 056c480eb.
+            if expected_checksum != data_checksum
+                && expected_checksum.0 != data_checksum.legacy_value()
+            {
+                return Err(NeedleError::CrcMismatch {
+                    needle_id: self.id,
+                    got: data_checksum.0,
+                    want: expected_checksum.0,
+                });
+            }
+            self.checksum = data_checksum;
+        } else {
+            self.checksum = expected_checksum;
+        }
+
+        if version == VERSION_3 {
+            let ts_offset = NEEDLE_CHECKSUM_SIZE;
+            if tail_bytes.len() < ts_offset + TIMESTAMP_SIZE {
+                return Err(NeedleError::TailTooShort);
+            }
+            self.append_at_ns = u64::from_be_bytes([
+                tail_bytes[ts_offset],
+                tail_bytes[ts_offset + 1],
+                tail_bytes[ts_offset + 2],
+                tail_bytes[ts_offset + 3],
+                tail_bytes[ts_offset + 4],
+                tail_bytes[ts_offset + 5],
+                tail_bytes[ts_offset + 6],
+                tail_bytes[ts_offset + 7],
+            ]);
+        }
+
+        Ok(())
+    }
+
+    // ---- Full read from bytes ----
+
+    /// Read a complete needle from its raw bytes (header + body + tail).
+    pub fn read_bytes(
+        &mut self,
+        bytes: &[u8],
+        offset: i64,
+        expected_size: Size,
+        version: Version,
+    ) -> Result<(), NeedleError> {
+        self.read_header(bytes);
+
+        if self.size != expected_size {
+            return Err(NeedleError::SizeMismatch {
+                offset,
+                id: self.id,
+                found: self.size,
+                expected: expected_size,
+            });
+        }
+
+        let body_start = NEEDLE_HEADER_SIZE;
+        let body_end = body_start + self.size.0 as usize;
+
+        if version == VERSION_1 {
+            self.data = bytes[body_start..body_end].to_vec();
+        } else {
+            self.read_body_v2(&bytes[body_start..body_end])?;
+        }
+
+        self.read_tail(&bytes[body_end..], version)?;
+        Ok(())
+    }
+
+    // ---- Write (serialize) ----
+
+    /// Serialize the needle to bytes for writing to a .dat file (Version 2/3).
+    pub fn write_bytes(&mut self, version: Version) -> Vec<u8> {
+        let mut buf = Vec::with_capacity(256);
+
+        // Compute sizes (matching Go writeNeedleCommon)
+        if self.name.len() >= 255 {
+            self.name_size = 255;
+        } else {
+            self.name_size = self.name.len() as u8;
+        }
+        self.data_size = self.data.len() as u32;
+        self.mime_size = self.mime.len() as u8;
+
+        // Compute n.Size (body size, excluding header)
+        if self.data_size > 0 {
+            let mut s: i32 = 4 + self.data_size as i32 + 1; // DataSize + Data + Flags
+            if self.has_name() {
+                s += 1 + self.name_size as i32;
+            }
+            if self.has_mime() {
+                s += 1 + self.mime_size as i32;
+            }
+            if self.has_last_modified_date() {
+                s += LAST_MODIFIED_BYTES_LENGTH as i32;
+            }
+            if self.has_ttl() {
+                s += TTL_BYTES_LENGTH as i32;
+            }
+            if self.has_pairs() {
+                s += 2 + self.pairs_size as i32;
+            }
+            self.size = Size(s);
+        } else {
+            self.size = Size(0);
+        }
+
+        // Header: Cookie(4) + NeedleId(8) + Size(4) = 16 bytes
+        let mut header = [0u8; NEEDLE_HEADER_SIZE];
+        self.cookie.to_bytes(&mut header[0..COOKIE_SIZE]);
+        self.id
+            .to_bytes(&mut header[COOKIE_SIZE..COOKIE_SIZE + NEEDLE_ID_SIZE]);
+        self.size
+            .to_bytes(&mut header[COOKIE_SIZE + NEEDLE_ID_SIZE..NEEDLE_HEADER_SIZE]);
+        buf.extend_from_slice(&header);
+
+        // Body
+        if self.data_size > 0 {
+            buf.extend_from_slice(&self.data_size.to_be_bytes());
+            buf.extend_from_slice(&self.data);
+            buf.push(self.flags);
+            if self.has_name() {
+                buf.push(self.name_size);
+                buf.extend_from_slice(&self.name[..self.name_size as usize]);
+            }
+            if self.has_mime() {
+                buf.push(self.mime_size);
+                buf.extend_from_slice(&self.mime);
+            }
+            if self.has_last_modified_date() {
+                // Write 5 bytes of last_modified (lower 5 bytes of u64 big-endian)
+                let lm_bytes = self.last_modified.to_be_bytes();
+                buf.extend_from_slice(&lm_bytes[8 - LAST_MODIFIED_BYTES_LENGTH..8]);
+            }
+            if self.has_ttl() {
+                if let Some(ref ttl) = self.ttl {
+                    let mut ttl_buf = [0u8; 2];
+                    ttl.to_bytes(&mut ttl_buf);
+                    buf.extend_from_slice(&ttl_buf);
+                } else {
+                    buf.extend_from_slice(&[0u8; 2]);
+                }
+            }
+            if self.has_pairs() {
+                buf.extend_from_slice(&self.pairs_size.to_be_bytes());
+                buf.extend_from_slice(&self.pairs);
+            }
+        }
+
+        // Compute checksum
+        self.checksum = CRC::new(&self.data);
+
+        // Tail: Checksum + [V3: AppendAtNs] + Padding
+        buf.extend_from_slice(&self.checksum.0.to_be_bytes());
+        if version == VERSION_3 {
+            buf.extend_from_slice(&self.append_at_ns.to_be_bytes());
+        }
+
+        // Padding to 8-byte alignment
+        let padding = padding_length(self.size, version).0 as usize;
+        buf.extend(std::iter::repeat(0u8).take(padding));
+
+        buf
+    }
+
+    /// Total disk size of this needle including header, body, checksum, timestamp, and padding.
+    pub fn disk_size(&self, version: Version) -> i64 {
+        get_actual_size(self.size, version)
+    }
+
+    /// Compute ETag string from checksum (matching Go).
+    pub fn etag(&self) -> String {
+        etag_from_checksum(self.checksum.0)
+    }
+}
+
+// ============================================================================
+// Helper functions (matching Go)
+// ============================================================================
+
+/// Compute padding to align needle to NEEDLE_PADDING_SIZE (8 bytes).
+pub fn padding_length(needle_size: Size, version: Version) -> Size {
+    if version == VERSION_3 {
+        Size(
+            NEEDLE_PADDING_SIZE as i32
+                - ((NEEDLE_HEADER_SIZE as i32
+                    + needle_size.0
+                    + NEEDLE_CHECKSUM_SIZE as i32
+                    + TIMESTAMP_SIZE as i32)
+                    % NEEDLE_PADDING_SIZE as i32),
+        )
+    } else {
+        Size(
+            NEEDLE_PADDING_SIZE as i32
+                - ((NEEDLE_HEADER_SIZE as i32 + needle_size.0 + NEEDLE_CHECKSUM_SIZE as i32)
+                    % NEEDLE_PADDING_SIZE as i32),
+        )
+    }
+}
+
+/// Body length = Size + Checksum + [Timestamp] + Padding.
+pub fn needle_body_length(needle_size: Size, version: Version) -> i64 {
+    if version == VERSION_3 {
+        needle_size.0 as i64
+            + NEEDLE_CHECKSUM_SIZE as i64
+            + TIMESTAMP_SIZE as i64
+            + padding_length(needle_size, version).0 as i64
+    } else {
+        needle_size.0 as i64
+            + NEEDLE_CHECKSUM_SIZE as i64
+            + padding_length(needle_size, version).0 as i64
+    }
+}
+
+/// Total actual size on disk: Header + Body.
+pub fn get_actual_size(size: Size, version: Version) -> i64 {
+    NEEDLE_HEADER_SIZE as i64 + needle_body_length(size, version)
+}
+
+/// Read 5 bytes as a u64 (big-endian, zero-padded high bytes).
+fn bytes_to_u64_5(bytes: &[u8]) -> u64 {
+    assert!(bytes.len() >= 5);
+    // The 5 bytes are the LOWER 5 bytes of a big-endian u64.
+    // In Go: util.BytesToUint64(bytes[index : index+5]) reads into a uint64
+    // Go's BytesToUint64 copies into the LAST 5 bytes of an 8-byte array (big-endian).
+    let mut buf = [0u8; 8];
+    buf[3..8].copy_from_slice(&bytes[..5]);
+    u64::from_be_bytes(buf)
+}
+
+/// ETag formatted as Go: hex of big-endian u32 bytes.
+pub fn etag_from_checksum(checksum: u32) -> String {
+    let bits = checksum.to_be_bytes();
+    format!(
+        "{:02x}{:02x}{:02x}{:02x}",
+        bits[0], bits[1], bits[2], bits[3]
+    )
+}
+
+// ============================================================================
+// FileId
+// ============================================================================
+
+/// FileId = VolumeId + NeedleId + Cookie.
+/// String format: "<volume_id>,<needle_id_hex><cookie_hex>"
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FileId {
+    pub volume_id: VolumeId,
+    pub key: NeedleId,
+    pub cookie: Cookie,
+}
+
+impl FileId {
+    pub fn new(volume_id: VolumeId, key: NeedleId, cookie: Cookie) -> Self {
+        FileId {
+            volume_id,
+            key,
+            cookie,
+        }
+    }
+
+    /// Parse "volume_id,needle_id_cookie" or "volume_id/needle_id_cookie".
+    pub fn parse(s: &str) -> Result<Self, String> {
+        let (vid_str, rest) = if let Some(pos) = s.find(',') {
+            (&s[..pos], &s[pos + 1..])
+        } else if let Some(pos) = s.find('/') {
+            (&s[..pos], &s[pos + 1..])
+        } else {
+            return Err(format!("invalid file id: {}", s));
+        };
+
+        let volume_id =
+            VolumeId::parse(vid_str).map_err(|e| format!("invalid volume id: {}", e))?;
+        let (key, cookie) = parse_needle_id_cookie(rest)?;
+        Ok(FileId {
+            volume_id,
+            key,
+            cookie,
+        })
+    }
+
+    /// Format the needle_id + cookie part as a hex string (stripping leading zeros).
+    pub fn needle_id_cookie_string(&self) -> String {
+        format_needle_id_cookie(self.key, self.cookie)
+    }
+}
+
+impl std::fmt::Display for FileId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{},{}", self.volume_id, self.needle_id_cookie_string())
+    }
+}
+
+/// Format NeedleId + Cookie as hex, stripping leading zero bytes from NeedleId only.
+/// Matches Go: strips leading zero bytes up to NeedleIdSize (8), so cookie is always present.
+fn format_needle_id_cookie(key: NeedleId, cookie: Cookie) -> String {
+    // Encode 12 bytes: 8 for NeedleId + 4 for Cookie
+    let mut bytes = [0u8; 12];
+    key.to_bytes(&mut bytes[0..8]);
+    cookie.to_bytes(&mut bytes[8..12]);
+
+    // Strip leading zero bytes, but only within NeedleId portion (first 8 bytes)
+    let mut nonzero_index = 0;
+    while nonzero_index < NEEDLE_ID_SIZE && bytes[nonzero_index] == 0 {
+        nonzero_index += 1;
+    }
+    hex::encode(&bytes[nonzero_index..])
+}
+
+/// Parse "needle_id_cookie_hex" or "needle_id_cookie_hex_delta" into (NeedleId, Cookie).
+/// Matches Go's ParsePath + ParseNeedleIdCookie: supports an optional `_delta` suffix
+/// where delta is a decimal number added to the NeedleId (used for sub-file addressing).
+/// Rejects strings that are too short or too long.
+pub fn parse_needle_id_cookie(s: &str) -> Result<(NeedleId, Cookie), String> {
+    // Go ParsePath: check for "_" suffix containing a decimal delta
+    let (hex_part, delta) = if let Some(underscore_pos) = s.rfind('_') {
+        if underscore_pos > 0 {
+            let delta_str = &s[underscore_pos + 1..];
+            let d: u64 = delta_str
+                .parse()
+                .map_err(|e| format!("Parse delta error: {}", e))?;
+            (&s[..underscore_pos], Some(d))
+        } else {
+            (s, None)
+        }
+    } else {
+        (s, None)
+    };
+
+    // Go: len(key_hash_string) <= CookieSize*2 => error (must be > 8 hex chars)
+    if hex_part.len() <= COOKIE_SIZE * 2 {
+        return Err("KeyHash is too short.".to_string());
+    }
+    // Go: len(key_hash_string) > (NeedleIdSize+CookieSize)*2 => error (must be <= 24 hex chars)
+    if hex_part.len() > (NEEDLE_ID_SIZE + COOKIE_SIZE) * 2 {
+        return Err("KeyHash is too long.".to_string());
+    }
+
+    // Split: last CookieSize*2 hex chars are cookie, rest is needle id
+    let split = hex_part.len() - COOKIE_SIZE * 2;
+    let needle_id_hex = &hex_part[..split];
+    let cookie_hex = &hex_part[split..];
+
+    let needle_id_bytes = hex::decode(needle_id_hex).map_err(|e| format!("Parse needleId error: {}", e))?;
+    let cookie_bytes = hex::decode(cookie_hex).map_err(|e| format!("Parse cookie error: {}", e))?;
+
+    // Pad needle id to 8 bytes
+    let mut nid_buf = [0u8; 8];
+    if needle_id_bytes.len() > 8 {
+        return Err(format!("KeyHash is too long."));
+    }
+    let start = 8 - needle_id_bytes.len();
+    nid_buf[start..].copy_from_slice(&needle_id_bytes);
+
+    let mut key = NeedleId::from_bytes(&nid_buf[0..8]);
+    let cookie = Cookie::from_bytes(&cookie_bytes[0..4]);
+
+    // Apply delta if present (Go: n.Id += Uint64ToNeedleId(d))
+    if let Some(d) = delta {
+        key = NeedleId(key.0.wrapping_add(d));
+    }
+
+    Ok((key, cookie))
+}
+
+// ============================================================================
+// Errors
+// ============================================================================
+
+#[derive(Debug, thiserror::Error)]
+pub enum NeedleError {
+    #[error("size mismatch at offset {offset}: found id={id} size={found:?}, expected size={expected:?}")]
+    SizeMismatch {
+        offset: i64,
+        id: NeedleId,
+        found: Size,
+        expected: Size,
+    },
+
+    #[error("CRC mismatch for needle {needle_id}: got {got:08x}, want {want:08x}")]
+    CrcMismatch {
+        needle_id: NeedleId,
+        got: u32,
+        want: u32,
+    },
+
+    #[error("index out of range ({0})")]
+    IndexOutOfRange(u32),
+
+    #[error("needle tail too short")]
+    TailTooShort,
+
+    #[error("unsupported version: {0}")]
+    UnsupportedVersion(u8),
+
+    #[error("IO error: {0}")]
+    Io(#[from] std::io::Error),
+}
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_header() {
+        let mut buf = [0u8; NEEDLE_HEADER_SIZE];
+        let cookie = Cookie(0xdeadbeef);
+        let id = NeedleId(0x123456789abcdef0);
+        let size = Size(1024);
+        cookie.to_bytes(&mut buf[0..4]);
+        id.to_bytes(&mut buf[4..12]);
+        size.to_bytes(&mut buf[12..16]);
+
+        let (c, i, s) = Needle::parse_header(&buf);
+        assert_eq!(c, cookie);
+        assert_eq!(i, id);
+        assert_eq!(s, size);
+    }
+
+    #[test]
+    fn test_needle_write_read_round_trip_v3() {
+        let mut n = Needle::default();
+        n.cookie = Cookie(42);
+        n.id = NeedleId(100);
+        n.data = b"hello world".to_vec();
+        n.flags = 0;
+        n.set_has_name();
+        n.name = b"test.txt".to_vec();
+        n.name_size = 8;
+        n.set_has_mime();
+        n.mime = b"text/plain".to_vec();
+        n.mime_size = 10;
+        n.set_has_last_modified_date();
+        n.last_modified = 1234567890;
+        n.set_has_ttl();
+        n.ttl = Some(TTL {
+            count: 5,
+            unit: super::super::ttl::TTL_UNIT_DAY,
+        });
+        n.append_at_ns = 999999999;
+
+        let bytes = n.write_bytes(VERSION_3);
+
+        // Verify total size matches expected
+        let expected_size = get_actual_size(n.size, VERSION_3);
+        assert_eq!(bytes.len() as i64, expected_size);
+
+        // Read it back
+        let mut n2 = Needle::default();
+        n2.read_bytes(&bytes, 0, n.size, VERSION_3).unwrap();
+
+        assert_eq!(n2.cookie, n.cookie);
+        assert_eq!(n2.id, n.id);
+        assert_eq!(n2.data, n.data);
+        assert_eq!(n2.name, n.name);
+        assert_eq!(n2.mime, n.mime);
+        assert_eq!(n2.last_modified, n.last_modified);
+        assert_eq!(n2.ttl, n.ttl);
+        assert_eq!(n2.checksum, n.checksum);
+        assert_eq!(n2.append_at_ns, n.append_at_ns);
+    }
+
+    #[test]
+    fn test_needle_write_read_round_trip_v2() {
+        let mut n = Needle::default();
+        n.cookie = Cookie(77);
+        n.id = NeedleId(200);
+        n.data = b"data v2".to_vec();
+        n.flags = 0;
+
+        let bytes = n.write_bytes(VERSION_2);
+        let expected_size = get_actual_size(n.size, VERSION_2);
+        assert_eq!(bytes.len() as i64, expected_size);
+
+        let mut n2 = Needle::default();
+        n2.read_bytes(&bytes, 0, n.size, VERSION_2).unwrap();
+
+        assert_eq!(n2.data, n.data);
+        assert_eq!(n2.checksum, n.checksum);
+    }
+
+    #[test]
+    fn test_read_bytes_meta_only_handles_tombstone_v3() {
+        let mut tombstone = Needle::default();
+        tombstone.cookie = Cookie(0x1234abcd);
+        tombstone.id = NeedleId(300);
+        tombstone.append_at_ns = 999_999;
+
+        let bytes = tombstone.write_bytes(VERSION_3);
+
+        let mut meta = Needle::default();
+        meta.read_bytes_meta_only(&bytes, 0, Size(0), VERSION_3)
+            .unwrap();
+
+        assert_eq!(meta.cookie, tombstone.cookie);
+        assert_eq!(meta.id, tombstone.id);
+        assert_eq!(meta.size, Size(0));
+        assert_eq!(meta.data_size, 0);
+        assert_eq!(meta.append_at_ns, tombstone.append_at_ns);
+        assert_eq!(meta.checksum, tombstone.checksum);
+    }
+
+    #[test]
+    fn test_padding_alignment() {
+        // All actual sizes should be multiples of 8
+        for size_val in 0..50 {
+            let s = Size(size_val);
+            let actual_v2 = get_actual_size(s, VERSION_2);
+            let actual_v3 = get_actual_size(s, VERSION_3);
+            assert_eq!(actual_v2 % 8, 0, "V2 size {} not aligned", size_val);
+            assert_eq!(actual_v3 % 8, 0, "V3 size {} not aligned", size_val);
+        }
+    }
+
+    #[test]
+    fn test_file_id_parse() {
+        let fid = FileId::parse("3,01637037d6").unwrap();
+        assert_eq!(fid.volume_id, VolumeId(3));
+        // The hex "01637037d6" is 5 bytes = 0x0163..., padded to 12 bytes
+        assert!(!fid.key.is_empty() || !fid.cookie.0 == 0);
+    }
+
+    #[test]
+    fn test_file_id_round_trip() {
+        let fid = FileId::new(VolumeId(5), NeedleId(0x123456), Cookie(0xabcd));
+        let s = fid.to_string();
+        let fid2 = FileId::parse(&s).unwrap();
+        assert_eq!(fid, fid2);
+    }
+
+    #[test]
+    fn test_needle_id_cookie_format() {
+        let s = format_needle_id_cookie(NeedleId(1), Cookie(0x12345678));
+        let (key, cookie) = parse_needle_id_cookie(&s).unwrap();
+        assert_eq!(key, NeedleId(1));
+        assert_eq!(cookie, Cookie(0x12345678));
+    }
+}

seaweed-volume/src/storage/needle/ttl.rs

@@ -0,0 +1,302 @@
+//! Time-to-live encoding for needles.
+//!
+//! TTL is stored as 2 bytes: Count(1) + Unit(1).
+//! Supported units: minute(m), hour(h), day(d), week(w), month(M), year(y).
+
+use std::fmt;
+
+/// TTL unit constants (matching Go).
+pub const TTL_UNIT_EMPTY: u8 = 0;
+pub const TTL_UNIT_MINUTE: u8 = 1;
+pub const TTL_UNIT_HOUR: u8 = 2;
+pub const TTL_UNIT_DAY: u8 = 3;
+pub const TTL_UNIT_WEEK: u8 = 4;
+pub const TTL_UNIT_MONTH: u8 = 5;
+pub const TTL_UNIT_YEAR: u8 = 6;
+
+pub const TTL_BYTES_LENGTH: usize = 2;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct TTL {
+    pub count: u8,
+    pub unit: u8,
+}
+
+impl TTL {
+    pub const EMPTY: TTL = TTL { count: 0, unit: 0 };
+
+    pub fn is_empty(&self) -> bool {
+        self.count == 0 && self.unit == 0
+    }
+
+    /// Load from 2 bytes.
+    pub fn from_bytes(input: &[u8]) -> Self {
+        if input.len() < 2 {
+            return TTL::EMPTY;
+        }
+        TTL {
+            count: input[0],
+            unit: input[1],
+        }
+    }
+
+    /// Serialize to 2 bytes.
+    pub fn to_bytes(&self, output: &mut [u8]) {
+        assert!(output.len() >= 2);
+        output[0] = self.count;
+        output[1] = self.unit;
+    }
+
+    /// Encode as u32: (count << 8) | unit.
+    pub fn to_u32(&self) -> u32 {
+        if self.count == 0 {
+            return 0;
+        }
+        ((self.count as u32) << 8) + (self.unit as u32)
+    }
+
+    /// Decode from u32.
+    pub fn from_u32(v: u32) -> Self {
+        if v == 0 {
+            return TTL::EMPTY;
+        }
+        TTL {
+            count: (v >> 8) as u8,
+            unit: (v & 0xFF) as u8,
+        }
+    }
+
+    /// Convert to total seconds.
+    pub fn to_seconds(&self) -> u64 {
+        unit_to_seconds(self.count as u64, self.unit)
+    }
+
+    /// Parse from string like "3m", "4h", "5d", "6w", "7M", "8y".
+    /// If the string is all digits (no unit suffix), defaults to minutes.
+    /// Matches Go's ReadTTL which calls fitTtlCount to normalize:
+    /// e.g. "120m" -> 2h, "7d" -> 1w, "24h" -> 1d.
+    pub fn read(s: &str) -> Result<Self, String> {
+        let s = s.trim();
+        if s.is_empty() {
+            return Ok(TTL::EMPTY);
+        }
+        let last_byte = s.as_bytes()[s.len() - 1];
+        let (num_str, unit_byte) = if last_byte >= b'0' && last_byte <= b'9' {
+            // All digits — default to minutes (matching Go)
+            (s, b'm')
+        } else {
+            (&s[..s.len() - 1], last_byte)
+        };
+        let count: u32 = num_str
+            .parse()
+            .map_err(|e| format!("invalid TTL count: {}", e))?;
+        let unit = match unit_byte {
+            b'm' => TTL_UNIT_MINUTE,
+            b'h' => TTL_UNIT_HOUR,
+            b'd' => TTL_UNIT_DAY,
+            b'w' => TTL_UNIT_WEEK,
+            b'M' => TTL_UNIT_MONTH,
+            b'y' => TTL_UNIT_YEAR,
+            _ => return Err(format!("unknown TTL unit: {}", unit_byte as char)),
+        };
+        // Match Go's ReadTTL: normalize via fitTtlCount
+        Ok(fit_ttl_count(count, unit))
+    }
+
+    /// Minutes representation.
+    pub fn minutes(&self) -> u32 {
+        (self.to_seconds() / 60) as u32
+    }
+}
+
+fn unit_to_seconds(count: u64, unit: u8) -> u64 {
+    match unit {
+        TTL_UNIT_EMPTY => 0,
+        TTL_UNIT_MINUTE => count * 60,
+        TTL_UNIT_HOUR => count * 60 * 60,
+        TTL_UNIT_DAY => count * 60 * 60 * 24,
+        TTL_UNIT_WEEK => count * 60 * 60 * 24 * 7,
+        TTL_UNIT_MONTH => count * 60 * 60 * 24 * 30,
+        TTL_UNIT_YEAR => count * 60 * 60 * 24 * 365,
+        _ => 0,
+    }
+}
+
+/// Fit a count+unit into a TTL that fits in a single byte count.
+/// Converts to seconds first, then finds the coarsest unit that fits.
+/// Matches Go's fitTtlCount called from ReadTTL.
+fn fit_ttl_count(count: u32, unit: u8) -> TTL {
+    if count == 0 || unit == TTL_UNIT_EMPTY {
+        return TTL::EMPTY;
+    }
+
+    // Always convert to seconds and normalize (matches Go).
+    let seconds = unit_to_seconds(count as u64, unit);
+    if seconds == 0 {
+        return TTL::EMPTY;
+    }
+
+    const YEAR_SECS: u64 = 3600 * 24 * 365;
+    const MONTH_SECS: u64 = 3600 * 24 * 30;
+    const WEEK_SECS: u64 = 3600 * 24 * 7;
+    const DAY_SECS: u64 = 3600 * 24;
+    const HOUR_SECS: u64 = 3600;
+    const MINUTE_SECS: u64 = 60;
+
+    // First pass: try exact fits from largest to smallest
+    if seconds % YEAR_SECS == 0 && seconds / YEAR_SECS < 256 {
+        return TTL { count: (seconds / YEAR_SECS) as u8, unit: TTL_UNIT_YEAR };
+    }
+    if seconds % MONTH_SECS == 0 && seconds / MONTH_SECS < 256 {
+        return TTL { count: (seconds / MONTH_SECS) as u8, unit: TTL_UNIT_MONTH };
+    }
+    if seconds % WEEK_SECS == 0 && seconds / WEEK_SECS < 256 {
+        return TTL { count: (seconds / WEEK_SECS) as u8, unit: TTL_UNIT_WEEK };
+    }
+    if seconds % DAY_SECS == 0 && seconds / DAY_SECS < 256 {
+        return TTL { count: (seconds / DAY_SECS) as u8, unit: TTL_UNIT_DAY };
+    }
+    if seconds % HOUR_SECS == 0 && seconds / HOUR_SECS < 256 {
+        return TTL { count: (seconds / HOUR_SECS) as u8, unit: TTL_UNIT_HOUR };
+    }
+    // Minutes: truncating division
+    if seconds / MINUTE_SECS < 256 {
+        return TTL { count: (seconds / MINUTE_SECS) as u8, unit: TTL_UNIT_MINUTE };
+    }
+    // Second pass: truncating division from smallest to largest
+    if seconds / HOUR_SECS < 256 {
+        return TTL { count: (seconds / HOUR_SECS) as u8, unit: TTL_UNIT_HOUR };
+    }
+    if seconds / DAY_SECS < 256 {
+        return TTL { count: (seconds / DAY_SECS) as u8, unit: TTL_UNIT_DAY };
+    }
+    if seconds / WEEK_SECS < 256 {
+        return TTL { count: (seconds / WEEK_SECS) as u8, unit: TTL_UNIT_WEEK };
+    }
+    if seconds / MONTH_SECS < 256 {
+        return TTL { count: (seconds / MONTH_SECS) as u8, unit: TTL_UNIT_MONTH };
+    }
+    if seconds / YEAR_SECS < 256 {
+        return TTL { count: (seconds / YEAR_SECS) as u8, unit: TTL_UNIT_YEAR };
+    }
+    TTL::EMPTY
+}
+
+fn unit_to_char(unit: u8) -> char {
+    match unit {
+        TTL_UNIT_MINUTE => 'm',
+        TTL_UNIT_HOUR => 'h',
+        TTL_UNIT_DAY => 'd',
+        TTL_UNIT_WEEK => 'w',
+        TTL_UNIT_MONTH => 'M',
+        TTL_UNIT_YEAR => 'y',
+        _ => ' ',
+    }
+}
+
+impl fmt::Display for TTL {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if self.count == 0 || self.unit == TTL_UNIT_EMPTY {
+            return write!(f, "");
+        }
+        write!(f, "{}{}", self.count, unit_to_char(self.unit))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_ttl_parse() {
+        let ttl = TTL::read("3m").unwrap();
+        assert_eq!(
+            ttl,
+            TTL {
+                count: 3,
+                unit: TTL_UNIT_MINUTE
+            }
+        );
+        assert_eq!(ttl.to_seconds(), 180);
+    }
+
+    #[test]
+    fn test_ttl_parse_hours() {
+        // 24h normalizes to 1d via fitTtlCount
+        let ttl = TTL::read("24h").unwrap();
+        assert_eq!(ttl.to_seconds(), 86400);
+        assert_eq!(ttl, TTL { count: 1, unit: TTL_UNIT_DAY });
+    }
+
+    #[test]
+    fn test_ttl_display() {
+        let ttl = TTL {
+            count: 5,
+            unit: TTL_UNIT_DAY,
+        };
+        assert_eq!(ttl.to_string(), "5d");
+    }
+
+    #[test]
+    fn test_ttl_bytes_round_trip() {
+        let ttl = TTL {
+            count: 10,
+            unit: TTL_UNIT_WEEK,
+        };
+        let mut buf = [0u8; 2];
+        ttl.to_bytes(&mut buf);
+        let ttl2 = TTL::from_bytes(&buf);
+        assert_eq!(ttl, ttl2);
+    }
+
+    #[test]
+    fn test_ttl_u32_round_trip() {
+        let ttl = TTL {
+            count: 42,
+            unit: TTL_UNIT_HOUR,
+        };
+        let v = ttl.to_u32();
+        let ttl2 = TTL::from_u32(v);
+        assert_eq!(ttl, ttl2);
+    }
+
+    #[test]
+    fn test_ttl_empty() {
+        assert!(TTL::EMPTY.is_empty());
+        assert_eq!(TTL::EMPTY.to_seconds(), 0);
+        assert_eq!(TTL::EMPTY.to_u32(), 0);
+    }
+
+    #[test]
+    fn test_ttl_overflow_normalizes() {
+        // Go's ReadTTL calls fitTtlCount: 300m = 18000s = 5h (exact fit)
+        let ttl = TTL::read("300m").unwrap();
+        assert_eq!(ttl, TTL { count: 5, unit: TTL_UNIT_HOUR });
+
+        // 256h = 921600s. Doesn't fit in hours (256 >= 256), doesn't fit exact in days.
+        // Second pass: 921600/86400 = 10 (truncated) < 256 -> 10d
+        let ttl = TTL::read("256h").unwrap();
+        assert_eq!(ttl, TTL { count: 10, unit: TTL_UNIT_DAY });
+    }
+
+    #[test]
+    fn test_ttl_normalizes_unit() {
+        // Go's ReadTTL calls fitTtlCount which normalizes to coarsest unit.
+        // 120m -> 2h, 7d -> 1w, 24h -> 1d.
+        let ttl = TTL::read("120m").unwrap();
+        assert_eq!(ttl, TTL { count: 2, unit: TTL_UNIT_HOUR });
+
+        let ttl = TTL::read("7d").unwrap();
+        assert_eq!(ttl, TTL { count: 1, unit: TTL_UNIT_WEEK });
+
+        let ttl = TTL::read("24h").unwrap();
+        assert_eq!(ttl, TTL { count: 1, unit: TTL_UNIT_DAY });
+
+        // Values that don't simplify stay as-is
+        let ttl = TTL::read("5d").unwrap();
+        assert_eq!(ttl, TTL { count: 5, unit: TTL_UNIT_DAY });
+
+        let ttl = TTL::read("3m").unwrap();
+        assert_eq!(ttl, TTL { count: 3, unit: TTL_UNIT_MINUTE });
+    }
+}

seaweed-volume/src/storage/needle_map/compact_map.rs

@@ -0,0 +1,375 @@
+//! CompactMap: memory-efficient in-memory map of NeedleId -> (Offset, Size).
+//!
+//! Port of Go's CompactMap from weed/storage/needle_map/compact_map.go.
+//! Uses segmented sorted arrays with compressed keys (u16 instead of u64)
+//! to achieve ~10 bytes per entry instead of ~40-48 bytes with HashMap.
+//!
+//! NeedleId is split into: chunk = id / SEGMENT_CHUNK_SIZE, compact_key = id % SEGMENT_CHUNK_SIZE.
+//! Each segment stores up to SEGMENT_CHUNK_SIZE entries in a sorted Vec, searched via binary search.
+//! Best case (ordered inserts): O(1). Worst case: O(log n) per segment.
+
+use std::collections::HashMap;
+
+use super::NeedleValue;
+use crate::storage::types::*;
+
+/// Maximum entries per segment. Must be <= u16::MAX (65535).
+const SEGMENT_CHUNK_SIZE: u64 = 50_000;
+
+/// Compact key: only the low bits of NeedleId within a segment.
+type CompactKey = u16;
+
+/// Segment chunk identifier: NeedleId / SEGMENT_CHUNK_SIZE.
+type Chunk = u64;
+
+/// Compact entry: 10 bytes (2 + 4 + 4) vs 16 bytes for full NeedleId + NeedleValue.
+#[derive(Clone, Copy)]
+struct CompactEntry {
+    key: CompactKey,           // 2 bytes
+    offset: [u8; OFFSET_SIZE], // 4 bytes
+    size: Size,                // 4 bytes
+}
+
+impl CompactEntry {
+    fn to_needle_value(&self) -> NeedleValue {
+        NeedleValue {
+            offset: Offset::from_bytes(&self.offset),
+            size: self.size,
+        }
+    }
+}
+
+/// A sorted segment of compact entries for a given chunk.
+struct Segment {
+    list: Vec<CompactEntry>,
+    chunk: Chunk,
+    first_key: CompactKey,
+    last_key: CompactKey,
+}
+
+impl Segment {
+    fn new(chunk: Chunk) -> Self {
+        Segment {
+            list: Vec::new(),
+            chunk,
+            first_key: u16::MAX,
+            last_key: 0,
+        }
+    }
+
+    fn compact_key(&self, id: NeedleId) -> CompactKey {
+        (id.0 - SEGMENT_CHUNK_SIZE * self.chunk) as CompactKey
+    }
+
+    /// Binary search for a compact key. Returns (index, found).
+    /// If not found, index is the insertion point.
+    fn bsearch(&self, id: NeedleId) -> (usize, bool) {
+        let ck = self.compact_key(id);
+
+        if self.list.is_empty() {
+            return (0, false);
+        }
+        if ck == self.first_key {
+            return (0, true);
+        }
+        if ck < self.first_key {
+            return (0, false);
+        }
+        if ck == self.last_key {
+            return (self.list.len() - 1, true);
+        }
+        if ck > self.last_key {
+            return (self.list.len(), false);
+        }
+
+        let i = self.list.partition_point(|e| e.key < ck);
+        if i < self.list.len() && self.list[i].key == ck {
+            (i, true)
+        } else {
+            (i, false)
+        }
+    }
+
+    /// Insert or update. Returns old NeedleValue if updating.
+    fn set(&mut self, id: NeedleId, offset: Offset, size: Size) -> Option<NeedleValue> {
+        let (i, found) = self.bsearch(id);
+
+        if found {
+            let old = self.list[i].to_needle_value();
+            let mut offset_bytes = [0u8; OFFSET_SIZE];
+            offset.to_bytes(&mut offset_bytes);
+            self.list[i].offset = offset_bytes;
+            self.list[i].size = size;
+            return Some(old);
+        }
+
+        // Insert at sorted position
+        let ck = self.compact_key(id);
+        let mut offset_bytes = [0u8; OFFSET_SIZE];
+        offset.to_bytes(&mut offset_bytes);
+
+        let entry = CompactEntry {
+            key: ck,
+            offset: offset_bytes,
+            size,
+        };
+
+        // Match Go panic: don't exceed segment capacity
+        if self.list.len() >= SEGMENT_CHUNK_SIZE as usize {
+            panic!(
+                "attempted to write more than {} entries on CompactMapSegment",
+                SEGMENT_CHUNK_SIZE
+            );
+        }
+
+        if self.list.len() == SEGMENT_CHUNK_SIZE as usize - 1 {
+            // Pin capacity to exact size when maxing out
+            let mut new_list = Vec::with_capacity(SEGMENT_CHUNK_SIZE as usize);
+            new_list.extend_from_slice(&self.list[..i]);
+            new_list.push(entry);
+            new_list.extend_from_slice(&self.list[i..]);
+            self.list = new_list;
+        } else {
+            self.list.insert(i, entry);
+        }
+
+        if ck < self.first_key {
+            self.first_key = ck;
+        }
+        if ck > self.last_key {
+            self.last_key = ck;
+        }
+
+        None
+    }
+
+    fn get(&self, id: NeedleId) -> Option<NeedleValue> {
+        let (i, found) = self.bsearch(id);
+        if found {
+            Some(self.list[i].to_needle_value())
+        } else {
+            None
+        }
+    }
+
+    /// Mark as deleted by negating size. Returns previous size if not already deleted.
+    /// Matches Go behavior: checks !IsDeleted() (i.e., size >= 0).
+    fn delete(&mut self, id: NeedleId) -> Option<Size> {
+        let (i, found) = self.bsearch(id);
+        if found && !self.list[i].size.is_deleted() {
+            let old_size = self.list[i].size;
+            if self.list[i].size.0 == 0 {
+                self.list[i].size = TOMBSTONE_FILE_SIZE;
+            } else {
+                self.list[i].size = Size(-self.list[i].size.0);
+            }
+            Some(old_size)
+        } else {
+            None
+        }
+    }
+}
+
+/// Memory-efficient map of NeedleId -> (Offset, Size).
+/// Segments NeedleIds into chunks of 50,000 and stores compact 10-byte entries
+/// in sorted arrays, using only 2 bytes for the key within each segment.
+pub struct CompactMap {
+    segments: HashMap<Chunk, Segment>,
+}
+
+impl CompactMap {
+    pub fn new() -> Self {
+        CompactMap {
+            segments: HashMap::new(),
+        }
+    }
+
+    fn _segment_for_key(&mut self, id: NeedleId) -> &mut Segment {
+        let chunk = id.0 / SEGMENT_CHUNK_SIZE;
+        self.segments
+            .entry(chunk)
+            .or_insert_with(|| Segment::new(chunk))
+    }
+
+    /// Insert or update. Returns old NeedleValue if updating.
+    pub fn set(&mut self, id: NeedleId, offset: Offset, size: Size) -> Option<NeedleValue> {
+        let chunk = id.0 / SEGMENT_CHUNK_SIZE;
+        let segment = self
+            .segments
+            .entry(chunk)
+            .or_insert_with(|| Segment::new(chunk));
+        segment.set(id, offset, size)
+    }
+
+    pub fn get(&self, id: NeedleId) -> Option<NeedleValue> {
+        let chunk = id.0 / SEGMENT_CHUNK_SIZE;
+        self.segments.get(&chunk)?.get(id)
+    }
+
+    /// Mark as deleted. Returns previous size if was valid.
+    pub fn delete(&mut self, id: NeedleId) -> Option<Size> {
+        let chunk = id.0 / SEGMENT_CHUNK_SIZE;
+        self.segments.get_mut(&chunk)?.delete(id)
+    }
+
+    /// Remove entry entirely (used during idx loading).
+    pub fn remove(&mut self, id: NeedleId) -> Option<NeedleValue> {
+        let chunk = id.0 / SEGMENT_CHUNK_SIZE;
+        let segment = self.segments.get_mut(&chunk)?;
+        let (i, found) = segment.bsearch(id);
+        if found {
+            let entry = segment.list.remove(i);
+            // Update first/last keys
+            if segment.list.is_empty() {
+                segment.first_key = u16::MAX;
+                segment.last_key = 0;
+            } else {
+                segment.first_key = segment.list[0].key;
+                segment.last_key = segment.list[segment.list.len() - 1].key;
+            }
+            Some(entry.to_needle_value())
+        } else {
+            None
+        }
+    }
+
+    /// Iterate all entries in ascending NeedleId order.
+    pub fn ascending_visit<F, E>(&self, mut f: F) -> Result<(), E>
+    where
+        F: FnMut(NeedleId, &NeedleValue) -> Result<(), E>,
+    {
+        let mut chunks: Vec<Chunk> = self.segments.keys().copied().collect();
+        chunks.sort_unstable();
+
+        for chunk in chunks {
+            let segment = &self.segments[&chunk];
+            for entry in &segment.list {
+                let id = NeedleId(SEGMENT_CHUNK_SIZE * segment.chunk + entry.key as u64);
+                let nv = entry.to_needle_value();
+                f(id, &nv)?;
+            }
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn offset(v: u32) -> Offset {
+        Offset::from_actual_offset(v as i64 * NEEDLE_PADDING_SIZE as i64)
+    }
+
+    #[test]
+    fn test_compact_map_basic() {
+        let mut m = CompactMap::new();
+
+        // Insert
+        assert!(m.set(NeedleId(1), offset(100), Size(50)).is_none());
+        assert!(m.set(NeedleId(2), offset(200), Size(60)).is_none());
+
+        // Get
+        let nv = m.get(NeedleId(1)).unwrap();
+        assert_eq!(nv.size, Size(50));
+
+        // Update returns old value
+        let old = m.set(NeedleId(1), offset(300), Size(70)).unwrap();
+        assert_eq!(old.size, Size(50));
+
+        // Get updated value
+        let nv = m.get(NeedleId(1)).unwrap();
+        assert_eq!(nv.size, Size(70));
+
+        // Miss
+        assert!(m.get(NeedleId(999)).is_none());
+    }
+
+    #[test]
+    fn test_compact_map_delete() {
+        let mut m = CompactMap::new();
+        m.set(NeedleId(1), offset(100), Size(50));
+
+        // Delete returns old size
+        let old = m.delete(NeedleId(1)).unwrap();
+        assert_eq!(old, Size(50));
+
+        // Get returns deleted (negative size)
+        let nv = m.get(NeedleId(1)).unwrap();
+        assert!(nv.size.is_deleted());
+
+        // Delete again returns None (already deleted)
+        assert!(m.delete(NeedleId(1)).is_none());
+    }
+
+    #[test]
+    fn test_compact_map_zero_size_delete() {
+        let mut m = CompactMap::new();
+        m.set(NeedleId(1), offset(100), Size(0));
+
+        let old = m.delete(NeedleId(1)).unwrap();
+        assert_eq!(old, Size(0));
+
+        let nv = m.get(NeedleId(1)).unwrap();
+        assert_eq!(nv.size, TOMBSTONE_FILE_SIZE);
+    }
+
+    #[test]
+    fn test_compact_map_cross_segment() {
+        let mut m = CompactMap::new();
+
+        // Insert across multiple segments
+        m.set(NeedleId(1), offset(1), Size(1));
+        m.set(NeedleId(50_000), offset(2), Size(2));
+        m.set(NeedleId(100_000), offset(3), Size(3));
+
+        assert_eq!(m.get(NeedleId(1)).unwrap().size, Size(1));
+        assert_eq!(m.get(NeedleId(50_000)).unwrap().size, Size(2));
+        assert_eq!(m.get(NeedleId(100_000)).unwrap().size, Size(3));
+    }
+
+    #[test]
+    fn test_compact_map_ascending_visit() {
+        let mut m = CompactMap::new();
+        m.set(NeedleId(100_005), offset(3), Size(3));
+        m.set(NeedleId(5), offset(1), Size(1));
+        m.set(NeedleId(50_005), offset(2), Size(2));
+
+        let mut visited = Vec::new();
+        m.ascending_visit(|id, nv| {
+            visited.push((id, nv.size));
+            Ok::<_, String>(())
+        })
+        .unwrap();
+
+        assert_eq!(visited.len(), 3);
+        assert_eq!(visited[0].0, NeedleId(5));
+        assert_eq!(visited[1].0, NeedleId(50_005));
+        assert_eq!(visited[2].0, NeedleId(100_005));
+    }
+
+    #[test]
+    fn test_compact_map_remove() {
+        let mut m = CompactMap::new();
+        m.set(NeedleId(1), offset(100), Size(50));
+        m.set(NeedleId(2), offset(200), Size(60));
+
+        let removed = m.remove(NeedleId(1)).unwrap();
+        assert_eq!(removed.size, Size(50));
+
+        assert!(m.get(NeedleId(1)).is_none());
+        assert_eq!(m.get(NeedleId(2)).unwrap().size, Size(60));
+    }
+
+    #[test]
+    fn test_compact_map_reverse_insert_order() {
+        let mut m = CompactMap::new();
+        // Insert in reverse order to test sorted insert
+        for i in (0..100).rev() {
+            m.set(NeedleId(i), offset(i as u32), Size(i as i32));
+        }
+        for i in 0..100 {
+            assert_eq!(m.get(NeedleId(i)).unwrap().size, Size(i as i32));
+        }
+    }
+}

seaweed-volume/src/storage/super_block.rs

@@ -0,0 +1,289 @@
+//! SuperBlock: the 8-byte (+ optional extra) header at the start of every .dat file.
+//!
+//! Byte layout:
+//!   [0]     Version
+//!   [1]     ReplicaPlacement byte
+//!   [2..4]  TTL (2 bytes)
+//!   [4..6]  CompactionRevision (u16 big-endian)
+//!   [6..8]  ExtraSize (u16 big-endian)
+//!   [8..]   Extra data (protobuf, ExtraSize bytes) — only for Version 2/3
+
+use crate::storage::needle::ttl::TTL;
+use crate::storage::types::Version;
+
+pub const SUPER_BLOCK_SIZE: usize = 8;
+
+/// SuperBlock metadata at the start of a volume .dat file.
+#[derive(Debug, Clone)]
+pub struct SuperBlock {
+    pub version: Version,
+    pub replica_placement: ReplicaPlacement,
+    pub ttl: TTL,
+    pub compaction_revision: u16,
+    pub extra_size: u16,
+    pub extra_data: Vec<u8>, // raw protobuf bytes (SuperBlockExtra)
+}
+
+impl SuperBlock {
+    /// Total block size on disk (base 8 + extra).
+    pub fn block_size(&self) -> usize {
+        match self.version.0 {
+            2 | 3 => SUPER_BLOCK_SIZE + self.extra_size as usize,
+            _ => SUPER_BLOCK_SIZE,
+        }
+    }
+
+    /// Serialize to bytes.
+    pub fn to_bytes(&self) -> Vec<u8> {
+        let mut header = vec![0u8; SUPER_BLOCK_SIZE];
+        header[0] = self.version.0;
+        header[1] = self.replica_placement.to_byte();
+        self.ttl.to_bytes(&mut header[2..4]);
+        header[4..6].copy_from_slice(&self.compaction_revision.to_be_bytes());
+
+        if !self.extra_data.is_empty() {
+            // Go checks extraSize > 256*256-2 and calls glog.Fatalf; guard against u16 overflow.
+            assert!(
+                self.extra_data.len() <= 65534,
+                "super block extra data too large: {} > 65534",
+                self.extra_data.len()
+            );
+            let extra_size = self.extra_data.len() as u16;
+            header[6..8].copy_from_slice(&extra_size.to_be_bytes());
+            header.extend_from_slice(&self.extra_data);
+        }
+
+        header
+    }
+
+    /// Parse from bytes (must be at least SUPER_BLOCK_SIZE bytes).
+    pub fn from_bytes(bytes: &[u8]) -> Result<Self, SuperBlockError> {
+        if bytes.len() < SUPER_BLOCK_SIZE {
+            return Err(SuperBlockError::TooShort(bytes.len()));
+        }
+
+        let version = Version(bytes[0]);
+        let replica_placement = ReplicaPlacement::from_byte(bytes[1])?;
+        let ttl = TTL::from_bytes(&bytes[2..4]);
+        let compaction_revision = u16::from_be_bytes([bytes[4], bytes[5]]);
+        let extra_size = u16::from_be_bytes([bytes[6], bytes[7]]);
+
+        let extra_data = if extra_size > 0 && bytes.len() >= SUPER_BLOCK_SIZE + extra_size as usize
+        {
+            bytes[SUPER_BLOCK_SIZE..SUPER_BLOCK_SIZE + extra_size as usize].to_vec()
+        } else {
+            vec![]
+        };
+
+        Ok(SuperBlock {
+            version,
+            replica_placement,
+            ttl,
+            compaction_revision,
+            extra_size,
+            extra_data,
+        })
+    }
+
+    pub fn initialized(&self) -> bool {
+        true // ReplicaPlacement and TTL are always valid after construction
+    }
+}
+
+impl Default for SuperBlock {
+    fn default() -> Self {
+        SuperBlock {
+            version: Version::current(),
+            replica_placement: ReplicaPlacement::default(),
+            ttl: TTL::EMPTY,
+            compaction_revision: 0,
+            extra_size: 0,
+            extra_data: vec![],
+        }
+    }
+}
+
+// ============================================================================
+// ReplicaPlacement
+// ============================================================================
+
+/// Replication strategy encoded as a single byte.
+///
+/// Byte value = DiffDataCenterCount * 100 + DiffRackCount * 10 + SameRackCount
+///
+/// Examples:
+///   "000" → no replication (1 copy total)
+///   "010" → 1 copy in different rack (2 copies total)
+///   "100" → 1 copy in different datacenter
+///   "200" → 2 copies in different datacenters
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct ReplicaPlacement {
+    pub same_rack_count: u8,
+    pub diff_rack_count: u8,
+    pub diff_data_center_count: u8,
+}
+
+impl ReplicaPlacement {
+    /// Parse from a string like "000", "010", "100".
+    /// Accepts 0-3 character strings, padding with leading zeros to match Go behavior.
+    /// E.g. "" -> "000", "1" -> "001", "01" -> "001", "010" -> "010"
+    pub fn from_string(s: &str) -> Result<Self, SuperBlockError> {
+        let s = s.trim();
+        if s.is_empty() {
+            return Ok(ReplicaPlacement::default());
+        }
+        // Pad with leading zeros to 3 chars, matching Go's NewReplicaPlacementFromString
+        let padded = match s.len() {
+            1 => format!("00{}", s),
+            2 => format!("0{}", s),
+            3 => s.to_string(),
+            _ => return Err(SuperBlockError::InvalidReplicaPlacement(s.to_string())),
+        };
+        let chars: Vec<char> = padded.chars().collect();
+        let dc = chars[0]
+            .to_digit(10)
+            .ok_or_else(|| SuperBlockError::InvalidReplicaPlacement(s.to_string()))?
+            as u8;
+        let rack = chars[1]
+            .to_digit(10)
+            .ok_or_else(|| SuperBlockError::InvalidReplicaPlacement(s.to_string()))?
+            as u8;
+        let same = chars[2]
+            .to_digit(10)
+            .ok_or_else(|| SuperBlockError::InvalidReplicaPlacement(s.to_string()))?
+            as u8;
+        // Go validates: value = dc*100 + rack*10 + same must fit in a byte
+        let value = dc as u16 * 100 + rack as u16 * 10 + same as u16;
+        if value > 255 {
+            return Err(SuperBlockError::InvalidReplicaPlacement(s.to_string()));
+        }
+        Ok(ReplicaPlacement {
+            diff_data_center_count: dc,
+            diff_rack_count: rack,
+            same_rack_count: same,
+        })
+    }
+
+    /// Parse from a single byte.
+    pub fn from_byte(b: u8) -> Result<Self, SuperBlockError> {
+        Ok(ReplicaPlacement {
+            diff_data_center_count: b / 100,
+            diff_rack_count: (b % 100) / 10,
+            same_rack_count: b % 10,
+        })
+    }
+
+    /// Encode as a single byte.
+    pub fn to_byte(&self) -> u8 {
+        self.diff_data_center_count * 100 + self.diff_rack_count * 10 + self.same_rack_count
+    }
+
+    /// Total number of copies (including the original).
+    pub fn get_copy_count(&self) -> u8 {
+        self.diff_data_center_count + self.diff_rack_count + self.same_rack_count + 1
+    }
+
+    /// Whether this placement requires replication (more than 1 copy).
+    pub fn has_replication(&self) -> bool {
+        self.get_copy_count() > 1
+    }
+}
+
+impl std::fmt::Display for ReplicaPlacement {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}{}{}",
+            self.diff_data_center_count, self.diff_rack_count, self.same_rack_count
+        )
+    }
+}
+
+// ============================================================================
+// Errors
+// ============================================================================
+
+#[derive(Debug, thiserror::Error)]
+pub enum SuperBlockError {
+    #[error("super block too short: {0} bytes")]
+    TooShort(usize),
+
+    #[error("invalid replica placement: {0}")]
+    InvalidReplicaPlacement(String),
+}
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::storage::types::*;
+
+    #[test]
+    fn test_super_block_round_trip() {
+        let sb = SuperBlock {
+            version: VERSION_3,
+            replica_placement: ReplicaPlacement::from_string("010").unwrap(),
+            ttl: TTL { count: 5, unit: 3 },
+            compaction_revision: 42,
+            extra_size: 0,
+            extra_data: vec![],
+        };
+
+        let bytes = sb.to_bytes();
+        assert_eq!(bytes.len(), SUPER_BLOCK_SIZE);
+
+        let sb2 = SuperBlock::from_bytes(&bytes).unwrap();
+        assert_eq!(sb2.version, sb.version);
+        assert_eq!(sb2.replica_placement, sb.replica_placement);
+        assert_eq!(sb2.ttl, sb.ttl);
+        assert_eq!(sb2.compaction_revision, sb.compaction_revision);
+    }
+
+    #[test]
+    fn test_super_block_with_extra() {
+        let sb = SuperBlock {
+            version: VERSION_3,
+            replica_placement: ReplicaPlacement::default(),
+            ttl: TTL::EMPTY,
+            compaction_revision: 0,
+            extra_size: 3,
+            extra_data: vec![1, 2, 3],
+        };
+
+        let bytes = sb.to_bytes();
+        assert_eq!(bytes.len(), SUPER_BLOCK_SIZE + 3);
+
+        let sb2 = SuperBlock::from_bytes(&bytes).unwrap();
+        assert_eq!(sb2.extra_data, vec![1, 2, 3]);
+    }
+
+    #[test]
+    fn test_replica_placement_byte_round_trip() {
+        let rp = ReplicaPlacement::from_string("123").unwrap();
+        assert_eq!(rp.diff_data_center_count, 1);
+        assert_eq!(rp.diff_rack_count, 2);
+        assert_eq!(rp.same_rack_count, 3);
+        assert_eq!(rp.to_byte(), 123);
+        assert_eq!(rp.get_copy_count(), 7); // 1+2+3+1
+
+        let rp2 = ReplicaPlacement::from_byte(123).unwrap();
+        assert_eq!(rp, rp2);
+    }
+
+    #[test]
+    fn test_replica_placement_no_replication() {
+        let rp = ReplicaPlacement::from_string("000").unwrap();
+        assert!(!rp.has_replication());
+        assert_eq!(rp.get_copy_count(), 1);
+    }
+
+    #[test]
+    fn test_replica_placement_display() {
+        let rp = ReplicaPlacement::from_string("010").unwrap();
+        assert_eq!(rp.to_string(), "010");
+        assert!(rp.has_replication());
+    }
+}

seaweed-volume/src/storage/types.rs

@@ -0,0 +1,679 @@
+//! Core storage types: NeedleId, Offset, Size, Cookie, DiskType.
+//!
+//! These types define the binary-compatible on-disk format matching the Go implementation.
+//! CRITICAL: Byte layout must match exactly for cross-compatibility.
+
+use std::fmt;
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+pub const NEEDLE_ID_SIZE: usize = 8;
+pub const NEEDLE_ID_EMPTY: u64 = 0;
+pub const COOKIE_SIZE: usize = 4;
+pub const SIZE_SIZE: usize = 4;
+pub const NEEDLE_HEADER_SIZE: usize = COOKIE_SIZE + NEEDLE_ID_SIZE + SIZE_SIZE; // 16
+pub const DATA_SIZE_SIZE: usize = 4;
+pub const TIMESTAMP_SIZE: usize = 8;
+pub const NEEDLE_PADDING_SIZE: usize = 8;
+pub const NEEDLE_CHECKSUM_SIZE: usize = 4;
+
+/// 5-byte offset mode (matching Go production builds with `-tags 5BytesOffset`).
+/// Max volume size: 8TB. Index entry: 17 bytes (8 + 5 + 4).
+#[cfg(feature = "5bytes")]
+pub const OFFSET_SIZE: usize = 5;
+#[cfg(feature = "5bytes")]
+pub const MAX_POSSIBLE_VOLUME_SIZE: u64 = 4 * 1024 * 1024 * 1024 * 8 * 256; // 8TB
+
+/// 4-byte offset mode (matching Go default build without `5BytesOffset`).
+/// Max volume size: 32GB. Index entry: 16 bytes (8 + 4 + 4).
+#[cfg(not(feature = "5bytes"))]
+pub const OFFSET_SIZE: usize = 4;
+#[cfg(not(feature = "5bytes"))]
+pub const MAX_POSSIBLE_VOLUME_SIZE: u64 = 4 * 1024 * 1024 * 1024 * 8; // 32GB
+
+pub const NEEDLE_MAP_ENTRY_SIZE: usize = NEEDLE_ID_SIZE + OFFSET_SIZE + SIZE_SIZE;
+
+// ============================================================================
+// NeedleId
+// ============================================================================
+
+/// 64-bit unique identifier for a needle within a volume.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Default)]
+pub struct NeedleId(pub u64);
+
+impl NeedleId {
+    pub fn to_bytes(&self, bytes: &mut [u8]) {
+        assert!(bytes.len() >= NEEDLE_ID_SIZE);
+        bytes[0..8].copy_from_slice(&self.0.to_be_bytes());
+    }
+
+    pub fn from_bytes(bytes: &[u8]) -> Self {
+        assert!(bytes.len() >= NEEDLE_ID_SIZE);
+        NeedleId(u64::from_be_bytes([
+            bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7],
+        ]))
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0 == 0
+    }
+
+    /// Parse a hex string into a NeedleId.
+    pub fn parse(s: &str) -> Result<Self, std::num::ParseIntError> {
+        u64::from_str_radix(s, 16).map(NeedleId)
+    }
+}
+
+impl fmt::Display for NeedleId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:x}", self.0)
+    }
+}
+
+impl From<u64> for NeedleId {
+    fn from(v: u64) -> Self {
+        NeedleId(v)
+    }
+}
+
+impl From<NeedleId> for u64 {
+    fn from(v: NeedleId) -> Self {
+        v.0
+    }
+}
+
+// ============================================================================
+// Cookie
+// ============================================================================
+
+/// Random 32-bit value to mitigate brute-force lookups.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+pub struct Cookie(pub u32);
+
+impl Cookie {
+    pub fn to_bytes(&self, bytes: &mut [u8]) {
+        assert!(bytes.len() >= COOKIE_SIZE);
+        bytes[0..4].copy_from_slice(&self.0.to_be_bytes());
+    }
+
+    pub fn from_bytes(bytes: &[u8]) -> Self {
+        assert!(bytes.len() >= COOKIE_SIZE);
+        Cookie(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]))
+    }
+
+    /// Parse a hex string into a Cookie.
+    pub fn parse(s: &str) -> Result<Self, std::num::ParseIntError> {
+        u32::from_str_radix(s, 16).map(Cookie)
+    }
+}
+
+impl fmt::Display for Cookie {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:x}", self.0)
+    }
+}
+
+impl From<u32> for Cookie {
+    fn from(v: u32) -> Self {
+        Cookie(v)
+    }
+}
+
+// ============================================================================
+// Size
+// ============================================================================
+
+/// Needle size as stored in the index. Negative = deleted.
+///
+/// - Positive: valid needle with that many bytes of body content
+/// - TombstoneFileSize (-1): tombstone marker
+/// - Other negative: deleted, absolute value was the original size
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+pub struct Size(pub i32);
+
+/// Special marker for a tombstone (deletion marker) entry.
+pub const TOMBSTONE_FILE_SIZE: Size = Size(-1);
+
+impl Size {
+    pub fn is_tombstone(&self) -> bool {
+        self.0 == TOMBSTONE_FILE_SIZE.0
+    }
+
+    pub fn is_deleted(&self) -> bool {
+        self.0 < 0 || self.0 == TOMBSTONE_FILE_SIZE.0
+    }
+
+    pub fn is_valid(&self) -> bool {
+        self.0 > 0 && !self.is_tombstone()
+    }
+
+    /// Raw storage size. For tombstones returns 0; for negative returns abs value.
+    pub fn raw(&self) -> u32 {
+        if self.is_tombstone() {
+            return 0;
+        }
+        if self.0 < 0 {
+            return (self.0 * -1) as u32;
+        }
+        self.0 as u32
+    }
+
+    pub fn to_bytes(&self, bytes: &mut [u8]) {
+        assert!(bytes.len() >= SIZE_SIZE);
+        bytes[0..4].copy_from_slice(&(self.0 as u32).to_be_bytes());
+    }
+
+    pub fn from_bytes(bytes: &[u8]) -> Self {
+        assert!(bytes.len() >= SIZE_SIZE);
+        let v = u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]);
+        Size(v as i32)
+    }
+}
+
+impl From<i32> for Size {
+    fn from(v: i32) -> Self {
+        Size(v)
+    }
+}
+
+impl From<Size> for i32 {
+    fn from(v: Size) -> Self {
+        v.0
+    }
+}
+
+// ============================================================================
+// Offset
+// ============================================================================
+
+/// Offset encoding for needle positions in .dat files.
+///
+/// The offset is stored divided by NEEDLE_PADDING_SIZE (8).
+///
+/// With `5bytes` feature (default, matching Go production builds):
+///   5 bytes can address up to 8TB.
+///   On-disk layout: [b3][b2][b1][b0][b4] (big-endian 4 bytes + 1 high byte)
+///
+/// Without `5bytes` feature (matching Go default build):
+///   4 bytes can address up to 32GB.
+///   On-disk layout: [b3][b2][b1][b0] (big-endian 4 bytes)
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+pub struct Offset {
+    pub b0: u8,
+    pub b1: u8,
+    pub b2: u8,
+    pub b3: u8,
+    #[cfg(feature = "5bytes")]
+    pub b4: u8,
+}
+
+impl Offset {
+    /// Convert to the actual byte offset in the .dat file.
+    pub fn to_actual_offset(&self) -> i64 {
+        let stored = self.b0 as i64
+            + (self.b1 as i64) * 256
+            + (self.b2 as i64) * 65536
+            + (self.b3 as i64) * 16777216;
+        #[cfg(feature = "5bytes")]
+        let stored = stored + (self.b4 as i64) * 4294967296; // 1 << 32
+        stored * NEEDLE_PADDING_SIZE as i64
+    }
+
+    /// Create an Offset from an actual byte offset.
+    pub fn from_actual_offset(offset: i64) -> Self {
+        let smaller = offset / NEEDLE_PADDING_SIZE as i64;
+        Offset {
+            b0: smaller as u8,
+            b1: (smaller >> 8) as u8,
+            b2: (smaller >> 16) as u8,
+            b3: (smaller >> 24) as u8,
+            #[cfg(feature = "5bytes")]
+            b4: (smaller >> 32) as u8,
+        }
+    }
+
+    /// Serialize to bytes in the .idx file format.
+    /// 5-byte layout: [b3][b2][b1][b0][b4]
+    /// 4-byte layout: [b3][b2][b1][b0]
+    pub fn to_bytes(&self, bytes: &mut [u8]) {
+        assert!(bytes.len() >= OFFSET_SIZE);
+        bytes[0] = self.b3;
+        bytes[1] = self.b2;
+        bytes[2] = self.b1;
+        bytes[3] = self.b0;
+        #[cfg(feature = "5bytes")]
+        {
+            bytes[4] = self.b4;
+        }
+    }
+
+    /// Deserialize from bytes in the .idx file format.
+    pub fn from_bytes(bytes: &[u8]) -> Self {
+        assert!(bytes.len() >= OFFSET_SIZE);
+        Offset {
+            b3: bytes[0],
+            b2: bytes[1],
+            b1: bytes[2],
+            b0: bytes[3],
+            #[cfg(feature = "5bytes")]
+            b4: bytes[4],
+        }
+    }
+
+    pub fn is_zero(&self) -> bool {
+        #[cfg(feature = "5bytes")]
+        {
+            self.b0 == 0 && self.b1 == 0 && self.b2 == 0 && self.b3 == 0 && self.b4 == 0
+        }
+        #[cfg(not(feature = "5bytes"))]
+        {
+            self.b0 == 0 && self.b1 == 0 && self.b2 == 0 && self.b3 == 0
+        }
+    }
+}
+
+impl fmt::Display for Offset {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.to_actual_offset())
+    }
+}
+
+// ============================================================================
+// DiskType
+// ============================================================================
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum DiskType {
+    HardDrive,
+    Ssd,
+    Custom(String),
+}
+
+impl DiskType {
+    pub fn from_string(s: &str) -> Self {
+        match s.to_lowercase().as_str() {
+            "" | "hdd" => DiskType::HardDrive,
+            "ssd" => DiskType::Ssd,
+            other => DiskType::Custom(other.to_string()),
+        }
+    }
+
+    pub fn readable_string(&self) -> &str {
+        match self {
+            DiskType::HardDrive => "hdd",
+            DiskType::Ssd => "ssd",
+            DiskType::Custom(s) => s,
+        }
+    }
+}
+
+impl fmt::Display for DiskType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            DiskType::HardDrive => write!(f, ""),
+            DiskType::Ssd => write!(f, "ssd"),
+            DiskType::Custom(s) => write!(f, "{}", s),
+        }
+    }
+}
+
+impl Default for DiskType {
+    fn default() -> Self {
+        DiskType::HardDrive
+    }
+}
+
+// ============================================================================
+// VolumeId
+// ============================================================================
+
+/// Volume identifier, stored as u32.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Default)]
+pub struct VolumeId(pub u32);
+
+impl VolumeId {
+    pub fn parse(s: &str) -> Result<Self, std::num::ParseIntError> {
+        s.parse::<u32>().map(VolumeId)
+    }
+
+    pub fn next(&self) -> VolumeId {
+        VolumeId(self.0 + 1)
+    }
+}
+
+impl fmt::Display for VolumeId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl From<u32> for VolumeId {
+    fn from(v: u32) -> Self {
+        VolumeId(v)
+    }
+}
+
+// ============================================================================
+// Version
+// ============================================================================
+
+/// Needle storage format version.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Version(pub u8);
+
+pub const VERSION_1: Version = Version(1);
+pub const VERSION_2: Version = Version(2);
+pub const VERSION_3: Version = Version(3);
+
+impl Version {
+    pub fn current() -> Self {
+        VERSION_3
+    }
+
+    pub fn is_supported(&self) -> bool {
+        self.0 >= 1 && self.0 <= 3
+    }
+}
+
+impl Default for Version {
+    fn default() -> Self {
+        VERSION_3
+    }
+}
+
+impl From<u8> for Version {
+    fn from(v: u8) -> Self {
+        Version(v)
+    }
+}
+
+// ============================================================================
+// ReadOption
+// ============================================================================
+
+/// Options controlling needle read behavior, matching Go's `ReadOption` in store.go.
+///
+/// Fields are split into request-side options (set by the caller) and response-side
+/// flags (set during the read to communicate status back).
+#[derive(Debug, Clone)]
+pub struct ReadOption {
+    // -- request --
+    /// If true, allow reading needles that have been soft-deleted.
+    pub read_deleted: bool,
+    /// If true, attempt to read only metadata for large needles (> PagedReadLimit).
+    pub attempt_meta_only: bool,
+    /// If true, the caller requires metadata only (no data payload).
+    pub must_meta_only: bool,
+
+    // -- response --
+    /// Set to true when the read actually returned metadata only.
+    pub is_meta_only: bool,
+    /// Compaction revision at the time of the read (for consistency during streaming).
+    pub volume_revision: u16,
+    /// Set to true when the offset exceeded MaxPossibleVolumeSize (4-byte offset wrap).
+    pub is_out_of_range: bool,
+
+    // -- slow-read / streaming --
+    /// When true, the read lock is acquired and released per chunk instead of held
+    /// for the entire read, reducing write latency at the cost of higher read P99.
+    pub has_slow_read: bool,
+    /// Buffer size for chunked streaming reads (used with `has_slow_read`).
+    pub read_buffer_size: i32,
+}
+
+impl Default for ReadOption {
+    fn default() -> Self {
+        ReadOption {
+            read_deleted: false,
+            attempt_meta_only: false,
+            must_meta_only: false,
+            is_meta_only: false,
+            volume_revision: 0,
+            is_out_of_range: false,
+            has_slow_read: false,
+            read_buffer_size: 0,
+        }
+    }
+}
+
+// ============================================================================
+// NeedleMapEntry helpers (for .idx file)
+// ============================================================================
+
+/// Parse a single .idx file entry (17 bytes) into (NeedleId, Offset, Size).
+pub fn idx_entry_from_bytes(bytes: &[u8]) -> (NeedleId, Offset, Size) {
+    assert!(bytes.len() >= NEEDLE_MAP_ENTRY_SIZE);
+    let key = NeedleId::from_bytes(&bytes[..NEEDLE_ID_SIZE]);
+    let offset = Offset::from_bytes(&bytes[NEEDLE_ID_SIZE..NEEDLE_ID_SIZE + OFFSET_SIZE]);
+    let size = Size::from_bytes(
+        &bytes[NEEDLE_ID_SIZE + OFFSET_SIZE..NEEDLE_ID_SIZE + OFFSET_SIZE + SIZE_SIZE],
+    );
+    (key, offset, size)
+}
+
+/// Write a single .idx file entry (17 bytes).
+pub fn idx_entry_to_bytes(bytes: &mut [u8], key: NeedleId, offset: Offset, size: Size) {
+    assert!(bytes.len() >= NEEDLE_MAP_ENTRY_SIZE);
+    key.to_bytes(&mut bytes[..NEEDLE_ID_SIZE]);
+    offset.to_bytes(&mut bytes[NEEDLE_ID_SIZE..NEEDLE_ID_SIZE + OFFSET_SIZE]);
+    size.to_bytes(
+        &mut bytes[NEEDLE_ID_SIZE + OFFSET_SIZE..NEEDLE_ID_SIZE + OFFSET_SIZE + SIZE_SIZE],
+    );
+}
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_needle_id_round_trip() {
+        let id = NeedleId(0x123456789abcdef0);
+        let mut buf = [0u8; 8];
+        id.to_bytes(&mut buf);
+        let id2 = NeedleId::from_bytes(&buf);
+        assert_eq!(id, id2);
+    }
+
+    #[test]
+    fn test_needle_id_display() {
+        let id = NeedleId(255);
+        assert_eq!(id.to_string(), "ff");
+    }
+
+    #[test]
+    fn test_needle_id_parse() {
+        let id = NeedleId::parse("ff").unwrap();
+        assert_eq!(id, NeedleId(255));
+    }
+
+    #[test]
+    fn test_cookie_round_trip() {
+        let cookie = Cookie(0xdeadbeef);
+        let mut buf = [0u8; 4];
+        cookie.to_bytes(&mut buf);
+        let cookie2 = Cookie::from_bytes(&buf);
+        assert_eq!(cookie, cookie2);
+    }
+
+    #[test]
+    fn test_size_semantics() {
+        assert!(Size(100).is_valid());
+        assert!(!Size(100).is_deleted());
+        assert!(!Size(100).is_tombstone());
+        assert_eq!(Size(100).raw(), 100);
+
+        assert!(Size(-50).is_deleted());
+        assert!(!Size(-50).is_tombstone());
+        assert_eq!(Size(-50).raw(), 50);
+
+        assert!(TOMBSTONE_FILE_SIZE.is_deleted());
+        assert!(TOMBSTONE_FILE_SIZE.is_tombstone());
+        assert_eq!(TOMBSTONE_FILE_SIZE.raw(), 0);
+
+        assert!(!Size(0).is_valid());
+        assert!(!Size(0).is_deleted());
+    }
+
+    #[test]
+    fn test_size_round_trip() {
+        let size = Size(12345);
+        let mut buf = [0u8; 4];
+        size.to_bytes(&mut buf);
+        let size2 = Size::from_bytes(&buf);
+        assert_eq!(size, size2);
+    }
+
+    #[test]
+    fn test_size_negative_round_trip() {
+        // Negative sizes round-trip through u32 bit pattern
+        let size = Size(-50);
+        let mut buf = [0u8; 4];
+        size.to_bytes(&mut buf);
+        let size2 = Size::from_bytes(&buf);
+        assert_eq!(size, size2);
+    }
+
+    #[test]
+    fn test_offset_round_trip() {
+        // Test with a known actual offset
+        let actual_offset: i64 = 8 * 1000000; // must be multiple of 8
+        let offset = Offset::from_actual_offset(actual_offset);
+        assert_eq!(offset.to_actual_offset(), actual_offset);
+
+        // Test byte serialization
+        let mut buf = [0u8; 5];
+        offset.to_bytes(&mut buf);
+        let offset2 = Offset::from_bytes(&buf);
+        assert_eq!(offset.to_actual_offset(), offset2.to_actual_offset());
+    }
+
+    #[test]
+    fn test_offset_zero() {
+        let offset = Offset::default();
+        assert!(offset.is_zero());
+        assert_eq!(offset.to_actual_offset(), 0);
+    }
+
+    #[test]
+    fn test_offset_max() {
+        // Max stored value depends on offset size
+        #[cfg(feature = "5bytes")]
+        let max_stored: i64 = (1i64 << 40) - 1; // 5-byte max
+        #[cfg(not(feature = "5bytes"))]
+        let max_stored: i64 = (1i64 << 32) - 1; // 4-byte max
+        let max_actual = max_stored * NEEDLE_PADDING_SIZE as i64;
+        let offset = Offset::from_actual_offset(max_actual);
+        assert_eq!(offset.to_actual_offset(), max_actual);
+    }
+
+    #[test]
+    fn test_offset_size_constants() {
+        #[cfg(feature = "5bytes")]
+        {
+            assert_eq!(OFFSET_SIZE, 5);
+            assert_eq!(NEEDLE_MAP_ENTRY_SIZE, 17); // 8 + 5 + 4
+            assert_eq!(MAX_POSSIBLE_VOLUME_SIZE, 4 * 1024 * 1024 * 1024 * 8 * 256);
+            // 8TB
+        }
+        #[cfg(not(feature = "5bytes"))]
+        {
+            assert_eq!(OFFSET_SIZE, 4);
+            assert_eq!(NEEDLE_MAP_ENTRY_SIZE, 16); // 8 + 4 + 4
+            assert_eq!(MAX_POSSIBLE_VOLUME_SIZE, 4 * 1024 * 1024 * 1024 * 8); // 32GB
+        }
+    }
+
+    #[test]
+    fn test_idx_entry_round_trip() {
+        let key = NeedleId(0xdeadbeef12345678);
+        let offset = Offset::from_actual_offset(8 * 999);
+        let size = Size(4096);
+
+        let mut buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
+        idx_entry_to_bytes(&mut buf, key, offset, size);
+
+        let (key2, offset2, size2) = idx_entry_from_bytes(&buf);
+        assert_eq!(key, key2);
+        assert_eq!(offset.to_actual_offset(), offset2.to_actual_offset());
+        assert_eq!(size, size2);
+    }
+
+    #[test]
+    fn test_volume_id() {
+        let vid = VolumeId::parse("42").unwrap();
+        assert_eq!(vid, VolumeId(42));
+        assert_eq!(vid.to_string(), "42");
+        assert_eq!(vid.next(), VolumeId(43));
+    }
+
+    #[test]
+    fn test_version() {
+        assert!(VERSION_1.is_supported());
+        assert!(VERSION_2.is_supported());
+        assert!(VERSION_3.is_supported());
+        assert!(!Version(0).is_supported());
+        assert!(!Version(4).is_supported());
+        assert_eq!(Version::current(), VERSION_3);
+    }
+
+    #[test]
+    fn test_disk_type() {
+        assert_eq!(DiskType::from_string(""), DiskType::HardDrive);
+        assert_eq!(DiskType::from_string("hdd"), DiskType::HardDrive);
+        assert_eq!(DiskType::from_string("SSD"), DiskType::Ssd);
+        assert_eq!(
+            DiskType::from_string("nvme"),
+            DiskType::Custom("nvme".to_string())
+        );
+        assert_eq!(DiskType::HardDrive.readable_string(), "hdd");
+        assert_eq!(DiskType::Ssd.readable_string(), "ssd");
+    }
+
+    #[test]
+    fn test_read_option_default() {
+        let ro = ReadOption::default();
+        assert!(!ro.read_deleted);
+        assert!(!ro.attempt_meta_only);
+        assert!(!ro.must_meta_only);
+        assert!(!ro.is_meta_only);
+        assert_eq!(ro.volume_revision, 0);
+        assert!(!ro.is_out_of_range);
+        assert!(!ro.has_slow_read);
+        assert_eq!(ro.read_buffer_size, 0);
+    }
+
+    #[test]
+    fn test_read_option_custom() {
+        let ro = ReadOption {
+            read_deleted: true,
+            attempt_meta_only: true,
+            has_slow_read: true,
+            read_buffer_size: 1024 * 1024,
+            ..ReadOption::default()
+        };
+        assert!(ro.read_deleted);
+        assert!(ro.attempt_meta_only);
+        assert!(!ro.must_meta_only);
+        assert!(!ro.is_meta_only);
+        assert!(ro.has_slow_read);
+        assert_eq!(ro.read_buffer_size, 1024 * 1024);
+    }
+
+    #[test]
+    fn test_read_option_clone() {
+        let ro = ReadOption {
+            is_out_of_range: true,
+            volume_revision: 42,
+            ..ReadOption::default()
+        };
+        let ro2 = ro.clone();
+        assert!(ro2.is_out_of_range);
+        assert_eq!(ro2.volume_revision, 42);
+    }
+}

seaweed-volume/src/version.rs

@@ -0,0 +1,79 @@
+//! Version helpers aligned with Go's util/version package.
+
+use std::sync::OnceLock;
+
+#[cfg(feature = "5bytes")]
+const SIZE_LIMIT: &str = "8000GB"; // Matches Go production builds (5BytesOffset)
+#[cfg(not(feature = "5bytes"))]
+const SIZE_LIMIT: &str = "30GB"; // Matches Go default build (!5BytesOffset)
+
+pub fn size_limit() -> &'static str {
+    SIZE_LIMIT
+}
+
+pub fn commit() -> &'static str {
+    option_env!("SEAWEEDFS_COMMIT")
+        .or(option_env!("GIT_COMMIT"))
+        .or(option_env!("GIT_SHA"))
+        .unwrap_or("")
+}
+
+pub fn version_number() -> &'static str {
+    static VERSION_NUMBER: OnceLock<String> = OnceLock::new();
+    VERSION_NUMBER
+        .get_or_init(|| {
+            parse_go_version_number().unwrap_or_else(|| env!("CARGO_PKG_VERSION").to_string())
+        })
+        .as_str()
+}
+
+pub fn version() -> &'static str {
+    static VERSION: OnceLock<String> = OnceLock::new();
+    VERSION
+        .get_or_init(|| format!("{} {}", size_limit(), version_number()))
+        .as_str()
+}
+
+pub fn full_version() -> &'static str {
+    static FULL: OnceLock<String> = OnceLock::new();
+    FULL.get_or_init(|| format!("{} {}", version(), commit()))
+        .as_str()
+}
+
+pub fn server_header() -> &'static str {
+    static HEADER: OnceLock<String> = OnceLock::new();
+    HEADER
+        .get_or_init(|| format!("SeaweedFS Volume {}", version()))
+        .as_str()
+}
+
+fn parse_go_version_number() -> Option<String> {
+    let src = include_str!(concat!(
+        env!("CARGO_MANIFEST_DIR"),
+        "/../weed/util/version/constants.go"
+    ));
+    let mut major: Option<u32> = None;
+    let mut minor: Option<u32> = None;
+    for line in src.lines() {
+        let l = line.trim();
+        if l.starts_with("MAJOR_VERSION") {
+            major = parse_int32_line(l);
+        } else if l.starts_with("MINOR_VERSION") {
+            minor = parse_int32_line(l);
+        }
+        if major.is_some() && minor.is_some() {
+            break;
+        }
+    }
+    match (major, minor) {
+        (Some(maj), Some(min)) => Some(format!("{}.{}", maj, format!("{:02}", min))),
+        _ => None,
+    }
+}
+
+fn parse_int32_line(line: &str) -> Option<u32> {
+    let start = line.find("int32(")? + "int32(".len();
+    let rest = &line[start..];
+    let end = rest.find(')')?;
+    rest[..end].trim().parse::<u32>().ok()
+}

seaweed-volume/tests/http_integration.rs

@@ -0,0 +1,677 @@
+//! Integration tests for the volume server HTTP handlers.
+//!
+//! Uses axum's Router with tower::ServiceExt::oneshot to test
+//! end-to-end without starting a real TCP server.
+
+use std::sync::{Arc, RwLock};
+
+use axum::body::Body;
+use axum::extract::connect_info::ConnectInfo;
+use axum::http::{Request, StatusCode};
+use tower::ServiceExt; // for `oneshot`
+
+use seaweed_volume::security::{Guard, SigningKey};
+use seaweed_volume::server::volume_server::{
+    build_admin_router, build_admin_router_with_ui, build_metrics_router, build_public_router,
+    VolumeServerState,
+};
+use seaweed_volume::storage::needle_map::NeedleMapKind;
+use seaweed_volume::storage::store::Store;
+use seaweed_volume::storage::types::{DiskType, Version, VolumeId};
+
+use tempfile::TempDir;
+
+/// Create a test VolumeServerState with a temp directory, a single disk
+/// location, and one pre-created volume (VolumeId 1).
+fn test_state() -> (Arc<VolumeServerState>, TempDir) {
+    test_state_with_guard(Vec::new(), Vec::new())
+}
+
+fn test_state_with_signing_key(signing_key: Vec<u8>) -> (Arc<VolumeServerState>, TempDir) {
+    test_state_with_guard(Vec::new(), signing_key)
+}
+
+fn test_state_with_whitelist(whitelist: Vec<String>) -> (Arc<VolumeServerState>, TempDir) {
+    test_state_with_guard(whitelist, Vec::new())
+}
+
+fn test_state_with_guard(
+    whitelist: Vec<String>,
+    signing_key: Vec<u8>,
+) -> (Arc<VolumeServerState>, TempDir) {
+    let tmp = TempDir::new().expect("failed to create temp dir");
+    let dir = tmp.path().to_str().unwrap();
+
+    let mut store = Store::new(NeedleMapKind::InMemory);
+    store
+        .add_location(
+            dir,
+            dir,
+            10,
+            DiskType::HardDrive,
+            seaweed_volume::config::MinFreeSpace::Percent(1.0),
+            Vec::new(),
+        )
+        .expect("failed to add location");
+    store
+        .add_volume(
+            VolumeId(1),
+            "",
+            None,
+            None,
+            0,
+            DiskType::HardDrive,
+            Version::current(),
+        )
+        .expect("failed to create volume");
+
+    let guard = Guard::new(
+        &whitelist,
+        SigningKey(signing_key),
+        0,
+        SigningKey(vec![]),
+        0,
+    );
+    let state = Arc::new(VolumeServerState {
+        store: RwLock::new(store),
+        guard: RwLock::new(guard),
+        is_stopping: RwLock::new(false),
+        maintenance: std::sync::atomic::AtomicBool::new(false),
+        state_version: std::sync::atomic::AtomicU32::new(0),
+        concurrent_upload_limit: 0,
+        concurrent_download_limit: 0,
+        inflight_upload_data_timeout: std::time::Duration::from_secs(60),
+        inflight_download_data_timeout: std::time::Duration::from_secs(60),
+        inflight_upload_bytes: std::sync::atomic::AtomicI64::new(0),
+        inflight_download_bytes: std::sync::atomic::AtomicI64::new(0),
+        upload_notify: tokio::sync::Notify::new(),
+        download_notify: tokio::sync::Notify::new(),
+        data_center: String::new(),
+        rack: String::new(),
+        file_size_limit_bytes: 0,
+        maintenance_byte_per_second: 0,
+        is_heartbeating: std::sync::atomic::AtomicBool::new(true),
+        has_master: false,
+        pre_stop_seconds: 0,
+        volume_state_notify: tokio::sync::Notify::new(),
+        write_queue: std::sync::OnceLock::new(),
+        s3_tier_registry: std::sync::RwLock::new(
+            seaweed_volume::remote_storage::s3_tier::S3TierRegistry::new(),
+        ),
+        read_mode: seaweed_volume::config::ReadMode::Local,
+        master_url: String::new(),
+        master_urls: Vec::new(),
+        self_url: String::new(),
+        http_client: reqwest::Client::new(),
+        outgoing_http_scheme: "http".to_string(),
+        outgoing_grpc_tls: None,
+        metrics_runtime: std::sync::RwLock::new(
+            seaweed_volume::server::volume_server::RuntimeMetricsConfig::default(),
+        ),
+        metrics_notify: tokio::sync::Notify::new(),
+        fix_jpg_orientation: false,
+        has_slow_read: false,
+        read_buffer_size_bytes: 1024 * 1024,
+        security_file: String::new(),
+        cli_white_list: vec![],
+        state_file_path: String::new(),
+    });
+    (state, tmp)
+}
+
+/// Helper: read the entire response body as bytes.
+async fn body_bytes(response: axum::response::Response) -> Vec<u8> {
+    let body = response.into_body();
+    axum::body::to_bytes(body, usize::MAX)
+        .await
+        .expect("failed to read body")
+        .to_vec()
+}
+
+fn with_remote_addr(request: Request<Body>, remote_addr: &str) -> Request<Body> {
+    let mut request = request;
+    let remote_addr = remote_addr
+        .parse::<std::net::SocketAddr>()
+        .expect("invalid socket address");
+    request.extensions_mut().insert(ConnectInfo(remote_addr));
+    request
+}
+
+// ============================================================================
+// 1. GET /healthz returns 200 when server is running
+// ============================================================================
+
+#[tokio::test]
+async fn healthz_returns_200_when_running() {
+    let (state, _tmp) = test_state();
+    let app = build_admin_router(state);
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/healthz")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+}
+
+// ============================================================================
+// 2. GET /healthz returns 503 when is_stopping=true
+// ============================================================================
+
+#[tokio::test]
+async fn healthz_returns_503_when_stopping() {
+    let (state, _tmp) = test_state();
+    *state.is_stopping.write().unwrap() = true;
+    let app = build_admin_router(state);
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/healthz")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+}
+
+// ============================================================================
+// 3. GET /status returns JSON with version and volumes array
+// ============================================================================
+
+#[tokio::test]
+async fn status_returns_json_with_version_and_volumes() {
+    let (state, _tmp) = test_state();
+    let app = build_admin_router(state);
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/status")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+
+    let body = body_bytes(response).await;
+    let json: serde_json::Value =
+        serde_json::from_slice(&body).expect("response is not valid JSON");
+
+    assert!(json.get("Version").is_some(), "missing 'Version' field");
+    assert!(json["Version"].is_string(), "'Version' should be a string");
+
+    assert!(json.get("Volumes").is_some(), "missing 'Volumes' field");
+    assert!(json["Volumes"].is_array(), "'Volumes' should be an array");
+
+    // We created one volume in test_state, so the array should have one entry
+    let volumes = json["Volumes"].as_array().unwrap();
+    assert_eq!(volumes.len(), 1, "expected 1 volume");
+    assert_eq!(volumes[0]["Id"], 1);
+}
+
+#[tokio::test]
+async fn admin_router_does_not_expose_metrics() {
+    let (state, _tmp) = test_state();
+    let app = build_admin_router(state);
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/metrics")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+}
+
+#[tokio::test]
+async fn metrics_router_serves_metrics() {
+    let app = build_metrics_router();
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/metrics")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+}
+
+#[tokio::test]
+async fn admin_router_rejects_non_whitelisted_uploads() {
+    let (state, _tmp) = test_state_with_whitelist(vec!["127.0.0.1".to_string()]);
+    let app = build_admin_router(state);
+
+    let response = app
+        .oneshot(with_remote_addr(
+            Request::builder()
+                .method("POST")
+                .uri("/1,000000000000000001")
+                .body(Body::from("blocked"))
+                .unwrap(),
+            "10.0.0.9:12345",
+        ))
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
+}
+
+#[tokio::test]
+async fn admin_router_rejects_non_whitelisted_deletes() {
+    let (state, _tmp) = test_state_with_whitelist(vec!["127.0.0.1".to_string()]);
+    let app = build_admin_router(state);
+
+    let response = app
+        .oneshot(with_remote_addr(
+            Request::builder()
+                .method("DELETE")
+                .uri("/1,000000000000000001")
+                .body(Body::empty())
+                .unwrap(),
+            "10.0.0.9:12345",
+        ))
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
+}
+
+// Go's volume_server.go has /stats/* endpoints commented out (L130-134).
+// Requests to /stats/counter fall through to the store handler which returns 400.
+#[tokio::test]
+async fn admin_router_does_not_expose_stats_routes() {
+    let (state, _tmp) = test_state_with_whitelist(vec!["127.0.0.1".to_string()]);
+    let app = build_admin_router_with_ui(state, true);
+
+    let response = app
+        .oneshot(with_remote_addr(
+            Request::builder()
+                .uri("/stats/counter")
+                .body(Body::empty())
+                .unwrap(),
+            "127.0.0.1:12345",
+        ))
+        .await
+        .unwrap();
+
+    // Falls through to store handler → 400 (bad volume id)
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+}
+
+// ============================================================================
+// 4. POST writes data, then GET reads it back
+// ============================================================================
+
+#[tokio::test]
+async fn write_then_read_needle() {
+    let (state, _tmp) = test_state();
+
+    // The fid "01637037d6" encodes NeedleId=0x01, Cookie=0x637037d6
+    let uri = "/1,01637037d6";
+    let payload = b"hello, seaweedfs!";
+
+    // --- POST (write) ---
+    let app = build_admin_router(state.clone());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri(uri)
+                .body(Body::from(payload.to_vec()))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(
+        response.status(),
+        StatusCode::CREATED,
+        "POST should return 201 Created"
+    );
+
+    let body = body_bytes(response).await;
+    let json: serde_json::Value =
+        serde_json::from_slice(&body).expect("POST response is not valid JSON");
+    assert_eq!(json["size"], payload.len() as u64);
+
+    // --- GET (read back) ---
+    let app = build_admin_router(state.clone());
+    let response = app
+        .oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap())
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK, "GET should return 200");
+
+    let body = body_bytes(response).await;
+    assert_eq!(body, payload, "GET body should match written data");
+}
+
+// ============================================================================
+// 5. DELETE deletes a needle, subsequent GET returns 404
+// ============================================================================
+
+#[tokio::test]
+async fn delete_then_get_returns_404() {
+    let (state, _tmp) = test_state();
+    let uri = "/1,01637037d6";
+    let payload = b"to be deleted";
+
+    // Write the needle first
+    let app = build_admin_router(state.clone());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri(uri)
+                .body(Body::from(payload.to_vec()))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::CREATED);
+
+    // Delete
+    let app = build_admin_router(state.clone());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .method("DELETE")
+                .uri(uri)
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(
+        response.status(),
+        StatusCode::ACCEPTED,
+        "DELETE should return 202 Accepted"
+    );
+
+    // GET should now return 404
+    let app = build_admin_router(state.clone());
+    let response = app
+        .oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap())
+        .await
+        .unwrap();
+    assert_eq!(
+        response.status(),
+        StatusCode::NOT_FOUND,
+        "GET after DELETE should return 404"
+    );
+}
+
+// ============================================================================
+// 6. HEAD returns headers without body
+// ============================================================================
+
+#[tokio::test]
+async fn head_returns_headers_without_body() {
+    let (state, _tmp) = test_state();
+    let uri = "/1,01637037d6";
+    let payload = b"head test data";
+
+    // Write needle
+    let app = build_admin_router(state.clone());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri(uri)
+                .body(Body::from(payload.to_vec()))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::CREATED);
+
+    // HEAD
+    let app = build_admin_router(state.clone());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .method("HEAD")
+                .uri(uri)
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK, "HEAD should return 200");
+
+    // Content-Length header should be present
+    let content_length = response
+        .headers()
+        .get("content-length")
+        .expect("HEAD should include Content-Length header");
+    let len: usize = content_length
+        .to_str()
+        .unwrap()
+        .parse()
+        .expect("Content-Length should be a number");
+    assert_eq!(
+        len,
+        payload.len(),
+        "Content-Length should match payload size"
+    );
+
+    // Body should be empty for HEAD
+    let body = body_bytes(response).await;
+    assert!(body.is_empty(), "HEAD body should be empty");
+}
+
+// ============================================================================
+// 7. Invalid URL path returns 400
+// ============================================================================
+
+#[tokio::test]
+async fn invalid_url_path_returns_400() {
+    let (state, _tmp) = test_state();
+    let app = build_admin_router(state);
+
+    // "invalidpath" has no comma or slash separator so parse_url_path returns None
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/invalidpath")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(
+        response.status(),
+        StatusCode::BAD_REQUEST,
+        "invalid URL path should return 400"
+    );
+}
+
+#[tokio::test]
+async fn deep_invalid_url_path_returns_400() {
+    let (state, _tmp) = test_state();
+    let app = build_admin_router(state);
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/not/a/valid/volume/path")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+}
+
+#[tokio::test]
+async fn admin_root_get_returns_400() {
+    let (state, _tmp) = test_state();
+    let app = build_admin_router(state);
+
+    let response = app
+        .oneshot(Request::builder().uri("/").body(Body::empty()).unwrap())
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+}
+
+#[tokio::test]
+async fn public_root_get_returns_400() {
+    let (state, _tmp) = test_state();
+    let app = build_public_router(state);
+
+    let response = app
+        .oneshot(Request::builder().uri("/").body(Body::empty()).unwrap())
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+}
+
+#[tokio::test]
+async fn public_router_does_not_expose_healthz() {
+    let (state, _tmp) = test_state();
+    let app = build_public_router(state);
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/healthz")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+}
+
+// Go's volume_server.go has /stats/* endpoints commented out (L130-134).
+#[tokio::test]
+async fn admin_router_stats_routes_not_registered() {
+    let (state, _tmp) = test_state();
+    let app = build_admin_router(state);
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/stats/counter")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    // Falls through to store handler → 400 (bad volume id)
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+}
+
+#[tokio::test]
+async fn admin_router_hides_ui_when_write_jwt_is_configured() {
+    let (state, _tmp) = test_state_with_signing_key(b"secret".to_vec());
+    let app = build_admin_router(state);
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/ui/index.html")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+}
+
+#[tokio::test]
+async fn admin_router_can_expose_ui_with_explicit_override() {
+    let (state, _tmp) = test_state_with_signing_key(b"secret".to_vec());
+    let app = build_admin_router_with_ui(state, true);
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/ui/index.html")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+    let body = body_bytes(response).await;
+    let html = String::from_utf8(body).unwrap();
+    assert!(html.contains("Disk Stats"));
+    assert!(html.contains("System Stats"));
+    assert!(html.contains("Volumes"));
+}
+
+#[tokio::test]
+async fn admin_router_ui_override_ignores_read_jwt_checks() {
+    let (state, _tmp) = test_state_with_signing_key(b"write-secret".to_vec());
+    state.guard.write().unwrap().read_signing_key = SigningKey(b"read-secret".to_vec());
+    let app = build_admin_router_with_ui(state, true);
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/ui/index.html")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+}
+
+#[tokio::test]
+async fn admin_router_serves_volume_ui_static_assets() {
+    let (state, _tmp) = test_state();
+    let app = build_admin_router(state);
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/seaweedfsstatic/bootstrap/3.3.1/css/bootstrap.min.css")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+    assert_eq!(
+        response
+            .headers()
+            .get("content-type")
+            .and_then(|value| value.to_str().ok()),
+        Some("text/css; charset=utf-8")
+    );
+    let body = body_bytes(response).await;
+    assert!(body.len() > 1000);
+}