4.31

* fix(seaweed-volume): bound request body and stored-content expansion to prevent OOM

The Rust volume server buffered the entire upload body with
to_bytes(usize::MAX) and only checked the file-size limit afterward, so a
single large upload — or many concurrent uploads, since the in-flight byte
throttle defaults to 0 (unlimited) — could exhaust memory and get the process
OOM-killed under load. The read path had two more single-request OOM vectors:
`vec![0u8; manifest.size]` allocated from an attacker-controlled chunk-manifest
size, and gzip decompression was unbounded (gzip bomb).

- Bound the upload body read by file_size_limit_bytes (plus a margin for
  multipart framing), mirroring Go's io.LimitReader(sizeLimit+1), and reject
  oversize before the whole body is buffered.
- Validate manifest.size (reject negative / oversized) before allocating.
- Cap gzip output in maybe_decompress_gzip and route the inline GzDecoder sites
  through it.

* fix(seaweed-volume): address review - chunk offset, 32-bit cast, decompress errors

- Validate chunk.offset before indexing in chunk-manifest expansion: a negative
  offset wrapped to a huge usize and underflowed `end - offset` (panic from a
  crafted manifest). Reject negative, skip out-of-range, use saturating math.
- Use usize::try_from for the upload body limit instead of `as usize`, so a
  >usize::MAX file_size_limit on 32-bit caps at usize::MAX rather than silently
  truncating to a tiny value.
- maybe_decompress_gzip now returns Result<_, GunzipError> distinguishing a
  decode failure (callers fall back to raw bytes, as before) from hitting the
  size cap (TooLarge), which now returns 413 instead of silently serving the
  still-compressed bytes.

* fix(seaweed-volume): inflate manifest chunks into the result window to cap peak memory

The chunk-manifest expansion still doubled memory: `result` was already allocated
at manifest.size (<=2 GiB) and each compressed chunk was inflated into a separate
Vec (also up to 2 GiB), so a single request could peak near 4 GiB.

Decompress compressed chunks directly into their result[offset..] window (bounded
by the remaining space) so a chunk never allocates a second large buffer; peak
stays at ~manifest.size. Bytes past the window are dropped (matching the prior
truncation), and a fully-undecodable chunk still falls back to its raw bytes.

* fix(seaweed-volume): fall back to raw chunk bytes on any decode failure

Per review: the gzip fallback must run on any decode error, not only when no
bytes were decoded. Clear the partially-written output and copy the chunk's raw
bytes (truncated to the window), restoring the prior decode-failure behavior.

.github/workflows/container_release_unified.yml

@@ -118,8 +118,9 @@ jobs:
     needs: [build-rust-binaries]
     runs-on: ubuntu-latest
     strategy:
-      # Build sequentially to avoid rate limits
-      max-parallel: 2
+      # All variants at once: pulls hit mirror.gcr.io and pushes go to GHCR only,
+      # so docker.io limits don't apply. Watch the 10 GB gha cache budget.
+      max-parallel: 5
       matrix:
         include:
           # Normal volume - multi-arch
@@ -272,93 +273,52 @@ jobs:
             BRANCH=${{ github.sha }}
             ${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
             
-      - name: Clean up build artifacts
-        if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
-        run: |
-          sudo docker system prune -f
-          sudo rm -rf /tmp/go-build*
-
-  copy-to-dockerhub:
-    runs-on: ubuntu-latest
-    needs: [build]
-    if: github.event_name != 'pull_request'
-    strategy:
-      matrix:
-        variant: [normal, large_disk, full, large_disk_full, rocksdb]
-        include:
-          - variant: normal
-            tag_suffix: ""
-          - variant: large_disk
-            tag_suffix: _large_disk
-          - variant: full
-            tag_suffix: _full
-          - variant: large_disk_full
-            tag_suffix: _large_disk_full
-          - variant: rocksdb
-            tag_suffix: _large_disk_rocksdb
-            
-    steps:
-      - name: Login to Docker Hub
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-        uses: docker/login-action@v4.2.0
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          
-      - name: Login to GHCR
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-        uses: docker/login-action@v4.2.0
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-          
+      # Copy GHCR -> Docker Hub here, per variant, so it overlaps with the other
+      # variants still building instead of waiting on the whole matrix.
       - name: Install crane
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
         run: |
           cd $(mktemp -d)
           curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
           sudo mv crane /usr/local/bin/
           crane version
-          
-      - name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+
+      - name: Copy ${{ matrix.variant }} to Docker Hub
+        if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
         run: |
-          # Function to retry with exponential backoff
           retry_with_backoff() {
             local max_attempts=5
             local timeout=1
             local attempt=1
             local exit_code=0
-            
             while [ $attempt -le $max_attempts ]; do
               if "$@"; then
                 return 0
               else
                 exit_code=$?
               fi
-              
               if [ $attempt -lt $max_attempts ]; then
                 echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
                 sleep $timeout
                 timeout=$((timeout * 2))
               fi
-              
               attempt=$((attempt + 1))
             done
-            
             echo "Command failed after $max_attempts attempts" >&2
             return $exit_code
           }
-          
-          # Copy multi-arch image from GHCR to Docker Hub with retry
-          # This is much more efficient than pulling/pushing individual arch images
+
           echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
           retry_with_backoff crane copy \
             ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
             chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
-          
-          echo "Successfully copied ${{ matrix.variant }} to Docker Hub"
+          echo "Copied ${{ matrix.variant }} to Docker Hub"
+
+      - name: Clean up build artifacts
+        if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
+        run: |
+          sudo docker system prune -f
+          sudo rm -rf /tmp/go-build*
 
   # Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
   # Security for visibility, but never blocks the release. Releases (including
@@ -418,7 +378,7 @@ jobs:
   # trivy-scan: vuln findings are reported but do not block `latest`.
   tag-latest:
     runs-on: ubuntu-latest
-    needs: [copy-to-dockerhub]
+    needs: [build]
     if: github.event_name == 'push'
     strategy:
       matrix:
@@ -483,7 +443,7 @@ jobs:
 
   helm-release:
     runs-on: ubuntu-latest
-    needs: [copy-to-dockerhub]
+    needs: [build]
     if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     permissions:
       contents: write

.github/workflows/nfs-tests.yml

@@ -1,137 +0,0 @@
-name: "NFS Integration Tests"
-
-on:
-  push:
-    branches: [ master, main ]
-    paths:
-      - 'weed/server/nfs/**'
-      - 'weed/command/nfs.go'
-      - 'weed/filer/filer_inode.go'
-      - 'weed/filer/filer_inode_index.go'
-      - 'weed/filer/filerstore_wrapper.go'
-      - 'weed/server/filer_grpc_server_rename.go'
-      - 'test/nfs/**'
-      - '.github/workflows/nfs-tests.yml'
-  pull_request:
-    branches: [ master, main ]
-    paths:
-      - 'weed/server/nfs/**'
-      - 'weed/command/nfs.go'
-      - 'weed/filer/filer_inode.go'
-      - 'weed/filer/filer_inode_index.go'
-      - 'weed/filer/filerstore_wrapper.go'
-      - 'weed/server/filer_grpc_server_rename.go'
-      - 'test/nfs/**'
-      - '.github/workflows/nfs-tests.yml'
-
-concurrency:
-  group: ${{ github.head_ref }}/nfs-tests
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-env:
-  TEST_TIMEOUT: '15m'
-
-jobs:
-  nfs-integration:
-    name: NFS Integration Testing
-    runs-on: ubuntu-22.04
-    timeout-minutes: 20
-
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v6
-
-    - name: Set up Go
-      uses: actions/setup-go@v6
-      with:
-        go-version-file: 'go.mod'
-
-    - name: Build SeaweedFS
-      run: |
-        cd weed
-        go build -o weed .
-        chmod +x weed
-        ./weed version
-
-    - name: Run NFS Integration Tests
-      run: |
-        cd test/nfs
-
-        echo "Running NFS integration tests..."
-        echo "============================================"
-
-        # Install test dependencies
-        go mod download
-
-        # Run the protocol-layer tests. The kernel-mount tests require root
-        # for mount(2) and are exercised in their own privileged step below;
-        # skip them here so a "skipped because not root" line doesn't show
-        # up as noise on every CI run.
-        go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./...
-
-        echo "============================================"
-        echo "NFS integration tests completed"
-
-    - name: Install kernel NFS client
-      run: |
-        # nfs-common provides mount.nfs; netbase provides /etc/protocols
-        # which mount.nfs's protocol-name lookups (`tcp`, `udp`) need.
-        sudo apt-get update
-        sudo apt-get install -y nfs-common netbase
-
-    - name: Run kernel-mount E2E tests
-      run: |
-        cd test/nfs
-
-        echo "Running kernel-mount end-to-end tests..."
-        echo "These mount the running 'weed nfs' subprocess via the actual"
-        echo "Linux NFS client to catch protocol regressions invisible to"
-        echo "the go-nfs-client-based tests above."
-        echo "============================================"
-
-        # mount(2) is privileged. Preserve PATH so 'go' (and the weed
-        # binary that test/nfs/framework.go locates via $PATH) resolve
-        # correctly under sudo, and pass through the Go module/cache dirs
-        # so we don't redownload modules under root.
-        sudo env "PATH=$PATH" \
-          GOMODCACHE="$(go env GOMODCACHE)" \
-          GOCACHE="$(go env GOCACHE)" \
-          go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./...
-
-        echo "============================================"
-        echo "Kernel-mount E2E tests completed"
-
-    - name: Test Summary
-      if: always()
-      run: |
-        echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
-        echo "" >> $GITHUB_STEP_SUMMARY
-        echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
-        echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
-        echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
-        echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
-        echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
-        echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
-        echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
-        echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
-        echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
-        echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
-        echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
-        echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
-        echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
-        echo "" >> $GITHUB_STEP_SUMMARY
-        echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY
-        echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY
-        echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY
-        echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY
-        echo "" >> $GITHUB_STEP_SUMMARY
-        echo "### Harness" >> $GITHUB_STEP_SUMMARY
-        echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
-        echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
-        echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY
-        echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY
-        echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY
-        echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY

.github/workflows/terraform_ci.yml

@@ -0,0 +1,80 @@
+name: "terraform: validate and test modules"
+
+on:
+  push:
+    branches: [ master ]
+    paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
+  pull_request:
+    branches: [ master ]
+    paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    name: fmt, validate, plan-level tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up OpenTofu
+        uses: opentofu/setup-opentofu@v1
+        with:
+          tofu_version: 1.12.1
+
+      - name: fmt check
+        working-directory: terraform
+        run: tofu fmt -recursive -check -diff
+
+      - name: validate core
+        working-directory: terraform/modules/core
+        run: |
+          tofu init -backend=false -input=false
+          tofu validate
+
+      - name: validate security
+        working-directory: terraform/modules/security
+        run: |
+          tofu init -backend=false -input=false
+          tofu validate
+
+      - name: plan-level tests (core)
+        working-directory: terraform/modules/core
+        run: tofu test
+
+      - name: validate examples
+        run: |
+          set -e
+          for ex in terraform/examples/*/; do
+            echo "== validate $ex =="
+            tofu -chdir="$ex" init -backend=false -input=false
+            tofu -chdir="$ex" validate
+          done
+
+  smoke:
+    name: local cluster smoke test (real weed)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+
+      - name: Build weed
+        run: go build -o "$RUNNER_TEMP/weed" ./weed
+
+      - name: Set up OpenTofu
+        uses: opentofu/setup-opentofu@v1
+        with:
+          tofu_version: 1.12.1
+
+      - name: Run local cluster harness
+        working-directory: terraform/test/local
+        run: WEED="$RUNNER_TEMP/weed" ./run_local_cluster.sh
+
+      - name: Run local mTLS cluster harness
+        working-directory: terraform/test/local-secure
+        run: WEED="$RUNNER_TEMP/weed" ./run_local_secure.sh

README.md

@@ -35,6 +35,7 @@ Your support will be really appreciated by me and other supporters!
 [![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com)
 [![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com)
 [![keepsec](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/keepsec.png)](https://www.keepsec.ca)
+[![zyner](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_zyner.png)](https://zyner.org)
 
 ---
 
@@ -654,7 +655,8 @@ Cluster Total: 3302.88 MiB/s, 550.51 obj/s over 43s.
 ## Enterprise ##
 
 For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition, 
-which has a self-healing storage format with better data protection.
+which has advanced features, including data recovery, self-healing storage, 
+customizable erasure coding, EC vacuum and repair, etc.
 
 [Back to TOC](#table-of-contents)
 

docker/Dockerfile.go_build

@@ -1,4 +1,6 @@
-FROM golang:1.25-alpine AS builder
+# Pin the builder to the host arch and cross-compile the (CGO-free) Go binary,
+# so arm64/arm/386 targets skip QEMU emulation of the whole compile.
+FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/seaweedfs/
 ARG BRANCH=${BRANCH:-master}
@@ -12,9 +14,12 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs && \
       git checkout $BRANCH) || \
      (echo "ERROR: Branch/commit $BRANCH not found in repository" && \
       echo "Available branches:" && git branch -a && exit 1))
+ARG TARGETOS TARGETARCH TARGETVARIANT
 RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
   && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
-  && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
+  && export GOOS=$TARGETOS GOARCH=$TARGETARCH \
+  && case "$TARGETARCH" in arm) export GOARM="${TARGETVARIANT#v}";; esac \
+  && CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}" -o /go/bin/weed .
 
 # Rust volume server: use pre-built binary from CI when available (placed in
 # weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile

docker/Makefile

@@ -127,6 +127,9 @@ test_tarantool: tags = tarantool
 test_tarantool: build_tarantool_dev_env build
 	docker compose -f compose/test-tarantool-filer.yml -p seaweedfs up
 
+test_keycloak_s3: build
+	docker compose -f compose/test-keycloak-s3.yml -p seaweedfs up
+
 clean:
 	rm ./weed
 

docker/compose/test-keycloak-s3.yml

@@ -0,0 +1,46 @@
+services:
+  keycloak:
+    image: quay.io/keycloak/keycloak:26.0.7
+    command: ["start-dev", "--import-realm"]
+    environment:
+      KC_BOOTSTRAP_ADMIN_USERNAME: admin
+      KC_BOOTSTRAP_ADMIN_PASSWORD: admin
+      KC_HTTP_ENABLED: "true"
+      KC_HOSTNAME: "http://keycloak:8080"
+      KC_HOSTNAME_STRICT: "false"
+    ports:
+      - "8080:8080"
+    volumes:
+      - ../../test/s3/iam/seaweedfs-test-realm.json:/opt/keycloak/data/import/seaweedfs-test-realm.json:ro
+    healthcheck:
+      test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/8080"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 60s
+
+  s3:
+    image: chrislusf/seaweedfs:local
+    command:
+      - -v=9
+      - server
+      - -ip=s3
+      - -filer
+      - -master.volumeSizeLimitMB=16
+      - -volume
+      - -volume.max=0
+      - -volume.preStopSeconds=1
+      - -s3
+      - -s3.port=8333
+      - -s3.config=/etc/seaweedfs/s3_config.json
+      - -s3.iam.config=/etc/seaweedfs/iam_config.json
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+    volumes:
+      - ../../test/s3/iam/test_config.json:/etc/seaweedfs/s3_config.json:ro
+      - ../../test/s3/iam/iam_config_docker.json:/etc/seaweedfs/iam_config.json:ro
+    ports:
+      - "8333:8333"
+    depends_on:
+      keycloak:
+        condition: service_healthy

go.mod

@@ -129,7 +129,6 @@ require (
 	github.com/cognusion/imaging v1.0.3
 	github.com/fluent/fluent-logger-golang v1.10.1
 	github.com/getsentry/sentry-go v0.44.1
-	github.com/go-git/go-billy/v5 v5.9.0
 	github.com/go-ldap/ldap/v3 v3.4.13
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
@@ -152,8 +151,6 @@ require (
 	github.com/tarantool/go-tarantool/v2 v2.4.2
 	github.com/testcontainers/testcontainers-go v0.40.0
 	github.com/tikv/client-go/v2 v2.0.7
-	github.com/willscott/go-nfs v0.0.4
-	github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2
@@ -211,6 +208,7 @@ require (
 	github.com/dromara/dongle v1.0.1 // indirect
 	github.com/gin-gonic/gin v1.11.0 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
+	github.com/go-git/go-billy/v5 v5.9.0 // indirect
 	github.com/goccy/go-yaml v1.18.0 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
@@ -257,7 +255,6 @@ require (
 	github.com/pquerna/otp v1.5.0 // indirect
 	github.com/pterm/pterm v0.12.82 // indirect
 	github.com/quic-go/qpack v0.6.0 // indirect
-	github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
 	github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
 	github.com/rclone/go-proton-api v1.0.2 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect

go.sum

@@ -1783,8 +1783,6 @@ github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SA
 github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
 github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8=
 github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
-github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
-github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
 github.com/rclone/Proton-API-Bridge v1.0.3 h1:Bs7RC4xCFSN0BPIYVda/BNxp0qo3NV0gB2VZqx2KIew=
 github.com/rclone/Proton-API-Bridge v1.0.3/go.mod h1:26RAest751Ofk+F/d8xtl4UyWXrZvMQwn39U8rm/WKM=
 github.com/rclone/go-proton-api v1.0.2 h1:cJtJUab0MGJ3C6q5kiEJs3pbyhSLnOKMyYOQehA0PBc=
@@ -2028,10 +2026,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
-github.com/willscott/go-nfs v0.0.4 h1:1vpOPAdECmoT2KmZ8u+ukO/jfvDjMEUNYhA2F1jGJtI=
-github.com/willscott/go-nfs v0.0.4/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg=
-github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
-github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
 github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
 github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.29"
+appVersion: "4.31"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.29.0
+version: 4.31.0

k8s/charts/seaweedfs/README.md

@@ -384,4 +384,4 @@ helm install seaweedfs seaweedfs/seaweedfs \
 ## Enterprise
 
 For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition, 
-which has a self-healing storage format with better data protection.
+which has advanced features, including data recovery, self-healing storage, customizable erasure coding, EC vacuum and repair, etc.

seaweed-volume/src/server/handlers.rs

@@ -23,6 +23,27 @@ use crate::pb::volume_server_pb;
 use crate::storage::needle::needle::Needle;
 use crate::storage::types::*;
 
+/// Slack added over the configured file-size limit when bounding the raw
+/// request body, to allow for multipart/form-data framing overhead. The exact
+/// per-file limit is still enforced on the parsed data after multipart parsing.
+const UPLOAD_BODY_OVERHEAD: usize = 16 * 1024 * 1024; // 16 MiB
+
+/// Upper bound on bytes we will materialize in memory for a single request when
+/// expanding stored content (gzip decompression or chunk-manifest assembly).
+/// Guards against gzip bombs and crafted/oversized manifest sizes OOM-killing
+/// the server; legitimate large objects are read via client-side chunk fetches.
+const MAX_EXPANSION_BYTES: u64 = 2 * 1024 * 1024 * 1024; // 2 GiB
+
+/// Why `maybe_decompress_gzip` failed, so callers can distinguish a recoverable
+/// "not valid gzip, use the raw bytes" from "the bomb cap was hit, reject it".
+#[derive(Debug)]
+enum GunzipError {
+    /// Input was not valid gzip / decode failed — callers may fall back to raw.
+    Decode,
+    /// Decompressed output would exceed `MAX_EXPANSION_BYTES` — must be rejected.
+    TooLarge,
+}
+
 // ============================================================================
 // Inflight Throttle Guard
 // ============================================================================
@@ -1452,12 +1473,16 @@ async fn get_or_head_handler_inner(
     if is_compressed {
         if needs_image_ops {
             // Always decompress for image operations (Go decompresses before resize/crop)
-            use flate2::read::GzDecoder;
-            use std::io::Read as _;
-            let mut decoder = GzDecoder::new(&data[..]);
-            let mut decompressed = Vec::new();
-            if decoder.read_to_end(&mut decompressed).is_ok() {
-                data = decompressed;
+            match maybe_decompress_gzip(&data) {
+                Ok(decompressed) => data = decompressed,
+                Err(GunzipError::TooLarge) => {
+                    return (
+                        StatusCode::PAYLOAD_TOO_LARGE,
+                        "compressed object exceeds decompression limit",
+                    )
+                        .into_response()
+                }
+                Err(GunzipError::Decode) => {} // not valid gzip; keep raw bytes
             }
         } else {
             let accept_encoding = headers
@@ -1474,12 +1499,16 @@ async fn get_or_head_handler_inner(
                 response_headers.insert(header::CONTENT_ENCODING, "gzip".parse().unwrap());
             } else {
                 // Decompress for client
-                use flate2::read::GzDecoder;
-                use std::io::Read as _;
-                let mut decoder = GzDecoder::new(&data[..]);
-                let mut decompressed = Vec::new();
-                if decoder.read_to_end(&mut decompressed).is_ok() {
-                    data = decompressed;
+                match maybe_decompress_gzip(&data) {
+                    Ok(decompressed) => data = decompressed,
+                    Err(GunzipError::TooLarge) => {
+                        return (
+                            StatusCode::PAYLOAD_TOO_LARGE,
+                            "compressed object exceeds decompression limit",
+                        )
+                            .into_response()
+                    }
+                    Err(GunzipError::Decode) => {} // not valid gzip; keep raw bytes
                 }
             }
         }
@@ -2134,15 +2163,30 @@ pub async fn post_handler(
         .and_then(|v| v.to_str().ok())
         .map(|s| s.to_string());
 
-    // Read body
-    let body = match axum::body::to_bytes(request.into_body(), usize::MAX).await {
+    // Read body, bounded by the configured file-size limit so a single upload
+    // cannot buffer unbounded memory and OOM-kill the server (mirrors Go's
+    // io.LimitReader(r.Body, sizeLimit+1)). A margin covers multipart framing;
+    // the exact per-file limit is still enforced on the parsed data below.
+    let body_limit = if state.file_size_limit_bytes > 0 {
+        // try_from (not `as usize`) so a >usize::MAX limit on 32-bit caps at
+        // usize::MAX instead of silently truncating/wrapping to a tiny value.
+        usize::try_from(state.file_size_limit_bytes)
+            .unwrap_or(usize::MAX)
+            .saturating_add(UPLOAD_BODY_OVERHEAD)
+    } else {
+        usize::MAX
+    };
+    let body = match axum::body::to_bytes(request.into_body(), body_limit).await {
         Ok(b) => b,
         Err(e) => {
-            return json_error_with_query(
-                StatusCode::BAD_REQUEST,
-                format!("read body: {}", e),
-                Some(&query),
-            )
+            // With a limit configured, an error here means the body exceeded it
+            // before we buffered the whole thing; report it like the size check.
+            let msg = if state.file_size_limit_bytes > 0 {
+                format!("file over the limited {} bytes", state.file_size_limit_bytes)
+            } else {
+                format!("read body: {}", e)
+            };
+            return json_error_with_query(StatusCode::BAD_REQUEST, msg, Some(&query));
         }
     };
 
@@ -2282,7 +2326,17 @@ pub async fn post_handler(
     };
 
     let uncompressed_data = if is_gzipped {
-        maybe_decompress_gzip(&body_data_raw).unwrap_or_else(|| body_data_raw.clone())
+        match maybe_decompress_gzip(&body_data_raw) {
+            Ok(d) => d,
+            Err(GunzipError::TooLarge) => {
+                return json_error_with_query(
+                    StatusCode::PAYLOAD_TOO_LARGE,
+                    "compressed object exceeds decompression limit",
+                    Some(&query),
+                );
+            }
+            Err(GunzipError::Decode) => body_data_raw.clone(),
+        }
     } else {
         body_data_raw.clone()
     };
@@ -2761,14 +2815,16 @@ pub async fn delete_handler(
     // If this is a chunk manifest, delete child chunks first
     if n.is_chunk_manifest() {
         let manifest_data = if n.is_compressed() {
-            use flate2::read::GzDecoder;
-            use std::io::Read as _;
-            let mut decoder = GzDecoder::new(&n.data[..]);
-            let mut decompressed = Vec::new();
-            if decoder.read_to_end(&mut decompressed).is_ok() {
-                decompressed
-            } else {
-                n.data.clone()
+            match maybe_decompress_gzip(&n.data) {
+                Ok(d) => d,
+                Err(GunzipError::TooLarge) => {
+                    return json_error_with_query(
+                        StatusCode::PAYLOAD_TOO_LARGE,
+                        "compressed manifest exceeds decompression limit",
+                        Some(&del_query),
+                    );
+                }
+                Err(GunzipError::Decode) => n.data.clone(),
             }
         } else {
             n.data.clone()
@@ -3124,14 +3180,19 @@ fn try_expand_chunk_manifest(
     last_modified_str: &Option<String>,
 ) -> Option<Response> {
     let data = if n.is_compressed() {
-        use flate2::read::GzDecoder;
-        use std::io::Read as _;
-        let mut decoder = GzDecoder::new(&n.data[..]);
-        let mut decompressed = Vec::new();
-        if decoder.read_to_end(&mut decompressed).is_err() {
-            return None;
+        match maybe_decompress_gzip(&n.data) {
+            Ok(d) => d,
+            Err(GunzipError::TooLarge) => {
+                return Some(
+                    (
+                        StatusCode::PAYLOAD_TOO_LARGE,
+                        "compressed manifest exceeds decompression limit",
+                    )
+                        .into_response(),
+                )
+            }
+            Err(GunzipError::Decode) => return None,
         }
-        decompressed
     } else {
         n.data.clone()
     };
@@ -3141,6 +3202,13 @@ fn try_expand_chunk_manifest(
         Err(_) => return None,
     };
 
+    // Guard the attacker-controlled manifest size before allocating: a negative
+    // value would wrap to a huge usize (capacity-overflow panic) and an oversized
+    // one would OOM-kill the server.
+    if manifest.size < 0 || manifest.size as u64 > MAX_EXPANSION_BYTES {
+        return None;
+    }
+
     // Read and concatenate all chunks
     let mut result = vec![0u8; manifest.size as usize];
     let store = state.store.read().unwrap();
@@ -3174,24 +3242,54 @@ fn try_expand_chunk_manifest(
                 )
             }
         }
-        let chunk_data = if chunk_needle.is_compressed() {
+        // Validate the attacker-controlled chunk offset before indexing: a
+        // negative value would wrap to a huge usize, and an out-of-range one has
+        // nowhere to land.
+        if chunk.offset < 0 {
+            return Some(
+                (
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    format!("invalid negative chunk offset in {}", chunk.fid),
+                )
+                    .into_response(),
+            );
+        }
+        let offset = chunk.offset as usize;
+        if offset >= result.len() {
+            continue;
+        }
+        // Write the chunk into its window in `result`. Compressed chunks inflate
+        // directly into the destination slice (bounded by the remaining window),
+        // so a chunk never allocates a second large buffer — peak memory stays at
+        // ~manifest.size instead of doubling. Bytes past the window are dropped,
+        // matching the prior truncation behavior.
+        if chunk_needle.is_compressed() {
             use flate2::read::GzDecoder;
             use std::io::Read as _;
             let mut decoder = GzDecoder::new(&chunk_needle.data[..]);
-            let mut decompressed = Vec::new();
-            if decoder.read_to_end(&mut decompressed).is_ok() {
-                decompressed
-            } else {
-                chunk_needle.data.clone()
+            let window = &mut result[offset..];
+            let mut written = 0usize;
+            let mut decode_failed = false;
+            while written < window.len() {
+                match decoder.read(&mut window[written..]) {
+                    Ok(0) => break,
+                    Ok(n) => written += n,
+                    Err(_) => {
+                        decode_failed = true;
+                        break;
+                    }
+                }
+            }
+            // On any decode failure, drop the partial output and fall back to the
+            // chunk's raw bytes (truncated to the window), preserving prior behavior.
+            if decode_failed {
+                window[..written].fill(0);
+                let copy_len = chunk_needle.data.len().min(window.len());
+                window[..copy_len].copy_from_slice(&chunk_needle.data[..copy_len]);
             }
         } else {
-            chunk_needle.data.clone()
-        };
-        let offset = chunk.offset as usize;
-        let end = std::cmp::min(offset + chunk_data.len(), result.len());
-        let copy_len = end - offset;
-        if copy_len > 0 {
-            result[offset..offset + copy_len].copy_from_slice(&chunk_data[..copy_len]);
+            let copy_len = chunk_needle.data.len().min(result.len() - offset);
+            result[offset..offset + copy_len].copy_from_slice(&chunk_needle.data[..copy_len]);
         }
     }
 
@@ -3564,13 +3662,22 @@ fn try_gzip_data(data: &[u8]) -> Option<Vec<u8>> {
     encoder.finish().ok()
 }
 
-fn maybe_decompress_gzip(data: &[u8]) -> Option<Vec<u8>> {
+fn maybe_decompress_gzip(data: &[u8]) -> Result<Vec<u8>, GunzipError> {
     use flate2::read::GzDecoder;
     use std::io::Read;
-    let mut decoder = GzDecoder::new(data);
+    // Cap the output so a crafted, highly-compressible (gzip-bomb) needle cannot
+    // OOM the server when decompressed on read or upload. take(limit+1) lets us
+    // tell "exactly at the limit" apart from "over it", which we reject as
+    // TooLarge so callers fail the request instead of silently using raw bytes.
+    let mut decoder = GzDecoder::new(data).take(MAX_EXPANSION_BYTES + 1);
     let mut decompressed = Vec::new();
-    decoder.read_to_end(&mut decompressed).ok()?;
-    Some(decompressed)
+    decoder
+        .read_to_end(&mut decompressed)
+        .map_err(|_| GunzipError::Decode)?;
+    if decompressed.len() as u64 > MAX_EXPANSION_BYTES {
+        return Err(GunzipError::TooLarge);
+    }
+    Ok(decompressed)
 }
 
 fn compute_md5_base64(data: &[u8]) -> String {
@@ -3812,7 +3919,11 @@ mod tests {
         let compressed = try_gzip_data(data).unwrap();
         let decompressed = maybe_decompress_gzip(&compressed).unwrap();
         assert_eq!(decompressed, data);
-        assert!(maybe_decompress_gzip(data).is_none());
+        // Non-gzip input is reported as a decode error (callers fall back to raw).
+        assert!(matches!(
+            maybe_decompress_gzip(data),
+            Err(GunzipError::Decode)
+        ));
     }
 
     #[test]

seaweed-volume/src/storage/disk_location.rs

@@ -131,8 +131,8 @@ impl DiskLocation {
                     volume_id = vid.0,
                     "volume was not completed: {}, removing files", note
                 );
-                remove_volume_files(&volume_name);
-                remove_volume_files(&idx_name);
+                remove_volume_files(&volume_name, false);
+                remove_volume_files(&idx_name, false);
                 continue;
             }
 

seaweed-volume/src/storage/store.rs

@@ -598,8 +598,16 @@ impl Store {
                     as i32;
                 let mut max_count = vol_count + ec_equivalent;
 
-                if unclaimed > volume_size_limit as i64 {
-                    max_count += (unclaimed as u64 / volume_size_limit) as i32 - 1;
+                // One slot per full volume that fits in the unclaimed space.
+                // A "- 1" here used to zero the count when the disk had room for
+                // exactly one volume (free between 1x and 2x the limit), stranding
+                // auto-sized disks at max_volume_count 0 with no writable volume.
+                if unclaimed > 0 {
+                    max_count += (unclaimed as u64 / volume_size_limit) as i32;
+                }
+                // An auto-sized disk with free space always hosts at least one volume.
+                if max_count < 1 {
+                    max_count = 1;
                 }
 
                 loc.max_volume_count.store(max_count, Ordering::Relaxed);
@@ -1462,6 +1470,47 @@ mod tests {
         assert!(with_preallocate > without_preallocate);
     }
 
+    #[test]
+    fn test_maybe_adjust_volume_max_no_dead_zone() {
+        // With auto max (original_max_volume_count == 0) and a large volume size
+        // limit, a disk with room for at least one volume must not report 0
+        // slots. It once did so for disks sized between 1x and 2x the limit,
+        // leaving no writable volume and timing out every assign.
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+
+        let (_, free) = crate::storage::disk_location::get_disk_stats(dir);
+        if free < 4 {
+            return; // cannot determine free disk space
+        }
+
+        let mut store = Store::new(NeedleMapKind::InMemory);
+        store
+            .add_location(
+                dir,
+                dir,
+                0, // auto
+                DiskType::HardDrive,
+                MinFreeSpace::Percent(1.0),
+                Vec::new(),
+            )
+            .unwrap();
+
+        // free disk is 1.5x the limit: room for one full volume, less than two.
+        let volume_size_limit = free * 2 / 3;
+        store
+            .volume_size_limit
+            .store(volume_size_limit, Ordering::Relaxed);
+
+        store.maybe_adjust_volume_max();
+
+        let max = store.locations[0].max_volume_count.load(Ordering::Relaxed);
+        assert!(
+            max >= 1,
+            "auto-sized disk with room for one volume reported max_volume_count={max}; want >= 1"
+        );
+    }
+
     #[test]
     fn test_find_free_location_predicate_prefers_more_capacity_and_skips_low_disk() {
         let tmp1 = TempDir::new().unwrap();

seaweed-volume/src/storage/volume.rs

@@ -3071,11 +3071,15 @@ impl Volume {
                 let bytes = fake_del_needle.write_bytes(version);
                 dst_dat.write_all(&bytes)?;
 
+                // Record the tombstone's real .dat offset, like the normal delete path,
+                // so a deletion left at the .dat tail stays visible to the integrity
+                // check on reload. Offset 0 hid the trailing tombstone and falsely
+                // flipped the volume read-only.
                 let mut idx_entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
                 crate::storage::types::idx_entry_to_bytes(
                     &mut idx_entry_buf,
                     key,
-                    Offset::from_actual_offset(0),
+                    Offset::from_actual_offset(dat_offset as i64),
                     Size(crate::storage::types::TOMBSTONE_FILE_SIZE.into()),
                 );
                 dst_idx.write_all(&idx_entry_buf)?;
@@ -3157,12 +3161,34 @@ impl Volume {
             }
         }
 
+        // A regular volume and an EC volume for the same id share <base>.vif.
+        // When EC artefacts coexist on this disk (e.g. shards distributed onto
+        // a source replica before it is deleted), keep the .vif so removing the
+        // regular volume does not strip the EC volume's info file.
+        let keep_vif = self.shares_vif_with_ec_volume();
         self.close();
-        remove_volume_files(&self.data_file_name());
-        remove_volume_files(&self.index_file_name());
+        remove_volume_files(&self.data_file_name(), keep_vif);
+        remove_volume_files(&self.index_file_name(), keep_vif);
         Ok(())
     }
 
+    /// Reports whether an EC volume for this id has a sealed .ecx on the same
+    /// disk, in which case its .vif is the same file as the regular volume's
+    /// and must outlive the regular volume's deletion. Mirrors the on-disk
+    /// portion of Go's Volume.sharesVifWithEcVolume / HasEcxFileOnDisk.
+    fn shares_vif_with_ec_volume(&self) -> bool {
+        let has_ecx = |base: &str| -> bool {
+            fs::metadata(format!("{}.ecx", base))
+                .map(|m| !m.is_dir() && m.len() > 0)
+                .unwrap_or(false)
+        };
+        if has_ecx(&volume_file_name(&self.dir_idx, &self.collection, self.id)) {
+            return true;
+        }
+        self.dir != self.dir_idx
+            && has_ecx(&volume_file_name(&self.dir, &self.collection, self.id))
+    }
+
     /// Check if an I/O error is EIO (errno 5) and record it for health monitoring.
     /// On success (None), clears any previously recorded EIO error.
     /// Matches Go's `checkReadWriteError` in volume_write.go.
@@ -3231,10 +3257,13 @@ fn get_append_at_ns(last: u64) -> u64 {
 
 /// Remove all files associated with a volume.
 /// .dat/.idx removals log at info level so destructive calls are traceable.
-pub(crate) fn remove_volume_files(base: &str) {
+pub(crate) fn remove_volume_files(base: &str, keep_vif: bool) {
     for ext in &[
         ".dat", ".idx", ".vif", ".sdx", ".cpd", ".cpx", ".note", ".rdb",
     ] {
+        if *ext == ".vif" && keep_vif {
+            continue;
+        }
         let path = format!("{}{}", base, ext);
         let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
         let existed = fs::remove_file(&path).is_ok();
@@ -4740,4 +4769,43 @@ mod tests {
             ".vif removed from data dir"
         );
     }
+
+    /// When an EC volume for the same id has a sealed .ecx on the same disk, the
+    /// .vif is shared with it and must survive the regular volume's deletion.
+    #[test]
+    fn test_destroy_keeps_vif_when_ec_coexists() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+
+        let mut v = make_test_volume(dir);
+        let mut n = Needle {
+            id: NeedleId(1),
+            cookie: Cookie(1),
+            data: b"test".to_vec(),
+            data_size: 4,
+            ..Needle::default()
+        };
+        v.write_needle(&mut n, true).unwrap();
+
+        let vif_path = format!("{}/1.vif", dir);
+        std::fs::write(&vif_path, r#"{"version":3}"#).unwrap();
+        // A sealed .ecx marks a coexisting EC volume for the same id.
+        let ecx_path = format!("{}/1.ecx", dir);
+        std::fs::write(&ecx_path, b"ec-index").unwrap();
+
+        v.destroy(false, false).unwrap();
+
+        let dat_path = format!("{}/1.dat", dir);
+        let idx_path = format!("{}/1.idx", dir);
+        assert!(!std::path::Path::new(&dat_path).exists(), ".dat removed");
+        assert!(!std::path::Path::new(&idx_path).exists(), ".idx removed");
+        assert!(
+            std::path::Path::new(&vif_path).exists(),
+            ".vif kept: shared with the coexisting EC volume"
+        );
+        assert!(
+            std::path::Path::new(&ecx_path).exists(),
+            ".ecx is an EC sidecar, never touched here"
+        );
+    }
 }

terraform/.gitignore

@@ -0,0 +1,15 @@
+# Terraform / OpenTofu working files
+.terraform/
+.terraform.lock.hcl
+*.tfstate
+*.tfstate.*
+crash.log
+crash.*.log
+*.tfvars
+!*.tfvars.example
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+.terraformrc
+terraform.rc

terraform/README.md

@@ -0,0 +1,128 @@
+# SeaweedFS on Terraform
+
+> **Experimental.** This Terraform support is an early scaffold under active
+> development. Interfaces (variables, outputs, module layout) may change without
+> notice, and not every tier is implemented yet. Not recommended for production
+> without your own review and testing.
+
+Self-contained Terraform/OpenTofu modules to deploy SeaweedFS on cloud VMs
+running the `weed` binary directly under systemd. No Helm and no Kubernetes
+required.
+
+What works today is verified end-to-end against a real `weed` cluster (see
+"Test it locally" below).
+
+## Layout
+
+```text
+terraform/
+  modules/
+    core/        cloud-agnostic renderer (ZERO cloud resources): turns an
+                 address map + config into per-node weed argv, systemd units,
+                 config files, disk-mount + secret-fetch scripts, and cloud-init.
+                 Both the cloud wrappers and the local harnesses consume `nodes`.
+    security/    cloud-agnostic CA + per-component mTLS certs (distinct CNs) +
+                 JWT signing keys (tls/random providers). Emits a `core_security`
+                 object ready for the core, plus PEMs for secret-store delivery.
+    aws/         thin AWS wrapper: reserves stable ENIs (fixed private IPs)
+                 first, feeds them to core, creates instances + protected EBS
+                 data disks + SG; with enable_security it generates certs/JWT,
+                 stores them in SSM SecureString, grants an instance role, and
+                 renders a boot fetch-secrets.sh. Keyed for_each throughout.
+  examples/
+    aws-ha-distributed/   3 masters + 3 volumes (1/AZ) + 2 filers + 1 S3,
+                          secure-by-default (mTLS via SSM-delivered certs)
+    aws-all-in-one/       single `weed server` instance via core directly
+  test/
+    local/         render a cluster with core and run it as real weed processes
+                   on 127.0.0.1 (no cloud, no docker), then assert it works
+    local-secure/  generate certs/JWT with the security module, run a real
+                   mTLS cluster, and assert it forms + enforces JWT auth
+```
+
+## Design in one paragraph
+
+The chart is the structural reference, not a dependency. A cloud-agnostic
+**core** renders everything portable; thin per-cloud wrappers provision infra.
+Addressing is an **input** to the core (wrapper reserves static IPs first), so
+the wrapper -> core dependency is one-way with no apply-time cycle. Stateful
+tiers (master/volume/filer) are keyed `for_each` maps, never `count`, so a
+middle node can be replaced without reindexing its peers or reattaching the
+wrong disk. Flag names are verified against the real `weed` binary
+(notably: volume uses `-mserver` for the master list; gRPC is `-port.grpc`,
+auto = http+10000; `minFreeSpacePercent` is a string).
+
+## Test it locally
+
+Requires `tofu` (or `terraform`), `jq`, `curl`, and a `weed` binary. Renders a
+3-master + volume + filer + S3 cluster from the core module and runs it as real
+processes, asserting quorum, volume registration, and filer/S3 round-trips:
+
+```bash
+cd terraform/test/local
+WEED=/path/to/weed ./run_local_cluster.sh
+# => 7 passed, 0 failed
+```
+
+The harness uses a high port range (29333/28080/28888/28333) so it does not
+collide with a SeaweedFS cluster already running on the machine, and aborts if a
+required port is taken. `KEEP=1 ./run_local_cluster.sh` leaves the cluster up.
+
+### mTLS end-to-end
+
+```bash
+cd terraform/test/local-secure
+WEED=/path/to/weed ./run_local_secure.sh
+# => generates a CA + component certs + JWT, renders security.toml, runs a real
+#    mTLS cluster, asserts master/volume/filer form over mutual TLS and that the
+#    filer enforces JWT signing (unsigned writes get 401). 5 passed.
+```
+
+## Plan-level tests (no cloud)
+
+```bash
+cd terraform/modules/core && tofu test
+# => 11 passed: peers list, -mserver vs -master, metrics gating,
+#    security.toml conditions, all-in-one inheritance, ...
+```
+
+## Validate the cloud wrappers
+
+```bash
+cd terraform/examples/aws-ha-distributed && tofu init && tofu validate
+```
+
+`apply` needs AWS credentials, a VPC, subnets, and an AMI with `weed` installed
+(bake with Packer, or install at boot).
+
+## Status
+
+Implemented and verified:
+- Tiers: master / volume / filer / s3 / all-in-one rendered by the core.
+- Disk mount: cloud-init runs a `mount-disks.sh` that auto-discovers (or takes
+  explicit candidate devices), `blkid`-guards `mkfs`, mounts, and persists to
+  `/etc/fstab` by UUID. The AWS wrapper wires the protected EBS disk to `/data`.
+- mTLS + JWT: the `security` module generates the CA + per-component certs
+  (distinct CNs) + JWT keys; the AWS wrapper stores them in SSM SecureString,
+  grants an instance role, and the core renders a boot fetch script so secrets
+  stay OUT of user_data. Proven end-to-end by `test/local-secure`.
+
+Not yet implemented:
+- sftp / admin / worker tiers (core does not render them yet).
+- GCP/Azure wrappers, the data-plane provider, and the K8s-native module.
+- CA in Vault PKI (currently TF-generated, so the CA key lives in state). KMS
+  decrypt in the IAM policy is scoped by
+  `ViaService` but uses `Resource="*"`; tighten to the SSM CMK in production.
+
+## Gotchas
+
+- This `weed` build starts an Iceberg REST catalog on `:8181` by default; set
+  `s3.iceberg_port = 0` to disable, or a port to relocate it.
+- Disk auto-discovery assumes a single attached data disk per node. For multiple
+  data disks, pass explicit `devices` candidates in `disk_mounts`.
+
+## Security note
+
+`tofu output`/state can contain secrets. Generate secrets outside Terraform
+(cloud secret manager / Vault) and fetch them at boot; never commit secret
+material.

terraform/examples/aws-all-in-one/main.tf

@@ -0,0 +1,133 @@
+# SeaweedFS all-in-one on a single AWS instance.
+#
+# Demonstrates the layered design directly: call the cloud-agnostic core to
+# render cloud-init for a `weed server` node, then attach it to one instance
+# with a protected data disk. No wrapper module needed for the simple case.
+#
+#   tofu init && tofu validate
+#   tofu apply   # requires AWS credentials + a weed AMI
+
+terraform {
+  required_version = ">= 1.3.0"
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.40"
+    }
+  }
+}
+
+provider "aws" {
+  region = var.region
+}
+
+variable "region" {
+  type    = string
+  default = "us-east-1"
+}
+variable "vpc_id" {
+  type = string
+}
+variable "subnet_id" {
+  type = string
+}
+variable "availability_zone" {
+  type    = string
+  default = "us-east-1a"
+}
+variable "ami_id" {
+  type = string
+}
+variable "private_ip" {
+  type    = string
+  default = "10.0.1.50"
+}
+
+module "core" {
+  source      = "../../modules/core"
+  weed_binary = "/usr/bin/weed"
+
+  # disable the distributed tiers; run a single all-in-one process
+  master = { enabled = false }
+  volume = { enabled = false }
+  filer  = { enabled = false }
+
+  all_in_one = {
+    enabled = true
+    nodes   = { a0 = { address = var.private_ip, data_dir = "/data" } }
+    s3      = { enabled = true }
+  }
+
+  s3_identities = [{
+    name       = "anonymous"
+    access_key = ""
+    secret_key = ""
+    actions    = ["Read", "List"]
+  }]
+}
+
+resource "aws_network_interface" "aio" {
+  subnet_id       = var.subnet_id
+  private_ips     = [var.private_ip]
+  security_groups = [aws_security_group.aio.id]
+  tags            = { Name = "seaweedfs-all-in-one" }
+}
+
+resource "aws_security_group" "aio" {
+  name_prefix = "seaweedfs-aio-"
+  vpc_id      = var.vpc_id
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_vpc_security_group_ingress_rule" "s3" {
+  security_group_id = aws_security_group.aio.id
+  cidr_ipv4         = "10.0.0.0/8"
+  ip_protocol       = "tcp"
+  from_port         = 8333
+  to_port           = 8333
+}
+
+resource "aws_vpc_security_group_egress_rule" "all" {
+  security_group_id = aws_security_group.aio.id
+  cidr_ipv4         = "0.0.0.0/0"
+  ip_protocol       = "-1"
+}
+
+resource "aws_instance" "aio" {
+  ami           = var.ami_id
+  instance_type = "m5.large"
+  user_data     = module.core.cloud_init_by_node["all-in-one-a0"]
+
+  network_interface {
+    network_interface_id = aws_network_interface.aio.id
+    device_index         = 0
+  }
+  metadata_options {
+    http_tokens = "required"
+  }
+  tags = { Name = "seaweedfs-all-in-one", Role = "all-in-one" }
+}
+
+resource "aws_ebs_volume" "data" {
+  availability_zone = var.availability_zone
+  size              = 200
+  type              = "gp3"
+  encrypted         = true
+  tags              = { Name = "seaweedfs-all-in-one-data" }
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+resource "aws_volume_attachment" "data" {
+  device_name                    = "/dev/xvdf"
+  volume_id                      = aws_ebs_volume.data.id
+  instance_id                    = aws_instance.aio.id
+  stop_instance_before_detaching = true
+}
+
+output "instance_id" {
+  value = aws_instance.aio.id
+}

terraform/examples/aws-ha-distributed/main.tf

@@ -0,0 +1,117 @@
+# SeaweedFS HA on AWS: 3-master quorum + 3 volume servers (one per AZ) + 2
+# filers (leveldb2-replicated HA) + 1 standalone S3 gateway.
+#
+#   tofu init && tofu validate
+#   tofu apply   # requires AWS credentials, a VPC, subnets, and a weed AMI
+#
+# This is a scaffold: it provisions instances, protected EBS data disks, and the
+# security group. Mounting the EBS disk at /data and secret-store cert delivery
+# are documented follow-ups (see terraform/README.md).
+
+terraform {
+  required_version = ">= 1.3.0"
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.40"
+    }
+  }
+}
+
+provider "aws" {
+  region = var.region
+}
+
+variable "region" {
+  type    = string
+  default = "us-east-1"
+}
+
+variable "vpc_id" {
+  type = string
+}
+
+variable "ami_id" {
+  description = "AMI with the weed binary at /usr/bin/weed."
+  type        = string
+}
+
+# subnet per AZ
+variable "subnet_a" { type = string }
+variable "subnet_b" { type = string }
+variable "subnet_c" { type = string }
+
+# AZ defaults assume region us-east-1. When you change `region`, override az_a/az_b/az_c
+# (and the matching subnets) with AZs that belong to that region, or the volume/filer
+# EBS volumes will fail to create in a mismatched AZ.
+variable "az_a" {
+  type    = string
+  default = "us-east-1a"
+}
+variable "az_b" {
+  type    = string
+  default = "us-east-1b"
+}
+variable "az_c" {
+  type    = string
+  default = "us-east-1c"
+}
+
+variable "client_ingress_cidrs" {
+  type    = list(string)
+  default = ["10.0.0.0/8"]
+}
+
+variable "ssh_ingress_cidrs" {
+  type    = list(string)
+  default = []
+}
+
+module "seaweedfs" {
+  source = "../../modules/aws"
+
+  name   = "seaweedfs"
+  vpc_id = var.vpc_id
+  ami_id = var.ami_id
+
+  # secure-by-default flagship: mTLS (certs + JWT generated by the security
+  # submodule, delivered via SSM and fetched at boot) plus monitoring.
+  enable_security    = true
+  monitoring_enabled = true
+
+  masters = {
+    m0 = { subnet_id = var.subnet_a, private_ip = "10.0.1.10" }
+    m1 = { subnet_id = var.subnet_b, private_ip = "10.0.2.10" }
+    m2 = { subnet_id = var.subnet_c, private_ip = "10.0.3.10" }
+  }
+
+  volumes = {
+    v0 = { subnet_id = var.subnet_a, availability_zone = var.az_a, private_ip = "10.0.1.20", rack = var.az_a, data_center = var.region, data_volume_size_gb = 500 }
+    v1 = { subnet_id = var.subnet_b, availability_zone = var.az_b, private_ip = "10.0.2.20", rack = var.az_b, data_center = var.region, data_volume_size_gb = 500 }
+    v2 = { subnet_id = var.subnet_c, availability_zone = var.az_c, private_ip = "10.0.3.20", rack = var.az_c, data_center = var.region, data_volume_size_gb = 500 }
+  }
+
+  filers = {
+    f0 = { subnet_id = var.subnet_a, availability_zone = var.az_a, private_ip = "10.0.1.30" }
+    f1 = { subnet_id = var.subnet_b, availability_zone = var.az_b, private_ip = "10.0.2.30" }
+  }
+
+  s3_nodes = {
+    s0 = { subnet_id = var.subnet_a, private_ip = "10.0.1.40" }
+  }
+
+  client_ingress_cidrs = var.client_ingress_cidrs
+  ssh_ingress_cidrs    = var.ssh_ingress_cidrs
+}
+
+output "master_peers" {
+  value = module.seaweedfs.master_peers
+}
+
+output "instance_ids" {
+  value = module.seaweedfs.instance_ids
+}
+
+output "security_group_id" {
+  value = module.seaweedfs.security_group_id
+}

terraform/modules/aws/main.tf

@@ -0,0 +1,205 @@
+# ---- step 1: reserve stable addressing (ENIs with fixed private IPs) -------
+# These exist BEFORE the core renders config, so the address map is known and
+# the core <- wrapper dependency is one-way (no apply-time cycle).
+
+resource "aws_network_interface" "master" {
+  for_each        = var.masters
+  subnet_id       = each.value.subnet_id
+  private_ips     = [each.value.private_ip]
+  security_groups = [aws_security_group.cluster.id]
+  tags            = merge(var.tags, { Name = "${var.name}-master-${each.key}", Role = "master" })
+}
+
+resource "aws_network_interface" "volume" {
+  for_each        = var.volumes
+  subnet_id       = each.value.subnet_id
+  private_ips     = [each.value.private_ip]
+  security_groups = [aws_security_group.cluster.id]
+  tags            = merge(var.tags, { Name = "${var.name}-volume-${each.key}", Role = "volume" })
+}
+
+resource "aws_network_interface" "filer" {
+  for_each        = var.filers
+  subnet_id       = each.value.subnet_id
+  private_ips     = [each.value.private_ip]
+  security_groups = [aws_security_group.cluster.id]
+  tags            = merge(var.tags, { Name = "${var.name}-filer-${each.key}", Role = "filer" })
+}
+
+resource "aws_network_interface" "s3" {
+  for_each        = var.s3_nodes
+  subnet_id       = each.value.subnet_id
+  private_ips     = [each.value.private_ip]
+  security_groups = [aws_security_group.cluster.id]
+  tags            = merge(var.tags, { Name = "${var.name}-s3-${each.key}", Role = "s3" })
+}
+
+# ---- step 2: render config with the cloud-agnostic core --------------------
+module "core" {
+  source = "../core"
+
+  weed_binary        = var.weed_binary
+  monitoring_enabled = var.monitoring_enabled
+  enable_security    = var.enable_security
+  security           = local.core_security
+
+  # Keep secrets out of cloud-init user_data when security is on; deliver them
+  # from SSM via the boot fetch script instead.
+  render_secret_files = !var.enable_security
+  boot_fetch_script   = local.fetch_script
+
+  master = {
+    nodes = { for k, v in var.masters : k => { address = v.private_ip } }
+  }
+
+  volume = {
+    enabled = length(var.volumes) > 0
+    nodes = { for k, v in var.volumes : k => {
+      address     = v.private_ip
+      rack        = v.rack
+      data_center = v.data_center
+      data_dirs   = [{ path = "/data", max_volumes = 0 }]
+      # auto-discover the single attached data disk and mount it at /data
+      disk_mounts = [{ mountpoint = "/data", fstype = "xfs" }]
+    } }
+  }
+
+  filer = {
+    enabled = length(var.filers) > 0
+    nodes = { for k, v in var.filers : k => {
+      address     = v.private_ip
+      data_dir    = "/data/filerldb2"
+      disk_mounts = [{ mountpoint = "/data", fstype = "xfs" }]
+    } }
+    s3 = var.embedded_s3
+  }
+
+  s3 = {
+    enabled = length(var.s3_nodes) > 0
+    nodes   = { for k, v in var.s3_nodes : k => { address = v.private_ip } }
+  }
+
+  s3_identities = var.s3_identities
+}
+
+# ---- step 3: instances (keyed for_each; user_data = rendered cloud-init) ----
+resource "aws_instance" "master" {
+  for_each                = var.masters
+  ami                     = var.ami_id
+  instance_type           = each.value.instance_type
+  key_name                = var.key_name
+  user_data               = module.core.cloud_init_by_node["master-${each.key}"]
+  iam_instance_profile    = var.enable_security ? aws_iam_instance_profile.node[0].name : null
+  disable_api_termination = var.termination_protection
+
+  network_interface {
+    network_interface_id = aws_network_interface.master[each.key].id
+    device_index         = 0
+  }
+  metadata_options {
+    http_tokens = "required" # IMDSv2
+  }
+  tags = merge(var.tags, { Name = "${var.name}-master-${each.key}", Role = "master" })
+}
+
+resource "aws_instance" "volume" {
+  for_each                = var.volumes
+  ami                     = var.ami_id
+  instance_type           = each.value.instance_type
+  key_name                = var.key_name
+  user_data               = module.core.cloud_init_by_node["volume-${each.key}"]
+  iam_instance_profile    = var.enable_security ? aws_iam_instance_profile.node[0].name : null
+  disable_api_termination = var.termination_protection
+
+  network_interface {
+    network_interface_id = aws_network_interface.volume[each.key].id
+    device_index         = 0
+  }
+  metadata_options {
+    http_tokens = "required"
+  }
+  tags = merge(var.tags, { Name = "${var.name}-volume-${each.key}", Role = "volume" })
+}
+
+resource "aws_instance" "filer" {
+  for_each                = var.filers
+  ami                     = var.ami_id
+  instance_type           = each.value.instance_type
+  key_name                = var.key_name
+  user_data               = module.core.cloud_init_by_node["filer-${each.key}"]
+  iam_instance_profile    = var.enable_security ? aws_iam_instance_profile.node[0].name : null
+  disable_api_termination = var.termination_protection
+
+  network_interface {
+    network_interface_id = aws_network_interface.filer[each.key].id
+    device_index         = 0
+  }
+  metadata_options {
+    http_tokens = "required"
+  }
+  tags = merge(var.tags, { Name = "${var.name}-filer-${each.key}", Role = "filer" })
+}
+
+resource "aws_instance" "s3" {
+  for_each             = var.s3_nodes
+  ami                  = var.ami_id
+  instance_type        = each.value.instance_type
+  key_name             = var.key_name
+  user_data            = module.core.cloud_init_by_node["s3-${each.key}"]
+  iam_instance_profile = var.enable_security ? aws_iam_instance_profile.node[0].name : null
+
+  network_interface {
+    network_interface_id = aws_network_interface.s3[each.key].id
+    device_index         = 0
+  }
+  metadata_options {
+    http_tokens = "required"
+  }
+  tags = merge(var.tags, { Name = "${var.name}-s3-${each.key}", Role = "s3" })
+}
+
+# ---- step 4: protected data disks (decoupled; survive instance replacement) -
+# The core's mount-disks.sh (wired via disk_mounts above) mkfs+mounts these at
+# /data before the weed unit starts.
+resource "aws_ebs_volume" "volume_data" {
+  for_each          = var.volumes
+  availability_zone = each.value.availability_zone
+  size              = each.value.data_volume_size_gb
+  type              = each.value.data_volume_type
+  iops              = each.value.data_volume_iops
+  encrypted         = true
+  tags              = merge(var.tags, { Name = "${var.name}-volume-${each.key}-data", Role = "volume" })
+
+  lifecycle {
+    prevent_destroy = true # break-glass: remove to allow destroy
+  }
+}
+
+resource "aws_volume_attachment" "volume_data" {
+  for_each                       = var.volumes
+  device_name                    = "/dev/xvdf"
+  volume_id                      = aws_ebs_volume.volume_data[each.key].id
+  instance_id                    = aws_instance.volume[each.key].id
+  stop_instance_before_detaching = true
+}
+
+resource "aws_ebs_volume" "filer_data" {
+  for_each          = var.filers
+  availability_zone = each.value.availability_zone
+  size              = each.value.data_volume_size_gb
+  type              = "gp3"
+  encrypted         = true
+  tags              = merge(var.tags, { Name = "${var.name}-filer-${each.key}-data", Role = "filer" })
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+resource "aws_volume_attachment" "filer_data" {
+  for_each                       = var.filers
+  device_name                    = "/dev/xvdf"
+  volume_id                      = aws_ebs_volume.filer_data[each.key].id
+  instance_id                    = aws_instance.filer[each.key].id
+  stop_instance_before_detaching = true
+}

terraform/modules/aws/outputs.tf

@@ -0,0 +1,34 @@
+output "security_group_id" {
+  description = "Cluster security group id."
+  value       = aws_security_group.cluster.id
+}
+
+output "master_private_ips" {
+  description = "Master private IPs keyed by node id."
+  value       = { for k, v in var.masters : k => v.private_ip }
+}
+
+output "master_peers" {
+  description = "Master peer list passed to the weed processes."
+  value       = module.core.master_peers
+}
+
+output "filer_private_ips" {
+  description = "Filer private IPs keyed by node id."
+  value       = { for k, v in var.filers : k => v.private_ip }
+}
+
+output "s3_private_ips" {
+  description = "Standalone S3 gateway private IPs keyed by node id."
+  value       = { for k, v in var.s3_nodes : k => v.private_ip }
+}
+
+output "instance_ids" {
+  description = "All instance ids keyed by role-node."
+  value = merge(
+    { for k, v in aws_instance.master : "master-${k}" => v.id },
+    { for k, v in aws_instance.volume : "volume-${k}" => v.id },
+    { for k, v in aws_instance.filer : "filer-${k}" => v.id },
+    { for k, v in aws_instance.s3 : "s3-${k}" => v.id },
+  )
+}

terraform/modules/aws/security.tf

@@ -0,0 +1,136 @@
+# =============================================================================
+# mTLS + JWT generation and secret delivery (active when enable_security=true).
+#
+# Generates the CA + per-component certs + JWT keys with the security submodule,
+# stores them (plus the rendered security.toml / S3 config) in SSM Parameter
+# Store as SecureString, and renders a boot fetch-secrets.sh that pulls them via
+# the instance role. This keeps secrets OUT of cloud-init user_data / IMDS.
+# Secrets still live in Terraform state (TF-generated); prefer Vault PKI for the
+# CA in production.
+# =============================================================================
+
+locals {
+  all_private_ips = concat(
+    [for k, v in var.masters : v.private_ip],
+    [for k, v in var.volumes : v.private_ip],
+    [for k, v in var.filers : v.private_ip],
+    [for k, v in var.s3_nodes : v.private_ip],
+  )
+
+  ssm_prefix        = "/seaweedfs/${var.name}"
+  deliver_s3_config = length(var.s3_nodes) > 0 || var.embedded_s3.enabled
+
+  # security.toml is identical across nodes; take it from the first master.
+  first_master       = var.enable_security ? sort(keys(var.masters))[0] : ""
+  security_toml_path = "/etc/seaweedfs/security.toml"
+  s3_config_path     = "/etc/seaweedfs/s3_config.json"
+
+  # Boot fetch entries: cert files + security.toml + (optional) S3 config.
+  fetch_entries = var.enable_security ? concat(
+    [for m in module.security[0].secret_manifest : { param = "${local.ssm_prefix}/certs/${m.key}", path = m.path, mode = m.mode }],
+    [{ param = "${local.ssm_prefix}/security.toml", path = local.security_toml_path, mode = "0600" }],
+    local.deliver_s3_config ? [{ param = "${local.ssm_prefix}/s3_config.json", path = local.s3_config_path, mode = "0600" }] : [],
+  ) : []
+
+  fetch_script = var.enable_security ? templatefile("${path.module}/templates/fetch-secrets.sh.tftpl", {
+    run_as_user = "seaweedfs"
+    entries     = local.fetch_entries
+  }) : ""
+
+  core_security = var.enable_security ? module.security[0].core_security : var.security
+}
+
+module "security" {
+  count           = var.enable_security ? 1 : 0
+  source          = "../security"
+  internal_domain = var.internal_domain
+  cert_dir        = var.cert_dir
+  ip_sans         = local.all_private_ips
+}
+
+# ---- SSM SecureString parameters -------------------------------------------
+resource "aws_ssm_parameter" "cert" {
+  for_each = var.enable_security ? { for m in module.security[0].secret_manifest : m.key => m } : {}
+  name     = "${local.ssm_prefix}/certs/${each.key}"
+  type     = "SecureString"
+  value    = module.security[0].secret_contents[each.key]
+  tags     = var.tags
+}
+
+resource "aws_ssm_parameter" "security_toml" {
+  count = var.enable_security ? 1 : 0
+  name  = "${local.ssm_prefix}/security.toml"
+  type  = "SecureString"
+  value = module.core.secret_files_by_node["master-${local.first_master}"][local.security_toml_path]
+  tags  = var.tags
+}
+
+resource "aws_ssm_parameter" "s3_config" {
+  count = var.enable_security && local.deliver_s3_config ? 1 : 0
+  name  = "${local.ssm_prefix}/s3_config.json"
+  type  = "SecureString"
+  # identical content across s3/filer nodes; render once via the security module's
+  # JSON is not available here, so read it back from any node that carries it.
+  value = local.s3_config_source
+  tags  = var.tags
+}
+
+locals {
+  # Pull the rendered S3 identity JSON from whichever node carries it.
+  s3_config_source = !local.deliver_s3_config ? "" : (
+    length(var.s3_nodes) > 0 ?
+    module.core.secret_files_by_node["s3-${sort(keys(var.s3_nodes))[0]}"][local.s3_config_path] :
+    (length(var.filers) > 0 ? module.core.secret_files_by_node["filer-${sort(keys(var.filers))[0]}"][local.s3_config_path] : "")
+  )
+}
+
+# ---- IAM: instance role allowed to read the SSM params ----------------------
+data "aws_iam_policy_document" "assume" {
+  count = var.enable_security ? 1 : 0
+  statement {
+    actions = ["sts:AssumeRole"]
+    principals {
+      type        = "Service"
+      identifiers = ["ec2.amazonaws.com"]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "ssm_read" {
+  count = var.enable_security ? 1 : 0
+  statement {
+    sid       = "ReadSeaweedfsParams"
+    actions   = ["ssm:GetParameter", "ssm:GetParameters"]
+    resources = ["arn:aws:ssm:*:*:parameter${local.ssm_prefix}/*"]
+  }
+  statement {
+    sid       = "DecryptSecureStrings"
+    actions   = ["kms:Decrypt"]
+    resources = [var.kms_key_arn] # defaults to "*"; set a CMK ARN in production
+    condition {
+      test     = "StringEquals"
+      variable = "kms:ViaService"
+      values   = ["ssm.*.amazonaws.com"]
+    }
+  }
+}
+
+resource "aws_iam_role" "node" {
+  count              = var.enable_security ? 1 : 0
+  name_prefix        = "${var.name}-node-"
+  assume_role_policy = data.aws_iam_policy_document.assume[0].json
+  tags               = var.tags
+}
+
+resource "aws_iam_role_policy" "ssm_read" {
+  count  = var.enable_security ? 1 : 0
+  name   = "ssm-read"
+  role   = aws_iam_role.node[0].id
+  policy = data.aws_iam_policy_document.ssm_read[0].json
+}
+
+resource "aws_iam_instance_profile" "node" {
+  count       = var.enable_security ? 1 : 0
+  name_prefix = "${var.name}-node-"
+  role        = aws_iam_role.node[0].name
+}

terraform/modules/aws/security_group.tf

@@ -0,0 +1,62 @@
+# Cluster security group + the SeaweedFS port matrix.
+# Intra-cluster: all TCP allowed within the SG (master 9333/19333, volume
+# 8080/18080, filer 8888/18888, admin 33646, ...). Client-facing: only S3 (8333)
+# and filer (8888). Metrics ports (9327) are never opened to clients.
+
+resource "aws_security_group" "cluster" {
+  name_prefix = "${var.name}-cluster-"
+  description = "SeaweedFS cluster"
+  vpc_id      = var.vpc_id
+  tags        = merge(var.tags, { Name = "${var.name}-cluster" })
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+# All intra-cluster TCP via SG self-reference.
+resource "aws_vpc_security_group_ingress_rule" "intra_cluster" {
+  security_group_id            = aws_security_group.cluster.id
+  referenced_security_group_id = aws_security_group.cluster.id
+  ip_protocol                  = "tcp"
+  from_port                    = 0
+  to_port                      = 65535
+  description                  = "intra-cluster (http + gRPC for all roles)"
+}
+
+resource "aws_vpc_security_group_ingress_rule" "s3" {
+  for_each          = toset(var.client_ingress_cidrs)
+  security_group_id = aws_security_group.cluster.id
+  cidr_ipv4         = each.value
+  ip_protocol       = "tcp"
+  from_port         = 8333
+  to_port           = 8333
+  description       = "S3 gateway"
+}
+
+resource "aws_vpc_security_group_ingress_rule" "filer" {
+  for_each          = toset(var.client_ingress_cidrs)
+  security_group_id = aws_security_group.cluster.id
+  cidr_ipv4         = each.value
+  ip_protocol       = "tcp"
+  from_port         = 8888
+  to_port           = 8888
+  description       = "filer HTTP"
+}
+
+resource "aws_vpc_security_group_ingress_rule" "ssh" {
+  for_each          = toset(var.ssh_ingress_cidrs)
+  security_group_id = aws_security_group.cluster.id
+  cidr_ipv4         = each.value
+  ip_protocol       = "tcp"
+  from_port         = 22
+  to_port           = 22
+  description       = "SSH"
+}
+
+resource "aws_vpc_security_group_egress_rule" "all" {
+  security_group_id = aws_security_group.cluster.id
+  cidr_ipv4         = "0.0.0.0/0"
+  ip_protocol       = "-1"
+  description       = "all egress"
+}

terraform/modules/aws/templates/fetch-secrets.sh.tftpl

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Rendered by terraform-aws-seaweedfs. Pulls certs + secret config from SSM
+# Parameter Store (SecureString) at boot using the instance role. Region is
+# auto-resolved by the AWS CLI from IMDS, so no region is baked in. Requires
+# the AWS CLI and a working instance profile (see security.tf).
+set -euo pipefail
+
+run_as_user="${run_as_user}"
+
+fetch() {
+  local param="$1" dest="$2" mode="$3"
+  mkdir -p "$(dirname "$dest")"
+  aws ssm get-parameter --with-decryption --name "$param" \
+    --query 'Parameter.Value' --output text > "$dest"
+  chmod "$mode" "$dest"
+  chown "$run_as_user:$run_as_user" "$dest" 2>/dev/null || true
+}
+
+%{ for e in entries ~}
+fetch "${e.param}" "${e.path}" "${e.mode}"
+%{ endfor ~}

terraform/modules/aws/variables.tf

@@ -0,0 +1,171 @@
+# =============================================================================
+# terraform-aws-seaweedfs - thin AWS wrapper around the cloud-agnostic core.
+#
+# Reserves stable per-node addressing (ENIs with fixed private IPs) FIRST, feeds
+# that address map to the core to render cloud-init, then creates instances and
+# protected EBS data disks. All tiers are keyed for_each (never count), so a
+# middle node can be replaced without reindexing its peers/disks.
+# =============================================================================
+
+variable "name" {
+  description = "Name prefix for all resources."
+  type        = string
+  default     = "seaweedfs"
+}
+
+variable "vpc_id" {
+  description = "VPC to deploy into."
+  type        = string
+}
+
+variable "ami_id" {
+  description = "AMI with the weed binary installed (e.g. baked with Packer)."
+  type        = string
+}
+
+variable "weed_binary" {
+  description = "Path to weed on the AMI."
+  type        = string
+  default     = "/usr/bin/weed"
+}
+
+variable "key_name" {
+  description = "EC2 key pair for SSH (optional)."
+  type        = string
+  default     = null
+}
+
+variable "monitoring_enabled" {
+  description = "Bind metrics ports and pass them through to the core."
+  type        = bool
+  default     = false
+}
+
+variable "enable_security" {
+  description = "Enable mTLS (security.toml). Cert material must be supplied via the security variable / secret store."
+  type        = bool
+  default     = false
+}
+
+variable "security" {
+  description = "Passed through to the core security variable when enable_security=false. When enable_security=true this is ignored and certs/JWT are generated by the security submodule."
+  type        = any
+  default     = {}
+}
+
+variable "internal_domain" {
+  description = "Internal domain for generated component cert CNs / peer-auth wildcard (enable_security=true)."
+  type        = string
+  default     = "seaweedfs.internal"
+}
+
+variable "cert_dir" {
+  description = "On-host directory where certs are fetched/placed."
+  type        = string
+  default     = "/usr/local/share/ca-certificates"
+}
+
+variable "kms_key_arn" {
+  description = "KMS key ARN the instance role may use to decrypt SSM SecureStrings. Defaults to \"*\" (the SSM default key); set to a specific CMK ARN in production."
+  type        = string
+  default     = "*"
+}
+
+variable "termination_protection" {
+  description = "Set disable_api_termination on stateful instances (masters, volumes)."
+  type        = bool
+  default     = true
+}
+
+# ---- stateful tiers: keyed node maps ---------------------------------------
+variable "masters" {
+  description = "Master nodes keyed by stable id (m0/m1/m2). Quorum size must be odd."
+  type = map(object({
+    subnet_id     = string
+    private_ip    = string
+    instance_type = optional(string, "t3.medium")
+  }))
+  validation {
+    condition     = length(var.masters) % 2 == 1
+    error_message = "Master quorum must be an odd number of nodes (1, 3, 5)."
+  }
+}
+
+variable "volumes" {
+  description = "Volume nodes keyed by stable id. Each owns one protected EBS data disk."
+  type = map(object({
+    subnet_id           = string
+    availability_zone   = string
+    private_ip          = string
+    instance_type       = optional(string, "m5.large")
+    rack                = optional(string)
+    data_center         = optional(string)
+    data_volume_size_gb = optional(number, 100)
+    data_volume_type    = optional(string, "gp3")
+    data_volume_iops    = optional(number, null)
+  }))
+  default = {}
+}
+
+variable "filers" {
+  description = "Filer nodes keyed by stable id. Each owns a protected EBS disk for its leveldb2 store."
+  type = map(object({
+    subnet_id           = string
+    availability_zone   = string
+    private_ip          = string
+    instance_type       = optional(string, "t3.medium")
+    data_volume_size_gb = optional(number, 50)
+  }))
+  default = {}
+}
+
+variable "s3_nodes" {
+  description = "Standalone S3 gateway nodes keyed by stable id (stateless)."
+  type = map(object({
+    subnet_id     = string
+    private_ip    = string
+    instance_type = optional(string, "t3.medium")
+  }))
+  default = {}
+}
+
+variable "embedded_s3" {
+  description = "Embed the S3 gateway in the filer instead of standalone nodes."
+  type = object({
+    enabled     = optional(bool, false)
+    port        = optional(number, 8333)
+    domain_name = optional(string, "")
+  })
+  default = {}
+}
+
+variable "s3_identities" {
+  description = "Non-admin S3 identities for the gateway config (admin key goes via EnvironmentFile)."
+  type = list(object({
+    name       = string
+    access_key = string
+    secret_key = string
+    actions    = list(string)
+  }))
+  default   = []
+  sensitive = true
+}
+
+# ---- networking ------------------------------------------------------------
+variable "client_ingress_cidrs" {
+  description = "CIDRs allowed to reach the client-facing S3 (8333) and filer (8888) ports."
+  type        = list(string)
+  default     = []
+}
+
+variable "ssh_ingress_cidrs" {
+  description = "CIDRs allowed SSH (22)."
+  type        = list(string)
+  default     = []
+}
+
+variable "tags" {
+  description = "Extra tags applied to all resources."
+  type        = map(string)
+  default     = {}
+}

terraform/modules/aws/versions.tf

@@ -0,0 +1,12 @@
+terraform {
+  required_version = ">= 1.3.0"
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+      # >= 5.40 for the modern aws_vpc_security_group_*_rule resources.
+      # Provider >= 6.0 is recommended for the EBS-attachment no-replace behavior;
+      # bump the floor here once you standardize on it.
+      version = ">= 5.40"
+    }
+  }
+}

terraform/modules/core/main.tf

@@ -0,0 +1,373 @@
+# =============================================================================
+# SeaweedFS Terraform core - pure renderer (zero cloud resources).
+#
+# Computes, per node: the verified `weed` argv, the full systemd ExecStart,
+# config files, environment, and rendered systemd unit + cloud-init.
+# =============================================================================
+
+locals {
+  # ---- shared security.toml (rendered when mTLS on OR any JWT key set) ----
+  jwt_any = anytrue([
+    var.security.jwt_signing_key != "",
+    var.security.jwt_signing_read_key != "",
+    var.security.jwt_filer_signing_key != "",
+    var.security.jwt_filer_signing_read_key != "",
+  ])
+  render_security = var.enable_security || local.jwt_any
+  security_toml = local.render_security ? templatefile("${path.module}/templates/security.toml.tftpl", {
+    enable_security            = var.enable_security
+    cert_dir                   = var.security.cert_dir
+    allowed_wildcard_domain    = var.security.allowed_wildcard_domain
+    allowed_common_names       = var.security.allowed_common_names
+    jwt_signing_key            = var.security.jwt_signing_key
+    jwt_signing_read_key       = var.security.jwt_signing_read_key
+    jwt_filer_signing_key      = var.security.jwt_filer_signing_key
+    jwt_filer_signing_read_key = var.security.jwt_filer_signing_read_key
+  }) : ""
+  security_files = local.render_security ? { "/etc/seaweedfs/security.toml" = local.security_toml } : {}
+
+  # ---- S3 identity JSON (non-admin identities only) ----
+  # Empty access_key => anonymous identity (no credentials block), per SeaweedFS
+  # convention where the identity literally named "anonymous" grants anon access.
+  s3_config_json = jsonencode({
+    identities = [for id in var.s3_identities : {
+      name        = id.name
+      credentials = id.access_key != "" ? [{ accessKey = id.access_key, secretKey = id.secret_key }] : []
+      actions     = id.actions
+    }]
+  })
+
+  # ---- global flags (before the subcommand) ----
+  global_flags = compact([
+    format("-v=%d", var.log_level),
+    var.log_to_stderr ? "-logtostderr=true" : "",
+  ])
+
+  # ---- master peers (sorted by key for determinism) ----
+  master_keys = sort(keys(var.master.nodes))
+  master_peer_list = [for k in local.master_keys :
+    format("%s:%d", var.master.nodes[k].address, coalesce(var.master.nodes[k].port, var.master.port))
+  ]
+  master_peers = join(",", local.master_peer_list)
+
+  # ---- first filer (default s3 target + cluster env) ----
+  filer_keys  = sort(keys(var.filer.nodes))
+  first_filer = length(local.filer_keys) > 0 ? format("%s:%d", var.filer.nodes[local.filer_keys[0]].address, coalesce(var.filer.nodes[local.filer_keys[0]].port, var.filer.port)) : ""
+
+  # ---- cluster discovery env (WEED_CLUSTER_*) ----
+  cluster_env = merge(
+    { WEED_CLUSTER_DEFAULT = var.cluster_name },
+    local.master_peers != "" ? { "WEED_CLUSTER_${replace(upper(var.cluster_name), "/[^A-Z0-9_]/", "_")}_MASTER" = local.master_peers } : {},
+    local.first_filer != "" ? { "WEED_CLUSTER_${replace(upper(var.cluster_name), "/[^A-Z0-9_]/", "_")}_FILER" = local.first_filer } : {},
+  )
+
+  metrics_flags_master = var.monitoring_enabled ? compact([
+    format("-metricsPort=%d", var.master.metrics_port),
+    var.master.metrics_ip != "" ? format("-metricsIp=%s", var.master.metrics_ip) : "",
+  ]) : []
+  metrics_flags_volume = var.monitoring_enabled ? compact([
+    format("-metricsPort=%d", var.volume.metrics_port),
+    var.volume.metrics_ip != "" ? format("-metricsIp=%s", var.volume.metrics_ip) : "",
+  ]) : []
+  metrics_flags_filer = var.monitoring_enabled ? compact([
+    format("-metricsPort=%d", var.filer.metrics_port),
+    var.filer.metrics_ip != "" ? format("-metricsIp=%s", var.filer.metrics_ip) : "",
+  ]) : []
+  metrics_flags_s3 = var.monitoring_enabled ? [format("-metricsPort=%d", var.s3.metrics_port)] : []
+
+  # ===========================================================================
+  # MASTER nodes
+  # ===========================================================================
+  master_nodes = var.master.enabled ? {
+    for k in local.master_keys : "master-${k}" => {
+      role        = "master"
+      name        = "master-${k}"
+      address     = var.master.nodes[k].address
+      disk_mounts = var.master.nodes[k].disk_mounts
+      data_dir    = coalesce(var.master.nodes[k].data_dir, var.master.data_dir)
+      data_dirs   = [coalesce(var.master.nodes[k].data_dir, var.master.data_dir)]
+      ports = {
+        http    = coalesce(var.master.nodes[k].port, var.master.port)
+        metrics = var.master.metrics_port
+      }
+      env          = local.cluster_env
+      config_files = local.security_files
+      argv = concat(
+        local.global_flags,
+        ["master"],
+        [
+          format("-port=%d", coalesce(var.master.nodes[k].port, var.master.port)),
+          format("-mdir=%s", coalesce(var.master.nodes[k].data_dir, var.master.data_dir)),
+          format("-ip=%s", var.master.nodes[k].address),
+          format("-ip.bind=%s", var.master.ip_bind),
+          format("-peers=%s", local.master_peers),
+          format("-defaultReplication=%s", var.master.default_replication),
+          format("-volumeSizeLimitMB=%d", var.master.volume_size_limit_mb),
+          format("-electionTimeout=%s", var.master.election_timeout),
+          format("-heartbeatInterval=%s", var.master.heartbeat_interval),
+        ],
+        var.master.nodes[k].grpc_port != null ? [format("-port.grpc=%d", var.master.nodes[k].grpc_port)] : (
+        var.master.grpc_port != null ? [format("-port.grpc=%d", var.master.grpc_port)] : []),
+        var.master.volume_preallocate ? ["-volumePreallocate"] : [],
+        var.master.garbage_threshold != null ? [format("-garbageThreshold=%s", tostring(var.master.garbage_threshold))] : [],
+        var.master.raft_hashicorp ? ["-raftHashicorp"] : [],
+        var.master.resume_state ? ["-resumeState"] : [],
+        var.master.disable_http ? ["-disableHttp"] : [],
+        var.master.white_list != "" ? [format("-whiteList=%s", var.master.white_list)] : [],
+        local.metrics_flags_master,
+        var.master.extra_args,
+      )
+    }
+  } : {}
+
+  # ===========================================================================
+  # VOLUME nodes
+  # ===========================================================================
+  volume_keys = sort(keys(var.volume.nodes))
+  volume_nodes = var.volume.enabled ? {
+    for k in local.volume_keys : "volume-${k}" => {
+      role        = "volume"
+      name        = "volume-${k}"
+      address     = var.volume.nodes[k].address
+      disk_mounts = var.volume.nodes[k].disk_mounts
+      data_dirs   = [for d in(var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs) : d.path]
+      data_dir    = (var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs)[0].path
+      ports = {
+        http    = coalesce(var.volume.nodes[k].port, var.volume.port)
+        metrics = var.volume.metrics_port
+      }
+      env          = local.cluster_env
+      config_files = local.security_files
+      argv = concat(
+        local.global_flags,
+        ["volume"],
+        [
+          format("-port=%d", coalesce(var.volume.nodes[k].port, var.volume.port)),
+          format("-dir=%s", join(",", [for d in(var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs) : d.path])),
+          format("-max=%s", join(",", [for d in(var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs) : tostring(d.max_volumes)])),
+          format("-ip=%s", var.volume.nodes[k].address),
+          format("-ip.bind=%s", var.volume.ip_bind),
+          format("-mserver=%s", local.master_peers),
+          format("-readMode=%s", var.volume.read_mode),
+          format("-compactionMBps=%d", var.volume.compaction_mbps),
+          format("-minFreeSpacePercent=%s", var.volume.min_free_space_percent),
+          format("-index=%s", var.volume.index),
+        ],
+        var.volume.nodes[k].grpc_port != null ? [format("-port.grpc=%d", var.volume.nodes[k].grpc_port)] : [],
+        (var.volume.nodes[k].idx_dir != null && var.volume.nodes[k].idx_dir != "") ? [format("-dir.idx=%s", var.volume.nodes[k].idx_dir)] : (var.volume.idx_dir != "" ? [format("-dir.idx=%s", var.volume.idx_dir)] : []),
+        var.volume.nodes[k].data_center != null ? [format("-dataCenter=%s", var.volume.nodes[k].data_center)] : [],
+        var.volume.nodes[k].rack != null ? [format("-rack=%s", var.volume.nodes[k].rack)] : [],
+        var.volume.nodes[k].public_url != null ? [format("-publicUrl=%s", var.volume.nodes[k].public_url)] : [],
+        var.volume.file_size_limit_mb != null ? [format("-fileSizeLimitMB=%d", var.volume.file_size_limit_mb)] : [],
+        var.volume.images_fix_orientation ? ["-images.fix.orientation"] : [],
+        var.volume.white_list != "" ? [format("-whiteList=%s", var.volume.white_list)] : [],
+        local.metrics_flags_volume,
+        var.volume.extra_args,
+      )
+    }
+  } : {}
+
+  # ===========================================================================
+  # FILER nodes (+ optional embedded S3)
+  # ===========================================================================
+  filer_s3_files = var.filer.s3.enabled ? { (var.filer.s3.config_path) = local.s3_config_json } : {}
+  filer_nodes = var.filer.enabled ? {
+    for k in local.filer_keys : "filer-${k}" => {
+      role        = "filer"
+      name        = "filer-${k}"
+      address     = var.filer.nodes[k].address
+      disk_mounts = var.filer.nodes[k].disk_mounts
+      data_dir    = coalesce(var.filer.nodes[k].data_dir, var.filer.data_dir)
+      data_dirs   = [coalesce(var.filer.nodes[k].data_dir, var.filer.data_dir)]
+      ports = {
+        http    = coalesce(var.filer.nodes[k].port, var.filer.port)
+        metrics = var.filer.metrics_port
+      }
+      env          = merge(local.cluster_env, var.filer.extra_env)
+      config_files = merge(local.security_files, local.filer_s3_files)
+      argv = concat(
+        local.global_flags,
+        ["filer"],
+        [
+          format("-port=%d", coalesce(var.filer.nodes[k].port, var.filer.port)),
+          format("-ip=%s", var.filer.nodes[k].address),
+          format("-ip.bind=%s", var.filer.ip_bind),
+          format("-master=%s", local.master_peers),
+          format("-defaultReplicaPlacement=%s", var.filer.default_replica_placement),
+          format("-dirListLimit=%d", var.filer.dir_list_limit),
+          format("-defaultStoreDir=%s", coalesce(var.filer.nodes[k].data_dir, var.filer.data_dir)),
+        ],
+        var.filer.nodes[k].grpc_port != null ? [format("-port.grpc=%d", var.filer.nodes[k].grpc_port)] : [],
+        var.filer.max_mb != null ? [format("-maxMB=%d", var.filer.max_mb)] : [],
+        var.filer.disable_dir_listing ? ["-disableDirListing"] : [],
+        var.filer.disable_http ? ["-disableHttp"] : [],
+        var.filer.encrypt_volume_data ? ["-encryptVolumeData"] : [],
+        var.filer.rack != "" ? [format("-rack=%s", var.filer.rack)] : [],
+        var.filer.data_center != "" ? [format("-dataCenter=%s", var.filer.data_center)] : [],
+        var.filer.filer_group != "" ? [format("-filerGroup=%s", var.filer.filer_group)] : [],
+        var.filer.s3.enabled ? [
+          "-s3",
+          format("-s3.port=%d", var.filer.s3.port),
+          format("-s3.config=%s", var.filer.s3.config_path),
+        ] : [],
+        (var.filer.s3.enabled && var.filer.s3.https_port > 0) ? [format("-s3.port.https=%d", var.filer.s3.https_port)] : [],
+        (var.filer.s3.enabled && var.filer.s3.domain_name != "") ? [format("-s3.domainName=%s", var.filer.s3.domain_name)] : [],
+        local.metrics_flags_filer,
+        var.filer.extra_args,
+      )
+    }
+  } : {}
+
+  # ===========================================================================
+  # S3 standalone nodes
+  # ===========================================================================
+  s3_keys      = sort(keys(var.s3.nodes))
+  s3_target    = var.s3.filer_address != "" ? var.s3.filer_address : local.first_filer
+  s3_std_files = merge(local.security_files, { (var.s3.config_path) = local.s3_config_json })
+  s3_nodes = var.s3.enabled ? {
+    for k in local.s3_keys : "s3-${k}" => {
+      role        = "s3"
+      name        = "s3-${k}"
+      address     = var.s3.nodes[k].address
+      disk_mounts = []
+      data_dir    = ""
+      data_dirs   = []
+      ports = {
+        http    = coalesce(var.s3.nodes[k].port, var.s3.port)
+        metrics = var.s3.metrics_port
+      }
+      env          = local.cluster_env
+      config_files = local.s3_std_files
+      argv = concat(
+        local.global_flags,
+        ["s3"],
+        [
+          format("-port=%d", coalesce(var.s3.nodes[k].port, var.s3.port)),
+          format("-ip.bind=%s", var.s3.ip_bind),
+          format("-filer=%s", local.s3_target),
+          format("-config=%s", var.s3.config_path),
+        ],
+        var.s3.https_port > 0 ? [format("-port.https=%d", var.s3.https_port)] : [],
+        var.s3.iceberg_port != null ? [format("-port.iceberg=%d", var.s3.iceberg_port)] : [],
+        var.s3.domain_name != "" ? [format("-domainName=%s", var.s3.domain_name)] : [],
+        var.s3.audit_log_config_path != "" ? [format("-auditLogConfig=%s", var.s3.audit_log_config_path)] : [],
+        var.s3.cert_file != "" ? [format("-cert.file=%s", var.s3.cert_file)] : [],
+        var.s3.key_file != "" ? [format("-key.file=%s", var.s3.key_file)] : [],
+        var.s3.cacert_file != "" ? [format("-cacert.file=%s", var.s3.cacert_file)] : [],
+        var.s3.verify_client_cert ? ["-tlsVerifyClientCert"] : [],
+        local.metrics_flags_s3,
+        var.s3.extra_args,
+      )
+    }
+  } : {}
+
+  # ===========================================================================
+  # ALL-IN-ONE nodes (weed server)
+  # ===========================================================================
+  aio_keys     = sort(keys(var.all_in_one.nodes))
+  aio_s3_files = var.all_in_one.s3.enabled ? { (var.all_in_one.s3.config_path) = local.s3_config_json } : {}
+  aio_nodes = var.all_in_one.enabled ? {
+    for k in local.aio_keys : "all-in-one-${k}" => {
+      role        = "server"
+      name        = "all-in-one-${k}"
+      address     = var.all_in_one.nodes[k].address
+      disk_mounts = var.all_in_one.nodes[k].disk_mounts
+      data_dir    = coalesce(var.all_in_one.nodes[k].data_dir, var.all_in_one.data_dir)
+      data_dirs   = [coalesce(var.all_in_one.nodes[k].data_dir, var.all_in_one.data_dir)]
+      ports = {
+        http    = var.all_in_one.master_port
+        metrics = var.all_in_one.metrics_port
+      }
+      env          = local.cluster_env
+      config_files = merge(local.security_files, local.aio_s3_files)
+      argv = concat(
+        local.global_flags,
+        ["server"],
+        [
+          format("-dir=%s", coalesce(var.all_in_one.nodes[k].data_dir, var.all_in_one.data_dir)),
+          format("-ip=%s", var.all_in_one.nodes[k].address),
+          format("-ip.bind=%s", var.all_in_one.ip_bind),
+          "-master",
+          format("-master.port=%d", var.all_in_one.master_port),
+          format("-master.peers=%s:%d", var.all_in_one.nodes[k].address, var.all_in_one.master_port),
+          format("-master.defaultReplication=%s", var.all_in_one.default_replication),
+          format("-master.volumeSizeLimitMB=%d", var.all_in_one.volume_size_limit_mb),
+          "-volume",
+          format("-volume.port=%d", var.all_in_one.volume_port),
+          "-filer",
+          format("-filer.port=%d", var.all_in_one.filer_port),
+          format("-idleTimeout=%d", var.all_in_one.idle_timeout),
+        ],
+        var.all_in_one.disable_http ? [format("-disableHttp=%t", var.all_in_one.disable_http)] : [],
+        var.all_in_one.s3.enabled ? [
+          "-s3",
+          format("-s3.port=%d", var.all_in_one.s3.port),
+          format("-s3.config=%s", var.all_in_one.s3.config_path),
+        ] : [],
+        (var.all_in_one.s3.enabled && var.all_in_one.s3.domain_name != "") ? [format("-s3.domainName=%s", var.all_in_one.s3.domain_name)] : [],
+        var.monitoring_enabled ? [format("-metricsPort=%d", var.all_in_one.metrics_port)] : [],
+        var.all_in_one.extra_args,
+      )
+    }
+  } : {}
+
+  # ===========================================================================
+  # Merge + render systemd unit and cloud-init per node
+  # ===========================================================================
+  nodes_base = merge(local.master_nodes, local.volume_nodes, local.filer_nodes, local.s3_nodes, local.aio_nodes)
+
+  # Disk-mount script per node (empty unless the node declares disk_mounts).
+  mount_scripts = {
+    for name, n in local.nodes_base : name => length(n.disk_mounts) > 0 ? templatefile("${path.module}/templates/mount-disks.sh.tftpl", {
+      run_as_user = var.hardening.run_as_user
+      mounts      = n.disk_mounts
+    }) : ""
+  }
+
+  nodes = {
+    for name, n in local.nodes_base : name => merge(n, {
+      exec_start = "${var.weed_binary} ${join(" ", n.argv)}"
+      # secret_files: the secret-bearing config a wrapper should deliver from a
+      # secret store when render_secret_files=false (else they go in cloud-init).
+      secret_files = n.config_files
+      mount_script = local.mount_scripts[name]
+      file_modes   = { for p in keys(n.config_files) : p => "0600" }
+      systemd_unit = templatefile("${path.module}/templates/weed.service.tftpl", {
+        role                = n.role
+        name                = n.name
+        run_as_user         = var.hardening.run_as_user
+        exec_start          = "${var.weed_binary} ${join(" ", n.argv)}"
+        no_new_privileges   = var.hardening.no_new_privileges
+        protect_system      = var.hardening.protect_system
+        cap_drop_all        = var.hardening.cap_drop_all
+        read_write_paths    = n.data_dirs
+        requires_mounts_for = n.data_dirs
+        env_file            = var.env_file
+        environment         = n.env
+      })
+      cloud_init = templatefile("${path.module}/templates/cloud-init.yaml.tftpl", {
+        role         = n.role
+        name         = n.name
+        run_as_user  = var.hardening.run_as_user
+        config_files = var.render_secret_files ? n.config_files : {}
+        file_modes   = { for p in keys(n.config_files) : p => "0600" }
+        pre_runcmd   = [for p in n.data_dirs : "install -d -o ${var.hardening.run_as_user} -g ${var.hardening.run_as_user} ${p}"]
+        mount_script = local.mount_scripts[name]
+        fetch_script = var.boot_fetch_script
+        systemd_unit = templatefile("${path.module}/templates/weed.service.tftpl", {
+          role                = n.role
+          name                = n.name
+          run_as_user         = var.hardening.run_as_user
+          exec_start          = "${var.weed_binary} ${join(" ", n.argv)}"
+          no_new_privileges   = var.hardening.no_new_privileges
+          protect_system      = var.hardening.protect_system
+          cap_drop_all        = var.hardening.cap_drop_all
+          read_write_paths    = n.data_dirs
+          requires_mounts_for = n.data_dirs
+          env_file            = var.env_file
+          environment         = n.env
+        })
+      })
+    })
+  }
+}

terraform/modules/core/outputs.tf

@@ -0,0 +1,32 @@
+output "nodes" {
+  description = "Per-node rendered artifacts keyed by node name. Each value: role, name, address, ports, data_dir(s), argv (list, after the weed binary), exec_start, env, config_files, systemd_unit, cloud_init."
+  value       = local.nodes
+  sensitive   = true # config_files/env may carry JWT keys or S3 identities
+}
+
+output "master_peers" {
+  description = "Comma-separated master peer list (address:port), as passed to -peers / -mserver / -master."
+  value       = local.master_peers
+}
+
+output "first_filer" {
+  description = "address:port of the lexically-first filer node (default S3 target)."
+  value       = local.first_filer
+}
+
+output "node_names" {
+  description = "List of rendered node names (non-sensitive)."
+  value       = sort(keys(local.nodes))
+}
+
+output "cloud_init_by_node" {
+  description = "Map of node name -> rendered cloud-init user_data (for the per-cloud wrappers)."
+  value       = { for name, n in local.nodes : name => n.cloud_init }
+  sensitive   = true
+}
+
+output "secret_files_by_node" {
+  description = "Map of node name -> {path => content} of secret-bearing files (security.toml, S3 identity JSON). Wrappers push these to a secret store and fetch them at boot when render_secret_files=false."
+  value       = { for name, n in local.nodes : name => n.secret_files }
+  sensitive   = true
+}

terraform/modules/core/templates/cloud-init.yaml.tftpl

@@ -0,0 +1,44 @@
+#cloud-config
+# Rendered by the SeaweedFS Terraform core module for node ${name} (role ${role}).
+# When render_secret_files=false, secret-bearing files are NOT here; they are
+# fetched at boot by fetch-secrets.sh from the cloud secret store.
+users:
+  - name: ${run_as_user}
+    system: true
+    shell: /usr/sbin/nologin
+write_files:
+%{ for path, content in config_files ~}
+  - path: ${path}
+    owner: "${run_as_user}:${run_as_user}"
+    permissions: '${lookup(file_modes, path, "0644")}'
+    content: |
+      ${indent(6, content)}
+%{ endfor ~}
+%{ if mount_script != "" ~}
+  - path: /opt/seaweedfs/mount-disks.sh
+    permissions: '0755'
+    content: |
+      ${indent(6, mount_script)}
+%{ endif ~}
+%{ if fetch_script != "" ~}
+  - path: /opt/seaweedfs/fetch-secrets.sh
+    permissions: '0755'
+    content: |
+      ${indent(6, fetch_script)}
+%{ endif ~}
+  - path: /etc/systemd/system/weed-${role}.service
+    permissions: '0644'
+    content: |
+      ${indent(6, systemd_unit)}
+runcmd:
+%{ if mount_script != "" ~}
+  - /opt/seaweedfs/mount-disks.sh
+%{ endif ~}
+%{ if fetch_script != "" ~}
+  - /opt/seaweedfs/fetch-secrets.sh
+%{ endif ~}
+%{ for cmd in pre_runcmd ~}
+  - ${cmd}
+%{ endfor ~}
+  - systemctl daemon-reload
+  - systemctl enable --now weed-${role}.service

terraform/modules/core/templates/mount-disks.sh.tftpl

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# Rendered by the SeaweedFS Terraform core module. Formats (only if blank) and
+# mounts each data disk, persisting by UUID to /etc/fstab. Safe to re-run.
+set -euo pipefail
+
+run_as_user="${run_as_user}"
+
+# Base block device backing "/" so auto-discovery never touches the root disk.
+root_src="$(findmnt -no SOURCE / 2>/dev/null || true)"
+root_base="$(lsblk -no pkname "$root_src" 2>/dev/null || true)"
+[ -n "$root_base" ] && root_base="/dev/$root_base"
+
+is_mounted() { findmnt -rno TARGET "$1" >/dev/null 2>&1; }
+
+# Pick the first unmounted, non-root whole disk (single-data-disk node).
+discover_dev() {
+  local d base
+  for d in /dev/nvme*n1 /dev/xvd[b-z] /dev/sd[b-z] /dev/vd[b-z]; do
+    [ -b "$d" ] || continue
+    [ "$d" = "$root_src" ] && continue
+    [ "$d" = "$root_base" ] && continue
+    base="/dev/$(lsblk -no pkname "$d" 2>/dev/null || true)"
+    [ "$base" = "$root_base" ] && continue
+    is_mounted "$d" && continue
+    echo "$d"; return 0
+  done
+  return 1
+}
+
+# Wait up to ~120s for one of the explicit candidate devices to appear.
+wait_dev() {
+  local i d
+  for i in $(seq 1 60); do
+    for d in "$@"; do [ -b "$d" ] && { echo "$d"; return 0; }; done
+    sleep 2
+  done
+  return 1
+}
+
+mount_one() {
+  local mountpoint="$1" fstype="$2"; shift 2
+  local dev=""
+  if [ "$#" -gt 0 ]; then
+    dev="$(wait_dev "$@" || true)"
+  fi
+  if [ -z "$dev" ]; then
+    dev="$(discover_dev || true)"
+  fi
+  if [ -z "$dev" ]; then
+    echo "mount-disks: no device found for $mountpoint" >&2
+    return 1
+  fi
+  if ! blkid "$dev" >/dev/null 2>&1; then
+    echo "mount-disks: formatting $dev as $fstype"
+    "mkfs.$fstype" -q "$dev"
+  fi
+  mkdir -p "$mountpoint"
+  local uuid; uuid="$(blkid -s UUID -o value "$dev")"
+  if [ -z "$uuid" ]; then
+    echo "mount-disks: failed to read a filesystem UUID for $dev" >&2
+    return 1
+  fi
+  if ! grep -q "$uuid" /etc/fstab; then
+    printf 'UUID=%s %s %s defaults,nofail 0 2\n' "$uuid" "$mountpoint" "$fstype" >> /etc/fstab
+  fi
+  is_mounted "$mountpoint" || mount "$mountpoint"
+  chown -R "$run_as_user:$run_as_user" "$mountpoint"
+  echo "mount-disks: $dev mounted at $mountpoint"
+}
+
+%{ for m in mounts ~}
+mount_one "${m.mountpoint}" "${m.fstype}"${length(m.devices) > 0 ? " ${join(" ", m.devices)}" : ""}
+%{ endfor ~}

terraform/modules/core/templates/security.toml.tftpl

@@ -0,0 +1,53 @@
+# SeaweedFS security.toml rendered by the Terraform core module.
+# Rendered when enable_security OR any non-default JWT signing option is set.
+# Presence of this file does NOT imply mTLS: the [grpc] TLS block is emitted
+# only under enable_security; a JWT-only config carries [jwt.*] and no TLS.
+
+%{ if enable_security ~}
+[grpc]
+ca = "${cert_dir}/ca/tls.crt"
+# Peer-identity authorization: without an allow-list, mTLS collapses to
+# "trusted the CA" with no authZ. Set allowed_wildcard_domain / commonNames.
+%{ if allowed_wildcard_domain != "" ~}
+allowed_wildcard_domain = "${allowed_wildcard_domain}"
+%{ endif ~}
+
+[grpc.master]
+cert = "${cert_dir}/master/tls.crt"
+key  = "${cert_dir}/master/tls.key"
+%{ if allowed_common_names != "" ~}
+allowed_commonNames = "${allowed_common_names}"
+%{ endif ~}
+
+[grpc.volume]
+cert = "${cert_dir}/volume/tls.crt"
+key  = "${cert_dir}/volume/tls.key"
+
+[grpc.filer]
+cert = "${cert_dir}/filer/tls.crt"
+key  = "${cert_dir}/filer/tls.key"
+
+[grpc.client]
+cert = "${cert_dir}/client/tls.crt"
+key  = "${cert_dir}/client/tls.key"
+%{ endif ~}
+%{ if jwt_signing_key != "" ~}
+
+[jwt.signing]
+key = "${jwt_signing_key}"
+%{ endif ~}
+%{ if jwt_signing_read_key != "" ~}
+
+[jwt.signing.read]
+key = "${jwt_signing_read_key}"
+%{ endif ~}
+%{ if jwt_filer_signing_key != "" ~}
+
+[jwt.filer_signing]
+key = "${jwt_filer_signing_key}"
+%{ endif ~}
+%{ if jwt_filer_signing_read_key != "" ~}
+
+[jwt.filer_signing.read]
+key = "${jwt_filer_signing_read_key}"
+%{ endif ~}

terraform/modules/core/templates/weed.service.tftpl

@@ -0,0 +1,39 @@
+[Unit]
+Description=SeaweedFS ${role} (${name})
+Documentation=https://github.com/seaweedfs/seaweedfs/wiki
+After=network-online.target
+Wants=network-online.target
+%{ for path in requires_mounts_for ~}
+RequiresMountsFor=${path}
+%{ endfor ~}
+
+[Service]
+Type=simple
+User=${run_as_user}
+Group=${run_as_user}
+ExecStart=${exec_start}
+Restart=always
+RestartSec=5
+LimitNOFILE=1048576
+TimeoutStopSec=60
+# Hardening (OpenShift SCC analogs; relax via the hardening variable)
+NoNewPrivileges=${no_new_privileges}
+%{ if protect_system ~}
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+%{ for path in read_write_paths ~}
+ReadWritePaths=${path}
+%{ endfor ~}
+%{ endif ~}
+%{ if cap_drop_all ~}
+CapabilityBoundingSet=
+AmbientCapabilities=
+%{ endif ~}
+EnvironmentFile=-${env_file}
+%{ for k, v in environment ~}
+Environment="${k}=${v}"
+%{ endfor ~}
+
+[Install]
+WantedBy=multi-user.target

terraform/modules/core/tests/render.tftest.hcl

@@ -0,0 +1,234 @@
+# Plan-level tests for the core renderer. No cloud credentials required.
+#   cd terraform/modules/core && tofu test
+
+variables {
+  weed_binary = "/usr/bin/weed"
+}
+
+# ---------------------------------------------------------------------------
+run "master_peers_and_count" {
+  command = plan
+  variables {
+    master = {
+      nodes = {
+        m0 = { address = "10.0.0.10" }
+        m1 = { address = "10.0.0.11" }
+        m2 = { address = "10.0.0.12" }
+      }
+    }
+    volume = { enabled = false }
+    filer  = { enabled = false }
+  }
+  assert {
+    condition     = output.master_peers == "10.0.0.10:9333,10.0.0.11:9333,10.0.0.12:9333"
+    error_message = "master -peers list is wrong"
+  }
+  assert {
+    condition     = length(output.node_names) == 3
+    error_message = "expected exactly 3 master nodes"
+  }
+}
+
+# ---------------------------------------------------------------------------
+run "volume_uses_mserver_not_master" {
+  command = plan
+  variables {
+    master = { nodes = { m0 = { address = "10.0.0.10" } } }
+    volume = { nodes = { v0 = { address = "10.0.0.20", rack = "rack-a", data_center = "dc1" } } }
+    filer  = { enabled = false }
+  }
+  assert {
+    condition     = contains(output.nodes["volume-v0"].argv, "-mserver=10.0.0.10:9333")
+    error_message = "volume must pass the master list via -mserver (verified flag), not -master"
+  }
+  assert {
+    condition     = !contains(output.nodes["volume-v0"].argv, "-master=10.0.0.10:9333")
+    error_message = "volume should not use -master for the master list"
+  }
+  assert {
+    condition     = contains(output.nodes["volume-v0"].argv, "-rack=rack-a") && contains(output.nodes["volume-v0"].argv, "-dataCenter=dc1")
+    error_message = "per-node rack/dataCenter must render"
+  }
+}
+
+# ---------------------------------------------------------------------------
+run "filer_uses_master_and_store_dir" {
+  command = plan
+  variables {
+    master = { nodes = { m0 = { address = "10.0.0.10" } } }
+    volume = { enabled = false }
+    filer  = { nodes = { f0 = { address = "10.0.0.30", data_dir = "/srv/filer" } } }
+  }
+  assert {
+    condition     = contains(output.nodes["filer-f0"].argv, "-master=10.0.0.10:9333")
+    error_message = "filer must use -master for the master list"
+  }
+  assert {
+    condition     = contains(output.nodes["filer-f0"].argv, "-defaultStoreDir=/srv/filer")
+    error_message = "filer leveldb2 store dir must render via -defaultStoreDir"
+  }
+}
+
+# ---------------------------------------------------------------------------
+run "metrics_gated_off_by_default" {
+  command = plan
+  variables {
+    master = { nodes = { m0 = { address = "10.0.0.10" } } }
+    volume = { enabled = false }
+    filer  = { enabled = false }
+  }
+  assert {
+    condition     = length([for a in output.nodes["master-m0"].argv : a if startswith(a, "-metricsPort")]) == 0
+    error_message = "metrics port must NOT be bound when monitoring is disabled"
+  }
+}
+
+run "metrics_on_when_enabled" {
+  command = plan
+  variables {
+    monitoring_enabled = true
+    master             = { nodes = { m0 = { address = "10.0.0.10" } }, metrics_port = 9327 }
+    volume             = { enabled = false }
+    filer              = { enabled = false }
+  }
+  assert {
+    condition     = contains(output.nodes["master-m0"].argv, "-metricsPort=9327")
+    error_message = "metrics port must bind when monitoring is enabled"
+  }
+}
+
+# ---------------------------------------------------------------------------
+run "security_toml_off_by_default" {
+  command = plan
+  variables {
+    master = { nodes = { m0 = { address = "10.0.0.10" } } }
+    volume = { enabled = false }
+    filer  = { enabled = false }
+  }
+  assert {
+    condition     = length(keys(output.nodes["master-m0"].config_files)) == 0
+    error_message = "security.toml must not render when security is off and no JWT keys set"
+  }
+}
+
+run "security_toml_on_with_mtls" {
+  command = plan
+  variables {
+    enable_security = true
+    security = {
+      allowed_wildcard_domain = ".seaweedfs.internal"
+      jwt_signing_key         = "test-signing-key-not-a-real-secret-0123456789"
+    }
+    master = { nodes = { m0 = { address = "10.0.0.10" } } }
+    volume = { enabled = false }
+    filer  = { enabled = false }
+  }
+  assert {
+    condition     = contains(keys(output.nodes["master-m0"].config_files), "/etc/seaweedfs/security.toml")
+    error_message = "security.toml must render when enable_security is true"
+  }
+}
+
+# ---------------------------------------------------------------------------
+run "all_in_one_renders_server" {
+  command = plan
+  variables {
+    master = { enabled = false }
+    volume = { enabled = false }
+    filer  = { enabled = false }
+    all_in_one = {
+      enabled = true
+      nodes   = { a0 = { address = "10.0.0.40" } }
+      s3      = { enabled = true }
+    }
+  }
+  assert {
+    condition     = contains(output.nodes["all-in-one-a0"].argv, "server")
+    error_message = "all-in-one must invoke the weed server subcommand"
+  }
+  assert {
+    condition     = contains(output.nodes["all-in-one-a0"].argv, "-s3") && contains(output.nodes["all-in-one-a0"].argv, "-s3.port=8333")
+    error_message = "all-in-one S3 must render when enabled"
+  }
+}
+
+# ---------------------------------------------------------------------------
+run "security_toml_mtls_only_no_jwt" {
+  command = plan
+  variables {
+    enable_security = true
+    master          = { nodes = { m0 = { address = "10.0.0.10" } } }
+    volume          = { enabled = false }
+    filer           = { enabled = false }
+  }
+  assert {
+    condition     = contains(keys(output.nodes["master-m0"].config_files), "/etc/seaweedfs/security.toml")
+    error_message = "security.toml must render for mTLS even without JWT keys"
+  }
+  assert {
+    condition     = !strcontains(output.nodes["master-m0"].config_files["/etc/seaweedfs/security.toml"], "[jwt.signing]")
+    error_message = "the jwt.signing block must be omitted when no signing key is set"
+  }
+}
+
+run "mount_fetch_and_secret_delivery" {
+  command = plan
+  variables {
+    enable_security     = true
+    render_secret_files = false
+    boot_fetch_script   = "#!/usr/bin/env bash\necho fetch\n"
+    security = {
+      jwt_signing_key = "test-signing-key-not-a-real-secret-0123456789"
+    }
+    master = { nodes = { m0 = { address = "10.0.0.10", disk_mounts = [{ mountpoint = "/data", fstype = "xfs" }] } } }
+    volume = { enabled = false }
+    filer  = { enabled = false }
+  }
+  assert {
+    condition     = strcontains(output.nodes["master-m0"].cloud_init, "/opt/seaweedfs/mount-disks.sh")
+    error_message = "cloud-init must write+run the mount script when disk_mounts is set"
+  }
+  assert {
+    condition     = strcontains(output.nodes["master-m0"].cloud_init, "/opt/seaweedfs/fetch-secrets.sh")
+    error_message = "cloud-init must write+run the fetch script when boot_fetch_script is set"
+  }
+  assert {
+    condition     = !strcontains(output.nodes["master-m0"].cloud_init, "[jwt.signing]")
+    error_message = "security.toml must NOT be inlined in cloud-init when render_secret_files=false"
+  }
+  assert {
+    condition     = contains(keys(output.nodes["master-m0"].secret_files), "/etc/seaweedfs/security.toml")
+    error_message = "secret_files must expose security.toml for secret-store delivery"
+  }
+}
+
+run "mount_script_absent_without_disks" {
+  command = plan
+  variables {
+    master = { nodes = { m0 = { address = "10.0.0.10" } } }
+    volume = { enabled = false }
+    filer  = { enabled = false }
+  }
+  assert {
+    condition     = !strcontains(output.nodes["master-m0"].cloud_init, "mount-disks.sh")
+    error_message = "no mount script should render when the node declares no disk_mounts"
+  }
+}
+
+run "s3_standalone_targets_filer" {
+  command = plan
+  variables {
+    master = { nodes = { m0 = { address = "10.0.0.10" } } }
+    volume = { enabled = false }
+    filer  = { nodes = { f0 = { address = "10.0.0.30" } } }
+    s3     = { enabled = true, nodes = { s0 = { address = "10.0.0.50" } } }
+  }
+  assert {
+    condition     = contains(output.nodes["s3-s0"].argv, "-filer=10.0.0.30:8888")
+    error_message = "standalone S3 must point -filer at the first filer node"
+  }
+  assert {
+    condition     = contains(output.nodes["s3-s0"].argv, "-config=/etc/seaweedfs/s3_config.json")
+    error_message = "standalone S3 must reference the rendered config path"
+  }
+}

terraform/modules/core/variables.tf

@@ -0,0 +1,318 @@
+# =============================================================================
+# SeaweedFS Terraform core module - input variables
+#
+# This module is a PURE RENDERER. It creates zero cloud resources. It takes a
+# per-node address map plus configuration and emits, per node, the rendered
+# `weed` argv, systemd unit, cloud-init, and config files. Both the per-cloud
+# infra wrappers (consume cloud_init/systemd_unit) and the local test harness
+# (consume argv/config_files) read its `nodes` output.
+#
+# Flag names below are verified against the real `weed` binary, not the chart.
+# Notably: volume uses -mserver for the master list; gRPC is -port.grpc
+# (auto = http+10000); minFreeSpacePercent is a string.
+# =============================================================================
+
+variable "weed_binary" {
+  description = "Path to the weed executable on the target host."
+  type        = string
+  default     = "/usr/bin/weed"
+}
+
+variable "cluster_name" {
+  description = "Logical cluster name (WEED_CLUSTER_DEFAULT)."
+  type        = string
+  default     = "sw"
+}
+
+variable "log_level" {
+  description = "Global glog verbosity (-v)."
+  type        = number
+  default     = 1
+}
+
+variable "log_to_stderr" {
+  description = "Log to stderr (journald) instead of -logdir. Recommended on systemd."
+  type        = bool
+  default     = true
+}
+
+variable "monitoring_enabled" {
+  description = "Gate the -metricsPort/-metrics.address flags. When false the metrics port is not bound, matching the chart."
+  type        = bool
+  default     = false
+}
+
+variable "enable_security" {
+  description = "Emit the security.toml [grpc] mTLS block and reference per-component certs. Independent of JWT."
+  type        = bool
+  default     = false
+}
+
+# ----------------------------------------------------------------------------
+# Security / secrets (the core only references paths + renders security.toml;
+# cert/secret material is generated by the wrapper, never defaulted here).
+# ----------------------------------------------------------------------------
+variable "security" {
+  description = "mTLS + JWT signing configuration. Secret values must be supplied (never defaulted in module code)."
+  type = object({
+    cert_dir                   = optional(string, "/usr/local/share/ca-certificates")
+    allowed_wildcard_domain    = optional(string, "")
+    allowed_common_names       = optional(string, "")
+    jwt_signing_key            = optional(string, "")
+    jwt_signing_read_key       = optional(string, "")
+    jwt_filer_signing_key      = optional(string, "")
+    jwt_filer_signing_read_key = optional(string, "")
+  })
+  default   = {}
+  sensitive = true
+}
+
+variable "hardening" {
+  description = "systemd hardening directives (OpenShift SCC analogs). Relax per field if needed."
+  type = object({
+    run_as_user       = optional(string, "seaweedfs")
+    no_new_privileges = optional(bool, true)
+    protect_system    = optional(bool, true)
+    cap_drop_all      = optional(bool, true)
+  })
+  default = {}
+}
+
+variable "env_file" {
+  description = "Path to the systemd EnvironmentFile holding secret env (DB creds, S3 admin key) fetched at boot. Optional (-prefixed in the unit, so a missing file is tolerated)."
+  type        = string
+  default     = "/etc/seaweedfs/weed.env"
+}
+
+variable "render_secret_files" {
+  description = <<-EOT
+    When true (default), secret-bearing config (security.toml, S3 identity JSON)
+    is written directly into cloud-init user_data. Convenient for the local test
+    harness and simple deployments. Set to false to keep secrets OUT of user_data
+    (and out of the cloud metadata service): the files are then exposed only via
+    the secret_files_by_node output, and the wrapper is expected to deliver them
+    from a secret store via boot_fetch_script.
+  EOT
+  type        = bool
+  default     = true
+}
+
+variable "boot_fetch_script" {
+  description = "Optional shell script written to /opt/seaweedfs/fetch-secrets.sh (root, 0755) and run before the weed unit starts. Used by wrappers to pull certs/secrets from a cloud secret store at boot."
+  type        = string
+  default     = ""
+}
+
+# ----------------------------------------------------------------------------
+# Master (Raft quorum). Keyed `nodes` map -> for_each, never count.
+# ----------------------------------------------------------------------------
+variable "master" {
+  description = "Master tier. nodes is a map keyed by stable node id (m0/m1/m2)."
+  type = object({
+    enabled = optional(bool, true)
+    nodes = optional(map(object({
+      address   = string
+      port      = optional(number)
+      grpc_port = optional(number)
+      data_dir  = optional(string)
+      disk_mounts = optional(list(object({
+        mountpoint = string
+        fstype     = optional(string, "ext4")
+        devices    = optional(list(string), [])
+      })), [])
+    })), {})
+    port                 = optional(number, 9333)
+    grpc_port            = optional(number, null) # null => weed auto = port+10000
+    metrics_port         = optional(number, 9327)
+    metrics_ip           = optional(string, "")
+    ip_bind              = optional(string, "0.0.0.0")
+    data_dir             = optional(string, "/var/lib/seaweedfs/master")
+    default_replication  = optional(string, "000")
+    volume_size_limit_mb = optional(number, 1000)
+    volume_preallocate   = optional(bool, false)
+    garbage_threshold    = optional(number, null)
+    disable_http         = optional(bool, false)
+    raft_hashicorp       = optional(bool, false)
+    resume_state         = optional(bool, true) # deliberate deviation: binary default false
+    election_timeout     = optional(string, "10s")
+    heartbeat_interval   = optional(string, "300ms")
+    white_list           = optional(string, "")
+    extra_args           = optional(list(string), [])
+  })
+  default = {}
+}
+
+# ----------------------------------------------------------------------------
+# Volume (stateful disk owners). Keyed `nodes` map.
+# ----------------------------------------------------------------------------
+variable "volume" {
+  description = "Volume tier. Each node owns one or more data dirs (-> -dir / -max)."
+  type = object({
+    enabled = optional(bool, true)
+    nodes = optional(map(object({
+      address     = string
+      port        = optional(number)
+      grpc_port   = optional(number)
+      public_url  = optional(string)
+      rack        = optional(string)
+      data_center = optional(string)
+      data_dirs = optional(list(object({
+        path        = string
+        max_volumes = optional(number, 0) # 0 => auto on non-windows
+      })))
+      idx_dir = optional(string)
+      disk_mounts = optional(list(object({
+        mountpoint = string
+        fstype     = optional(string, "ext4")
+        devices    = optional(list(string), [])
+      })), [])
+    })), {})
+    port         = optional(number, 8080)
+    metrics_port = optional(number, 9327)
+    metrics_ip   = optional(string, "")
+    ip_bind      = optional(string, "0.0.0.0")
+    data_dirs = optional(list(object({
+      path        = string
+      max_volumes = optional(number, 0)
+    })), [{ path = "/data", max_volumes = 0 }])
+    idx_dir                = optional(string, "")
+    read_mode              = optional(string, "proxy")
+    compaction_mbps        = optional(number, 50)
+    min_free_space_percent = optional(string, "1")
+    file_size_limit_mb     = optional(number, null)
+    index                  = optional(string, "memory")
+    images_fix_orientation = optional(bool, false)
+    white_list             = optional(string, "")
+    extra_args             = optional(list(string), [])
+  })
+  default = {}
+}
+
+# ----------------------------------------------------------------------------
+# Filer (+ optional embedded S3). Keyed `nodes` map.
+# ----------------------------------------------------------------------------
+variable "filer" {
+  description = "Filer tier. Default leveldb2-replicated HA (each filer has its own store)."
+  type = object({
+    enabled = optional(bool, true)
+    nodes = optional(map(object({
+      address   = string
+      port      = optional(number)
+      grpc_port = optional(number)
+      data_dir  = optional(string)
+      disk_mounts = optional(list(object({
+        mountpoint = string
+        fstype     = optional(string, "ext4")
+        devices    = optional(list(string), [])
+      })), [])
+    })), {})
+    port                      = optional(number, 8888)
+    metrics_port              = optional(number, 9327)
+    metrics_ip                = optional(string, "")
+    ip_bind                   = optional(string, "0.0.0.0")
+    data_dir                  = optional(string, "/var/lib/seaweedfs/filer") # -defaultStoreDir for leveldb2
+    default_replica_placement = optional(string, "000")
+    dir_list_limit            = optional(number, 100000)
+    max_mb                    = optional(number, null)
+    disable_dir_listing       = optional(bool, false)
+    disable_http              = optional(bool, false)
+    encrypt_volume_data       = optional(bool, false)
+    rack                      = optional(string, "")
+    data_center               = optional(string, "")
+    filer_group               = optional(string, "")
+    extra_env                 = optional(map(string), {})
+    extra_args                = optional(list(string), [])
+    s3 = optional(object({
+      enabled     = optional(bool, false)
+      port        = optional(number, 8333)
+      https_port  = optional(number, 0)
+      domain_name = optional(string, "")
+      config_path = optional(string, "/etc/seaweedfs/s3_config.json")
+    }), {})
+  })
+  default = {}
+}
+
+# ----------------------------------------------------------------------------
+# S3 gateway (standalone). Keyed `nodes` map (stateless, but kept keyed for
+# parity; wrappers may instead drive an ASG).
+# ----------------------------------------------------------------------------
+variable "s3" {
+  description = "Standalone S3 gateway tier."
+  type = object({
+    enabled = optional(bool, false)
+    nodes = optional(map(object({
+      address   = string
+      port      = optional(number)
+      grpc_port = optional(number)
+    })), {})
+    port                  = optional(number, 8333)
+    https_port            = optional(number, 0)
+    iceberg_port          = optional(number, null)
+    metrics_port          = optional(number, 9327)
+    ip_bind               = optional(string, "0.0.0.0")
+    domain_name           = optional(string, "")
+    filer_address         = optional(string, "") # override; else first filer node
+    config_path           = optional(string, "/etc/seaweedfs/s3_config.json")
+    audit_log_config_path = optional(string, "")
+    cert_file             = optional(string, "")
+    key_file              = optional(string, "")
+    cacert_file           = optional(string, "")
+    verify_client_cert    = optional(bool, false)
+    extra_args            = optional(list(string), [])
+  })
+  default = {}
+}
+
+# ----------------------------------------------------------------------------
+# S3 identities -> rendered into the s3 config JSON (non-admin only; the admin
+# credential is delivered via EnvironmentFile, never the JSON).
+# ----------------------------------------------------------------------------
+variable "s3_identities" {
+  description = "Non-admin S3 identities rendered into the config JSON. Admin key goes in the EnvironmentFile, not here."
+  type = list(object({
+    name       = string
+    access_key = string
+    secret_key = string
+    actions    = list(string)
+  }))
+  default   = []
+  sensitive = true
+}
+
+# ----------------------------------------------------------------------------
+# All-in-one (single `weed server` process).
+# ----------------------------------------------------------------------------
+variable "all_in_one" {
+  description = "All-in-one tier (weed server). Mutually exclusive with the distributed tiers."
+  type = object({
+    enabled = optional(bool, false)
+    nodes = optional(map(object({
+      address  = string
+      data_dir = optional(string)
+      disk_mounts = optional(list(object({
+        mountpoint = string
+        fstype     = optional(string, "ext4")
+        devices    = optional(list(string), [])
+      })), [])
+    })), {})
+    data_dir             = optional(string, "/data")
+    ip_bind              = optional(string, "0.0.0.0")
+    master_port          = optional(number, 9333)
+    volume_port          = optional(number, 8080)
+    filer_port           = optional(number, 8888)
+    default_replication  = optional(string, "000")
+    volume_size_limit_mb = optional(number, 1000)
+    idle_timeout         = optional(number, 30)
+    metrics_port         = optional(number, 9324)
+    disable_http         = optional(bool, false)
+    s3 = optional(object({
+      enabled     = optional(bool, false)
+      port        = optional(number, 8333)
+      config_path = optional(string, "/etc/seaweedfs/s3/s3_config.json")
+      domain_name = optional(string, "")
+    }), {})
+    extra_args = optional(list(string), [])
+  })
+  default = {}
+}

terraform/modules/core/versions.tf

@@ -0,0 +1,6 @@
+terraform {
+  # 1.3+ is required for optional() object attributes with defaults.
+  # The published wrappers pin a single honest floor;
+  # the core itself only needs language features available since 1.3.
+  required_version = ">= 1.3.0"
+}

terraform/modules/security/main.tf

@@ -0,0 +1,68 @@
+locals {
+  components = ["master", "volume", "filer", "client"]
+  ip_sans    = distinct(concat(["127.0.0.1"], var.ip_sans))
+}
+
+# ---- CA ---------------------------------------------------------------------
+resource "tls_private_key" "ca" {
+  algorithm   = var.key_algorithm
+  ecdsa_curve = var.ecdsa_curve
+  rsa_bits    = var.rsa_bits
+}
+
+resource "tls_self_signed_cert" "ca" {
+  private_key_pem   = tls_private_key.ca.private_key_pem
+  is_ca_certificate = true
+
+  subject {
+    common_name  = "SeaweedFS CA"
+    organization = "SeaweedFS"
+  }
+
+  validity_period_hours = var.ca_validity_hours
+  allowed_uses          = ["cert_signing", "crl_signing", "digital_signature"]
+}
+
+# ---- per-component leaf certs (distinct CN per component) --------------------
+resource "tls_private_key" "component" {
+  for_each    = toset(local.components)
+  algorithm   = var.key_algorithm
+  ecdsa_curve = var.ecdsa_curve
+  rsa_bits    = var.rsa_bits
+}
+
+resource "tls_cert_request" "component" {
+  for_each        = toset(local.components)
+  private_key_pem = tls_private_key.component[each.key].private_key_pem
+
+  subject {
+    # Distinct CN per component so peer-identity authZ (allowed_wildcard_domain
+    # / allowed_commonNames) is meaningful. Do NOT use one CN for all.
+    common_name  = "${each.key}.${var.internal_domain}"
+    organization = "SeaweedFS"
+  }
+
+  dns_names = distinct(concat(
+    ["${each.key}.${var.internal_domain}", "localhost"],
+    var.extra_dns_sans,
+  ))
+  ip_addresses = local.ip_sans
+}
+
+resource "tls_locally_signed_cert" "component" {
+  for_each           = toset(local.components)
+  cert_request_pem   = tls_cert_request.component[each.key].cert_request_pem
+  ca_private_key_pem = tls_private_key.ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.ca.cert_pem
+
+  validity_period_hours = var.cert_validity_hours
+  early_renewal_hours   = var.cert_early_renewal_hours
+  allowed_uses          = ["digital_signature", "key_agreement", "server_auth", "client_auth"]
+}
+
+# ---- JWT signing keys -------------------------------------------------------
+resource "random_password" "jwt" {
+  for_each = var.generate_jwt ? toset(["signing", "signing_read", "filer_signing", "filer_signing_read"]) : toset([])
+  length   = var.jwt_length
+  special  = false # TOML-safe
+}

terraform/modules/security/outputs.tf

@@ -0,0 +1,65 @@
+output "ca_cert_pem" {
+  description = "CA certificate PEM."
+  value       = tls_self_signed_cert.ca.cert_pem
+}
+
+output "certs" {
+  description = "Per-component { cert_pem, private_key_pem }."
+  value = {
+    for c in local.components : c => {
+      cert_pem        = tls_locally_signed_cert.component[c].cert_pem
+      private_key_pem = tls_private_key.component[c].private_key_pem
+    }
+  }
+  sensitive = true
+}
+
+output "jwt" {
+  description = "Generated JWT keys (empty strings when generate_jwt = false)."
+  value = {
+    signing            = var.generate_jwt ? random_password.jwt["signing"].result : ""
+    signing_read       = var.generate_jwt ? random_password.jwt["signing_read"].result : ""
+    filer_signing      = var.generate_jwt ? random_password.jwt["filer_signing"].result : ""
+    filer_signing_read = var.generate_jwt ? random_password.jwt["filer_signing_read"].result : ""
+  }
+  sensitive = true
+}
+
+# Ready to pass straight to the core module's `security` variable.
+output "core_security" {
+  description = "Object shaped for the core module's `security` input (cert_dir, allowed_wildcard_domain, JWT keys)."
+  value = {
+    cert_dir                   = var.cert_dir
+    allowed_wildcard_domain    = ".${var.internal_domain}"
+    allowed_common_names       = ""
+    jwt_signing_key            = var.generate_jwt ? random_password.jwt["signing"].result : ""
+    jwt_signing_read_key       = var.generate_jwt ? random_password.jwt["signing_read"].result : ""
+    jwt_filer_signing_key      = var.generate_jwt ? random_password.jwt["filer_signing"].result : ""
+    jwt_filer_signing_read_key = var.generate_jwt ? random_password.jwt["filer_signing_read"].result : ""
+  }
+  sensitive = true
+}
+
+# Non-sensitive manifest (key, on-host path, mode) so a wrapper can for_each
+# over SSM/secret-store resources without tripping the sensitive-for_each rule.
+output "secret_manifest" {
+  description = "List of { key, path, mode } for the CA + component cert/key files (no content)."
+  value = concat(
+    [{ key = "ca/tls.crt", path = "${var.cert_dir}/ca/tls.crt", mode = "0644" }],
+    flatten([for c in local.components : [
+      { key = "${c}/tls.crt", path = "${var.cert_dir}/${c}/tls.crt", mode = "0644" },
+      { key = "${c}/tls.key", path = "${var.cert_dir}/${c}/tls.key", mode = "0600" },
+    ]]),
+  )
+}
+
+# Sensitive map keyed by the manifest key -> file content.
+output "secret_contents" {
+  description = "Map of manifest key -> PEM content."
+  value = merge(
+    { "ca/tls.crt" = tls_self_signed_cert.ca.cert_pem },
+    { for c in local.components : "${c}/tls.crt" => tls_locally_signed_cert.component[c].cert_pem },
+    { for c in local.components : "${c}/tls.key" => tls_private_key.component[c].private_key_pem },
+  )
+  sensitive = true
+}

terraform/modules/security/variables.tf

@@ -0,0 +1,89 @@
+# =============================================================================
+# SeaweedFS security material generator (cloud-agnostic).
+#
+# Generates the CA, per-component mTLS certs with DISTINCT CommonNames, and JWT
+# signing keys. Outputs a `core_security` object ready to feed the core module's
+# `security` variable, plus the raw PEMs for a wrapper to deliver via a secret
+# store. NOTE: TF-generated secrets live in state; prefer Vault PKI for the CA
+# in production.
+# =============================================================================
+
+variable "internal_domain" {
+  description = "Internal domain for component CNs (e.g. master.<domain>). Drives the allowed_wildcard_domain peer-auth check."
+  type        = string
+  default     = "seaweedfs.internal"
+}
+
+variable "ip_sans" {
+  description = "IP addresses to include as SANs on every component cert (all node private IPs + 127.0.0.1 is added automatically)."
+  type        = list(string)
+  default     = []
+}
+
+variable "extra_dns_sans" {
+  description = "Additional DNS SANs to include on every component cert."
+  type        = list(string)
+  default     = []
+}
+
+variable "key_algorithm" {
+  description = "CA/leaf key algorithm: ECDSA (default, modern) or RSA."
+  type        = string
+  default     = "ECDSA"
+  validation {
+    condition     = contains(["ECDSA", "RSA"], var.key_algorithm)
+    error_message = "key_algorithm must be ECDSA or RSA."
+  }
+}
+
+variable "ecdsa_curve" {
+  description = "ECDSA curve when key_algorithm = ECDSA."
+  type        = string
+  default     = "P256"
+}
+
+variable "rsa_bits" {
+  description = "RSA key size when key_algorithm = RSA."
+  type        = number
+  default     = 3072
+}
+
+variable "ca_validity_hours" {
+  description = "CA certificate validity (default 10 years)."
+  type        = number
+  default     = 87600
+}
+
+variable "cert_validity_hours" {
+  description = "Component certificate validity. Short by default; certs hot-reload so reissue is cheap."
+  type        = number
+  default     = 72
+}
+
+variable "cert_early_renewal_hours" {
+  description = "Renew component certs this many hours before expiry."
+  type        = number
+  default     = 24
+}
+
+variable "cert_dir" {
+  description = "On-host directory where certs are placed; surfaced in core_security.cert_dir."
+  type        = string
+  default     = "/usr/local/share/ca-certificates"
+}
+
+variable "generate_jwt" {
+  description = "Generate JWT signing keys (signing, signing.read, filer_signing, filer_signing.read)."
+  type        = bool
+  default     = true
+}
+
+variable "jwt_length" {
+  description = "Length of generated JWT keys (>= 32 hardened)."
+  type        = number
+  default     = 40
+  validation {
+    condition     = var.jwt_length >= 32
+    error_message = "jwt_length must be >= 32 (hardened minimum)."
+  }
+}

terraform/modules/security/versions.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_version = ">= 1.3.0"
+  required_providers {
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = ">= 3.5"
+    }
+  }
+}

terraform/test/local-secure/main.tf

@@ -0,0 +1,74 @@
+# =============================================================================
+# Local mTLS test harness: generate the CA + component certs + JWT keys with the
+# security submodule, render security.toml with the core, then run a real
+# master+volume+filer cluster with mTLS enabled on 127.0.0.1 and assert it works.
+# Proves the generated security material is valid and accepted by weed.
+# =============================================================================
+
+terraform {
+  required_version = ">= 1.3.0"
+}
+
+variable "weed_binary" {
+  type    = string
+  default = "/usr/bin/weed"
+}
+
+variable "workdir" {
+  type    = string
+  default = "/tmp/seaweedfs-tftest-secure"
+}
+
+module "security" {
+  source          = "../../modules/security"
+  internal_domain = "seaweedfs.internal"
+  ip_sans         = ["127.0.0.1"]
+  cert_dir        = "${var.workdir}/certs"
+}
+
+module "core" {
+  source          = "../../modules/core"
+  weed_binary     = var.weed_binary
+  enable_security = true
+  security        = module.security.core_security
+  # We read security.toml back from the output and place it ourselves (under
+  # $HOME/.seaweedfs), so keep it in the rendered file set.
+  render_secret_files = true
+
+  # High, distinct ports so this never collides with a running SeaweedFS or the
+  # non-secure harness.
+  master = {
+    nodes              = { m0 = { address = "127.0.0.1", port = 29336, data_dir = "${var.workdir}/master-m0" } }
+    election_timeout   = "3s"
+    heartbeat_interval = "200ms"
+  }
+  volume = {
+    nodes = { v0 = { address = "127.0.0.1", port = 28081, data_dirs = [{ path = "${var.workdir}/volume-v0", max_volumes = 20 }] } }
+  }
+  filer = {
+    nodes = { f0 = { address = "127.0.0.1", port = 28889, data_dir = "${var.workdir}/filer-f0" } }
+  }
+}
+
+output "cluster" {
+  sensitive = true
+  value = {
+    for name, n in module.core.nodes : name => {
+      role      = n.role
+      http_port = n.ports.http
+      data_dirs = n.data_dirs
+      argv      = n.argv
+      env       = n.env
+    }
+  }
+}
+
+output "security_toml" {
+  sensitive = true
+  value     = module.core.secret_files_by_node["master-m0"]["/etc/seaweedfs/security.toml"]
+}
+
+output "certs" {
+  sensitive = true
+  value     = [for m in module.security.secret_manifest : { path = m.path, mode = m.mode, content = module.security.secret_contents[m.key] }]
+}

terraform/test/local-secure/run_local_secure.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+# Generate mTLS material + security.toml with Terraform, then run a real
+# master+volume+filer cluster with mTLS enabled and assert it works.
+#   ./run_local_secure.sh        # render + run + assert + teardown
+#   KEEP=1 ./run_local_secure.sh
+set -u
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+export PATH="/opt/homebrew/bin:$PATH"
+TOFU="${TOFU:-tofu}"
+WEED="${WEED:-$(go env GOPATH 2>/dev/null || echo "$HOME/go")/bin/weed}"
+WORKDIR="${WORKDIR:-/tmp/seaweedfs-tftest-secure}"
+LOGDIR="$WORKDIR/logs"
+RUNDIR="$WORKDIR/run"
+
+PASS=0
+FAIL=0
+ok()   { echo "  PASS: $1"; PASS=$((PASS + 1)); }
+bad()  { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }
+info() { echo "==> $1"; }
+
+cleanup() {
+  info "tearing down"
+  if [ -d "$RUNDIR" ]; then
+    for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill "$(cat "$pf")" 2>/dev/null; done
+    sleep 1
+    for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill -9 "$(cat "$pf")" 2>/dev/null; done
+  fi
+}
+
+info "cleaning $WORKDIR"
+case "$WORKDIR" in "" | "/" | "$HOME") echo "refusing to delete '$WORKDIR'" >&2; exit 2 ;; esac
+rm -rf "$WORKDIR"
+mkdir -p "$LOGDIR" "$RUNDIR" "$WORKDIR/.seaweedfs"
+[ -x "$WEED" ] || { echo "weed not found at $WEED" >&2; exit 2; }
+
+info "generating certs + rendering config with OpenTofu"
+cd "$HERE"
+"$TOFU" init -backend=false -input=false -no-color >/dev/null 2>&1 || { echo "tofu init failed"; exit 2; }
+if ! "$TOFU" apply -auto-approve -input=false -no-color \
+       -var "weed_binary=$WEED" -var "workdir=$WORKDIR" >"$LOGDIR/tofu.log" 2>&1; then
+  echo "tofu apply failed"; tail -30 "$LOGDIR/tofu.log"; exit 2
+fi
+
+# write certs to their on-host paths
+"$TOFU" output -json certs | jq -c '.[]' | while IFS= read -r c; do
+  p="$(echo "$c" | jq -r '.path')"; m="$(echo "$c" | jq -r '.mode')"
+  mkdir -p "$(dirname "$p")"
+  echo "$c" | jq -r '.content' > "$p"
+  chmod "$m" "$p"
+done
+# place security.toml where weed searches ($HOME/.seaweedfs)
+"$TOFU" output -raw security_toml > "$WORKDIR/.seaweedfs/security.toml"
+info "security.toml + $(ls "$WORKDIR"/certs | wc -l | tr -d ' ') cert dirs written"
+
+OUT="$("$TOFU" output -json cluster)"
+
+# derive ports from the rendered config (high range; never hardcoded)
+port_of() { echo "$OUT" | jq -r --arg r "$1" '.[] | select(.role==$r) | .http_port' | head -1; }
+MPORT="$(port_of master)"; VPORT="$(port_of volume)"; FPORT="$(port_of filer)"
+
+busy=""
+for p in $(echo "$OUT" | jq -r '.[].http_port'); do
+  lsof -nP -iTCP:"$p" -sTCP:LISTEN >/dev/null 2>&1 && busy="$busy $p"
+done
+[ -n "$busy" ] && { echo "Required port(s) already in use:$busy -- is another SeaweedFS running?" >&2; exit 3; }
+
+[ "${KEEP:-0}" = "1" ] || trap cleanup EXIT INT TERM
+
+# launch a node with HOME pointed at $WORKDIR so weed loads security.toml
+launch() {
+  n="$1"
+  for d in $(echo "$OUT" | jq -r --arg n "$n" '.[$n].data_dirs[]?'); do mkdir -p "$d"; done
+  ENVS=("HOME=$WORKDIR")
+  while IFS= read -r e; do [ -n "$e" ] && ENVS+=("$e"); done \
+    < <(echo "$OUT" | jq -r --arg n "$n" '.[$n].env | to_entries[] | "\(.key)=\(.value)"')
+  ARGV=()
+  while IFS= read -r a; do ARGV+=("$a"); done \
+    < <(echo "$OUT" | jq -r --arg n "$n" '.[$n].argv[]')
+  info "launching $n (mTLS)"
+  env "${ENVS[@]}" "$WEED" "${ARGV[@]}" >"$LOGDIR/$n.log" 2>&1 &
+  echo "$!" > "$RUNDIR/$n.pid"
+}
+
+wait_http() {
+  url="$1"; t="${2:-30}"; i=0
+  while [ "$i" -lt "$t" ]; do
+    curl -fsS -o /dev/null --max-time 2 "$url" 2>/dev/null && return 0
+    i=$((i + 1)); sleep 1
+  done
+  return 1
+}
+
+launch master-m0
+QOK=0; i=0
+while [ "$i" -lt 40 ]; do
+  st="$(curl -fsS --max-time 2 "http://127.0.0.1:$MPORT/cluster/status" 2>/dev/null)" || { i=$((i+1)); sleep 1; continue; }
+  [ "$(echo "$st" | jq -r '.IsLeader // false')" = "true" ] && { QOK=1; break; }
+  i=$((i + 1)); sleep 1
+done
+[ "$QOK" -eq 1 ] && ok "master elected leader under mTLS" || bad "master leader (mTLS)"
+
+launch volume-v0
+# registration (master gRPC over mTLS) confirms the volume joined the cluster
+AOK=0; i=0
+while [ "$i" -lt 40 ]; do
+  fid="$(curl -fsS --max-time 2 "http://127.0.0.1:$MPORT/dir/assign" 2>/dev/null | jq -r '.fid // ""')"
+  [ -n "$fid" ] && { AOK=1; break; }
+  i=$((i + 1)); sleep 1
+done
+[ "$AOK" -eq 1 ] && ok "volume registered via mTLS gRPC (fid $fid)" || bad "volume registration over mTLS"
+# /healthz flips to 200 after the first heartbeat completes (slower under mTLS)
+wait_http "http://127.0.0.1:$VPORT/healthz" 60 && ok "volume /healthz up (mTLS)" || bad "volume /healthz"
+
+launch filer-f0
+wait_http "http://127.0.0.1:$FPORT/" 30 && ok "filer / up (mTLS)" || bad "filer /"
+# [jwt.filer_signing] is active, so the filer requires a signed JWT for writes.
+# An unsigned write MUST be rejected with 401 -- this proves the JWT signing key
+# rendered into security.toml is enforced (positive security assertion).
+echo "smoke" > "$WORKDIR/hello.txt"
+code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -X POST \
+  -F "file=@$WORKDIR/hello.txt" "http://127.0.0.1:$FPORT/smoke/hello.txt" 2>/dev/null)"
+[ "$code" = "401" ] && ok "filer rejects unsigned write (HTTP 401) => JWT signing enforced" \
+  || bad "filer JWT enforcement (expected 401, got HTTP $code)"
+
+echo
+info "RESULTS: $PASS passed, $FAIL failed"
+[ "$FAIL" -eq 0 ]

terraform/test/local/main.tf

@@ -0,0 +1,100 @@
+# =============================================================================
+# Local test harness: render a small SeaweedFS cluster with the core module
+# and run it as real `weed` processes on 127.0.0.1 (no cloud, no docker).
+#
+# 3 masters (quorum) + 1 volume + 1 filer + 1 standalone S3, each on a distinct
+# port. run_local_cluster.sh consumes the `cluster` output, launches the weed
+# processes from the rendered argv, and asserts the cluster actually works.
+# =============================================================================
+
+terraform {
+  required_version = ">= 1.3.0"
+}
+
+variable "weed_binary" {
+  description = "Path to the weed executable used for the local cluster."
+  type        = string
+  default     = "/usr/bin/weed"
+}
+
+variable "workdir" {
+  description = "Scratch directory for per-node data dirs and config files."
+  type        = string
+  default     = "/tmp/seaweedfs-tftest"
+}
+
+module "core" {
+  source = "../../modules/core"
+
+  weed_binary        = var.weed_binary
+  monitoring_enabled = false
+  enable_security    = false
+
+  # High port range so the harness does not collide with a SeaweedFS cluster
+  # that may already be running on this dev machine (default 9333/8080/8888/8333).
+  master = {
+    nodes = {
+      m0 = { address = "127.0.0.1", port = 29333, data_dir = "${var.workdir}/master-m0" }
+      m1 = { address = "127.0.0.1", port = 29334, data_dir = "${var.workdir}/master-m1" }
+      m2 = { address = "127.0.0.1", port = 29335, data_dir = "${var.workdir}/master-m2" }
+    }
+    # Tighten election so a 3-node local quorum converges quickly.
+    election_timeout   = "3s"
+    heartbeat_interval = "200ms"
+  }
+
+  volume = {
+    nodes = {
+      v0 = {
+        address   = "127.0.0.1"
+        port      = 28080
+        rack      = "rack-a"
+        data_dirs = [{ path = "${var.workdir}/volume-v0", max_volumes = 20 }]
+      }
+    }
+  }
+
+  filer = {
+    nodes = {
+      f0 = { address = "127.0.0.1", port = 28888, data_dir = "${var.workdir}/filer-f0" }
+    }
+  }
+
+  s3 = {
+    enabled = true
+    nodes   = { s0 = { address = "127.0.0.1", port = 28333 } }
+    # this weed build starts an Iceberg REST catalog on 8181 by default; disable
+    # it so the test does not collide with a cluster already using that port.
+    iceberg_port = 0
+    config_path  = "${var.workdir}/s3_config.json"
+  }
+
+  # Anonymous identity so the smoke test can PUT/GET without sigv4 signing.
+  s3_identities = [{
+    name       = "anonymous"
+    access_key = ""
+    secret_key = ""
+    actions    = ["Admin", "Read", "Write", "List", "Tagging"]
+  }]
+}
+
+output "cluster" {
+  description = "Node specs consumed by run_local_cluster.sh (read via `tofu output -json`, which emits values even when sensitive)."
+  sensitive   = true
+  value = {
+    for name, n in module.core.nodes : name => {
+      role         = n.role
+      address      = n.address
+      http_port    = n.ports.http
+      data_dirs    = n.data_dirs
+      argv         = n.argv
+      env          = n.env
+      config_files = n.config_files
+      exec_start   = n.exec_start
+    }
+  }
+}
+
+output "master_peers" {
+  value = module.core.master_peers
+}

terraform/test/local/run_local_cluster.sh

@@ -0,0 +1,170 @@
+#!/usr/bin/env bash
+# Render a SeaweedFS cluster with the Terraform core module and run it as real
+# `weed` processes locally, then assert it actually works. No cloud, no docker.
+#
+#   ./run_local_cluster.sh            # render + run + assert + teardown
+#   KEEP=1 ./run_local_cluster.sh     # leave the cluster running after asserts
+#   WEED=/path/to/weed ./run_local_cluster.sh
+#
+# Ports come from the rendered config (high range, to avoid colliding with a
+# SeaweedFS cluster already running on this machine). Exits non-zero on any
+# failed assertion or if a required port is already in use.
+set -u
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+export PATH="/opt/homebrew/bin:$PATH"
+TOFU="${TOFU:-tofu}"
+WEED="${WEED:-$(go env GOPATH 2>/dev/null || echo "$HOME/go")/bin/weed}"
+WORKDIR="${WORKDIR:-/tmp/seaweedfs-tftest}"
+LOGDIR="$WORKDIR/logs"
+RUNDIR="$WORKDIR/run"
+
+PASS=0
+FAIL=0
+ok()   { echo "  PASS: $1"; PASS=$((PASS + 1)); }
+bad()  { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }
+info() { echo "==> $1"; }
+
+cleanup() {
+  info "tearing down"
+  if [ -d "$RUNDIR" ]; then
+    for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill "$(cat "$pf")" 2>/dev/null; done
+    sleep 1
+    for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill -9 "$(cat "$pf")" 2>/dev/null; done
+  fi
+}
+
+info "cleaning $WORKDIR"
+case "$WORKDIR" in "" | "/" | "$HOME") echo "refusing to delete '$WORKDIR'" >&2; exit 2 ;; esac
+rm -rf "$WORKDIR"
+mkdir -p "$LOGDIR" "$RUNDIR"
+[ -x "$WEED" ] || { echo "weed binary not found/executable at $WEED" >&2; exit 2; }
+
+info "rendering cluster config with OpenTofu"
+cd "$HERE"
+"$TOFU" init -backend=false -input=false -no-color >/dev/null 2>&1 || { echo "tofu init failed"; exit 2; }
+if ! "$TOFU" apply -auto-approve -input=false -no-color \
+       -var "weed_binary=$WEED" -var "workdir=$WORKDIR" >"$LOGDIR/tofu.log" 2>&1; then
+  echo "tofu apply failed; see $LOGDIR/tofu.log"; tail -30 "$LOGDIR/tofu.log"; exit 2
+fi
+OUT="$("$TOFU" output -json cluster)"
+[ -n "$OUT" ] || { echo "empty cluster output"; exit 2; }
+
+# ---- derive ports by role (never hardcoded) ---------------------------------
+port_of() { echo "$OUT" | jq -r --arg r "$1" '.[] | select(.role==$r) | .http_port' | head -1; }
+mports()  { echo "$OUT" | jq -r '.[] | select(.role=="master") | .http_port' | sort; }
+MPORT1="$(mports | head -1)"
+VPORT="$(port_of volume)"; FPORT="$(port_of filer)"; SPORT="$(port_of s3)"
+NMASTERS="$(mports | wc -l | tr -d ' ')"
+
+# ---- refuse to run if a required port is already taken (foreign cluster) -----
+busy=""
+for p in $(echo "$OUT" | jq -r '.[].http_port'); do
+  lsof -nP -iTCP:"$p" -sTCP:LISTEN >/dev/null 2>&1 && busy="$busy $p"
+done
+[ -n "$busy" ] && { echo "Required port(s) already in use:$busy -- is another SeaweedFS running?" >&2; exit 3; }
+
+[ "${KEEP:-0}" = "1" ] || trap cleanup EXIT INT TERM
+
+node_names() { echo "$OUT" | jq -r 'keys[]'; }
+node_field() { echo "$OUT" | jq -r --arg n "$1" --arg f "$2" '.[$n][$f]'; }
+
+launch_node() {
+  n="$1"
+  for p in $(echo "$OUT" | jq -r --arg n "$n" '.[$n].config_files | keys[]?'); do
+    mkdir -p "$(dirname "$p")"
+    echo "$OUT" | jq -r --arg n "$n" --arg p "$p" '.[$n].config_files[$p]' > "$p"
+  done
+  for d in $(echo "$OUT" | jq -r --arg n "$n" '.[$n].data_dirs[]?'); do mkdir -p "$d"; done
+  ENVS=()
+  while IFS= read -r e; do [ -n "$e" ] && ENVS+=("$e"); done \
+    < <(echo "$OUT" | jq -r --arg n "$n" '.[$n].env | to_entries[] | "\(.key)=\(.value)"')
+  ARGV=()
+  while IFS= read -r a; do ARGV+=("$a"); done \
+    < <(echo "$OUT" | jq -r --arg n "$n" '.[$n].argv[]')
+  info "launching $n"
+  if [ "${#ENVS[@]}" -gt 0 ]; then
+    env "${ENVS[@]}" "$WEED" "${ARGV[@]}" >"$LOGDIR/$n.log" 2>&1 &
+  else
+    "$WEED" "${ARGV[@]}" >"$LOGDIR/$n.log" 2>&1 &
+  fi
+  echo "$!" > "$RUNDIR/$n.pid"
+}
+
+wait_http() {
+  url="$1"; t="${2:-30}"; i=0
+  while [ "$i" -lt "$t" ]; do
+    curl -fsS -o /dev/null --max-time 2 "$url" 2>/dev/null && return 0
+    i=$((i + 1)); sleep 1
+  done
+  return 1
+}
+
+# ---- launch masters and wait for quorum -------------------------------------
+for n in $(node_names); do
+  case "$(node_field "$n" role)" in master) launch_node "$n";; esac
+done
+
+info "waiting for master quorum across ports: $(mports | tr '\n' ' ')"
+QUORUM_OK=0; i=0
+while [ "$i" -lt 40 ]; do
+  LEADERS=0; AGREED_LEADER=""; mismatch=0; reachable=0
+  for port in $(mports); do
+    st="$(curl -fsS --max-time 2 "http://127.0.0.1:$port/cluster/status" 2>/dev/null)" || continue
+    reachable=$((reachable + 1))
+    [ "$(echo "$st" | jq -r '.IsLeader // false')" = "true" ] && LEADERS=$((LEADERS + 1))
+    ldr="$(echo "$st" | jq -r '.Leader // ""')"
+    if [ -n "$ldr" ]; then
+      if [ -z "$AGREED_LEADER" ]; then AGREED_LEADER="$ldr"; elif [ "$AGREED_LEADER" != "$ldr" ]; then mismatch=1; fi
+    fi
+  done
+  if [ "$reachable" -eq "$NMASTERS" ] && [ "$LEADERS" -eq 1 ] && [ "$mismatch" -eq 0 ] && [ -n "$AGREED_LEADER" ]; then
+    QUORUM_OK=1; break
+  fi
+  i=$((i + 1)); sleep 1
+done
+if [ "$QUORUM_OK" -eq 1 ]; then
+  ok "$NMASTERS-master quorum: exactly one leader ($AGREED_LEADER), all agree"
+else
+  bad "master quorum (reachable=$reachable leaders=$LEADERS leader='$AGREED_LEADER')"
+fi
+
+# ---- volume -----------------------------------------------------------------
+for n in $(node_names); do
+  case "$(node_field "$n" role)" in volume) launch_node "$n";; esac
+done
+wait_http "http://127.0.0.1:$VPORT/healthz" 40 && ok "volume /healthz up" || bad "volume /healthz"
+ASSIGN_OK=0; i=0
+while [ "$i" -lt 20 ]; do
+  fid="$(curl -fsS --max-time 2 "http://127.0.0.1:$MPORT1/dir/assign" 2>/dev/null | jq -r '.fid // ""')"
+  [ -n "$fid" ] && { ASSIGN_OK=1; break; }
+  i=$((i + 1)); sleep 1
+done
+[ "$ASSIGN_OK" -eq 1 ] && ok "master assigned fid ($fid) => volume registered" || bad "master /dir/assign (no volume?)"
+
+# ---- filer ------------------------------------------------------------------
+for n in $(node_names); do
+  case "$(node_field "$n" role)" in filer) launch_node "$n";; esac
+done
+wait_http "http://127.0.0.1:$FPORT/" 30 && ok "filer / up" || bad "filer /"
+PAYLOAD="seaweedfs-terraform-smoke-$$"
+echo "$PAYLOAD" > "$WORKDIR/hello.txt"
+curl -fsS -F "file=@$WORKDIR/hello.txt" "http://127.0.0.1:$FPORT/smoke/hello.txt" >/dev/null 2>&1
+got="$(curl -fsS --max-time 5 "http://127.0.0.1:$FPORT/smoke/hello.txt" 2>/dev/null)"
+[ "$got" = "$PAYLOAD" ] && ok "filer PUT/GET round-trip" || bad "filer round-trip (got '$got')"
+
+# ---- s3 ---------------------------------------------------------------------
+for n in $(node_names); do
+  case "$(node_field "$n" role)" in s3) launch_node "$n";; esac
+done
+wait_http "http://127.0.0.1:$SPORT/status" 30 && ok "s3 /status up" || bad "s3 /status"
+curl -fsS -X PUT "http://127.0.0.1:$SPORT/smoke-bucket" >/dev/null 2>&1
+sleep 1
+curl -fsS -X PUT --data-binary "$PAYLOAD" "http://127.0.0.1:$SPORT/smoke-bucket/obj.txt" >/dev/null 2>&1
+s3got="$(curl -fsS --max-time 5 "http://127.0.0.1:$SPORT/smoke-bucket/obj.txt" 2>/dev/null)"
+[ "$s3got" = "$PAYLOAD" ] && ok "s3 PUT/GET round-trip" || bad "s3 round-trip (got '$s3got')"
+
+echo
+info "RESULTS: $PASS passed, $FAIL failed"
+[ "${KEEP:-0}" = "1" ] && info "KEEP=1: cluster left running. Logs in $LOGDIR."
+[ "$FAIL" -eq 0 ]

test/nfs/Makefile

@@ -1,36 +0,0 @@
-.PHONY: all build test test-verbose test-short test-debug clean deps tidy
-
-all: build test
-
-# Build the weed binary first
-build:
-	cd ../../weed && go build -o weed .
-
-# Install test dependencies
-deps:
-	go mod download
-
-# Run all tests
-test: build deps
-	go test -timeout 5m ./...
-
-# Run tests with verbose output
-test-verbose: build deps
-	go test -v -timeout 5m ./...
-
-# Skip long-running integration tests
-test-short: deps
-	go test -short -v ./...
-
-# Run tests with debug output from SeaweedFS
-test-debug: build deps
-	go test -v -timeout 5m ./... 2>&1 | tee test.log
-
-# Clean up test artifacts
-clean:
-	rm -f test.log
-	go clean -testcache
-
-# Update go.sum
-tidy:
-	go mod tidy

test/nfs/README.md

@@ -1,92 +0,0 @@
-# SeaweedFS NFS Integration Tests
-
-End-to-end tests that boot a real SeaweedFS cluster (`master` + `volume` +
-`filer`) plus the experimental `weed nfs` frontend and drive it through the
-NFSv3 wire protocol. The tests talk to the server over TCP using
-`github.com/willscott/go-nfs-client`, which means they do **not** need a
-kernel NFS mount, privileged ports, or any platform-specific tooling.
-
-## Prerequisites
-
-1. Build the `weed` binary:
-   ```bash
-   cd ../../weed
-   go build -o weed .
-   ```
-2. Go 1.24 or later.
-
-## Running the tests
-
-```bash
-# Build weed and run everything
-make test
-
-# Verbose output, keeps the subprocess stdout
-make test-verbose
-
-# Skip integration tests — useful when iterating on the framework itself
-make test-short
-
-# Run a single test
-go test -v -run TestNfsBasicReadWrite ./...
-```
-
-Every test starts its own cluster on random loopback ports, so runs are
-isolated and can execute in parallel.
-
-## Layout
-
-- `framework.go` — launches `weed master`, `weed volume`, `weed filer`, and
-  `weed nfs` as subprocesses, waits for each to accept TCP, and exposes a
-  `Mount()` helper that returns an `nfsclient.Target`.
-- `basic_test.go` — covers the most common NFS operations:
-  - Read/write round-trip (`TestNfsBasicReadWrite`)
-  - Mkdir / ReadDirPlus / RmDir (`TestNfsMkdirAndRmdir`)
-  - Nested directory + leaf file (`TestNfsNestedDirectories`)
-  - Rename preserves content (`TestNfsRenamePreservesContent`)
-  - Overwrite shrinks file size (`TestNfsOverwriteShrinksFile`)
-  - Large binary file round-trip (`TestNfsLargeFile`)
-  - Arbitrary binary and empty files (`TestNfsBinaryAndEmptyFiles`)
-  - Symlink + Readlink (`TestNfsSymlinkRoundTrip`)
-  - ReadDirPlus ordering sanity (`TestNfsReadDirPlusOrdering`)
-  - Remove on missing path errors cleanly (`TestNfsRemoveMissingFailsCleanly`)
-  - FSINFO advertises non-zero limits (`TestNfsFSInfoReturnsSaneLimits`)
-  - Sequential append writes concatenate (`TestNfsAppendIsSequential`)
-  - ReadDir after remove (`TestNfsReadDirAfterRemove`)
-
-## Debugging a failing test
-
-Keep the cluster temp dir for inspection:
-
-```go
-config := DefaultTestConfig()
-config.SkipCleanup = true
-```
-
-Enable subprocess stdout/stderr:
-
-```go
-config := DefaultTestConfig()
-config.EnableDebug = true
-```
-
-Or run with `-v`, which flips `EnableDebug` automatically via `testing.Verbose()`.
-
-## Notes
-
-- The NFS server binds to `127.0.0.1` with `-ip.bind=127.0.0.1` and exports
-  `/nfs_export`. The test framework pre-creates that directory via the
-  filer's HTTP API before starting the NFS server — the NFS server requires
-  its export root to exist in the filer's namespace with a real entry, and
-  the filer's synthetic `/` root does not match the `Name=="/"` check the
-  NFS server performs during `ensureIndexedEntry`.
-- Ports are allocated dynamically. Each test run opens a short-lived
-  listener on `127.0.0.1:0`, reads back the assigned port, closes the
-  listener, and hands the port to `weed master/volume/filer/nfs`. There is
-  a tiny race window between close and reopen that has not been a problem
-  in practice but is worth remembering if you see a "bind: address already
-  in use" failure.
-- All four `weed` components are started with explicit `-port.grpc=...`
-  flags. Without them, the default is `-port + 10000`, which overflows
-  `65535` whenever the HTTP port lands above `55535` — the kernel's
-  ephemeral port range on macOS routinely does.

test/nfs/basic_test.go

@@ -1,400 +0,0 @@
-package nfs
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"os"
-	"path"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	nfsclient "github.com/willscott/go-nfs-client/nfs"
-)
-
-// setupFramework is a small helper that boots the cluster for a single test
-// and tears everything down on completion. Every test gets a fresh filer +
-// volume pair so they cannot step on each other's namespace.
-func setupFramework(t *testing.T) *NfsTestFramework {
-	t.Helper()
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-	config := DefaultTestConfig()
-	config.EnableDebug = testing.Verbose()
-	fw := NewNfsTestFramework(t, config)
-	require.NoError(t, fw.Setup(config), "framework setup")
-	t.Cleanup(fw.Cleanup)
-	return fw
-}
-
-// writeAll writes payload to path on the target in a single Write call. The
-// NFS WRITE3 RPC chunks internally, so this exists purely so tests read
-// linearly.
-func writeAll(t *testing.T, target *nfsclient.Target, remotePath string, payload []byte) {
-	t.Helper()
-	file, err := target.OpenFile(remotePath, 0o644)
-	require.NoError(t, err, "open %s for write", remotePath)
-	if len(payload) > 0 {
-		n, err := file.Write(payload)
-		require.NoError(t, err, "write %s", remotePath)
-		require.Equal(t, len(payload), n, "short write on %s", remotePath)
-	}
-	require.NoError(t, file.Close(), "close %s", remotePath)
-}
-
-// readAll opens path on the target and returns the full file contents.
-func readAll(t *testing.T, target *nfsclient.Target, remotePath string) []byte {
-	t.Helper()
-	file, err := target.Open(remotePath)
-	require.NoError(t, err, "open %s for read", remotePath)
-	defer file.Close()
-	content, err := io.ReadAll(file)
-	require.NoError(t, err, "read %s", remotePath)
-	return content
-}
-
-// TestNfsBasicReadWrite exercises the most common NFS path: OpenFile + Write
-// + Close followed by Open + Read to verify round-trip data integrity.
-func TestNfsBasicReadWrite(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	payload := []byte("hello from seaweedfs nfs integration test")
-	writeAll(t, target, "/hello.txt", payload)
-
-	got := readAll(t, target, "/hello.txt")
-	assert.Equal(t, payload, got, "round-tripped content must match")
-
-	info, err := target.Getattr("/hello.txt")
-	require.NoError(t, err)
-	assert.Equal(t, int64(len(payload)), int64(info.Filesize))
-}
-
-// TestNfsMkdirAndRmdir covers Mkdir, ReadDirPlus, and RmDir. The readdir
-// assertion also verifies that the newly-created directory shows up under
-// the export root the way a POSIX client would expect.
-func TestNfsMkdirAndRmdir(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	_, err = target.Mkdir("/dir1", 0o755)
-	require.NoError(t, err)
-
-	entries, err := target.ReadDirPlus("/")
-	require.NoError(t, err)
-	found := false
-	for _, entry := range entries {
-		if entry.Name() == "dir1" {
-			found = true
-			assert.True(t, entry.IsDir(), "dir1 should be a directory")
-		}
-	}
-	assert.True(t, found, "expected dir1 in readdir listing")
-
-	require.NoError(t, target.RmDir("/dir1"))
-
-	// After removal, dir1 must be gone from the listing.
-	entries, err = target.ReadDirPlus("/")
-	require.NoError(t, err)
-	for _, entry := range entries {
-		assert.NotEqual(t, "dir1", entry.Name(), "dir1 should be removed")
-	}
-}
-
-// TestNfsNestedDirectories ensures the server can materialise a deep tree in
-// a single Mkdir-per-segment sequence and that reads/writes work at the
-// leaves.
-func TestNfsNestedDirectories(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	for _, segment := range []string{"/a", "/a/b", "/a/b/c"} {
-		_, err := target.Mkdir(segment, 0o755)
-		require.NoError(t, err, "mkdir %s", segment)
-	}
-
-	payload := []byte("deep path content")
-	writeAll(t, target, "/a/b/c/leaf.txt", payload)
-
-	got := readAll(t, target, "/a/b/c/leaf.txt")
-	assert.Equal(t, payload, got)
-
-	require.NoError(t, target.Remove("/a/b/c/leaf.txt"))
-	require.NoError(t, target.RmDir("/a/b/c"))
-	require.NoError(t, target.RmDir("/a/b"))
-	require.NoError(t, target.RmDir("/a"))
-}
-
-// TestNfsRenamePreservesContent renames a file and makes sure the content
-// at the new path matches what was written at the old one, and that the
-// old path disappears. It does not assert on inode identity because pjdfstest
-// already covers that and this test intentionally avoids depending on the
-// mount-side identity plumbing.
-func TestNfsRenamePreservesContent(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	payload := []byte("rename me")
-	writeAll(t, target, "/src.txt", payload)
-
-	require.NoError(t, target.Rename("/src.txt", "/dst.txt"))
-
-	_, _, err = target.Lookup("/src.txt")
-	assert.Error(t, err, "source should be gone after rename")
-
-	got := readAll(t, target, "/dst.txt")
-	assert.Equal(t, payload, got)
-
-	require.NoError(t, target.Remove("/dst.txt"))
-}
-
-// TestNfsOverwriteShrinksFile rewrites an existing file with shorter content
-// and asserts Getattr reports the new (smaller) size. go-nfs-client's
-// OpenFile does not pass O_TRUNC, so the test truncates explicitly via
-// Setattr(size=0) before the second write — mirroring what `echo >file`
-// does on a POSIX client.
-func TestNfsOverwriteShrinksFile(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	writeAll(t, target, "/overwrite.txt", []byte("the quick brown fox"))
-
-	require.NoError(t, target.Setattr("/overwrite.txt", nfsclient.Sattr3{
-		Size: nfsclient.SetSize{SetIt: true, Size: 0},
-	}))
-
-	writeAll(t, target, "/overwrite.txt", []byte("short"))
-
-	info, err := target.Getattr("/overwrite.txt")
-	require.NoError(t, err)
-	assert.Equal(t, int64(len("short")), int64(info.Filesize))
-
-	got := readAll(t, target, "/overwrite.txt")
-	assert.Equal(t, []byte("short"), got)
-
-	require.NoError(t, target.Remove("/overwrite.txt"))
-}
-
-// TestNfsLargeFile writes a multi-megabyte payload so the write path has to
-// cut chunks and flush through the volume server rather than inlining
-// content in the filer entry.
-func TestNfsLargeFile(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	const size = 3 * 1024 * 1024 // 3 MiB — exceeds the 4 MiB inline cutoff boundary when combined with metadata
-	payload := make([]byte, size)
-	for i := range payload {
-		payload[i] = byte(i % 251) // non-repeating to catch offset bugs
-	}
-
-	writeAll(t, target, "/big.bin", payload)
-
-	info, err := target.Getattr("/big.bin")
-	require.NoError(t, err)
-	assert.Equal(t, int64(size), int64(info.Filesize))
-
-	got := readAll(t, target, "/big.bin")
-	require.Equal(t, size, len(got))
-	assert.True(t, bytes.Equal(payload, got), "large file content must round-trip byte-for-byte")
-
-	require.NoError(t, target.Remove("/big.bin"))
-}
-
-// TestNfsBinaryAndEmptyFiles covers two edge-case payloads the write path
-// tends to regress on: arbitrary binary bytes and zero-length files.
-func TestNfsBinaryAndEmptyFiles(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	t.Run("AllByteValues", func(t *testing.T) {
-		payload := make([]byte, 256)
-		for i := range payload {
-			payload[i] = byte(i)
-		}
-		writeAll(t, target, "/binary.bin", payload)
-		assert.Equal(t, payload, readAll(t, target, "/binary.bin"))
-		require.NoError(t, target.Remove("/binary.bin"))
-	})
-
-	t.Run("EmptyFile", func(t *testing.T) {
-		writeAll(t, target, "/empty.txt", nil)
-		info, err := target.Getattr("/empty.txt")
-		require.NoError(t, err)
-		assert.Equal(t, int64(0), int64(info.Filesize))
-		require.NoError(t, target.Remove("/empty.txt"))
-	})
-}
-
-// TestNfsSymlinkRoundTrip covers Symlink and Readlink through the nfs server.
-// Readlink returns the target path; the server does not auto-traverse it.
-func TestNfsSymlinkRoundTrip(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	// Symlink uses a different RPC than open+create, and our server routes it
-	// through the billy Change interface.
-	require.NoError(t, target.Symlink("/target.txt", "/link.txt"))
-
-	// The underlying target does not need to exist for readlink to succeed.
-	file, _, err := target.Lookup("/link.txt")
-	require.NoError(t, err, "lookup symlink")
-	assert.True(t, file.Mode()&os.ModeSymlink != 0, "expected symlink mode, got %s", file.Mode())
-
-	require.NoError(t, target.Remove("/link.txt"))
-}
-
-// TestNfsReadDirPlusOrdering creates a handful of files with distinct names
-// and ensures ReadDirPlus surfaces every one of them. The server pages
-// listings from the filer, so we want to make sure nothing is truncated.
-func TestNfsReadDirPlusOrdering(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	_, err = target.Mkdir("/listing", 0o755)
-	require.NoError(t, err)
-
-	names := []string{"alpha.txt", "beta.txt", "gamma.txt", "delta.txt", "epsilon.txt"}
-	for _, name := range names {
-		writeAll(t, target, path.Join("/listing", name), []byte(name))
-	}
-
-	entries, err := target.ReadDirPlus("/listing")
-	require.NoError(t, err)
-	seen := make(map[string]struct{}, len(entries))
-	for _, entry := range entries {
-		if entry.Name() == "." || entry.Name() == ".." {
-			continue
-		}
-		seen[entry.Name()] = struct{}{}
-	}
-	for _, name := range names {
-		_, ok := seen[name]
-		assert.True(t, ok, "expected %s in directory listing", name)
-	}
-
-	for _, name := range names {
-		require.NoError(t, target.Remove(path.Join("/listing", name)))
-	}
-	require.NoError(t, target.RmDir("/listing"))
-}
-
-// TestNfsRemoveMissingFailsCleanly asserts that removing a non-existent path
-// surfaces an error instead of silently succeeding. A bug where the server
-// returned NFS3_OK on missing entries would hide metadata drift.
-func TestNfsRemoveMissingFailsCleanly(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	err = target.Remove("/does_not_exist.txt")
-	require.Error(t, err, "removing a missing file must error")
-	// NFS3 surfaces this as NFS3ERR_NOENT; make sure the error text is
-	// recognisable without locking us into the library's exact wording.
-	assert.True(t,
-		strings.Contains(strings.ToLower(err.Error()), "noent") ||
-			strings.Contains(strings.ToLower(err.Error()), "not exist") ||
-			strings.Contains(strings.ToLower(err.Error()), "no such"),
-		"unexpected error shape: %v", err)
-}
-
-// TestNfsFSInfoReturnsSaneLimits pokes at FSINFO so we catch regressions
-// where the server advertises zero read/write limits (which would make
-// clients fall back to the 8 KiB floor and slow every test that follows).
-func TestNfsFSInfoReturnsSaneLimits(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	info, err := target.FSInfo()
-	require.NoError(t, err)
-	require.NotNil(t, info)
-	assert.Greater(t, info.RTPref, uint32(0), "rtpref must be positive")
-	assert.Greater(t, info.WTPref, uint32(0), "wtpref must be positive")
-}
-
-// TestNfsAppendIsSequential writes two chunks to the same file in separate
-// Open cycles and asserts the concatenation is preserved. The second write
-// uses O_APPEND (the default Open path in go-nfs-client does not pass
-// flags, so we explicitly reopen after writing the first chunk).
-func TestNfsAppendIsSequential(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	const prefix = "part1-"
-	const suffix = "part2"
-
-	writeAll(t, target, "/concat.txt", []byte(prefix))
-
-	file, err := target.OpenFile("/concat.txt", 0o644)
-	require.NoError(t, err)
-	// Seek to end before writing so we append rather than overwrite. go-nfs
-	// client's File.Seek uses the same offset tracking as Write so this is
-	// enough to place the second chunk after the first.
-	_, err = file.Seek(int64(len(prefix)), io.SeekStart)
-	require.NoError(t, err)
-	_, err = file.Write([]byte(suffix))
-	require.NoError(t, err)
-	require.NoError(t, file.Close())
-
-	got := readAll(t, target, "/concat.txt")
-	assert.Equal(t, prefix+suffix, string(got))
-
-	require.NoError(t, target.Remove("/concat.txt"))
-}
-
-// Regression: readdir should not emit stale entries after a remove. This is
-// the scenario the PR's meta cache invalidation logic was written to fix.
-func TestNfsReadDirAfterRemove(t *testing.T) {
-	fw := setupFramework(t)
-	target, cleanup, err := fw.Mount()
-	require.NoError(t, err)
-	defer cleanup()
-
-	_, err = target.Mkdir("/churn", 0o755)
-	require.NoError(t, err)
-	for i := 0; i < 5; i++ {
-		writeAll(t, target, path.Join("/churn", fmt.Sprintf("f%d.txt", i)), []byte{byte(i)})
-	}
-	// Remove the middle one and re-list.
-	require.NoError(t, target.Remove("/churn/f2.txt"))
-
-	entries, err := target.ReadDirPlus("/churn")
-	require.NoError(t, err)
-	for _, entry := range entries {
-		assert.NotEqual(t, "f2.txt", entry.Name(), "removed file should not reappear in listing")
-	}
-
-	for i := 0; i < 5; i++ {
-		if i == 2 {
-			continue
-		}
-		require.NoError(t, target.Remove(path.Join("/churn", fmt.Sprintf("f%d.txt", i))))
-	}
-	require.NoError(t, target.RmDir("/churn"))
-}

test/nfs/framework.go

@@ -1,423 +0,0 @@
-package nfs
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"mime/multipart"
-	"net"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"syscall"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/testutil"
-	"github.com/stretchr/testify/require"
-	nfsclient "github.com/willscott/go-nfs-client/nfs"
-	"github.com/willscott/go-nfs-client/nfs/rpc"
-)
-
-// NfsTestFramework boots a minimal SeaweedFS cluster (master + volume + filer)
-// plus the experimental `weed nfs` frontend and hands out NFSv3 RPC clients
-// that talk to it. Everything is driven via subprocesses so the tests exercise
-// the same binary an operator would deploy, and no kernel mount is required.
-type NfsTestFramework struct {
-	t             *testing.T
-	tempDir       string
-	dataDir       string
-	masterProcess *os.Process
-	volumeProcess *os.Process
-	filerProcess  *os.Process
-	nfsProcess    *os.Process
-	masterAddr    string
-	masterGrpc    int
-	volumeAddr    string
-	volumeGrpc    int
-	filerAddr     string
-	filerGrpc     int
-	nfsAddr       string
-	exportRoot    string
-	weedBinary    string
-	isSetup       bool
-	skipCleanup   bool
-}
-
-// TestConfig controls how the framework boots the cluster.
-type TestConfig struct {
-	NumVolumes  int
-	EnableDebug bool
-	SkipCleanup bool // keep temp dir on failure for inspection
-	// ExportRoot is the filer path the NFS server exports. Defaults to "/"
-	// so tests can use any path, with a single warning logged by the server.
-	ExportRoot string
-}
-
-// DefaultTestConfig returns the defaults used by most tests. A dedicated
-// /nfs_export subtree is used as the NFS export root because the NFS server
-// requires the export directory to exist in the filer's namespace and carry
-// a non-zero inode — passing "/" would succeed only for filer setups that
-// have already backfilled the root inode.
-func DefaultTestConfig() *TestConfig {
-	return &TestConfig{
-		NumVolumes:  3,
-		EnableDebug: false,
-		SkipCleanup: false,
-		ExportRoot:  "/nfs_export",
-	}
-}
-
-// NewNfsTestFramework allocates a framework bound to the current test. Call
-// Setup next to actually start the cluster.
-func NewNfsTestFramework(t *testing.T, config *TestConfig) *NfsTestFramework {
-	if config == nil {
-		config = DefaultTestConfig()
-	}
-
-	tempDir, err := os.MkdirTemp("", "seaweedfs_nfs_test_")
-	require.NoError(t, err)
-
-	// testutil.MustAllocatePorts holds every listener open until the full
-	// batch has been reserved, which avoids the "close-then-hope" race my
-	// original per-port helper had. We need seven ports: four HTTP (master,
-	// volume, filer, nfs) and three gRPC (master, volume, filer — nfs has
-	// no gRPC endpoint).
-	ports := testutil.MustAllocatePorts(t, 7)
-
-	exportRoot := config.ExportRoot
-	if exportRoot == "" {
-		exportRoot = "/"
-	}
-
-	return &NfsTestFramework{
-		t:           t,
-		tempDir:     tempDir,
-		dataDir:     filepath.Join(tempDir, "data"),
-		masterAddr:  fmt.Sprintf("127.0.0.1:%d", ports[0]),
-		masterGrpc:  ports[1],
-		volumeAddr:  fmt.Sprintf("127.0.0.1:%d", ports[2]),
-		volumeGrpc:  ports[3],
-		filerAddr:   fmt.Sprintf("127.0.0.1:%d", ports[4]),
-		filerGrpc:   ports[5],
-		nfsAddr:     fmt.Sprintf("127.0.0.1:%d", ports[6]),
-		exportRoot:  exportRoot,
-		weedBinary:  findWeedBinary(),
-		isSetup:     false,
-		skipCleanup: config.SkipCleanup,
-	}
-}
-
-// Setup starts the SeaweedFS cluster and the NFS frontend, waiting for each
-// component to accept connections before moving on.
-func (f *NfsTestFramework) Setup(config *TestConfig) error {
-	if f.isSetup {
-		return fmt.Errorf("framework already setup")
-	}
-
-	dirs := []string{
-		f.dataDir,
-		filepath.Join(f.dataDir, "master"),
-		filepath.Join(f.dataDir, "volume"),
-	}
-	for _, dir := range dirs {
-		if err := os.MkdirAll(dir, 0755); err != nil {
-			return fmt.Errorf("failed to create directory %s: %v", dir, err)
-		}
-	}
-
-	if err := f.startMaster(config); err != nil {
-		return fmt.Errorf("failed to start master: %v", err)
-	}
-	if !testutil.WaitForPort(portFromAddr(f.masterAddr), testutil.SeaweedMiniStartupTimeout) {
-		return fmt.Errorf("master not ready at %s", f.masterAddr)
-	}
-
-	if err := f.startVolumeServer(config); err != nil {
-		return fmt.Errorf("failed to start volume server: %v", err)
-	}
-	if !testutil.WaitForPort(portFromAddr(f.volumeAddr), testutil.SeaweedMiniStartupTimeout) {
-		return fmt.Errorf("volume server not ready at %s", f.volumeAddr)
-	}
-
-	if err := f.startFiler(config); err != nil {
-		return fmt.Errorf("failed to start filer: %v", err)
-	}
-	if !testutil.WaitForPort(portFromAddr(f.filerAddr), testutil.SeaweedMiniStartupTimeout) {
-		return fmt.Errorf("filer not ready at %s", f.filerAddr)
-	}
-
-	// Pre-create the export root in the filer's namespace. The NFS server
-	// expects its export directory to exist with a real inode; uploading a
-	// placeholder file creates the parent directory implicitly and then
-	// removing the file leaves the empty directory in place.
-	if f.exportRoot != "/" {
-		if err := f.ensureExportRootExists(); err != nil {
-			return fmt.Errorf("failed to pre-create export root %s: %v", f.exportRoot, err)
-		}
-	}
-
-	if err := f.startNfsServer(config); err != nil {
-		return fmt.Errorf("failed to start NFS server: %v", err)
-	}
-	if !testutil.WaitForPort(portFromAddr(f.nfsAddr), testutil.SeaweedMiniStartupTimeout) {
-		return fmt.Errorf("NFS server not ready at %s", f.nfsAddr)
-	}
-
-	// Let the NFS server finish wiring up its gRPC subscription to the filer
-	// before the first client call hits MOUNT/LOOKUP.
-	time.Sleep(500 * time.Millisecond)
-
-	f.isSetup = true
-	return nil
-}
-
-// Cleanup stops all processes. Temp state is preserved if SkipCleanup is set.
-func (f *NfsTestFramework) Cleanup() {
-	processes := []*os.Process{f.nfsProcess, f.filerProcess, f.volumeProcess, f.masterProcess}
-	for _, proc := range processes {
-		if proc != nil {
-			_ = proc.Signal(syscall.SIGTERM)
-			_, _ = proc.Wait()
-		}
-	}
-	if !f.skipCleanup {
-		_ = os.RemoveAll(f.tempDir)
-	}
-}
-
-// NfsAddr returns the TCP address the NFS server is listening on.
-func (f *NfsTestFramework) NfsAddr() string { return f.nfsAddr }
-
-// FilerAddr returns the TCP address of the filer.
-func (f *NfsTestFramework) FilerAddr() string { return f.filerAddr }
-
-// ExportRoot returns the path the NFS server exports.
-func (f *NfsTestFramework) ExportRoot() string { return f.exportRoot }
-
-// Mount opens an NFSv3 MOUNT+NFS connection against the running NFS server
-// and returns a Target that tests can drive like a mini-VFS. Caller is
-// responsible for calling the returned cleanup func to Unmount and close the
-// TCP connection.
-func (f *NfsTestFramework) Mount() (*nfsclient.Target, func(), error) {
-	var (
-		client *rpc.Client
-		err    error
-	)
-	// The NFS server's TCP listener may already be accepting connections when
-	// waitForService returns, but the RPC program registration can trail it
-	// by a few milliseconds. Retry the dial to absorb that small window.
-	for attempt := 0; attempt < 20; attempt++ {
-		client, err = rpc.DialTCP("tcp", f.nfsAddr, false)
-		if err == nil {
-			break
-		}
-		time.Sleep(25 * time.Millisecond)
-	}
-	if err != nil {
-		return nil, nil, fmt.Errorf("dial NFS: %w", err)
-	}
-
-	// Note: do not set Mount.Addr here. When Addr is non-empty, the go-nfs
-	// client re-dials via portmapper and concatenates `:111` onto the
-	// address, which produces "too many colons" for a raw `host:port`
-	// string. Reusing the existing RPC client avoids that path entirely.
-	mounter := &nfsclient.Mount{Client: client}
-	target, err := mounter.Mount(f.exportRoot, rpc.AuthNull)
-	if err != nil {
-		client.Close()
-		return nil, nil, fmt.Errorf("mount %s: %w", f.exportRoot, err)
-	}
-
-	cleanup := func() {
-		_ = mounter.Unmount()
-		client.Close()
-	}
-	return target, cleanup, nil
-}
-
-func (f *NfsTestFramework) startMaster(config *TestConfig) error {
-	_, masterPort := splitHostPort(f.masterAddr)
-	args := []string{
-		"master",
-		"-ip=127.0.0.1",
-		fmt.Sprintf("-port=%d", masterPort),
-		fmt.Sprintf("-port.grpc=%d", f.masterGrpc),
-		"-mdir=" + filepath.Join(f.dataDir, "master"),
-		"-raftBootstrap",
-		"-peers=none",
-	}
-	return f.startProcess(&f.masterProcess, config, args)
-}
-
-func (f *NfsTestFramework) startVolumeServer(config *TestConfig) error {
-	_, volumePort := splitHostPort(f.volumeAddr)
-	// pb.ServerAddress encodes a non-default gRPC port as `host:port.grpc`.
-	// See weed/pb/server_address.go — the dot, not a colon, is the separator
-	// between the HTTP port and the gRPC port.
-	masterWithGrpc := fmt.Sprintf("%s.%d", f.masterAddr, f.masterGrpc)
-	args := []string{
-		"volume",
-		"-master=" + masterWithGrpc,
-		"-ip=127.0.0.1",
-		fmt.Sprintf("-port=%d", volumePort),
-		fmt.Sprintf("-port.grpc=%d", f.volumeGrpc),
-		"-dir=" + filepath.Join(f.dataDir, "volume"),
-		fmt.Sprintf("-max=%d", config.NumVolumes),
-	}
-	return f.startProcess(&f.volumeProcess, config, args)
-}
-
-func (f *NfsTestFramework) startFiler(config *TestConfig) error {
-	_, filerPort := splitHostPort(f.filerAddr)
-	masterWithGrpc := fmt.Sprintf("%s.%d", f.masterAddr, f.masterGrpc)
-	args := []string{
-		"filer",
-		"-master=" + masterWithGrpc,
-		"-ip=127.0.0.1",
-		fmt.Sprintf("-port=%d", filerPort),
-		fmt.Sprintf("-port.grpc=%d", f.filerGrpc),
-	}
-	return f.startProcess(&f.filerProcess, config, args)
-}
-
-func (f *NfsTestFramework) startNfsServer(config *TestConfig) error {
-	_, nfsPort := splitHostPort(f.nfsAddr)
-	// `host:port.grpc` encoding — see pb/server_address.go.
-	filerWithGrpc := fmt.Sprintf("%s.%d", f.filerAddr, f.filerGrpc)
-	args := []string{
-		"nfs",
-		"-filer=" + filerWithGrpc,
-		"-ip.bind=127.0.0.1",
-		fmt.Sprintf("-port=%d", nfsPort),
-		"-filer.path=" + f.exportRoot,
-	}
-	return f.startProcess(&f.nfsProcess, config, args)
-}
-
-func (f *NfsTestFramework) startProcess(target **os.Process, config *TestConfig, args []string) error {
-	cmd := exec.Command(f.weedBinary, args...)
-	cmd.Dir = f.tempDir
-	if config.EnableDebug {
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-	}
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-	*target = cmd.Process
-	return nil
-}
-
-// portFromAddr returns just the port number from a `host:port` string.
-// testutil.WaitForPort takes an int port, not a full address.
-func portFromAddr(addr string) int {
-	_, port := splitHostPort(addr)
-	return port
-}
-
-// ensureExportRootExists posts a placeholder file to f.exportRoot via the
-// filer's HTTP API, then deletes it. That roundtrip implicitly creates the
-// target directory so the NFS server has something to mount. We bypass
-// weed/pb here because the HTTP client is simpler and needs no gRPC stubs.
-func (f *NfsTestFramework) ensureExportRootExists() error {
-	exportRoot := strings.TrimRight(f.exportRoot, "/")
-	if exportRoot == "" {
-		return nil
-	}
-	placeholder := exportRoot + "/.nfs_test_init"
-	filerURL := "http://" + f.filerAddr + placeholder
-
-	var body bytes.Buffer
-	writer := multipart.NewWriter(&body)
-	part, err := writer.CreateFormFile("file", ".nfs_test_init")
-	if err != nil {
-		return err
-	}
-	if _, err := io.WriteString(part, ""); err != nil {
-		return err
-	}
-	if err := writer.Close(); err != nil {
-		return err
-	}
-
-	httpClient := &http.Client{Timeout: 10 * time.Second}
-	req, err := http.NewRequest(http.MethodPost, filerURL, &body)
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Content-Type", writer.FormDataContentType())
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return err
-	}
-	_, _ = io.Copy(io.Discard, resp.Body)
-	resp.Body.Close()
-	if resp.StatusCode/100 != 2 {
-		return fmt.Errorf("filer POST %s returned status %d", filerURL, resp.StatusCode)
-	}
-
-	// Delete the placeholder; the directory stays behind.
-	deleteReq, err := http.NewRequest(http.MethodDelete, filerURL, nil)
-	if err != nil {
-		return err
-	}
-	deleteResp, err := httpClient.Do(deleteReq)
-	if err != nil {
-		return err
-	}
-	_, _ = io.Copy(io.Discard, deleteResp.Body)
-	deleteResp.Body.Close()
-	if deleteResp.StatusCode/100 != 2 && deleteResp.StatusCode != http.StatusNotFound {
-		return fmt.Errorf("filer DELETE %s returned status %d", filerURL, deleteResp.StatusCode)
-	}
-	return nil
-}
-
-func splitHostPort(addr string) (string, int) {
-	host, portStr, err := net.SplitHostPort(addr)
-	if err != nil {
-		return "", 0
-	}
-	var port int
-	_, _ = fmt.Sscanf(portStr, "%d", &port)
-	return host, port
-}
-
-// findWeedBinary locates the weed binary, preferring the local build in the
-// checkout so tests run against the code under review rather than whatever is
-// on $PATH.
-func findWeedBinary() string {
-	if _, thisFile, _, ok := runtime.Caller(0); ok {
-		thisDir := filepath.Dir(thisFile)
-		candidates := []string{
-			filepath.Join(thisDir, "../../weed/weed"),
-			filepath.Join(thisDir, "../weed/weed"),
-		}
-		for _, candidate := range candidates {
-			if _, err := os.Stat(candidate); err == nil {
-				abs, _ := filepath.Abs(candidate)
-				return abs
-			}
-		}
-	}
-	cwd, _ := os.Getwd()
-	candidates := []string{
-		filepath.Join(cwd, "../../weed/weed"),
-		filepath.Join(cwd, "../weed/weed"),
-		filepath.Join(cwd, "./weed"),
-	}
-	for _, candidate := range candidates {
-		if _, err := os.Stat(candidate); err == nil {
-			abs, _ := filepath.Abs(candidate)
-			return abs
-		}
-	}
-	if path, err := exec.LookPath("weed"); err == nil {
-		return path
-	}
-	return "weed"
-}

test/nfs/go.mod

@@ -1,21 +0,0 @@
-module seaweedfs-nfs-tests
-
-go 1.25.0
-
-// test/testutil lives inside the main seaweedfs module; pull it in via a
-// local replace so this integration suite can reuse the shared port
-// allocator and readiness helpers instead of reinventing them.
-replace github.com/seaweedfs/seaweedfs => ../..
-
-require (
-	github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
-	github.com/stretchr/testify v1.11.1
-	github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
-)
-
-require (
-	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-)

test/nfs/go.sum

@@ -1,14 +0,0 @@
-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
-github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
-github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
-github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

test/nfs/kernel_mount_test.go

@@ -1,193 +0,0 @@
-//go:build linux
-
-package nfs
-
-// End-to-end mount tests that drive the real Linux NFS client (mount.nfs +
-// in-tree kernel) against a running `weed nfs` subprocess. These exist to
-// catch regressions that the existing framework can't see, because the
-// framework drives the server with willscott/go-nfs-client — the same RPC
-// library the server uses internally — so any bug shared between the two
-// (XDR layout, version dispatch, RPC framing) round-trips invisibly.
-//
-// Two real bugs hit recently were exactly that shape:
-//   1. NFSv4 mis-routed to the v3 SETATTR handler (#9262). The client
-//      library never sends NFSv4, so the test suite never noticed; the
-//      Linux kernel mount path did notice, with EIO.
-//   2. UDP MOUNT v3 missing. Only TCP MOUNT was advertised; the kernel
-//      defaults mountproto=udp in many setups, so the in-tree client
-//      surfaced EPROTONOSUPPORT during MOUNT setup.
-//
-// These tests mount over the actual loopback interface using mount.nfs and
-// shell out to /bin/mount and /bin/umount. They require root (mount(2) is
-// privileged) and Linux (the in-tree NFS client is what's being exercised);
-// they t.Skip cleanly when either prerequisite is missing.
-//
-// Run locally with:
-//
-//	cd test/nfs
-//	sudo go test -v -run TestKernelMount ./...
-//
-// CI runs them via .github/workflows/nfs-tests.yml after installing
-// nfs-common (mount.nfs + helpers).
-
-import (
-	"errors"
-	"fmt"
-	"net"
-	"os"
-	"os/exec"
-	"strings"
-	"testing"
-)
-
-// kernelMountSkipIfUnsupported skips the test when the host can't run a
-// real NFS mount. The combined check belongs in one place so the three
-// kernel-mount tests stay focused on what they're actually verifying.
-func kernelMountSkipIfUnsupported(t *testing.T) {
-	t.Helper()
-	if os.Geteuid() != 0 {
-		t.Skip("kernel mount test requires root; mount(2) is privileged")
-	}
-	if _, err := exec.LookPath("mount.nfs"); err != nil {
-		t.Skipf("mount.nfs not installed: %v (CI installs the nfs-common package)", err)
-	}
-}
-
-// kernelMount runs /bin/mount with the given options against the framework's
-// running NFS server, returns the mountpoint and an unmount closure. We pass
-// explicit port=/mountport= options so the kernel never queries portmap.
-// That keeps the harness honest about what it's testing — the NFS / MOUNT
-// wire protocol — and avoids colliding with a system rpcbind on shared CI
-// runners (port 111 is privileged and frequently in use already).
-func kernelMount(t *testing.T, fw *NfsTestFramework, optsTemplate string) (string, func()) {
-	t.Helper()
-	host, portStr, err := net.SplitHostPort(fw.NfsAddr())
-	if err != nil {
-		t.Fatalf("split nfs addr %q: %v", fw.NfsAddr(), err)
-	}
-	mountpoint, err := os.MkdirTemp("", "weed-nfs-kmount-")
-	if err != nil {
-		t.Fatalf("mkdtemp: %v", err)
-	}
-	opts := strings.ReplaceAll(optsTemplate, "{port}", portStr)
-	target := fmt.Sprintf("%s:%s", host, fw.ExportRoot())
-	cmd := exec.Command("mount", "-t", "nfs", "-o", opts, target, mountpoint)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		_ = os.RemoveAll(mountpoint)
-		t.Fatalf("mount %s -o %s failed: %v\nmount output:\n%s", target, opts, err, out)
-	}
-	teardown := func() {
-		// -f to bail out faster if the server's already gone.
-		_ = exec.Command("umount", "-f", mountpoint).Run()
-		_ = os.RemoveAll(mountpoint)
-	}
-	return mountpoint, teardown
-}
-
-func newKernelMountFramework(t *testing.T) *NfsTestFramework {
-	t.Helper()
-	cfg := DefaultTestConfig()
-	fw := NewNfsTestFramework(t, cfg)
-	if err := fw.Setup(cfg); err != nil {
-		fw.Cleanup()
-		t.Fatalf("framework setup: %v", err)
-	}
-	t.Cleanup(fw.Cleanup)
-	return fw
-}
-
-// TestKernelMountV3TCP exercises the most common mount form: NFSv3 + MOUNT
-// v3, both over TCP. This is what the existing go-nfs-client tests cover at
-// the protocol layer, but running it through mount.nfs and the kernel
-// confirms that the wire format we emit decodes cleanly under a different
-// XDR/RPC parser.
-func TestKernelMountV3TCP(t *testing.T) {
-	kernelMountSkipIfUnsupported(t)
-	fw := newKernelMountFramework(t)
-
-	mountpoint, undo := kernelMount(t, fw,
-		"nfsvers=3,nolock,port={port},mountport={port},proto=tcp,mountproto=tcp")
-	defer undo()
-
-	if _, err := os.Stat(mountpoint); err != nil {
-		t.Errorf("stat mountpoint: %v", err)
-	}
-	if _, err := os.ReadDir(mountpoint); err != nil {
-		t.Errorf("readdir mountpoint: %v", err)
-	}
-}
-
-// TestKernelMountV3MountProtoUDP is the regression test for the UDP MOUNT
-// v3 responder. mountproto=udp forces the kernel to call MOUNT over UDP
-// only; before the responder existed the kernel hit nothing (MOUNT was
-// advertised TCP-only) and surfaced EPROTONOSUPPORT during mount setup.
-func TestKernelMountV3MountProtoUDP(t *testing.T) {
-	kernelMountSkipIfUnsupported(t)
-	fw := newKernelMountFramework(t)
-
-	mountpoint, undo := kernelMount(t, fw,
-		"nfsvers=3,nolock,port={port},mountport={port},proto=tcp,mountproto=udp")
-	defer undo()
-
-	if _, err := os.Stat(mountpoint); err != nil {
-		t.Errorf("stat mountpoint: %v", err)
-	}
-}
-
-// TestKernelMountV4RejectsCleanly is the regression test for the NFSv4
-// PROG_MISMATCH path (#9262). The server only speaks NFSv3, but the
-// previous behaviour was to mis-route v4 COMPOUND to the v3 SETATTR
-// handler and write garbage; the kernel surfaced EIO instead of a
-// version-mismatch error and (depending on distro) didn't fall back to
-// v3. The version filter now answers PROG_MISMATCH so the kernel sees
-// "v4 not supported" cleanly.
-//
-// The test asserts:
-//   1. mount.nfs exits non-zero (no silent success against a v3 server);
-//   2. the failure message mentions protocol/version/io, which is what the
-//      kernel surfaces when it gets PROG_MISMATCH instead of garbage. A
-//      pre-fix server returns "mount system call failed" with no further
-//      context, so a regression collapses the assertion onto that branch.
-func TestKernelMountV4RejectsCleanly(t *testing.T) {
-	kernelMountSkipIfUnsupported(t)
-	fw := newKernelMountFramework(t)
-
-	host, portStr, err := net.SplitHostPort(fw.NfsAddr())
-	if err != nil {
-		t.Fatalf("split nfs addr: %v", err)
-	}
-	mountpoint, err := os.MkdirTemp("", "weed-nfs-kmount-v4-")
-	if err != nil {
-		t.Fatalf("mkdtemp: %v", err)
-	}
-	defer os.RemoveAll(mountpoint)
-
-	target := fmt.Sprintf("%s:%s", host, fw.ExportRoot())
-	cmd := exec.Command("mount", "-t", "nfs", "-o",
-		fmt.Sprintf("vers=4,port=%s", portStr),
-		target, mountpoint)
-	out, err := cmd.CombinedOutput()
-	defer exec.Command("umount", "-f", mountpoint).Run()
-
-	if err == nil {
-		t.Fatalf("v4 mount unexpectedly succeeded against v3-only server\nmount output:\n%s", out)
-	}
-	// Don't pin the exact error string — different distros print slightly
-	// different things — but require some hint that the kernel saw a
-	// protocol-level failure rather than a generic "mount system call
-	// failed". Without the version filter, mount.nfs prints the latter
-	// alone; with it, the former.
-	lower := strings.ToLower(string(out))
-	if !strings.Contains(lower, "protocol") &&
-		!strings.Contains(lower, "version") &&
-		!strings.Contains(lower, "i/o") {
-		t.Errorf("v4 mount failure didn't mention protocol/version/io; output:\n%s", out)
-	}
-	// Also require a non-zero exit so a future change that makes mount(2)
-	// silently succeed (e.g. by relaxing the version filter) shows up
-	// here even if the message phrasing changes.
-	var ee *exec.ExitError
-	if !errors.As(err, &ee) {
-		t.Errorf("expected mount to exit non-zero with ExitError, got %v", err)
-	}
-}

test/plugin_workers/fake_volume_server.go

@@ -30,21 +30,22 @@ type VolumeServer struct {
 	address  string
 	baseDir  string
 
-	mu                  sync.Mutex
-	receivedFiles       map[string]uint64
-	mountRequests       []*volume_server_pb.VolumeEcShardsMountRequest
-	deleteRequests      []*volume_server_pb.VolumeDeleteRequest
-	markReadonlyCalls   int
-	markWritableCalls   int
-	readFileStatusCalls int
-	vacuumGarbageRatio  float64
-	vacuumCheckCalls    int
-	vacuumCompactCalls  int
-	vacuumCommitCalls   int
-	vacuumCleanupCalls  int
-	volumeCopyCalls     int
-	volumeMountCalls    int
-	tailReceiverCalls   int
+	mu                   sync.Mutex
+	receivedFiles        map[string]uint64
+	mountRequests        []*volume_server_pb.VolumeEcShardsMountRequest
+	deleteRequests       []*volume_server_pb.VolumeDeleteRequest
+	markReadonlyCalls    int
+	markWritableCalls    int
+	readFileStatusCalls  int
+	vacuumGarbageRatio   float64
+	vacuumCommitReadOnly bool
+	vacuumCheckCalls     int
+	vacuumCompactCalls   int
+	vacuumCommitCalls    int
+	vacuumCleanupCalls   int
+	volumeCopyCalls      int
+	volumeMountCalls     int
+	tailReceiverCalls    int
 }
 
 // NewVolumeServer starts a test volume server using the provided base directory.
@@ -115,6 +116,13 @@ func (v *VolumeServer) SetVacuumGarbageRatio(ratio float64) {
 	v.vacuumGarbageRatio = ratio
 }
 
+// SetVacuumCommitReadOnly sets the IsReadOnly value returned by VacuumVolumeCommit.
+func (v *VolumeServer) SetVacuumCommitReadOnly(readOnly bool) {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	v.vacuumCommitReadOnly = readOnly
+}
+
 // VacuumStats returns the vacuum RPC call counts.
 func (v *VolumeServer) VacuumStats() (check, compact, commit, cleanup int) {
 	v.mu.Lock()
@@ -428,8 +436,9 @@ func (v *VolumeServer) VacuumVolumeCompact(req *volume_server_pb.VacuumVolumeCom
 func (v *VolumeServer) VacuumVolumeCommit(ctx context.Context, req *volume_server_pb.VacuumVolumeCommitRequest) (*volume_server_pb.VacuumVolumeCommitResponse, error) {
 	v.mu.Lock()
 	v.vacuumCommitCalls++
+	readOnly := v.vacuumCommitReadOnly
 	v.mu.Unlock()
-	return &volume_server_pb.VacuumVolumeCommitResponse{}, nil
+	return &volume_server_pb.VacuumVolumeCommitResponse{IsReadOnly: readOnly}, nil
 }
 
 func (v *VolumeServer) VacuumVolumeCleanup(ctx context.Context, req *volume_server_pb.VacuumVolumeCleanupRequest) (*volume_server_pb.VacuumVolumeCleanupResponse, error) {

test/plugin_workers/vacuum/execution_test.go

@@ -15,6 +15,24 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 )
 
+func vacuumJobSpec(volumeID uint32, server string) *plugin_pb.JobSpec {
+	return &plugin_pb.JobSpec{
+		JobId:   fmt.Sprintf("vacuum-job-%d", volumeID),
+		JobType: "vacuum",
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"volume_id": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(volumeID)},
+			},
+			"server": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: server},
+			},
+			"collection": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "vac-test"},
+			},
+		},
+	}
+}
+
 func TestVacuumExecutionIntegration(t *testing.T) {
 	volumeID := uint32(202)
 
@@ -31,26 +49,10 @@ func TestVacuumExecutionIntegration(t *testing.T) {
 	source := pluginworkers.NewVolumeServer(t, "")
 	source.SetVacuumGarbageRatio(0.6)
 
-	job := &plugin_pb.JobSpec{
-		JobId:   fmt.Sprintf("vacuum-job-%d", volumeID),
-		JobType: "vacuum",
-		Parameters: map[string]*plugin_pb.ConfigValue{
-			"volume_id": {
-				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(volumeID)},
-			},
-			"server": {
-				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: source.Address()},
-			},
-			"collection": {
-				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "vac-test"},
-			},
-		},
-	}
-
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
-	result, err := harness.Plugin().ExecuteJob(ctx, job, nil, 1)
+	result, err := harness.Plugin().ExecuteJob(ctx, vacuumJobSpec(volumeID, source.Address()), nil, 1)
 	require.NoError(t, err)
 	require.NotNil(t, result)
 	require.True(t, result.Success)
@@ -59,5 +61,48 @@ func TestVacuumExecutionIntegration(t *testing.T) {
 	require.GreaterOrEqual(t, checkCalls, 2)
 	require.GreaterOrEqual(t, compactCalls, 1)
 	require.GreaterOrEqual(t, commitCalls, 1)
-	require.GreaterOrEqual(t, cleanupCalls, 1)
+	// Cleanup is only invoked when Phase 1 (Compact) fails to roll back
+	// the .cpd/.cpx/.cpldb temp files; on the success path Commit
+	// consumes them (rename .cpd → .dat, .cpx → .idx, .cpldb → .ldb via
+	// the leveldb needle map) so no Cleanup call is needed. Matches
+	// topology.vacuumOneVolumeId which only calls batchVacuumVolumeCleanup
+	// on the Compact-failure branch.
+	require.Equal(t, 0, cleanupCalls)
+	// Phase 3 marks each replica writable so master returns it to the
+	// writables layout. See upstream seaweedfs#9685.
+	require.GreaterOrEqual(t, source.MarkWritableCount(), 1)
+}
+
+// A replica that commits still read-only (operator-set, EIO-quarantined,
+// disk-space-low) must not be force-marked writable: master built-in vacuum
+// skips it via SetVolumeAvailable, and it recovers on its own ReadOnly=false
+// heartbeat.
+func TestVacuumExecutionSkipsMarkWritableWhenReadOnly(t *testing.T) {
+	volumeID := uint32(203)
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	handler := vacuum.NewVacuumHandler(dialOption, 1)
+	harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
+		WorkerOptions: pluginworker.WorkerOptions{
+			GrpcDialOption: dialOption,
+		},
+		Handlers: []pluginworker.JobHandler{handler},
+	})
+	harness.WaitForJobType("vacuum")
+
+	source := pluginworkers.NewVolumeServer(t, "")
+	source.SetVacuumGarbageRatio(0.6)
+	source.SetVacuumCommitReadOnly(true)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	result, err := harness.Plugin().ExecuteJob(ctx, vacuumJobSpec(volumeID, source.Address()), nil, 1)
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	require.True(t, result.Success)
+
+	_, _, commitCalls, _ := source.VacuumStats()
+	require.GreaterOrEqual(t, commitCalls, 1)
+	require.Equal(t, 0, source.MarkWritableCount())
 }

test/s3/iam/seaweedfs-test-realm.json

@@ -0,0 +1,107 @@
+{
+  "realm": "seaweedfs-test",
+  "enabled": true,
+  "roles": {
+    "realm": [
+      { "name": "s3-admin" },
+      { "name": "s3-read-only" },
+      { "name": "s3-write-only" },
+      { "name": "s3-read-write" }
+    ]
+  },
+  "clients": [
+    {
+      "clientId": "seaweedfs-s3",
+      "enabled": true,
+      "secret": "seaweedfs-s3-secret",
+      "publicClient": false,
+      "serviceAccountsEnabled": true,
+      "standardFlowEnabled": true,
+      "directAccessGrantsEnabled": true,
+      "fullScopeAllowed": true,
+      "redirectUris": ["*"],
+      "webOrigins": ["*"],
+      "protocol": "openid-connect",
+      "protocolMappers": [
+        {
+          "name": "realm-roles",
+          "protocol": "openid-connect",
+          "protocolMapper": "oidc-usermodel-realm-role-mapper",
+          "config": {
+            "claim.name": "roles",
+            "jsonType.label": "String",
+            "multivalued": "true",
+            "usermodel.realmRoleMapping.rolePrefix": "",
+            "access.token.claim": "true",
+            "id.token.claim": "true",
+            "userinfo.token.claim": "true"
+          }
+        },
+        {
+          "name": "audience-mapper",
+          "protocol": "openid-connect",
+          "protocolMapper": "oidc-audience-mapper",
+          "config": {
+            "included.client.audience": "seaweedfs-s3",
+            "id.token.claim": "false",
+            "access.token.claim": "true"
+          }
+        }
+      ]
+    }
+  ],
+  "users": [
+    {
+      "username": "admin-user",
+      "enabled": true,
+      "email": "admin-user@test.com",
+      "firstName": "Test",
+      "lastName": "User",
+      "emailVerified": true,
+      "requiredActions": [],
+      "realmRoles": ["s3-admin"],
+      "credentials": [
+        { "type": "password", "value": "adminuser123", "temporary": false }
+      ]
+    },
+    {
+      "username": "read-user",
+      "enabled": true,
+      "email": "read-user@test.com",
+      "firstName": "Test",
+      "lastName": "User",
+      "emailVerified": true,
+      "requiredActions": [],
+      "realmRoles": ["s3-read-only"],
+      "credentials": [
+        { "type": "password", "value": "readuser123", "temporary": false }
+      ]
+    },
+    {
+      "username": "write-user",
+      "enabled": true,
+      "email": "write-user@test.com",
+      "firstName": "Test",
+      "lastName": "User",
+      "emailVerified": true,
+      "requiredActions": [],
+      "realmRoles": ["s3-read-write"],
+      "credentials": [
+        { "type": "password", "value": "writeuser123", "temporary": false }
+      ]
+    },
+    {
+      "username": "write-only-user",
+      "enabled": true,
+      "email": "write-only-user@test.com",
+      "firstName": "Test",
+      "lastName": "User",
+      "emailVerified": true,
+      "requiredActions": [],
+      "realmRoles": ["s3-write-only"],
+      "credentials": [
+        { "type": "password", "value": "writeonlyuser123", "temporary": false }
+      ]
+    }
+  ]
+}

test/volume_server/framework/cluster_multi.go

@@ -7,6 +7,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"sync"
 	"testing"
 
@@ -24,6 +25,7 @@ type MultiVolumeCluster struct {
 	logsDir           string
 	keepLogs          bool
 	volumeServerCount int
+	disksPerServer    int
 
 	masterPort     int
 	masterGrpcPort int
@@ -31,6 +33,7 @@ type MultiVolumeCluster struct {
 	volumePorts     []int
 	volumeGrpcPorts []int
 	volumePubPorts  []int
+	volumeDiskDirs  [][]string // per server: the data directories (one DiskLocation each)
 
 	masterCmd  *exec.Cmd
 	volumeCmds []*exec.Cmd
@@ -38,13 +41,25 @@ type MultiVolumeCluster struct {
 	cleanupOnce sync.Once
 }
 
-// StartMultiVolumeCluster starts a cluster with a specified number of volume servers
+// StartMultiVolumeCluster starts a cluster with serverCount volume servers, one
+// data directory (DiskLocation) each.
 func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount int) *MultiVolumeCluster {
+	return StartMultiVolumeClusterWithDisks(t, profile, serverCount, 1)
+}
+
+// StartMultiVolumeClusterWithDisks starts serverCount volume servers, each with
+// disksPerServer data directories passed to -dir as a comma list so every
+// directory becomes its own DiskLocation. Lets tests exercise per-disk EC
+// layouts (e.g. shards on one disk, a stub/index on a sibling disk).
+func StartMultiVolumeClusterWithDisks(t testing.TB, profile matrix.Profile, serverCount, disksPerServer int) *MultiVolumeCluster {
 	t.Helper()
 
 	if serverCount < 1 {
 		t.Fatalf("serverCount must be at least 1, got %d", serverCount)
 	}
+	if disksPerServer < 1 {
+		t.Fatalf("disksPerServer must be at least 1, got %d", disksPerServer)
+	}
 
 	weedBinary, err := FindOrBuildWeedBinary()
 	if err != nil {
@@ -60,10 +75,21 @@ func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount i
 	logsDir := filepath.Join(baseDir, "logs")
 	masterDataDir := filepath.Join(baseDir, "master")
 
-	// Create directories for master and all volume servers
+	// Create directories for master and all volume servers. With one disk the
+	// layout stays at baseDir/volumeN; multi-disk servers get baseDir/volumeN/diskD.
 	dirs := []string{configDir, logsDir, masterDataDir}
+	volumeDiskDirs := make([][]string, serverCount)
 	for i := 0; i < serverCount; i++ {
-		dirs = append(dirs, filepath.Join(baseDir, fmt.Sprintf("volume%d", i)))
+		serverDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
+		volumeDiskDirs[i] = make([]string, disksPerServer)
+		for d := 0; d < disksPerServer; d++ {
+			if disksPerServer == 1 {
+				volumeDiskDirs[i][d] = serverDir
+			} else {
+				volumeDiskDirs[i][d] = filepath.Join(serverDir, fmt.Sprintf("disk%d", d))
+			}
+			dirs = append(dirs, volumeDiskDirs[i][d])
+		}
 	}
 	for _, dir := range dirs {
 		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
@@ -98,11 +124,13 @@ func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount i
 		logsDir:           logsDir,
 		keepLogs:          keepLogs,
 		volumeServerCount: serverCount,
+		disksPerServer:    disksPerServer,
 		masterPort:        masterPort,
 		masterGrpcPort:    masterGrpcPort,
 		volumePorts:       make([]int, serverCount),
 		volumeGrpcPorts:   make([]int, serverCount),
 		volumePubPorts:    make([]int, serverCount),
+		volumeDiskDirs:    volumeDiskDirs,
 		volumeCmds:        make([]*exec.Cmd, serverCount),
 	}
 
@@ -133,8 +161,7 @@ func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount i
 
 	// Start all volume servers
 	for i := 0; i < serverCount; i++ {
-		volumeDataDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
-		if err = c.startVolume(i, volumeDataDir); err != nil {
+		if err = c.startVolume(i, c.volumeDiskDirs[i]); err != nil {
 			// Log current server's log for debugging startup failures
 			volumeLog := fmt.Sprintf("volume%d.log", i)
 			c.Stop()
@@ -207,13 +234,17 @@ func (c *MultiVolumeCluster) startMaster(dataDir string) error {
 	return c.masterCmd.Start()
 }
 
-func (c *MultiVolumeCluster) startVolume(index int, dataDir string) error {
+func (c *MultiVolumeCluster) startVolume(index int, dataDirs []string) error {
 	logName := fmt.Sprintf("volume%d.log", index)
 	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
 	if err != nil {
 		return err
 	}
 
+	maxPerDir := make([]string, len(dataDirs))
+	for i := range dataDirs {
+		maxPerDir[i] = "16"
+	}
 	args := []string{
 		"-config_dir=" + c.configDir,
 		"volume",
@@ -221,8 +252,8 @@ func (c *MultiVolumeCluster) startVolume(index int, dataDir string) error {
 		"-port=" + strconv.Itoa(c.volumePorts[index]),
 		"-port.grpc=" + strconv.Itoa(c.volumeGrpcPorts[index]),
 		"-port.public=" + strconv.Itoa(c.volumePubPorts[index]),
-		"-dir=" + dataDir,
-		"-max=16",
+		"-dir=" + strings.Join(dataDirs, ","),
+		"-max=" + strings.Join(maxPerDir, ","),
 		"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
 		"-readMode=" + c.profile.ReadMode,
 		"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
@@ -301,3 +332,17 @@ func (c *MultiVolumeCluster) VolumePublicURL(index int) string {
 func (c *MultiVolumeCluster) BaseDir() string {
 	return c.baseDir
 }
+
+// VolumeDiskDir returns the data directory backing disk diskIndex on volume
+// server serverIndex — i.e. the directory for that DiskLocation, for file
+// inspection in tests.
+func (c *MultiVolumeCluster) VolumeDiskDir(serverIndex, diskIndex int) string {
+	if serverIndex < 0 || serverIndex >= len(c.volumeDiskDirs) {
+		return ""
+	}
+	disks := c.volumeDiskDirs[serverIndex]
+	if diskIndex < 0 || diskIndex >= len(disks) {
+		return ""
+	}
+	return disks[diskIndex]
+}

unmaintained/stress/ec_stress_runner.go

@@ -301,7 +301,7 @@ func (r *runner) fetchVolumeList(ctx context.Context) (*master_pb.VolumeListResp
 		}
 
 		var response *master_pb.VolumeListResponse
-		err := pb.WithMasterClient(false, pb.ServerAddress(masterAddress), r.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
+		err := pb.WithMasterClient(context.Background(), false, pb.ServerAddress(masterAddress), r.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
 			callCtx, cancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
 			defer cancel()
 

weed/admin/dash/admin_server.go

@@ -1778,7 +1778,7 @@ func collectCollectionStats(topologyInfo *master_pb.TopologyInfo) map[string]col
 						shards := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo)
 						data := collectionMap[collection]
 						data.PhysicalSize += int64(shards.TotalSize())
-						data.LogicalSize += int64(shards.MinusParityShards().TotalSize())
+						data.LogicalSize += int64(shards.MinusParityShards(erasure_coding.DataShardsCount).TotalSize())
 						collectionMap[collection] = data
 
 						// fileCount is volume-wide (same .ecx on every shard

weed/admin/handlers/admin_handlers.go

@@ -246,6 +246,7 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
 	filesApi.HandleFunc("/download", h.fileBrowserHandlers.DownloadFile).Methods(http.MethodGet)
 	filesApi.HandleFunc("/view", h.fileBrowserHandlers.ViewFile).Methods(http.MethodGet)
 	filesApi.HandleFunc("/properties", h.fileBrowserHandlers.GetFileProperties).Methods(http.MethodGet)
+	filesApi.HandleFunc("/metadata", h.fileBrowserHandlers.ExportMetadata).Methods(http.MethodGet)
 
 	volumeApi := api.PathPrefix("/volumes").Subrouter()
 	volumeApi.Handle("/{id}/{server}/vacuum", wrapWrite(h.clusterHandlers.VacuumVolume)).Methods(http.MethodPost)

weed/admin/handlers/file_browser_grpc.go

@@ -9,8 +9,10 @@ import (
 	"net/http"
 	"path"
 	"strconv"
+	"strings"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/operation"
@@ -19,13 +21,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
-// Admin file browser upload chunk sizing — kept in sync with the values
-// s3api uses so files end up split into the same fid-sized pieces the rest of
-// the cluster expects.
-const (
-	adminUploadChunkSize      = 8 * 1024 * 1024
-	adminUploadSmallFileLimit = 256 * 1024
-)
+// Admin upload chunk size, matching s3api so files split into the same fid-sized pieces.
+const adminUploadChunkSize = 8 * 1024 * 1024
 
 // File browser handlers backed by the filer gRPC service. They bypass the
 // filer's HTTP listener so the UI keeps working when the filer is started
@@ -67,7 +64,7 @@ func (h *FileBrowserHandlers) fetchFileContentGrpc(ctx context.Context, filePath
 // response writer receives the canonical attachment headers and the raw
 // bytes; this replaces the HTTP-to-filer proxy that used to run in
 // DownloadFile.
-func (h *FileBrowserHandlers) downloadFileGrpc(ctx context.Context, filePath string, w http.ResponseWriter) error {
+func (h *FileBrowserHandlers) downloadFileGrpc(ctx context.Context, filePath string, w http.ResponseWriter, inline bool) error {
 	cleanFilePath, err := h.validateAndCleanFilePath(filePath)
 	if err != nil {
 		return err
@@ -84,27 +81,28 @@ func (h *FileBrowserHandlers) downloadFileGrpc(ctx context.Context, filePath str
 	size := int64(filer.FileSize(entry))
 
 	fileName := path.Base(cleanFilePath)
-	w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName}))
 
-	contentType := ""
-	if entry.Attributes != nil {
-		contentType = entry.Attributes.Mime
-	}
-	if contentType == "" {
-		contentType = "application/octet-stream"
+	// Resolve mime like the viewer does so Content-Type and the inline check agree.
+	contentType := dash.ResolveEntryMime(entry)
+
+	// Only inline images and PDFs; serve the rest as attachments so a hostile upload
+	// (HTML, SVG) can't run as same-origin script. nosniff locks the declared type.
+	disposition := "attachment"
+	if inline && (strings.HasPrefix(contentType, "image/") || contentType == "application/pdf") {
+		disposition = "inline"
 	}
+	w.Header().Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": fileName}))
 	w.Header().Set("Content-Type", contentType)
+	w.Header().Set("X-Content-Type-Options", "nosniff")
 	w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
 	w.WriteHeader(http.StatusOK)
 
 	return h.streamEntryContent(ctx, entry, size, w)
 }
 
-// uploadFileGrpc streams the upload to volume servers in 8 MiB chunks via the
-// shared chunked-upload helper, then registers the assembled entry through the
-// filer gRPC service. Bytes never enter the admin process's heap as a whole —
-// each chunk is sized to adminUploadChunkSize. Small files (< 256 KiB) are
-// stored inline on the entry, matching the S3 server's behaviour.
+// uploadFileGrpc streams the upload to volumes in 8 MiB chunks via the shared
+// chunked-upload helper, then registers the entry over the filer gRPC service.
+// Content always lands in volumes, never inlined on the entry.
 func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath string, fileName string, mimeType string, reader io.Reader) error {
 	cleanFilePath, err := h.validateAndCleanFilePath(filePath)
 	if err != nil {
@@ -152,11 +150,9 @@ func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath strin
 	}
 
 	chunkResult, err := operation.UploadReaderInChunks(ctx, reader, &operation.ChunkedUploadOption{
-		ChunkSize:       adminUploadChunkSize,
-		SmallFileLimit:  adminUploadSmallFileLimit,
-		SaveSmallInline: true,
-		MimeType:        mimeType,
-		AssignFunc:      assignFunc,
+		ChunkSize:  adminUploadChunkSize,
+		MimeType:   mimeType,
+		AssignFunc: assignFunc,
 	})
 	if err != nil {
 		// Partial chunks come back even on error so we can clean them up rather
@@ -178,11 +174,7 @@ func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath strin
 			Mime:     mimeType,
 		},
 	}
-	if len(chunkResult.SmallContent) > 0 {
-		entry.Content = chunkResult.SmallContent
-	} else {
-		entry.Chunks = chunkResult.FileChunks
-	}
+	entry.Chunks = chunkResult.FileChunks
 
 	err = h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 		_, createErr := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{

weed/admin/handlers/file_browser_handlers.go

@@ -1,8 +1,12 @@
 package handlers
 
 import (
+	"bufio"
+	"compress/gzip"
 	"context"
 	"fmt"
+	"io"
+	"mime"
 	"mime/multipart"
 	"net/http"
 	"os"
@@ -15,10 +19,12 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
 	"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
 	"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/http/client"
+	"google.golang.org/protobuf/proto"
 )
 
 type FileBrowserHandlers struct {
@@ -391,8 +397,9 @@ func (h *FileBrowserHandlers) DownloadFile(w http.ResponseWriter, r *http.Reques
 		writeJSONError(w, http.StatusBadRequest, "File path is required")
 		return
 	}
+	inline := r.URL.Query().Get("inline") == "true"
 	tracker := &responseWriteTracker{ResponseWriter: w}
-	if err := h.downloadFileGrpc(r.Context(), filePath, tracker); err != nil {
+	if err := h.downloadFileGrpc(r.Context(), filePath, tracker, inline); err != nil {
 		// Once bytes have been written we can't switch to a JSON error body
 		// without corrupting the partial response — log and stop. Before any
 		// write the response is still uncommitted, so a 502 with details is
@@ -628,6 +635,93 @@ func (h *FileBrowserHandlers) GetFileProperties(w http.ResponseWriter, r *http.R
 	writeJSON(w, http.StatusOK, properties)
 }
 
+// ExportMetadata streams a file or folder's metadata as a gzipped, length-prefixed
+// FullEntry stream — the format weed shell fs.meta.load reads. Directories are
+// walked recursively via the filer BFS metadata stream.
+func (h *FileBrowserHandlers) ExportMetadata(w http.ResponseWriter, r *http.Request) {
+	filePath := r.URL.Query().Get("path")
+	if filePath == "" {
+		writeJSONError(w, http.StatusBadRequest, "File path is required")
+		return
+	}
+	cleanPath, err := h.validateAndCleanFilePath(filePath)
+	if err != nil {
+		writeJSONError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+
+	tracker := &responseWriteTracker{ResponseWriter: w}
+
+	err = h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		stream, err := client.TraverseBfsMetadata(r.Context(), &filer_pb.TraverseBfsMetadataRequest{
+			Directory:        cleanPath,
+			ExcludedPrefixes: []string{filer.SystemLogDir},
+		})
+		if err != nil {
+			return err
+		}
+
+		// Read the first entry before sending headers so a bad path returns a clean error.
+		first, err := stream.Recv()
+		if err != nil {
+			return err
+		}
+
+		downloadName := path.Base(cleanPath)
+		if downloadName == "/" || downloadName == "." || downloadName == "" {
+			downloadName = "root"
+		}
+		tracker.Header().Set("Content-Type", "application/gzip")
+		tracker.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": downloadName + ".meta.gz"}))
+		tracker.WriteHeader(http.StatusOK)
+
+		bw := bufio.NewWriter(tracker)
+		gw := gzip.NewWriter(bw)
+
+		sizeBuf := make([]byte, 4)
+		writeEntry := func(resp *filer_pb.TraverseBfsMetadataResponse) error {
+			b, err := proto.Marshal(&filer_pb.FullEntry{Dir: resp.Directory, Entry: resp.Entry})
+			if err != nil {
+				return err
+			}
+			util.Uint32toBytes(sizeBuf, uint32(len(b)))
+			if _, err := gw.Write(sizeBuf); err != nil {
+				return err
+			}
+			_, err = gw.Write(b)
+			return err
+		}
+
+		if err := writeEntry(first); err != nil {
+			return err
+		}
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr == io.EOF {
+				break
+			}
+			if recvErr != nil {
+				return recvErr
+			}
+			if err := writeEntry(resp); err != nil {
+				return err
+			}
+		}
+
+		if err := gw.Close(); err != nil {
+			return err
+		}
+		return bw.Flush()
+	})
+	if err != nil {
+		if tracker.committed {
+			glog.Errorf("Error exporting metadata for %s: %v", cleanPath, err)
+			return
+		}
+		writeJSONError(w, http.StatusInternalServerError, "Failed to export metadata: "+err.Error())
+	}
+}
+
 // Helper function to format bytes
 func (h *FileBrowserHandlers) formatBytes(bytes int64) string {
 	const unit = 1024
@@ -685,4 +779,3 @@ func min(a, b int64) int64 {
 	}
 	return b
 }
-

weed/admin/static/js/admin.js

@@ -1788,7 +1788,7 @@ function createFileViewerContent(file, content) {
     if (file.mime.startsWith('image/')) {
         return `
             <div class="text-center">
-                <img src="${basePath('/api/files/download?path=' + encodeURIComponent(file.full_path))}"
+                <img src="${basePath('/api/files/download?path=' + encodeURIComponent(file.full_path) + '&inline=true')}"
                      class="img-fluid" alt="${file.name}" style="max-height: 500px;">
             </div>
         `;
@@ -1806,7 +1806,7 @@ function createFileViewerContent(file, content) {
     } else if (file.mime === 'application/pdf') {
         return `
             <div class="text-center">
-                <embed src="${basePath('/api/files/download?path=' + encodeURIComponent(file.full_path))}"
+                <embed src="${basePath('/api/files/download?path=' + encodeURIComponent(file.full_path) + '&inline=true')}"
                        type="application/pdf" width="100%" height="500px">
             </div>
         `;

weed/admin/view/app/file_browser.templ

@@ -198,6 +198,9 @@ templ FileBrowser(data dash.FileBrowserData) {
 											<button type="button" class="btn btn-outline-secondary btn-sm" title="Properties" data-action="properties" data-path={ entry.FullPath }>
 												<i class="fas fa-info-circle"></i>
 											</button>
+											<button type="button" class="btn btn-outline-success btn-sm" title="Export metadata" data-action="export-metadata" data-path={ entry.FullPath }>
+												<i class="fas fa-file-export"></i>
+											</button>
 											<button type="button" class="btn btn-outline-danger btn-sm" title="Delete" data-action="delete" data-path={ entry.FullPath }>
 												<i class="fas fa-trash"></i>
 											</button>
@@ -354,7 +357,7 @@ templ FileBrowser(data dash.FileBrowserData) {
 			}
 		});
 		
-		// Handle file browser action buttons (download, view, properties, delete)
+		// Handle file browser action buttons (download, view, properties, export-metadata, delete)
 		document.addEventListener('click', function(e) {
 			const button = e.target.closest('[data-action]');
 			if (!button) return;
@@ -374,6 +377,9 @@ templ FileBrowser(data dash.FileBrowserData) {
 				case 'properties':
 					showFileProperties(path);
 					break;
+				case 'export-metadata':
+					exportMetadata(path);
+					break;
 				case 'delete':
 					const fileName = path.split('/').pop();
 					showDeleteConfirm(fileName, function() {
@@ -400,6 +406,11 @@ templ FileBrowser(data dash.FileBrowserData) {
 		window.open((window.__BASE_PATH__ || '') + '/api/files/view?path=' + encodeURIComponent(path), '_blank');
 	}
 
+	function exportMetadata(path) {
+		// Download metadata as an fs.meta.load-compatible .meta.gz
+		window.open((window.__BASE_PATH__ || '') + '/api/files/metadata?path=' + encodeURIComponent(path), '_blank');
+	}
+
 	function showFileProperties(path) {
 		// Fetch file properties and show in modal
 		fetch((window.__BASE_PATH__ || '') + '/api/files/properties?path=' + encodeURIComponent(path))

weed/cluster/master_client.go

@@ -12,7 +12,7 @@ import (
 
 func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string, clientType string) (existingNodes []*master_pb.ClusterNodeUpdate) {
 
-	if grpcErr := pb.WithMasterClient(false, master, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
+	if grpcErr := pb.WithMasterClient(context.Background(), false, master, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
 		resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
 			ClientType: clientType,
 			FilerGroup: filerGroup,

weed/command/command.go

@@ -48,7 +48,6 @@ var Commands = []*Command{
 	cmdVolume,
 	cmdWebDav,
 	cmdSftp,
-	cmdNfs,
 	cmdWorker,
 }
 

weed/command/filer_copy.go

@@ -596,7 +596,7 @@ var _ = filer_pb.FilerClient(&FileCopyWorker{})
 func (worker *FileCopyWorker) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) {
 
 	filerGrpcAddress := worker.filerAddress.ToGrpcAddress()
-	err = pb.WithGrpcClient(streamingMode, worker.signature, func(grpcConnection *grpc.ClientConn) error {
+	err = pb.WithGrpcClient(context.Background(), streamingMode, worker.signature, func(grpcConnection *grpc.ClientConn) error {
 		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 		return fn(client)
 	}, filerGrpcAddress, false, worker.options.grpcDialOption)

weed/command/filer_sync_jobs_test.go

@@ -4,6 +4,7 @@ import (
 	"container/heap"
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -582,3 +583,30 @@ func BenchmarkConflictCheck(b *testing.B) {
 		})
 	}
 }
+
+// TestMetadataProcessorEmptyMarkerKeepsWatermarkStale: the MaxUnsyncedEvents
+// marker (empty EventNotification, fresh timestamp) is dropped by AddSyncJob and
+// does NOT advance processedTsWatermark, so offsetFunc keeps publishing the stale
+// offset. This is why the client must not drive sync_offset off the watermark
+// for these markers.
+func TestMetadataProcessorEmptyMarkerKeepsWatermarkStale(t *testing.T) {
+	const staleOffset = int64(1_000_000_000)
+	freshTs := staleOffset + int64(time.Hour) // a "now"-ish source timestamp
+
+	p := NewMetadataProcessor(func(*filer_pb.SubscribeMetadataResponse) error { return nil }, 4, staleOffset)
+
+	marker := &filer_pb.SubscribeMetadataResponse{
+		TsNs:              freshTs,
+		EventNotification: &filer_pb.EventNotification{},
+	}
+	if !filer_pb.IsEmpty(marker) {
+		t.Fatal("marker should be IsEmpty")
+	}
+
+	p.AddSyncJob(marker)
+
+	if got := p.processedTsWatermark.Load(); got != staleOffset {
+		t.Fatalf("empty marker advanced watermark to %d; want it to stay stale at %d", got, staleOffset)
+	}
+	t.Logf("marker carried fresh ts %d but watermark stayed stale at %d", freshTs, staleOffset)
+}

weed/command/filer_sync_verify.go

@@ -177,7 +177,7 @@ type simpleFilerClient struct {
 }
 
 func (c *simpleFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
-	return pb.WithGrpcClient(streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
+	return pb.WithGrpcClient(context.Background(), streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
 		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 		return fn(client)
 	}, c.grpcAddress.ToGrpcAddress(), false, c.grpcDialOption)

weed/command/fix.go

@@ -6,6 +6,7 @@ import (
 	"io/fs"
 	"os"
 	"path"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -36,14 +37,15 @@ var cmdFix = &Command{
 }
 
 var (
-	fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
-	fixVolumeId         = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
-	fixIncludeDeleted   = cmdFix.Flag.Bool("includeDeleted", true, "include deleted entries in the index file")
-	fixIgnoreError      = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors")
-	fixRemoteFile       = cmdFix.Flag.Bool("remoteFile", false, "an optional, if true will not try to load the local .dat file, but only the remote file")
-	fixGenerateEcx      = cmdFix.Flag.Bool("ecx", false, "regenerate a lost EC index (.ecx) — and the .vif when missing — from the local .ec## shards (missing shards are reconstructed from parity when enough survive). Run with the volume server stopped.")
-	fixEcDataShards     = cmdFix.Flag.Int("ecDataShards", 0, "EC data shard count for -ecx (0 = read from .vif, otherwise default 10)")
-	fixEcParityShards   = cmdFix.Flag.Int("ecParityShards", 0, "EC parity shard count for -ecx (0 = read from .vif, infer from shard count, otherwise default 4)")
+	fixVolumeCollection      = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
+	fixVolumeId              = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
+	fixIncludeDeleted        = cmdFix.Flag.Bool("includeDeleted", true, "include deleted entries in the index file")
+	fixIgnoreError           = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors")
+	fixRemoteFile            = cmdFix.Flag.Bool("remoteFile", false, "an optional, if true will not try to load the local .dat file, but only the remote file")
+	fixGenerateEcx           = cmdFix.Flag.Bool("ecx", false, "regenerate a lost EC index (.ecx) — and the .vif when missing — from the local .ec## shards (missing shards are reconstructed from parity when enough survive). Run with the volume server stopped.")
+	fixEcDataShards          = cmdFix.Flag.Int("ecDataShards", 0, "EC data shard count for -ecx (0 = read from .vif, otherwise default 10)")
+	fixEcParityShards        = cmdFix.Flag.Int("ecParityShards", 0, "EC parity shard count for -ecx (0 = read from .vif, infer from shard count, otherwise default 4)")
+	fixEcUnsafeIgnoreSidecar = cmdFix.Flag.Bool("ecUnsafeIgnoreSidecar", false, "for -ecx: proceed even when the EC bitrot checksum sidecar (.ecsum) is malformed or stale, instead of failing closed; the reconstructed shards are not verified against it")
 )
 
 type VolumeFileScanner4Fix struct {
@@ -190,15 +192,34 @@ func SaveToIdx(scaner *VolumeFileScanner4Fix, idxName string) (ret error) {
 		idxFile.Close()
 	}()
 
-	return scaner.nm.AscendingVisit(func(value needle_map.NeedleValue) error {
-		_, err := idxFile.Write(value.ToBytes())
-		if scaner.includeDeleted && err == nil {
-			if deleted, ok := scaner.nmDeleted.Get(value.Key); ok {
-				_, err = idxFile.Write(deleted.ToBytes())
-			}
-		}
+	// Emit entries in .dat offset (append) order so the .idx stays the
+	// append-ordered log the volume server writes at runtime — not sorted by
+	// key, which is the .sdx / .ecx shape. A key-sorted .idx puts the
+	// highest-key needle last instead of the .dat-tail needle, which used to
+	// flip volumes read-only on load. Every tombstone is included (including
+	// any whose live needle is gone) so the last entry is the real .dat tail.
+	var values []needle_map.NeedleValue
+	collect := func(value needle_map.NeedleValue) error {
+		values = append(values, value)
+		return nil
+	}
+	if err = scaner.nm.AscendingVisit(collect); err != nil {
 		return err
+	}
+	if scaner.includeDeleted {
+		if err = scaner.nmDeleted.AscendingVisit(collect); err != nil {
+			return err
+		}
+	}
+	sort.Slice(values, func(i, j int) bool {
+		return values[i].Offset.ToActualOffset() < values[j].Offset.ToActualOffset()
 	})
+	for _, value := range values {
+		if _, err = idxFile.Write(value.ToBytes()); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64, fixIncludeDeleted bool) {
@@ -361,7 +382,7 @@ func doFixEcxFromShards(basePath, baseFileName, collection string, volumeId int6
 	if !dataComplete {
 		ctx := &erasure_coding.ECContext{DataShards: dataShards, ParityShards: parityShards}
 		glog.Infof("volume %d: %d/%d shards present; reconstructing missing shards (%s) before index rebuild", volumeId, presentCount, dataShards+parityShards, ctx.String())
-		if _, err := erasure_coding.RebuildEcFilesWithContext(base, ctx); err != nil {
+		if _, err := erasure_coding.RebuildEcFiles(base, ctx, *fixEcUnsafeIgnoreSidecar); err != nil {
 			fail(fmt.Errorf("volume %d: reconstruct missing shards from %d survivors: %w", volumeId, presentCount, err))
 			return
 		}

weed/command/fix_ecx_test.go

@@ -92,7 +92,7 @@ func buildAndEncodeTestEcVolume(t *testing.T, dir, baseName string) (base string
 	idxFile.Close()
 	nm.Close()
 
-	if err := erasure_coding.WriteEcFiles(base); err != nil {
+	if _, err := erasure_coding.WriteEcFiles(base, erasure_coding.BackgroundECContext()); err != nil {
 		t.Fatalf("WriteEcFiles: %v", err)
 	}
 	if err := erasure_coding.WriteSortedFileFromIdx(base, ".ecx"); err != nil {

weed/command/fix_test.go

@@ -0,0 +1,75 @@
+package command
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// TestSaveToIdxWritesOffsetOrder guards that weed fix rebuilds the .idx as an
+// append-ordered log (sorted by .dat offset), not sorted by key. A key-sorted
+// .idx puts the highest-key needle last instead of the .dat-tail needle, which
+// flipped volumes read-only on load (issue #9688). It must also carry every
+// tombstone — including one whose live needle is gone — so the last entry is
+// the real .dat tail.
+func TestSaveToIdxWritesOffsetOrder(t *testing.T) {
+	nm := needle_map.NewMemDb()
+	defer nm.Close()
+	nmDeleted := needle_map.NewMemDb()
+	defer nmDeleted.Close()
+
+	// Live needles with high keys at low offsets (as if written first).
+	for _, e := range []struct {
+		key    uint64
+		offset int64
+	}{{30, 8}, {20, 128}, {10, 256}} {
+		if err := nm.Set(types.Uint64ToNeedleId(e.key), types.ToOffset(e.offset), types.Size(100)); err != nil {
+			t.Fatalf("nm.Set: %v", err)
+		}
+	}
+	// A tombstone at the .dat tail whose live needle is gone; the old SaveToIdx
+	// dropped these because the key was absent from the live map.
+	if err := nmDeleted.Set(types.Uint64ToNeedleId(5), types.ToOffset(384), types.TombstoneFileSize); err != nil {
+		t.Fatalf("nmDeleted.Set: %v", err)
+	}
+
+	scanner := &VolumeFileScanner4Fix{nm: nm, nmDeleted: nmDeleted, includeDeleted: true}
+
+	idxPath := filepath.Join(t.TempDir(), "v.idx")
+	if err := SaveToIdx(scanner, idxPath); err != nil {
+		t.Fatalf("SaveToIdx: %v", err)
+	}
+
+	f, err := os.Open(idxPath)
+	if err != nil {
+		t.Fatalf("open idx: %v", err)
+	}
+	defer f.Close()
+
+	var offsets []int64
+	var lastKey types.NeedleId
+	if err := idx.WalkIndexFile(f, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error {
+		offsets = append(offsets, offset.ToActualOffset())
+		lastKey = key
+		return nil
+	}); err != nil {
+		t.Fatalf("walk idx: %v", err)
+	}
+
+	if len(offsets) != 4 {
+		t.Fatalf("expected 4 entries (3 live + 1 tombstone), got %d", len(offsets))
+	}
+	for i := 1; i < len(offsets); i++ {
+		if offsets[i] < offsets[i-1] {
+			t.Fatalf("entries not in offset order: %v", offsets)
+		}
+	}
+	// The .dat-tail needle (the orphan tombstone at the highest offset) must be last.
+	if lastKey != types.Uint64ToNeedleId(5) {
+		t.Errorf("last entry should be the .dat-tail tombstone (key 5), got key %v", lastKey)
+	}
+}

weed/command/mini.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math/bits"
@@ -1787,8 +1788,19 @@ func printWelcomeMessage() {
 		fmt.Fprintf(&sb, "    Admin UI:        http://%s:%d\n", *miniIp, *miniAdminOptions.port)
 	}
 
-	fmt.Fprintf(&sb, "\n  Data Directory: %s\n\n", *miniDataFolders)
-	sb.WriteString("  Press Ctrl+C to stop all components")
+	fmt.Fprintf(&sb, "\n  Data Directory:   %s\n", *miniDataFolders)
+	firstDir := util.StringSplit(*miniDataFolders, ",")[0]
+	if ds := stats_collect.NewDiskStatus(firstDir); ds != nil && ds.All > 0 {
+		fmt.Fprintf(&sb, "  Free Space:       %s\n", util.BytesToHumanReadable(ds.Free))
+	}
+	if miniMasterOptions.volumeSizeLimitMB != nil {
+		fmt.Fprintf(&sb, "  Volume Size:      %s\n", util.BytesToHumanReadable(uint64(*miniMasterOptions.volumeSizeLimitMB)*bytesPerMB))
+	}
+	if max, free, ok := miniVolumeCounts(); ok {
+		fmt.Fprintf(&sb, "  Volume Count:     %d\n", max)
+		fmt.Fprintf(&sb, "  Free Volumes:     %d\n", free)
+	}
+	sb.WriteString("\n  Press Ctrl+C to stop all components")
 
 	switch {
 	case s3api.HasAnyIdentity():
@@ -1810,6 +1822,33 @@ func printWelcomeMessage() {
 	fmt.Println("")
 }
 
+// miniVolumeCounts asks the local master for the total volume slots the data
+// directory supports (Topology.Max) and how many are still free (Topology.Free).
+// Best-effort: any error returns ok=false so the welcome banner simply omits
+// the lines.
+func miniVolumeCounts() (max, free int64, ok bool) {
+	url := getHealthCheckAddr(fmt.Sprintf("http://%s:%d/dir/status", *miniIp, *miniMasterOptions.port))
+	client := &http.Client{Timeout: 2 * time.Second}
+	resp, err := client.Get(url)
+	if err != nil {
+		return 0, 0, false
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return 0, 0, false
+	}
+	var status struct {
+		Topology struct {
+			Max  int64
+			Free int64
+		}
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&status); err != nil {
+		return 0, 0, false
+	}
+	return status.Topology.Max, status.Topology.Free, true
+}
+
 // ensureMiniBuckets creates each named bucket on the embedded filer if it does
 // not already exist. bucketSpec is a comma-separated list (whitespace around
 // each name is trimmed); empty entries and an empty spec are no-ops so callers

weed/command/nfs.go

@@ -1,127 +0,0 @@
-package command
-
-import (
-	"fmt"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/security"
-	weed_server_nfs "github.com/seaweedfs/seaweedfs/weed/server/nfs"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
-)
-
-var (
-	nfsStandaloneOptions NfsOptions
-)
-
-type NfsOptions struct {
-	filer              *string
-	ipBind             *string
-	port               *int
-	filerRootPath      *string
-	readOnly           *bool
-	allowedClients     *string
-	volumeServerAccess *string
-	portmapBind        *string
-}
-
-func init() {
-	cmdNfs.Run = runNfs // break init cycle
-	nfsStandaloneOptions.filer = cmdNfs.Flag.String("filer", "localhost:8888", "filer server address")
-	nfsStandaloneOptions.ipBind = cmdNfs.Flag.String("ip.bind", "127.0.0.1", "ip address to bind to. Defaults to loopback; override explicitly to expose the experimental server to the network.")
-	nfsStandaloneOptions.port = cmdNfs.Flag.Int("port", 2049, "NFS server listen port")
-	nfsStandaloneOptions.filerRootPath = cmdNfs.Flag.String("filer.path", "", "remote path from filer server to export. Required: no default is provided so operators must opt in to exporting a namespace subtree.")
-	nfsStandaloneOptions.readOnly = cmdNfs.Flag.Bool("readOnly", false, "export the filer path as read only")
-	nfsStandaloneOptions.allowedClients = cmdNfs.Flag.String("allowedClients", "", "comma-separated client IPs, hostnames, or CIDRs allowed to connect")
-	nfsStandaloneOptions.volumeServerAccess = cmdNfs.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]")
-	nfsStandaloneOptions.portmapBind = cmdNfs.Flag.String("portmap.bind", "", "when set, bind a built-in portmap v2 responder on <ip>:111 so plain `mount -t nfs` works without client-side portmap bypass. Empty disables it. Binding port 111 requires root or CAP_NET_BIND_SERVICE and must not conflict with a system rpcbind.")
-}
-
-var cmdNfs = &Command{
-	UsageLine: "nfs -port=2049 -filer=<ip:port> -filer.path=<exported subtree>",
-	Short:     "start an experimental NFSv3 server backed by a filer",
-	Long: `start an experimental NFSv3 server backed by a filer.
-
-This command serves an experimental filer-native NFSv3 frontend with
-deterministic filehandles, filer-backed metadata operations, and direct
-volume-server data access for chunk reads and buffered writes.
-
-Safer defaults (since export ACLs are still not implemented):
-
-  - ip.bind defaults to 127.0.0.1, so the server is not reachable from
-    other hosts unless you override it explicitly.
-  - filer.path has no default; you must pick the subtree to export.
-
-Override -ip.bind to a routable address only after you have reviewed
--allowedClients and the readiness of the rest of your deployment.
-
-Mounting from a Linux client
-----------------------------
-The server does not run portmap/rpcbind by default. That means Linux
-mount.nfs, which queries portmap on port 111 first, will fail with
-"portmap query failed" against the plain form:
-
-    mount -t nfs -o nfsvers=3,nolock <host>:/export /mnt
-
-Either tell the client to bypass portmap:
-
-    mount -t nfs -o nfsvers=3,nolock,port=2049,mountport=2049,\
-        proto=tcp,mountproto=tcp <host>:/export /mnt
-
-or enable the built-in portmap responder on the server:
-
-    weed nfs ... -portmap.bind=0.0.0.0
-
-With the responder enabled MOUNT v3 is answered over both TCP and UDP,
-so the plain mount form above just works — no mountproto override is
-required even on clients whose default mountproto is UDP.
-
-Binding port 111 requires root or CAP_NET_BIND_SERVICE and must not
-collide with a system rpcbind.
-	`,
-}
-
-func runNfs(cmd *Command, args []string) bool {
-	util.LoadSecurityConfiguration()
-
-	if *nfsStandaloneOptions.ipBind == "" {
-		*nfsStandaloneOptions.ipBind = "127.0.0.1"
-	}
-
-	if *nfsStandaloneOptions.filerRootPath == "" {
-		glog.Errorf("-filer.path is required: pick an explicit subtree to export; exporting \"/\" is not a default")
-		return false
-	}
-	if *nfsStandaloneOptions.filerRootPath == "/" {
-		glog.Warningf("-filer.path=/ exports the entire filer namespace; ensure -allowedClients or -ip.bind constrains access")
-	}
-
-	listenAddress := fmt.Sprintf("%s:%d", *nfsStandaloneOptions.ipBind, *nfsStandaloneOptions.port)
-	glog.V(0).Infof("Starting Seaweed NFS Server %s at %s", version.Version(), listenAddress)
-
-	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
-
-	nfsServer, err := weed_server_nfs.NewServer(&weed_server_nfs.Option{
-		Filer:              pb.ServerAddress(*nfsStandaloneOptions.filer),
-		BindIp:             *nfsStandaloneOptions.ipBind,
-		Port:               *nfsStandaloneOptions.port,
-		FilerRootPath:      *nfsStandaloneOptions.filerRootPath,
-		ReadOnly:           *nfsStandaloneOptions.readOnly,
-		AllowedClients:     util.StringSplit(*nfsStandaloneOptions.allowedClients, ","),
-		VolumeServerAccess: *nfsStandaloneOptions.volumeServerAccess,
-		GrpcDialOption:     grpcDialOption,
-		PortmapBind:        *nfsStandaloneOptions.portmapBind,
-	})
-	if err != nil {
-		glog.Errorf("NFS Server startup error: %v", err)
-		return false
-	}
-
-	if err := nfsServer.Start(); err != nil {
-		glog.Errorf("NFS Server startup error: %v", err)
-		return false
-	}
-
-	return true
-}

weed/command/upload.go

@@ -145,7 +145,7 @@ func runUpload(cmd *Command, args []string) bool {
 }
 
 func readMasterConfiguration(grpcDialOption grpc.DialOption, masterAddress pb.ServerAddress) (replication string, err error) {
-	err = pb.WithMasterClient(false, masterAddress, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
+	err = pb.WithMasterClient(context.Background(), false, masterAddress, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
 		resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
 		if err != nil {
 			return fmt.Errorf("get master %s configuration: %v", masterAddress, err)

weed/command/version.go

@@ -21,7 +21,7 @@ func runVersion(cmd *Command, args []string) bool {
 
 	fmt.Printf("version %s %s %s\n", version.Version(), runtime.GOOS, runtime.GOARCH)
 	println()
-	println("For enterprise users, please visit https://seaweedfs.com for SeaweedFS Enterprise Edition,")
-	println("which has a self-healing storage format with better data protection.")
+	println("For enterprise users, please visit https://seaweedfs.com for the SeaweedFS Enterprise Edition,")
+	println("which has advanced features, including data recovery, self-healing storage, customizable erasure coding, EC vacuum and repair, etc.")
 	return true
 }

weed/command/volume.go

@@ -23,6 +23,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/server/constants"
 	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
@@ -139,6 +140,9 @@ var (
 	volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
 	minFreeSpacePercent   = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).")
 	minFreeSpace          = cmdVolume.Flag.String("minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
+
+	ecBitrotChecksum    = cmdVolume.Flag.Bool("ec.bitrotChecksum", true, "write a bitrot checksum sidecar (.ecsum) when generating EC shards, enabling silent-corruption detection on scrub/rebuild")
+	ecBitrotBlockSizeMB = cmdVolume.Flag.Int("ec.bitrotBlockSizeMB", 16, "EC bitrot checksum block granularity in MiB; must be a power of two and at least 1")
 )
 
 func runVolume(cmd *Command, args []string) bool {
@@ -171,6 +175,24 @@ func runVolume(cmd *Command, args []string) bool {
 		*v.mastersString = *v.mserverString
 	}
 
+	// Apply EC bitrot checksum settings.
+	erasure_coding.BitrotProtectionEnabled = *ecBitrotChecksum
+	// Bound-check before the multiply so a huge value cannot overflow int64 past
+	// the power-of-two check. Cap = shared MaxBitrotBlockSize, kept in sync with
+	// ValidateBitrotManifest.
+	const maxBitrotBlockSizeMB = erasure_coding.MaxBitrotBlockSize / (1024 * 1024)
+	if mb := *ecBitrotBlockSizeMB; mb >= 1 && mb <= maxBitrotBlockSizeMB {
+		if blockSize := int64(mb) * 1024 * 1024; blockSize&(blockSize-1) == 0 {
+			erasure_coding.BitrotBlockSize = blockSize
+		} else if mb != 16 {
+			glog.Warningf("ignoring invalid -ec.bitrotBlockSizeMB=%d (must be a power of two); using %d MiB",
+				mb, erasure_coding.BitrotBlockSize/(1024*1024))
+		}
+	} else if *ecBitrotBlockSizeMB != 16 {
+		glog.Warningf("ignoring out-of-range -ec.bitrotBlockSizeMB=%d (must be a power of two in [1, %d] MiB); using %d MiB",
+			*ecBitrotBlockSizeMB, maxBitrotBlockSizeMB, erasure_coding.BitrotBlockSize/(1024*1024))
+	}
+
 	minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
 	v.masters = pb.ServerAddresses(*v.mastersString).ToAddresses()
 	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces)

weed/credential/grpc/grpc_store.go

@@ -98,7 +98,7 @@ func (store *IamGrpcStore) withIamClient(ctx context.Context, fn func(ctx contex
 		}
 	}
 
-	return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+	return pb.WithGrpcClient(context.Background(), false, 0, func(conn *grpc.ClientConn) error {
 		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
 		return fn(ctx, client)
 	}, filerAddress.ToGrpcAddress(), false, dialOption)

weed/credential/propagating_store.go

@@ -86,7 +86,7 @@ func (s *PropagatingCredentialStore) propagateChange(ctx context.Context, fn fun
 		wg.Add(1)
 		go func(server string) {
 			defer wg.Done()
-			err := pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+			err := pb.WithGrpcClient(context.Background(), false, 0, func(conn *grpc.ClientConn) error {
 				glog.V(4).Infof("IAM: successfully connected to S3 server %s for propagation", server)
 				client := s3_pb.NewSeaweedS3IamCacheClient(conn)
 				return fn(propagateCtx, client)

weed/filer/abstract_sql/abstract_sql_store_kv.go

@@ -21,7 +21,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by
 
 	dirStr, dirHash, name := GenDirAndName(key)
 
-	res, err := db.ExecContext(ctx, store.GetSqlInsert(DEFAULT_TABLE), dirHash, name, dirStr, value)
+	_, err = db.ExecContext(ctx, store.GetSqlInsert(DEFAULT_TABLE), dirHash, name, dirStr, value)
 	if err == nil {
 		return
 	}
@@ -34,7 +34,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by
 	// now the insert failed possibly due to duplication constraints
 	glog.V(1).InfofCtx(ctx, "kv insert falls back to update: %s", err)
 
-	res, err = db.ExecContext(ctx, store.GetSqlUpdate(DEFAULT_TABLE), value, dirHash, name, dirStr)
+	res, err := db.ExecContext(ctx, store.GetSqlUpdate(DEFAULT_TABLE), value, dirHash, name, dirStr)
 	if err != nil {
 		return fmt.Errorf("kv upsert: %s", err)
 	}

weed/filer/filer.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/filer/empty_folder_cleanup"
-	"github.com/seaweedfs/seaweedfs/weed/sequence"
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -64,7 +63,6 @@ type Filer struct {
 	DeletionRetryQueue      *DeletionRetryQueue
 	EmptyFolderCleaner      *empty_folder_cleanup.EmptyFolderCleaner
 	EmptyFolderCleanupDelay time.Duration
-	inodeSequencer          sequence.Sequencer
 }
 
 func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer {
@@ -79,7 +77,6 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH
 		MaxFilenameLength:   maxFilenameLength,
 		deletionQuit:        make(chan struct{}),
 		DeletionRetryQueue:  NewDeletionRetryQueue(),
-		inodeSequencer:      newInodeSequencer(filerHost),
 	}
 	if f.UniqueFilerId < 0 {
 		f.UniqueFilerId = -f.UniqueFilerId

weed/filer/filer_conf.go

@@ -233,6 +233,35 @@ func ClonePathConf(src *filer_pb.FilerConf_PathConf) *filer_pb.FilerConf_PathCon
 	}
 }
 
+// ApplyBucketQuotaReadOnly sets read-only when usedSize exceeds quota and clears it
+// once back under, reporting whether the flag changed. A non-positive quota is left
+// untouched so a manually locked bucket is never reopened.
+func (fc *FilerConf) ApplyBucketQuotaReadOnly(locationPrefix string, usedSize, quota float64) (readOnly, changed bool) {
+	if quota <= 0 {
+		return fc.MatchStorageRule(locationPrefix).ReadOnly, false
+	}
+
+	locConf := ClonePathConf(fc.MatchStorageRule(locationPrefix))
+	locConf.LocationPrefix = locationPrefix
+	wasReadOnly := locConf.ReadOnly
+
+	if wasReadOnly {
+		if usedSize < quota {
+			locConf.ReadOnly = false
+		}
+	} else {
+		if usedSize > quota {
+			locConf.ReadOnly = true
+		}
+	}
+
+	if locConf.ReadOnly == wasReadOnly {
+		return wasReadOnly, false
+	}
+	fc.SetLocationConf(locConf)
+	return locConf.ReadOnly, true
+}
+
 func (fc *FilerConf) GetCollectionTtls(collection string) (ttls map[string]string) {
 	ttls = make(map[string]string)
 	fc.rules.Walk(func(key []byte, value *filer_pb.FilerConf_PathConf) bool {

weed/filer/filer_conf_test.go

@@ -128,3 +128,36 @@ func TestClonePathConfNil(t *testing.T) {
 	assert.NotNil(t, clone, "ClonePathConf(nil) should return a non-nil empty PathConf")
 	assert.Equal(t, "", clone.LocationPrefix, "ClonePathConf(nil) should return empty PathConf")
 }
+
+func TestApplyBucketQuotaReadOnly(t *testing.T) {
+	const prefix = "/buckets/b/"
+
+	// over quota: flips to read-only
+	fc := NewFilerConf()
+	readOnly, changed := fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
+	assert.True(t, changed)
+	assert.True(t, readOnly)
+	assert.True(t, fc.MatchStorageRule(prefix).ReadOnly)
+
+	// still over quota: no change
+	_, changed = fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
+	assert.False(t, changed)
+
+	// back under quota: flips to writable
+	readOnly, changed = fc.ApplyBucketQuotaReadOnly(prefix, 50, 100)
+	assert.True(t, changed)
+	assert.False(t, readOnly)
+	assert.False(t, fc.MatchStorageRule(prefix).ReadOnly)
+
+	// quota disabled leaves the flag untouched, so manual locks survive
+	fc = NewFilerConf()
+	fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
+	readOnly, changed = fc.ApplyBucketQuotaReadOnly(prefix, 150, -1)
+	assert.False(t, changed)
+	assert.True(t, readOnly)
+
+	// under quota and not read-only: no rule churn
+	fc = NewFilerConf()
+	_, changed = fc.ApplyBucketQuotaReadOnly(prefix, 50, 100)
+	assert.False(t, changed)
+}

weed/filer/filer_inode.go

@@ -1,51 +1,26 @@
 package filer
 
 import (
-	"os"
-	"strconv"
+	"time"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
-// newInodeSequencer constructs the inode sequencer used to assign object
-// identity for filer entries. The Snowflake node id defaults to a masked hash
-// of filerHost, which only has 1024 possible values; operators running a
-// multi-filer cluster should set SEAWEEDFS_FILER_SNOWFLAKE_ID to an explicit
-// per-filer value (1..1023) to avoid birthday-paradox collisions.
-//
-// Initialization failures are fatal: a process-local fallback allocator would
-// re-use inode values across restarts and violate the stable object identity
-// guarantee that NFS filehandles and the inode secondary index rely on.
-func newInodeSequencer(filerHost pb.ServerAddress) sequence.Sequencer {
-	snowflakeId := parseSnowflakeIdFromEnv()
-	seq, err := sequence.NewSnowflakeSequencer(string(filerHost), snowflakeId)
-	if err != nil {
-		glog.Fatalf("initialize inode sequencer for filer %s (snowflakeId=%d): %v", filerHost, snowflakeId, err)
-	}
-	return seq
-}
-
-func parseSnowflakeIdFromEnv() int {
-	raw := os.Getenv("SEAWEEDFS_FILER_SNOWFLAKE_ID")
-	if raw == "" {
-		return 0
-	}
-	id, err := strconv.Atoi(raw)
-	if err != nil || id < 0 || id > 0x3ff {
-		glog.Fatalf("SEAWEEDFS_FILER_SNOWFLAKE_ID must be an integer in [0,1023], got %q", raw)
-	}
-	return id
-}
-
+// ensureEntryInode derives a stable inode the same way the FUSE mount does, so
+// the filer-stored value matches what a mount would otherwise compute and no
+// per-object reverse index is required. Hard links hash their shared
+// HardLinkId, so every link resolves to one inode; other entries hash the path
+// and creation time.
 func (f *Filer) ensureEntryInode(entry *Entry) {
 	if entry == nil || entry.Attr.Inode != 0 {
 		return
 	}
-	entry.Attr.Inode = f.nextInode()
-}
-
-func (f *Filer) nextInode() uint64 {
-	return f.inodeSequencer.NextFileId(1)
+	if entry.Attr.Crtime.IsZero() {
+		entry.Attr.Crtime = time.Now()
+	}
+	if len(entry.HardLinkId) > 0 {
+		entry.Attr.Inode = uint64(util.HashStringToLong(string(entry.HardLinkId)))
+		return
+	}
+	entry.Attr.Inode = entry.FullPath.AsInode(entry.Attr.Crtime.Unix())
 }

weed/filer/filer_inode_index.go

@@ -1,300 +0,0 @@
-package filer
-
-import (
-	"context"
-	"encoding/binary"
-	"encoding/json"
-	"sort"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-)
-
-const inodeIndexKeyPrefix = "filer.inode.path."
-const InodeIndexInitialGeneration uint64 = 1
-
-type inodeIndexEntry struct {
-	path  util.FullPath
-	inode uint64
-}
-
-type InodeIndexRecord struct {
-	Generation uint64   `json:"generation,omitempty"`
-	Paths      []string `json:"paths,omitempty"`
-}
-
-func InodeIndexKey(inode uint64) []byte {
-	key := make([]byte, len(inodeIndexKeyPrefix)+8)
-	copy(key, inodeIndexKeyPrefix)
-	binary.BigEndian.PutUint64(key[len(inodeIndexKeyPrefix):], inode)
-	return key
-}
-
-func DecodeInodeIndexRecord(value []byte) (*InodeIndexRecord, error) {
-	if len(value) == 0 {
-		return &InodeIndexRecord{}, nil
-	}
-
-	// The first foundation slice stored the current path as raw bytes. Keep that
-	// format readable so existing records are transparently upgraded on write.
-	if value[0] != '{' {
-		record := &InodeIndexRecord{Generation: InodeIndexInitialGeneration}
-		record.addPath(util.FullPath(value))
-		return record, nil
-	}
-
-	record := &InodeIndexRecord{}
-	if err := json.Unmarshal(value, record); err != nil {
-		return nil, err
-	}
-	record.normalize()
-	return record, nil
-}
-
-func (record *InodeIndexRecord) Encode() ([]byte, error) {
-	record.normalize()
-	return json.Marshal(record)
-}
-
-func (record *InodeIndexRecord) normalize() {
-	if len(record.Paths) == 0 {
-		return
-	}
-	if record.Generation == 0 {
-		record.Generation = InodeIndexInitialGeneration
-	}
-
-	sanitized := make([]string, 0, len(record.Paths))
-	for _, path := range record.Paths {
-		if path == "" {
-			continue
-		}
-		sanitized = append(sanitized, path)
-	}
-	if len(sanitized) == 0 {
-		record.Paths = nil
-		return
-	}
-
-	sort.Strings(sanitized)
-	deduped := sanitized[:1]
-	for _, path := range sanitized[1:] {
-		if path == deduped[len(deduped)-1] {
-			continue
-		}
-		deduped = append(deduped, path)
-	}
-	record.Paths = deduped
-}
-
-func (record *InodeIndexRecord) addPath(path util.FullPath) bool {
-	if path == "" {
-		return false
-	}
-	record.normalize()
-	target := string(path)
-	index := sort.SearchStrings(record.Paths, target)
-	if index < len(record.Paths) && record.Paths[index] == target {
-		return false
-	}
-	record.Paths = append(record.Paths, "")
-	copy(record.Paths[index+1:], record.Paths[index:])
-	record.Paths[index] = target
-	return true
-}
-
-func (record *InodeIndexRecord) removePath(path util.FullPath) bool {
-	if len(record.Paths) == 0 || path == "" {
-		return false
-	}
-	record.normalize()
-	target := string(path)
-	index := sort.SearchStrings(record.Paths, target)
-	if index >= len(record.Paths) || record.Paths[index] != target {
-		return false
-	}
-	record.Paths = append(record.Paths[:index], record.Paths[index+1:]...)
-	if len(record.Paths) == 0 {
-		record.Paths = nil
-	}
-	return true
-}
-
-func (record *InodeIndexRecord) CanonicalPath() util.FullPath {
-	record.normalize()
-	if len(record.Paths) == 0 {
-		return ""
-	}
-	return util.FullPath(record.Paths[0])
-}
-
-func (record *InodeIndexRecord) FullPaths() []util.FullPath {
-	record.normalize()
-	if len(record.Paths) == 0 {
-		return nil
-	}
-	paths := make([]util.FullPath, 0, len(record.Paths))
-	for _, path := range record.Paths {
-		paths = append(paths, util.FullPath(path))
-	}
-	return paths
-}
-
-func (fsw *FilerStoreWrapper) lookupInodeIndex(ctx context.Context, inode uint64) (*InodeIndexRecord, error) {
-	if inode == 0 {
-		return nil, ErrKvNotFound
-	}
-
-	value, err := fsw.KvGet(ctx, InodeIndexKey(inode))
-	if err != nil {
-		return nil, err
-	}
-
-	return DecodeInodeIndexRecord(value)
-}
-
-func (fsw *FilerStoreWrapper) storeInodeIndex(ctx context.Context, path util.FullPath, inode uint64) error {
-	if inode == 0 || path == "" {
-		return nil
-	}
-
-	record, err := fsw.lookupInodeIndex(ctx, inode)
-	if err != nil {
-		if err != ErrKvNotFound {
-			return err
-		}
-		record = &InodeIndexRecord{Generation: InodeIndexInitialGeneration}
-	}
-	record.addPath(path)
-
-	value, err := record.Encode()
-	if err != nil {
-		return err
-	}
-	return fsw.KvPut(ctx, InodeIndexKey(inode), value)
-}
-
-func (fsw *FilerStoreWrapper) lookupInodePath(ctx context.Context, inode uint64) (util.FullPath, error) {
-	record, err := fsw.lookupInodeIndex(ctx, inode)
-	if err != nil {
-		return "", err
-	}
-
-	path := record.CanonicalPath()
-	if path == "" {
-		return "", ErrKvNotFound
-	}
-	return path, nil
-}
-
-func (fsw *FilerStoreWrapper) lookupInodePaths(ctx context.Context, inode uint64) ([]util.FullPath, error) {
-	record, err := fsw.lookupInodeIndex(ctx, inode)
-	if err != nil {
-		return nil, err
-	}
-
-	paths := record.FullPaths()
-	if len(paths) == 0 {
-		return nil, ErrKvNotFound
-	}
-	return paths, nil
-}
-
-func (fsw *FilerStoreWrapper) removePathFromInodeIndex(ctx context.Context, path util.FullPath, inode uint64) error {
-	if inode == 0 || path == "" {
-		return nil
-	}
-
-	record, err := fsw.lookupInodeIndex(ctx, inode)
-	if err != nil {
-		if err == ErrKvNotFound {
-			return nil
-		}
-		return err
-	}
-
-	if !record.removePath(path) {
-		return nil
-	}
-	if len(record.Paths) == 0 {
-		return fsw.KvDelete(ctx, InodeIndexKey(inode))
-	}
-
-	value, err := record.Encode()
-	if err != nil {
-		return err
-	}
-	return fsw.KvPut(ctx, InodeIndexKey(inode), value)
-}
-
-func (fsw *FilerStoreWrapper) collectInodeIndexEntries(ctx context.Context, dirPath util.FullPath) ([]inodeIndexEntry, error) {
-	// Honor caller cancellation during the walk: a DeleteFolderChildren on a
-	// pathological directory could otherwise loop indefinitely gathering
-	// entries even after the client has given up, turning into a DoS vector.
-	// If the walk is aborted, the caller treats the index cleanup as
-	// best-effort and drops the partial result.
-	var collected []inodeIndexEntry
-	if err := fsw.collectInodeIndexEntriesRecursive(ctx, dirPath, &collected); err != nil {
-		return nil, err
-	}
-	return collected, nil
-}
-
-func (fsw *FilerStoreWrapper) collectInodeIndexEntriesRecursive(ctx context.Context, dirPath util.FullPath, collected *[]inodeIndexEntry) error {
-	actualStore := fsw.getActualStore(dirPath + "/")
-
-	lastFileName := ""
-	includeStartFile := false
-	for {
-		page := make([]*Entry, 0, PaginationSize)
-		nextLastFileName, err := actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, includeStartFile, PaginationSize, func(entry *Entry) (bool, error) {
-			page = append(page, entry)
-			return true, nil
-		})
-		if err != nil {
-			return err
-		}
-
-		for _, entry := range page {
-			if entry.Attr.Inode != 0 {
-				*collected = append(*collected, inodeIndexEntry{path: entry.FullPath, inode: entry.Attr.Inode})
-			}
-			if entry.IsDirectory() {
-				if err := fsw.collectInodeIndexEntriesRecursive(ctx, entry.FullPath, collected); err != nil {
-					return err
-				}
-			}
-		}
-
-		if len(page) < PaginationSize {
-			return nil
-		}
-		lastFileName = nextLastFileName
-		includeStartFile = false
-	}
-}
-
-// recordInodeIndexWrite updates the inode→path secondary index after the
-// primary store mutation has already succeeded. The index is best-effort: a
-// failure here must not surface as an operation error, because the caller
-// would then observe a failed create/update even though the entry was
-// persisted, and a retry cannot heal the index (DeleteEntry exits early once
-// the entry is gone). We log and let later writes rebuild the record.
-func (fsw *FilerStoreWrapper) recordInodeIndexWrite(ctx context.Context, op string, path util.FullPath, inode uint64) {
-	if inode == 0 || path == "" {
-		return
-	}
-	if err := fsw.storeInodeIndex(ctx, path, inode); err != nil {
-		glog.WarningfCtx(ctx, "%s: update inode index for %s (inode %d): %v", op, path, inode, err)
-	}
-}
-
-// recordInodeIndexRemoval mirrors recordInodeIndexWrite for removals.
-func (fsw *FilerStoreWrapper) recordInodeIndexRemoval(ctx context.Context, op string, path util.FullPath, inode uint64) {
-	if inode == 0 || path == "" {
-		return
-	}
-	if err := fsw.removePathFromInodeIndex(ctx, path, inode); err != nil {
-		glog.WarningfCtx(ctx, "%s: clear inode index for %s (inode %d): %v", op, path, inode, err)
-	}
-}

weed/filer/filer_inode_index_test.go

@@ -1,206 +0,0 @@
-package filer
-
-import (
-	"context"
-	"os"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/weed/util"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestFilerStoreWrapperMaintainsInodeIndexLifecycle(t *testing.T) {
-	wrapper := NewFilerStoreWrapper(newStubFilerStore())
-	ctx := context.Background()
-
-	created := &Entry{
-		FullPath: util.FullPath("/docs/report.txt"),
-		Attr: Attr{
-			Mode:  0o644,
-			Inode: 42,
-		},
-	}
-
-	require.NoError(t, wrapper.InsertEntry(ctx, created))
-	path, err := wrapper.lookupInodePath(ctx, created.Attr.Inode)
-	require.NoError(t, err)
-	assert.Equal(t, created.FullPath, path)
-	paths, err := wrapper.lookupInodePaths(ctx, created.Attr.Inode)
-	require.NoError(t, err)
-	assert.Equal(t, []util.FullPath{created.FullPath}, paths)
-	record, err := wrapper.lookupInodeIndex(ctx, created.Attr.Inode)
-	require.NoError(t, err)
-	assert.Equal(t, InodeIndexInitialGeneration, record.Generation)
-
-	updated := &Entry{
-		FullPath: util.FullPath("/docs/report.txt"),
-		Attr: Attr{
-			Mode:  0o600,
-			Inode: 42,
-		},
-	}
-	require.NoError(t, wrapper.UpdateEntry(ctx, updated))
-	path, err = wrapper.lookupInodePath(ctx, updated.Attr.Inode)
-	require.NoError(t, err)
-	assert.Equal(t, updated.FullPath, path)
-
-	require.NoError(t, wrapper.DeleteEntry(ctx, created.FullPath))
-	_, err = wrapper.lookupInodePath(ctx, created.Attr.Inode)
-	require.ErrorIs(t, err, ErrKvNotFound)
-}
-
-func TestFilerStoreWrapperMaintainsMultiplePathsPerInode(t *testing.T) {
-	wrapper := NewFilerStoreWrapper(newStubFilerStore())
-	ctx := context.Background()
-	inode := uint64(88)
-	hardLinkId := NewHardLinkId()
-
-	require.NoError(t, wrapper.InsertEntry(ctx, &Entry{
-		FullPath: util.FullPath("/links/b.txt"),
-		Attr: Attr{
-			Mode:  0o644,
-			Inode: inode,
-		},
-		HardLinkId:      hardLinkId,
-		HardLinkCounter: 2,
-	}))
-	require.NoError(t, wrapper.InsertEntry(ctx, &Entry{
-		FullPath: util.FullPath("/links/a.txt"),
-		Attr: Attr{
-			Mode:  0o644,
-			Inode: inode,
-		},
-		HardLinkId:      hardLinkId,
-		HardLinkCounter: 2,
-	}))
-
-	paths, err := wrapper.lookupInodePaths(ctx, inode)
-	require.NoError(t, err)
-	assert.Equal(t, []util.FullPath{"/links/a.txt", "/links/b.txt"}, paths)
-	record, err := wrapper.lookupInodeIndex(ctx, inode)
-	require.NoError(t, err)
-	assert.Equal(t, InodeIndexInitialGeneration, record.Generation)
-
-	path, err := wrapper.lookupInodePath(ctx, inode)
-	require.NoError(t, err)
-	assert.Equal(t, util.FullPath("/links/a.txt"), path)
-
-	require.NoError(t, wrapper.DeleteEntry(ctx, util.FullPath("/links/a.txt")))
-
-	paths, err = wrapper.lookupInodePaths(ctx, inode)
-	require.NoError(t, err)
-	assert.Equal(t, []util.FullPath{"/links/b.txt"}, paths)
-
-	path, err = wrapper.lookupInodePath(ctx, inode)
-	require.NoError(t, err)
-	assert.Equal(t, util.FullPath("/links/b.txt"), path)
-}
-
-func TestFilerStoreWrapperUpgradesLegacySinglePathInodeIndexRecords(t *testing.T) {
-	wrapper := NewFilerStoreWrapper(newStubFilerStore())
-	ctx := context.Background()
-	inode := uint64(91)
-
-	require.NoError(t, wrapper.KvPut(ctx, InodeIndexKey(inode), []byte("/legacy/path.txt")))
-
-	path, err := wrapper.lookupInodePath(ctx, inode)
-	require.NoError(t, err)
-	assert.Equal(t, util.FullPath("/legacy/path.txt"), path)
-
-	paths, err := wrapper.lookupInodePaths(ctx, inode)
-	require.NoError(t, err)
-	assert.Equal(t, []util.FullPath{"/legacy/path.txt"}, paths)
-
-	require.NoError(t, wrapper.storeInodeIndex(ctx, util.FullPath("/legacy/second.txt"), inode))
-
-	paths, err = wrapper.lookupInodePaths(ctx, inode)
-	require.NoError(t, err)
-	assert.Equal(t, []util.FullPath{"/legacy/path.txt", "/legacy/second.txt"}, paths)
-
-	value, err := wrapper.KvGet(ctx, InodeIndexKey(inode))
-	require.NoError(t, err)
-	assert.JSONEq(t, `{"generation":1,"paths":["/legacy/path.txt","/legacy/second.txt"]}`, string(value))
-}
-
-func TestFilerStoreWrapperKeepsInodeIndexWhenDeleteArrivesAfterRenameInsert(t *testing.T) {
-	wrapper := NewFilerStoreWrapper(newStubFilerStore())
-	ctx := context.Background()
-	inode := uint64(77)
-
-	require.NoError(t, wrapper.InsertEntry(ctx, &Entry{
-		FullPath: util.FullPath("/old/name.txt"),
-		Attr: Attr{
-			Mode:  0o644,
-			Inode: inode,
-		},
-	}))
-	require.NoError(t, wrapper.InsertEntry(ctx, &Entry{
-		FullPath: util.FullPath("/new/name.txt"),
-		Attr: Attr{
-			Mode:  0o644,
-			Inode: inode,
-		},
-	}))
-	require.NoError(t, wrapper.DeleteEntry(ctx, util.FullPath("/old/name.txt")))
-
-	path, err := wrapper.lookupInodePath(ctx, inode)
-	require.NoError(t, err)
-	assert.Equal(t, util.FullPath("/new/name.txt"), path)
-
-	paths, err := wrapper.lookupInodePaths(ctx, inode)
-	require.NoError(t, err)
-	assert.Equal(t, []util.FullPath{"/new/name.txt"}, paths)
-}
-
-func TestRecursiveDeleteRemovesDescendantInodeIndexes(t *testing.T) {
-	f, store := newTestFilerWithStubStore()
-	ctx := context.Background()
-
-	entries := []*Entry{
-		{
-			FullPath: util.FullPath("/tree"),
-			Attr: Attr{
-				Mode:  os.ModeDir | 0o755,
-				Inode: 100,
-			},
-		},
-		{
-			FullPath: util.FullPath("/tree/file.txt"),
-			Attr: Attr{
-				Mode:  0o644,
-				Inode: 101,
-			},
-		},
-		{
-			FullPath: util.FullPath("/tree/subdir"),
-			Attr: Attr{
-				Mode:  os.ModeDir | 0o755,
-				Inode: 102,
-			},
-		},
-		{
-			FullPath: util.FullPath("/tree/subdir/nested.txt"),
-			Attr: Attr{
-				Mode:  0o644,
-				Inode: 103,
-			},
-		},
-	}
-
-	for _, entry := range entries {
-		require.NoError(t, f.Store.InsertEntry(ctx, entry))
-	}
-
-	require.NoError(t, f.DeleteEntryMetaAndData(ctx, util.FullPath("/tree"), true, false, false, false, nil, 0))
-
-	for _, inode := range []uint64{100, 101, 102, 103} {
-		_, err := f.Store.(*FilerStoreWrapper).lookupInodePath(ctx, inode)
-		require.ErrorIs(t, err, ErrKvNotFound)
-	}
-
-	for _, path := range []string{"/tree", "/tree/file.txt", "/tree/subdir", "/tree/subdir/nested.txt"} {
-		_, err := store.FindEntry(ctx, util.FullPath(path))
-		require.Error(t, err)
-	}
-}

weed/filer/filer_inode_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -11,6 +12,47 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestEnsureEntryInodeMatchesFuseDerivation(t *testing.T) {
+	f := &Filer{}
+	crtime := time.Unix(1700000000, 0)
+
+	entry := &Entry{
+		FullPath: util.FullPath("/dir/file.txt"),
+		Attr:     Attr{Crtime: crtime},
+	}
+	f.ensureEntryInode(entry)
+
+	// The filer stores exactly what the FUSE mount would compute for a
+	// non-hard-linked entry, and it is deterministic across calls.
+	assert.Equal(t, entry.FullPath.AsInode(crtime.Unix()), entry.Attr.Inode)
+	again := &Entry{FullPath: entry.FullPath, Attr: Attr{Crtime: crtime}}
+	f.ensureEntryInode(again)
+	assert.Equal(t, entry.Attr.Inode, again.Attr.Inode)
+}
+
+func TestEnsureEntryInodeSharesAcrossHardLinks(t *testing.T) {
+	f := &Filer{}
+	hardLinkId := NewHardLinkId()
+
+	a := &Entry{
+		FullPath:   util.FullPath("/links/a.txt"),
+		Attr:       Attr{Crtime: time.Unix(1700000000, 0)},
+		HardLinkId: hardLinkId,
+	}
+	b := &Entry{
+		FullPath:   util.FullPath("/links/b.txt"),
+		Attr:       Attr{Crtime: time.Unix(1800000000, 0)},
+		HardLinkId: hardLinkId,
+	}
+	f.ensureEntryInode(a)
+	f.ensureEntryInode(b)
+
+	// Every link to the same target resolves to one inode, independent of path
+	// or creation time.
+	assert.Equal(t, uint64(util.HashStringToLong(string(hardLinkId))), a.Attr.Inode)
+	assert.Equal(t, a.Attr.Inode, b.Attr.Inode)
+}
+
 func newTestFilerWithStubStore() (*Filer, *stubFilerStore) {
 	store := newStubFilerStore()
 	f := NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)

weed/filer/filer_lazy_remote_test.go

@@ -276,7 +276,6 @@ func newTestFiler(t *testing.T, store *stubFilerStore, rs *FilerRemoteStorage) *
 		FilerConf:           NewFilerConf(),
 		MaxFilenameLength:   255,
 		MasterClient:        mc,
-		inodeSequencer:      newInodeSequencer("test-filer"),
 		fileIdDeletionQueue: util.NewUnboundedQueue(),
 		deletionQuit:        make(chan struct{}),
 		LocalMetaLogBuffer: log_buffer.NewLogBuffer("test", time.Minute,

weed/filer/filerstore_wrapper.go

@@ -132,7 +132,6 @@ func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) err
 		return err
 	}
 	ctx = context.WithoutCancel(ctx)
-	fullPath := entry.FullPath
 	actualStore := fsw.getActualStore(entry.FullPath)
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "insert").Inc()
 	start := time.Now()
@@ -152,11 +151,7 @@ func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) err
 		return err
 	}
 
-	if err := actualStore.InsertEntry(ctx, entry); err != nil {
-		return err
-	}
-	fsw.recordInodeIndexWrite(ctx, "InsertEntry", fullPath, entry.Attr.Inode)
-	return nil
+	return actualStore.InsertEntry(ctx, entry)
 }
 
 // InsertEntryKnownAbsent skips the pre-insert FindEntry path when the caller has
@@ -166,7 +161,6 @@ func (fsw *FilerStoreWrapper) InsertEntryKnownAbsent(ctx context.Context, entry
 		return err
 	}
 	ctx = context.WithoutCancel(ctx)
-	fullPath := entry.FullPath
 	actualStore := fsw.getActualStore(entry.FullPath)
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "insert").Inc()
 	start := time.Now()
@@ -185,11 +179,7 @@ func (fsw *FilerStoreWrapper) InsertEntryKnownAbsent(ctx context.Context, entry
 		}
 	}
 
-	if err := actualStore.InsertEntry(ctx, entry); err != nil {
-		return err
-	}
-	fsw.recordInodeIndexWrite(ctx, "InsertEntryKnownAbsent", fullPath, entry.Attr.Inode)
-	return nil
+	return actualStore.InsertEntry(ctx, entry)
 }
 
 func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) error {
@@ -197,7 +187,6 @@ func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) err
 		return err
 	}
 	ctx = context.WithoutCancel(ctx)
-	fullPath := entry.FullPath
 	actualStore := fsw.getActualStore(entry.FullPath)
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "update").Inc()
 	start := time.Now()
@@ -217,11 +206,7 @@ func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) err
 		return err
 	}
 
-	if err := actualStore.UpdateEntry(ctx, entry); err != nil {
-		return err
-	}
-	fsw.recordInodeIndexWrite(ctx, "UpdateEntry", fullPath, entry.Attr.Inode)
-	return nil
+	return actualStore.UpdateEntry(ctx, entry)
 }
 
 func normalizeEntryMimeForStore(entry *Entry) {
@@ -273,8 +258,6 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath)
 	if findErr == filer_pb.ErrNotFound || existingEntry == nil {
 		return nil
 	}
-	inode := existingEntry.Attr.Inode
-	fullPath := existingEntry.FullPath
 	if len(existingEntry.HardLinkId) != 0 {
 		// remove hard link
 		op := ctx.Value("OP")
@@ -289,11 +272,7 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath)
 		}
 	}
 
-	if err := actualStore.DeleteEntry(ctx, fp); err != nil {
-		return err
-	}
-	fsw.recordInodeIndexRemoval(ctx, "DeleteEntry", fullPath, inode)
-	return nil
+	return actualStore.DeleteEntry(ctx, fp)
 }
 
 func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry *Entry) (err error) {
@@ -301,8 +280,6 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
 		return err
 	}
 	ctx = context.WithoutCancel(ctx)
-	fullPath := existingEntry.FullPath
-	inode := existingEntry.Attr.Inode
 	actualStore := fsw.getActualStore(existingEntry.FullPath)
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "delete").Inc()
 	start := time.Now()
@@ -325,11 +302,7 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
 		}
 	}
 
-	if err := actualStore.DeleteEntry(ctx, existingEntry.FullPath); err != nil {
-		return err
-	}
-	fsw.recordInodeIndexRemoval(ctx, "DeleteOneEntry", fullPath, inode)
-	return nil
+	return actualStore.DeleteEntry(ctx, existingEntry.FullPath)
 }
 
 func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
@@ -344,20 +317,7 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.
 		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Observe(time.Since(start).Seconds())
 	}()
 
-	collected, err := fsw.collectInodeIndexEntries(ctx, fp)
-	if err != nil {
-		// Index collection is best-effort: a failure here only prevents inode
-		// index housekeeping, not the directory removal itself.
-		glog.WarningfCtx(ctx, "collectInodeIndexEntries %s: %v; deleting folder children without index cleanup", fp, err)
-		collected = nil
-	}
-	if err := actualStore.DeleteFolderChildren(ctx, fp); err != nil {
-		return err
-	}
-	for _, entry := range collected {
-		fsw.recordInodeIndexRemoval(ctx, "DeleteFolderChildren", entry.path, entry.inode)
-	}
-	return nil
+	return actualStore.DeleteFolderChildren(ctx, fp)
 }
 
 func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) {

weed/filer/mysql/mysql_sql_gen.go

@@ -13,6 +13,13 @@ type SqlGenMysql struct {
 	UpsertQueryTemplate    string
 }
 
+// DefaultUpsertQuery keeps INSERTs idempotent so the inode-index KvPut
+// after every entry write does not emit a duplicate-key roundtrip on
+// every mutation. The VALUES() form works on MariaDB and MySQL >=5.7;
+// the newer "AS new" alias from MySQL 8.0.19 errors on MariaDB, so it
+// is left for users to opt into via an explicit upsertQuery.
+const DefaultUpsertQuery = "INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE `meta` = VALUES(`meta`)"
+
 var (
 	_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
 )

weed/filer/mysql/mysql_sql_gen_test.go

@@ -0,0 +1,28 @@
+package mysql
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDefaultUpsertQueryUsesOnDuplicateKey(t *testing.T) {
+	gen := &SqlGenMysql{UpsertQueryTemplate: DefaultUpsertQuery}
+	got := gen.GetSqlInsert("filemeta")
+	if !strings.Contains(got, "ON DUPLICATE KEY UPDATE") {
+		t.Fatalf("expected ON DUPLICATE KEY UPDATE in default upsert, got: %s", got)
+	}
+	if strings.Contains(got, "AS `new`") {
+		t.Fatalf("default should avoid MySQL 8.0.19 row-alias syntax for MariaDB compat, got: %s", got)
+	}
+	if !strings.Contains(got, "`filemeta`") {
+		t.Fatalf("expected backticked table name, got: %s", got)
+	}
+}
+
+func TestEmptyUpsertTemplateFallsBackToPlainInsert(t *testing.T) {
+	gen := &SqlGenMysql{}
+	got := gen.GetSqlInsert("filemeta")
+	if strings.Contains(got, "ON DUPLICATE KEY UPDATE") {
+		t.Fatalf("plain INSERT path should not contain ON DUPLICATE KEY UPDATE, got: %s", got)
+	}
+}

weed/filer/mysql/mysql_store.go

@@ -35,6 +35,9 @@ func (store *MysqlStore) GetName() string {
 func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
 	configuration.SetDefault(prefix+"connection_max_idle", 2)
+	// Default on so minimal configs avoid the duplicate-key roundtrip the
+	// inode-index KvPut would otherwise emit on every write.
+	configuration.SetDefault(prefix+"enableUpsert", true)
 	return store.initialize(
 		configuration.GetString(prefix+"dsn"),
 		configuration.GetString(prefix+"upsertQuery"),
@@ -64,6 +67,8 @@ func (store *MysqlStore) initialize(dsn string, upsertQuery string, enableUpsert
 	store.SupportBucketTable = false
 	if !enableUpsert {
 		upsertQuery = ""
+	} else if upsertQuery == "" {
+		upsertQuery = DefaultUpsertQuery
 	}
 	store.SqlGenerator = &SqlGenMysql{
 		CreateTableSqlTemplate: "",

weed/filer/mysql2/mysql2_store.go

@@ -35,6 +35,9 @@ func (store *MysqlStore2) GetName() string {
 func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
 	// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
 	configuration.SetDefault(prefix+"connection_max_idle", 2)
+	// Default on so minimal configs avoid the duplicate-key roundtrip the
+	// inode-index KvPut would otherwise emit on every write.
+	configuration.SetDefault(prefix+"enableUpsert", true)
 	return store.initialize(
 		configuration.GetString(prefix+"createTable"),
 		configuration.GetString(prefix+"upsertQuery"),
@@ -57,6 +60,8 @@ func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpse
 	store.SupportBucketTable = true
 	if !enableUpsert {
 		upsertQuery = ""
+	} else if upsertQuery == "" {
+		upsertQuery = mysql.DefaultUpsertQuery
 	}
 	store.SqlGenerator = &mysql.SqlGenMysql{
 		CreateTableSqlTemplate: createTable,

weed/filer/postgres/postgres_sql_gen.go

@@ -13,6 +13,11 @@ type SqlGenPostgres struct {
 	UpsertQueryTemplate    string
 }
 
+// DefaultUpsertQuery keeps INSERTs idempotent so a duplicate-key failure
+// (23505) cannot poison the surrounding transaction (25P02). Used when the
+// user enables upsert but does not provide their own template.
+const DefaultUpsertQuery = `INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash, name) DO UPDATE SET directory=EXCLUDED.directory, meta=EXCLUDED.meta`
+
 var (
 	_ = abstract_sql.SqlGenerator(&SqlGenPostgres{})
 )

weed/filer/postgres/postgres_sql_gen_test.go

@@ -0,0 +1,25 @@
+package postgres
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDefaultUpsertQueryIsConflictSafe(t *testing.T) {
+	gen := &SqlGenPostgres{UpsertQueryTemplate: DefaultUpsertQuery}
+	got := gen.GetSqlInsert("filemeta")
+	if !strings.Contains(got, "ON CONFLICT") {
+		t.Fatalf("expected ON CONFLICT in default upsert, got: %s", got)
+	}
+	if !strings.Contains(got, `"filemeta"`) {
+		t.Fatalf("expected quoted table name, got: %s", got)
+	}
+}
+
+func TestEmptyUpsertTemplateFallsBackToPlainInsert(t *testing.T) {
+	gen := &SqlGenPostgres{}
+	got := gen.GetSqlInsert("filemeta")
+	if strings.Contains(got, "ON CONFLICT") {
+		t.Fatalf("plain INSERT path should not contain ON CONFLICT, got: %s", got)
+	}
+}

weed/filer/postgres/postgres_store.go

@@ -30,6 +30,9 @@ func (store *PostgresStore) GetName() string {
 func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
 	configuration.SetDefault(prefix+"connection_max_idle", 2)
+	// Default on so minimal configs are not exposed to duplicate-key tx
+	// poisoning on Postgres; an explicit false still disables it.
+	configuration.SetDefault(prefix+"enableUpsert", true)
 	return store.initialize(
 		configuration.GetString(prefix+"upsertQuery"),
 		configuration.GetBool(prefix+"enableUpsert"),
@@ -56,6 +59,8 @@ func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, us
 	store.SupportBucketTable = false
 	if !enableUpsert {
 		upsertQuery = ""
+	} else if upsertQuery == "" {
+		upsertQuery = DefaultUpsertQuery
 	}
 	store.SqlGenerator = &SqlGenPostgres{
 		CreateTableSqlTemplate: "",

weed/filer/postgres2/postgres2_store.go

@@ -35,6 +35,9 @@ func (store *PostgresStore2) GetName() string {
 func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
 	// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
 	configuration.SetDefault(prefix+"connection_max_idle", 2)
+	// Default on so minimal configs are not exposed to duplicate-key tx
+	// poisoning on Postgres; an explicit false still disables it.
+	configuration.SetDefault(prefix+"enableUpsert", true)
 	return store.initialize(
 		configuration.GetString(prefix+"createTable"),
 		configuration.GetString(prefix+"upsertQuery"),
@@ -62,6 +65,8 @@ func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableU
 	store.SupportBucketTable = true
 	if !enableUpsert {
 		upsertQuery = ""
+	} else if upsertQuery == "" {
+		upsertQuery = postgres.DefaultUpsertQuery
 	}
 	store.SqlGenerator = &postgres.SqlGenPostgres{
 		CreateTableSqlTemplate: createTable,

weed/iamapi/iamapi_server.go

@@ -124,11 +124,23 @@ func (iama *IamApiServer) registerRouter(router *mux.Router) {
 
 	// apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_ADMIN), "LIST"))
 	apiRouter.Methods(http.MethodPost).Path("/").HandlerFunc(iama.iam.Auth(iama.DoActions, ACTION_ADMIN))
-	//
+
+	// Health probes
+	apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/healthz").HandlerFunc(iama.healthzHandler)
+	apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/readyz").HandlerFunc(iama.readyzHandler)
+
 	// NotFound
 	apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler)
 }
 
+func (iama *IamApiServer) healthzHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
+func (iama *IamApiServer) readyzHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
 // Shutdown gracefully stops the IAM API server and releases resources.
 // It cancels the master client connection goroutine and closes gRPC connections.
 // This method is safe to call multiple times.

weed/mount/wfs_filer_client.go

@@ -1,6 +1,7 @@
 package mount
 
 import (
+	"context"
 	"sync/atomic"
 
 	"google.golang.org/grpc"
@@ -21,7 +22,7 @@ func (wfs *WFS) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFile
 		for x := 0; x < n; x++ {
 
 			filerGrpcAddress := wfs.option.FilerAddresses[i].ToGrpcAddress()
-			err = pb.WithGrpcClient(streamingMode, wfs.signature, func(grpcConnection *grpc.ClientConn) error {
+			err = pb.WithGrpcClient(context.Background(), streamingMode, wfs.signature, func(grpcConnection *grpc.ClientConn) error {
 				client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 				return fn(client)
 			}, filerGrpcAddress, false, wfs.option.GrpcDialOption)

weed/operation/assign_file_id.go

@@ -91,7 +91,10 @@ func Assign(ctx context.Context, masterFn GetMasterFn, grpcDialOption grpc.DialO
 				// Per-attempt timeout to prevent a single slow RPC from consuming the entire retry budget
 				attemptCtx, attemptCancel := context.WithTimeout(deadlineCtx, 10*time.Second)
 				defer attemptCancel()
-				return WithMasterServerClient(false, masterFn(attemptCtx), grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
+				// Pass attemptCtx so its expiry is not mistaken for a dead shared
+				// connection: invalidating it would cancel every other in-flight
+				// assign with "the client connection is closing".
+				return WithMasterServerClient(attemptCtx, false, masterFn(attemptCtx), grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
 					req := &master_pb.AssignRequest{
 						Count:               request.Count,
 						Replication:         request.Replication,
@@ -151,7 +154,7 @@ func Assign(ctx context.Context, masterFn GetMasterFn, grpcDialOption grpc.DialO
 
 func LookupJwt(master pb.ServerAddress, grpcDialOption grpc.DialOption, fileId string) (token security.EncodedJwt) {
 
-	WithMasterServerClient(false, master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
+	WithMasterServerClient(context.Background(), false, master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
 
 		resp, grpcErr := masterClient.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{
 			VolumeOrFileIds: []string{fileId},