test(remote_storage/s3): assert WriteFile success in mime tests

filer.remote.sync was uploading every object without a Content-Type, so
S3-compatible backends (e.g. Backblaze B2) stored binary/octet-stream
and browsers refused to render HTML, CSS, etc.

Pass entry.Attributes.Mime through to UploadInput.ContentType, leaving
the header unset when no Mime is recorded so the remote keeps its own
default behavior.

.github/workflows/container_dev.yml

@@ -111,7 +111,7 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v4.1.0
+        uses: docker/setup-qemu-action@v4
 
       - name: Create BuildKit config
         run: |

.github/workflows/container_foundationdb_version.yml

@@ -126,7 +126,7 @@ jobs:
           echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v4.1.0
+        uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4

.github/workflows/container_latest.yml

@@ -207,7 +207,7 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       - name: Set up QEMU
         if: matrix.platform != 'amd64'
-        uses: docker/setup-qemu-action@v4.1.0
+        uses: docker/setup-qemu-action@v4
       - name: Create BuildKit config
         run: |
           cat > /tmp/buildkitd.toml <<EOF

.github/workflows/container_release_foundationdb.yml

@@ -43,7 +43,7 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v4.1.0
+        uses: docker/setup-qemu-action@v4
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4

.github/workflows/container_release_unified.yml

@@ -118,9 +118,8 @@ jobs:
     needs: [build-rust-binaries]
     runs-on: ubuntu-latest
     strategy:
-      # All variants at once: pulls hit mirror.gcr.io and pushes go to GHCR only,
-      # so docker.io limits don't apply. Watch the 10 GB gha cache budget.
-      max-parallel: 5
+      # Build sequentially to avoid rate limits
+      max-parallel: 2
       matrix:
         include:
           # Normal volume - multi-arch
@@ -220,7 +219,7 @@ jobs:
             
       - name: Set up QEMU
         if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
-        uses: docker/setup-qemu-action@v4.1.0
+        uses: docker/setup-qemu-action@v4
         
       - name: Create BuildKit config
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
@@ -273,52 +272,93 @@ jobs:
             BRANCH=${{ github.sha }}
             ${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
             
-      # Copy GHCR -> Docker Hub here, per variant, so it overlaps with the other
-      # variants still building instead of waiting on the whole matrix.
+      - name: Clean up build artifacts
+        if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
+        run: |
+          sudo docker system prune -f
+          sudo rm -rf /tmp/go-build*
+
+  copy-to-dockerhub:
+    runs-on: ubuntu-latest
+    needs: [build]
+    if: github.event_name != 'pull_request'
+    strategy:
+      matrix:
+        variant: [normal, large_disk, full, large_disk_full, rocksdb]
+        include:
+          - variant: normal
+            tag_suffix: ""
+          - variant: large_disk
+            tag_suffix: _large_disk
+          - variant: full
+            tag_suffix: _full
+          - variant: large_disk_full
+            tag_suffix: _large_disk_full
+          - variant: rocksdb
+            tag_suffix: _large_disk_rocksdb
+            
+    steps:
+      - name: Login to Docker Hub
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        uses: docker/login-action@v4.2.0
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          
+      - name: Login to GHCR
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        uses: docker/login-action@v4.2.0
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.GHCR_USERNAME }}
+          password: ${{ secrets.GHCR_TOKEN }}
+          
       - name: Install crane
-        if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         run: |
           cd $(mktemp -d)
           curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
           sudo mv crane /usr/local/bin/
           crane version
-
-      - name: Copy ${{ matrix.variant }} to Docker Hub
-        if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
+          
+      - name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         run: |
+          # Function to retry with exponential backoff
           retry_with_backoff() {
             local max_attempts=5
             local timeout=1
             local attempt=1
             local exit_code=0
+            
             while [ $attempt -le $max_attempts ]; do
               if "$@"; then
                 return 0
               else
                 exit_code=$?
               fi
+              
               if [ $attempt -lt $max_attempts ]; then
                 echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
                 sleep $timeout
                 timeout=$((timeout * 2))
               fi
+              
               attempt=$((attempt + 1))
             done
+            
             echo "Command failed after $max_attempts attempts" >&2
             return $exit_code
           }
-
+          
+          # Copy multi-arch image from GHCR to Docker Hub with retry
+          # This is much more efficient than pulling/pushing individual arch images
           echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
           retry_with_backoff crane copy \
             ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
             chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
-          echo "Copied ${{ matrix.variant }} to Docker Hub"
-
-      - name: Clean up build artifacts
-        if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
-        run: |
-          sudo docker system prune -f
-          sudo rm -rf /tmp/go-build*
+          
+          echo "Successfully copied ${{ matrix.variant }} to Docker Hub"
 
   # Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
   # Security for visibility, but never blocks the release. Releases (including
@@ -378,7 +418,7 @@ jobs:
   # trivy-scan: vuln findings are reported but do not block `latest`.
   tag-latest:
     runs-on: ubuntu-latest
-    needs: [build]
+    needs: [copy-to-dockerhub]
     if: github.event_name == 'push'
     strategy:
       matrix:
@@ -443,7 +483,7 @@ jobs:
 
   helm-release:
     runs-on: ubuntu-latest
-    needs: [build]
+    needs: [copy-to-dockerhub]
     if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     permissions:
       contents: write

.github/workflows/container_rocksdb_version.yml

@@ -82,7 +82,7 @@ jobs:
           echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v1
+        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1

.github/workflows/e2e.yml

@@ -31,11 +31,6 @@ jobs:
       with:
         go-version-file: 'go.mod'
 
-    - name: Configure Docker Hub mirror
-      run: |
-        echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-        sudo systemctl restart docker
-
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v4
       

.github/workflows/go.yml

@@ -37,21 +37,6 @@ jobs:
         # Fail only if there are actual vet errors (not counting the filtered lock warnings)
         if grep -q "vet:" vet-output.txt; then exit 1; fi
 
-  vet-32bit:
-    name: Go Vet 32-bit
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v6
-    - name: Set up Go
-      uses: actions/setup-go@v6
-      with:
-        go-version-file: 'go.mod'
-    - name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows)
-      run: |
-        GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt
-        if grep -q "vet:" vet-32bit-output.txt; then exit 1; fi
-
   build:
     name: Build
     runs-on: ubuntu-latest
@@ -77,19 +62,3 @@ jobs:
         go-version-file: 'go.mod'
     - name: Test
       run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...
-
-  test-32bit:
-    name: Test 32-bit
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v6
-    - name: Set up Go
-      uses: actions/setup-go@v6
-      with:
-        go-version-file: 'go.mod'
-    # 386 test binaries run natively on the amd64 runner. This catches what vet
-    # can't: unaligned 64-bit atomics and arithmetic that wraps at runtime.
-    # -short skips the e2e suites already covered on amd64.
-    - name: Test linux/386
-      run: cd weed; GOOS=linux GOARCH=386 go test -short ./...

.github/workflows/helm_ci.yml

@@ -51,71 +51,22 @@ jobs:
           
           echo "=== Testing default configuration ==="
           helm template test $CHART_DIR > /tmp/default.yaml
-          echo "Default configuration renders successfully"
+          echo "✓ Default configuration renders successfully"
           
           echo "=== Testing with S3 enabled ==="
           helm template test $CHART_DIR --set s3.enabled=true > /tmp/s3.yaml
           grep -q "kind: Deployment" /tmp/s3.yaml && grep -q "seaweedfs-s3" /tmp/s3.yaml
-          echo "S3 deployment renders correctly"
+          echo "✓ S3 deployment renders correctly"
           
           echo "=== Testing with all-in-one mode ==="
           helm template test $CHART_DIR --set allInOne.enabled=true > /tmp/allinone.yaml
           grep -q "seaweedfs-all-in-one" /tmp/allinone.yaml
-          echo "All-in-one deployment renders correctly"
+          echo "✓ All-in-one deployment renders correctly"
           
           echo "=== Testing with security enabled ==="
           helm template test $CHART_DIR --set global.seaweedfs.enableSecurity=true > /tmp/security.yaml
           grep -q "security-config" /tmp/security.yaml
-          echo "Security configuration renders correctly"
-
-          echo ""
-          echo "=== Testing JWT expiration overrides ==="
-          helm template test $CHART_DIR \
-            --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeWrite=11 \
-            > /tmp/jwt-volume-write-expiration.yaml
-          grep -q "security-config" /tmp/jwt-volume-write-expiration.yaml
-          grep -q "expires_after_seconds = 11" /tmp/jwt-volume-write-expiration.yaml
-
-          helm template test $CHART_DIR \
-            --set global.seaweedfs.securityConfig.jwtSigning.volumeRead=true \
-            --set global.seaweedfs.securityConfig.jwtSigning.filerWrite=true \
-            --set global.seaweedfs.securityConfig.jwtSigning.filerRead=true \
-            --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeWrite=11 \
-            --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeRead=22 \
-            --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.filerWrite=33 \
-            --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.filerRead=44 \
-            > /tmp/jwt-expiration.yaml
-
-          assert_jwt_expiration() {
-            local section="$1"
-            local seconds="$2"
-            awk -v section="[$section]" -v seconds="$seconds" '
-              /^[[:space:]]*\[.*\][[:space:]]*$/ {
-                in_section = index($0, section) > 0
-              }
-              in_section && $0 ~ "^[[:space:]]*expires_after_seconds = " seconds "$" {
-                found = 1
-              }
-              END { exit !found }
-            ' /tmp/jwt-expiration.yaml
-          }
-
-          assert_jwt_expiration jwt.signing 11
-          assert_jwt_expiration jwt.signing.read 22
-          assert_jwt_expiration jwt.filer_signing 33
-          assert_jwt_expiration jwt.filer_signing.read 44
-
-          helm template test $CHART_DIR \
-            --set global.seaweedfs.enableSecurity=true \
-            --set global.seaweedfs.securityConfig.jwtSigning.volumeRead=true \
-            --set global.seaweedfs.securityConfig.jwtSigning.filerWrite=true \
-            --set global.seaweedfs.securityConfig.jwtSigning.filerRead=true \
-            > /tmp/jwt-default-expiration.yaml
-          if grep -q "expires_after_seconds =" /tmp/jwt-default-expiration.yaml; then
-            echo "FAIL: zero JWT expiration values should preserve runtime defaults"
-            exit 1
-          fi
-          echo "JWT expiration overrides render correctly"
+          echo "✓ Security configuration renders correctly"
 
           echo ""
           echo "=== Testing IAM gRPC opt-in path ==="
@@ -170,7 +121,7 @@ jobs:
           if configmap(out, "test-seaweedfs-security-config") is not None:
               failed.append("defaults: security ConfigMap should not render")
           else:
-              print("defaults: no security-config ConfigMap (unchanged)")
+              print("✓ defaults: no security-config ConfigMap (unchanged)")
 
           # Case 2: filerWrite=true alone is the documented opt-in for
           # the Admin UI Users tab. Configmap must render with
@@ -190,7 +141,7 @@ jobs:
               if "[grpc" in toml:
                   failed.append("filerWrite=true: security.toml unexpectedly has [grpc.*] (would need cert mounts)")
               if "[jwt.filer_signing]" in toml and "[grpc" not in toml:
-                  print("filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]")
+                  print("✓ filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]")
 
           # Case 3: filer + admin pods must MOUNT the security ConfigMap
           # under filerWrite=true so the JWT key reaches both processes.
@@ -203,7 +154,7 @@ jobs:
               if "security-config" not in vols or "security-config" not in mounts:
                   failed.append(f"filerWrite=true: {wl} does not mount security-config (IAM gRPC would still fail)")
               else:
-                  print(f"filerWrite=true: {wl} mounts security-config")
+                  print(f"✓ filerWrite=true: {wl} mounts security-config")
               cert_vols = {v for v in vols if v.endswith("-cert")}
               if cert_vols:
                   failed.append(f"filerWrite=true: {wl} unexpectedly has cert volumes {sorted(cert_vols)}")
@@ -221,7 +172,7 @@ jobs:
               if missing:
                   failed.append(f"enableSecurity=true: security.toml missing {missing}")
               else:
-                  print("enableSecurity=true: security.toml has [jwt.signing] + [grpc.*] preserved")
+                  print("✓ enableSecurity=true: security.toml has [jwt.signing] + [grpc.*] preserved")
 
           # Case 5: helper must tolerate explicit nulls (gemini-code-assist
           # PR review). securityConfig=null was the parens-pattern crash
@@ -236,7 +187,7 @@ jobs:
               if configmap(out, "test-seaweedfs-security-config") is not None:
                   failed.append(f"{null_path}=null: should not render configmap")
               else:
-                  print(f"{null_path}=null: render tolerates explicit null")
+                  print(f"✓ {null_path}=null: render tolerates explicit null")
 
           if failed:
               print("\nFAIL:", file=sys.stderr)
@@ -244,14 +195,14 @@ jobs:
                   print(f"  - {f}", file=sys.stderr)
               sys.exit(1)
           PYEOF
-          echo "IAM gRPC decoupling tests passed"
+          echo "✓ IAM gRPC decoupling tests passed"
 
           echo "=== Testing with monitoring enabled ==="
           helm template test $CHART_DIR \
             --set global.seaweedfs.monitoring.enabled=true \
             --set global.seaweedfs.monitoring.gatewayHost=prometheus \
             --set global.seaweedfs.monitoring.gatewayPort=9091 > /tmp/monitoring.yaml
-          echo "Monitoring configuration renders correctly"
+          echo "✓ Monitoring configuration renders correctly"
           
           echo "=== Testing with PVC storage ==="
           helm template test $CHART_DIR \
@@ -259,25 +210,25 @@ jobs:
             --set master.data.size=10Gi \
             --set master.data.storageClass=standard > /tmp/pvc.yaml
           grep -q "PersistentVolumeClaim" /tmp/pvc.yaml
-          echo "PVC configuration renders correctly"
+          echo "✓ PVC configuration renders correctly"
           
           echo "=== Testing with custom replicas ==="
           helm template test $CHART_DIR \
             --set master.replicas=3 \
             --set filer.replicas=2 \
             --set volume.replicas=3 > /tmp/replicas.yaml
-          echo "Custom replicas configuration renders correctly"
+          echo "✓ Custom replicas configuration renders correctly"
           
           echo "=== Testing filer with S3 gateway ==="
           helm template test $CHART_DIR \
             --set filer.s3.enabled=true \
             --set filer.s3.enableAuth=true > /tmp/filer-s3.yaml
-          echo "Filer S3 gateway renders correctly"
+          echo "✓ Filer S3 gateway renders correctly"
           
           echo "=== Testing SFTP enabled ==="
           helm template test $CHART_DIR --set sftp.enabled=true > /tmp/sftp.yaml
           grep -q "seaweedfs-sftp" /tmp/sftp.yaml
-          echo "SFTP deployment renders correctly"
+          echo "✓ SFTP deployment renders correctly"
           
           echo "=== Testing ingress configurations ==="
           helm template test $CHART_DIR \
@@ -286,12 +237,12 @@ jobs:
             --set s3.enabled=true \
             --set s3.ingress.enabled=true > /tmp/ingress.yaml
           grep -q "kind: Ingress" /tmp/ingress.yaml
-          echo "Ingress configurations render correctly"
+          echo "✓ Ingress configurations render correctly"
           
           echo "=== Testing COSI driver ==="
           helm template test $CHART_DIR --set cosi.enabled=true > /tmp/cosi.yaml
           grep -q "seaweedfs-cosi" /tmp/cosi.yaml
-          echo "COSI driver renders correctly"
+          echo "✓ COSI driver renders correctly"
           
           echo ""
           echo "=== Testing long release name: service names match DNS references ==="
@@ -334,7 +285,7 @@ jobs:
           [ "$MASTER_SVC" = "$MASTER_ADDR_SVC" ] || { echo "FAIL: master service name mismatch"; exit 1; }
           [ "$FILER_CLIENT_SVC" = "$FILER_ADDR_SVC" ] || { echo "FAIL: filer-client service name mismatch"; exit 1; }
           [ "$FILER_CLIENT_SVC" = "$S3_FILER_SVC" ] || { echo "FAIL: S3 -filer= does not match filer-client service"; exit 1; }
-          echo "Normal mode: service names match DNS references with long release name"
+          echo "✓ Normal mode: service names match DNS references with long release name"
 
           # --- All-in-one mode: all-in-one service vs both helper addresses ---
           helm template "$LONG_RELEASE" $CHART_DIR \
@@ -354,7 +305,7 @@ jobs:
 
           [ "$AIO_SVC" = "$AIO_MASTER_ADDR_SVC" ] || { echo "FAIL: all-in-one master address mismatch"; exit 1; }
           [ "$AIO_SVC" = "$AIO_FILER_ADDR_SVC" ] || { echo "FAIL: all-in-one filer address mismatch"; exit 1; }
-          echo "All-in-one mode: service names match DNS references with long release name"
+          echo "✓ All-in-one mode: service names match DNS references with long release name"
 
           echo ""
           echo "=== Testing security+S3: no blank lines in shell command blocks ==="
@@ -392,7 +343,7 @@ jobs:
                   print(f"FAIL: {e}", file=sys.stderr)
               print("Rendered with: global.seaweedfs.enableSecurity=true, filer.s3.enabled=true, s3.enabled=true, allInOne.enabled=true", file=sys.stderr)
               sys.exit(1)
-          print("No blank lines in security+S3 command blocks")
+          print("✓ No blank lines in security+S3 command blocks")
           PYEOF
 
           echo ""
@@ -470,7 +421,7 @@ jobs:
                             f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
                       failed = True
                   else:
-                      print(f"{label}: cert/key/https args emitted together")
+                      print(f"✓ {label}: cert/key/https args emitted together")
               else:
                   if has_cert or has_key or has_https:
                       print(f"FAIL: {label}: expected none of {cert_flag}/{key_flag}/{https_flag}; "
@@ -478,7 +429,7 @@ jobs:
                             f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
                       failed = True
                   else:
-                      print(f"{label}: no TLS args emitted, main -port stays HTTP")
+                      print(f"✓ {label}: no TLS args emitted, main -port stays HTTP")
 
               # bash -n: pin down that the rendered script parses. Guards against
               # a future helper change that leaves a dangling `\` with nothing
@@ -495,49 +446,7 @@ jobs:
           sys.exit(1 if failed else 0)
           PYEOF
 
-          echo ""
-          echo "=== Testing all-in-one env: a key in both global and component renders once ==="
-          # Regression: all-in-one looped global and component extraEnvironmentVars
-          # in two separate ranges, emitting duplicate env entries for any key set
-          # in both maps. Render a shared key and assert it appears exactly once in
-          # the all-in-one container, with the component value winning (consistent
-          # with the merge helper the other components already use). pyyaml is
-          # installed by the earlier IAM gRPC block in this same step.
-          helm template test $CHART_DIR \
-            --set allInOne.enabled=true \
-            --set global.seaweedfs.extraEnvironmentVars.WEED_SHARED=fromGlobal \
-            --set allInOne.extraEnvironmentVars.WEED_SHARED=fromComponent > /tmp/aio-env.yaml
-          python3 - /tmp/aio-env.yaml <<'PYEOF'
-          import sys, yaml
-          from collections import Counter
-          docs = [d for d in yaml.safe_load_all(open(sys.argv[1])) if d]
-          dep = next(d for d in docs if d.get("kind") == "Deployment"
-                     and d["metadata"]["name"].endswith("all-in-one"))
-          envs = [e["name"] for c in dep["spec"]["template"]["spec"]["containers"]
-                  for e in c.get("env", [])]
-          dups = {k: v for k, v in Counter(envs).items() if v > 1}
-          if dups:
-              print(f"FAIL: duplicate env entries in all-in-one container: {dups}", file=sys.stderr)
-              sys.exit(1)
-          val = next(e.get("value") for c in dep["spec"]["template"]["spec"]["containers"]
-                     for e in c.get("env", []) if e["name"] == "WEED_SHARED")
-          if val != "fromComponent":
-              print(f"FAIL: WEED_SHARED should take the component value 'fromComponent', got '{val}'",
-                    file=sys.stderr)
-              sys.exit(1)
-          print("all-in-one env: shared key renders once, component value wins")
-          PYEOF
-
-          echo "=== Testing bucket versioning: YAML bool false suspends like string \"false\" ==="
-          # bool false used to be a silent no-op while string "false" suspended.
-          BOOL_FALSE=$(helm template test $CHART_DIR \
-            --set s3.enabled=true \
-            --set s3.createBuckets[0].name=verbucket \
-            --set s3.createBuckets[0].versioning=false | grep 's3.bucket.versioning -name verbucket' || true)
-          echo "$BOOL_FALSE" | grep -q -- '-status Suspended' || { echo "FAIL: bool false versioning did not Suspend the bucket"; exit 1; }
-          echo "Bucket versioning: YAML bool false suspends consistently with string \"false\""
-
-          echo "All template rendering tests passed!"
+          echo "✅ All template rendering tests passed!"
 
       - name: Create kind cluster
         uses: helm/kind-action@v1.14.0

.github/workflows/kafka-quicktest.yml

@@ -32,11 +32,6 @@ jobs:
           **/go.sum
       id: go
 
-    - name: Configure Docker Hub mirror
-      run: |
-        echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-        sudo systemctl restart docker
-
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v4
 

.github/workflows/nfs-tests.yml

@@ -0,0 +1,137 @@
+name: "NFS Integration Tests"
+
+on:
+  push:
+    branches: [ master, main ]
+    paths:
+      - 'weed/server/nfs/**'
+      - 'weed/command/nfs.go'
+      - 'weed/filer/filer_inode.go'
+      - 'weed/filer/filer_inode_index.go'
+      - 'weed/filer/filerstore_wrapper.go'
+      - 'weed/server/filer_grpc_server_rename.go'
+      - 'test/nfs/**'
+      - '.github/workflows/nfs-tests.yml'
+  pull_request:
+    branches: [ master, main ]
+    paths:
+      - 'weed/server/nfs/**'
+      - 'weed/command/nfs.go'
+      - 'weed/filer/filer_inode.go'
+      - 'weed/filer/filer_inode_index.go'
+      - 'weed/filer/filerstore_wrapper.go'
+      - 'weed/server/filer_grpc_server_rename.go'
+      - 'test/nfs/**'
+      - '.github/workflows/nfs-tests.yml'
+
+concurrency:
+  group: ${{ github.head_ref }}/nfs-tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  TEST_TIMEOUT: '15m'
+
+jobs:
+  nfs-integration:
+    name: NFS Integration Testing
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v6
+
+    - name: Set up Go
+      uses: actions/setup-go@v6
+      with:
+        go-version-file: 'go.mod'
+
+    - name: Build SeaweedFS
+      run: |
+        cd weed
+        go build -o weed .
+        chmod +x weed
+        ./weed version
+
+    - name: Run NFS Integration Tests
+      run: |
+        cd test/nfs
+
+        echo "Running NFS integration tests..."
+        echo "============================================"
+
+        # Install test dependencies
+        go mod download
+
+        # Run the protocol-layer tests. The kernel-mount tests require root
+        # for mount(2) and are exercised in their own privileged step below;
+        # skip them here so a "skipped because not root" line doesn't show
+        # up as noise on every CI run.
+        go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./...
+
+        echo "============================================"
+        echo "NFS integration tests completed"
+
+    - name: Install kernel NFS client
+      run: |
+        # nfs-common provides mount.nfs; netbase provides /etc/protocols
+        # which mount.nfs's protocol-name lookups (`tcp`, `udp`) need.
+        sudo apt-get update
+        sudo apt-get install -y nfs-common netbase
+
+    - name: Run kernel-mount E2E tests
+      run: |
+        cd test/nfs
+
+        echo "Running kernel-mount end-to-end tests..."
+        echo "These mount the running 'weed nfs' subprocess via the actual"
+        echo "Linux NFS client to catch protocol regressions invisible to"
+        echo "the go-nfs-client-based tests above."
+        echo "============================================"
+
+        # mount(2) is privileged. Preserve PATH so 'go' (and the weed
+        # binary that test/nfs/framework.go locates via $PATH) resolve
+        # correctly under sudo, and pass through the Go module/cache dirs
+        # so we don't redownload modules under root.
+        sudo env "PATH=$PATH" \
+          GOMODCACHE="$(go env GOMODCACHE)" \
+          GOCACHE="$(go env GOCACHE)" \
+          go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./...
+
+        echo "============================================"
+        echo "Kernel-mount E2E tests completed"
+
+    - name: Test Summary
+      if: always()
+      run: |
+        echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
+        echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
+        echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
+        echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
+        echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
+        echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
+        echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
+        echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
+        echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
+        echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
+        echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
+        echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
+        echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY
+        echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY
+        echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY
+        echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "### Harness" >> $GITHUB_STEP_SUMMARY
+        echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
+        echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
+        echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY
+        echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY
+        echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY
+        echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY

.github/workflows/postgres-tests.yml

@@ -31,11 +31,6 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v6
 
-    - name: Configure Docker Hub mirror
-      run: |
-        echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-        sudo systemctl restart docker
-
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v4
 

.github/workflows/rust_binaries_release.yml

@@ -66,42 +66,34 @@ jobs:
           SEAWEEDFS_COMMIT: ${{ github.sha }}
         run: |
           cd seaweed-volume
-          cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
+          cargo build --release --target ${{ matrix.target }}
 
       - name: Build Rust volume server (normal)
         env:
           SEAWEEDFS_COMMIT: ${{ github.sha }}
         run: |
           cd seaweed-volume
-          cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
+          cargo build --release --target ${{ matrix.target }} --no-default-features
 
       - name: Package binaries
         run: |
           # Large disk (default, 5bytes feature)
-          cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
+          cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
           tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
           rm weed-volume-large-disk
 
           # Normal volume size
-          cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
+          cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
           tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
           rm weed-volume-normal
 
-      - name: Generate md5 checksums
-        run: |
-          for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
-            md5sum "$f" > "$f.md5"
-          done
-
       - name: Upload release assets
         if: startsWith(github.ref, 'refs/tags/')
         uses: softprops/action-gh-release@v3
         with:
           files: |
             weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
-            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
             weed-volume_${{ matrix.asset_suffix }}.tar.gz
-            weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -112,9 +104,7 @@ jobs:
           name: rust-volume-${{ matrix.asset_suffix }}
           path: |
             weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
-            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
             weed-volume_${{ matrix.asset_suffix }}.tar.gz
-            weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
 
   build-rust-volume-darwin:
     permissions:
@@ -155,40 +145,32 @@ jobs:
           SEAWEEDFS_COMMIT: ${{ github.sha }}
         run: |
           cd seaweed-volume
-          cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
+          cargo build --release --target ${{ matrix.target }}
 
       - name: Build Rust volume server (normal)
         env:
           SEAWEEDFS_COMMIT: ${{ github.sha }}
         run: |
           cd seaweed-volume
-          cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
+          cargo build --release --target ${{ matrix.target }} --no-default-features
 
       - name: Package binaries
         run: |
-          cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
+          cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
           tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
           rm weed-volume-large-disk
 
-          cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
+          cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
           tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
           rm weed-volume-normal
 
-      - name: Generate md5 checksums
-        run: |
-          for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
-            md5 -r "$f" > "$f.md5"
-          done
-
       - name: Upload release assets
         if: startsWith(github.ref, 'refs/tags/')
         uses: softprops/action-gh-release@v3
         with:
           files: |
             weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
-            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
             weed-volume_${{ matrix.asset_suffix }}.tar.gz
-            weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -199,9 +181,7 @@ jobs:
           name: rust-volume-${{ matrix.asset_suffix }}
           path: |
             weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
-            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
             weed-volume_${{ matrix.asset_suffix }}.tar.gz
-            weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
 
   build-rust-volume-windows:
     permissions:
@@ -233,42 +213,33 @@ jobs:
           SEAWEEDFS_COMMIT: ${{ github.sha }}
         run: |
           cd seaweed-volume
-          cargo build --release --target-dir target/large-disk
+          cargo build --release
 
       - name: Build Rust volume server (normal)
         env:
           SEAWEEDFS_COMMIT: ${{ github.sha }}
         run: |
           cd seaweed-volume
-          cargo build --release --no-default-features --target-dir target/normal
+          cargo build --release --no-default-features
 
       - name: Package binaries
         shell: bash
         run: |
-          cp seaweed-volume/target/large-disk/release/weed-volume.exe weed-volume-large-disk.exe
+          cp seaweed-volume/target/release/weed-volume.exe weed-volume-large-disk.exe
           7z a weed-volume_large_disk_windows_amd64.zip weed-volume-large-disk.exe
           rm weed-volume-large-disk.exe
 
-          cp seaweed-volume/target/normal/release/weed-volume.exe weed-volume-normal.exe
+          cp seaweed-volume/target/release/weed-volume.exe weed-volume-normal.exe
           7z a weed-volume_windows_amd64.zip weed-volume-normal.exe
           rm weed-volume-normal.exe
 
-      - name: Generate md5 checksums
-        shell: bash
-        run: |
-          for f in weed-volume_large_disk_windows_amd64.zip weed-volume_windows_amd64.zip; do
-            md5sum "$f" > "$f.md5"
-          done
-
       - name: Upload release assets
         if: startsWith(github.ref, 'refs/tags/')
         uses: softprops/action-gh-release@v3
         with:
           files: |
             weed-volume_large_disk_windows_amd64.zip
-            weed-volume_large_disk_windows_amd64.zip.md5
             weed-volume_windows_amd64.zip
-            weed-volume_windows_amd64.zip.md5
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -279,6 +250,4 @@ jobs:
           name: rust-volume-windows_amd64
           path: |
             weed-volume_large_disk_windows_amd64.zip
-            weed-volume_large_disk_windows_amd64.zip.md5
             weed-volume_windows_amd64.zip
-            weed-volume_windows_amd64.zip.md5

.github/workflows/s3-proxy-signature-tests.yml

@@ -28,11 +28,6 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 

.github/workflows/s3-spark-tests.yml

@@ -33,19 +33,15 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v4
 
       - name: Install SeaweedFS
         run: |
           go install -buildvcs=false ./weed
 
       - name: Pre-pull Spark image
-        run: |
-          pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
-          pull apache/spark:3.5.1
+        run: docker pull apache/spark:3.5.8
 
       - name: Run S3 Spark integration tests
         working-directory: test/s3/spark

.github/workflows/s3-tables-tests.yml

@@ -83,11 +83,6 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
-
       - name: Run go mod tidy
         run: go mod tidy
 
@@ -144,15 +139,11 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v4
 
       - name: Pre-pull Trino image
-        run: |
-          pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
-          pull trinodb/trino:479
+        run: docker pull trinodb/trino:479
 
       - name: Run go mod tidy
         run: go mod tidy
@@ -214,16 +205,14 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v4
 
-      - name: Pre-pull images
-        run: |
-          pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
-          pull dremio/dremio-oss:25.2.0
-          pull python:3.11-slim
+      - name: Pre-pull Dremio image
+        run: docker pull dremio/dremio-oss:25.2.0
+
+      - name: Pre-pull Python image for PyIceberg writer
+        run: docker pull python:3.11-slim
 
       - name: Run go mod tidy
         run: go mod tidy
@@ -287,16 +276,14 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v4
 
-      - name: Pre-pull images
-        run: |
-          pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
-          pull apache/doris:doris-all-in-one-2.1.0
-          pull python:3.11-slim
+      - name: Pre-pull Doris image
+        run: docker pull apache/doris:doris-all-in-one-2.1.0
+
+      - name: Pre-pull Python image for PyIceberg writer
+        run: docker pull python:3.11-slim
 
       - name: Run go mod tidy
         run: go mod tidy
@@ -367,15 +354,8 @@ jobs:
         run: |
           go install -buildvcs=false ./weed
 
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
-
       - name: Pre-pull Polaris image
-        run: |
-          pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
-          pull apache/polaris:latest
+        run: docker pull apache/polaris:latest
 
       - name: Run Polaris Integration Tests
         timeout-minutes: 25
@@ -428,15 +408,11 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v4
 
       - name: Pre-pull Spark image
-        run: |
-          pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
-          pull apache/spark:3.5.1
+        run: docker pull apache/spark:3.5.1
 
       - name: Run go mod tidy
         run: go mod tidy
@@ -498,16 +474,13 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v4
 
       - name: Pre-pull RisingWave image
         run: |
-          pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
-          pull risingwavelabs/risingwave:v2.5.0
-          pull postgres:16-alpine
+          docker pull risingwavelabs/risingwave:v2.5.0
+          docker pull postgres:16-alpine
 
       - name: Run go mod tidy
         run: go mod tidy
@@ -569,39 +542,11 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v4
 
-      - name: Week stamp for image cache key
-        id: week
-        run: echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT"
-
-      - name: Restore python:3 image cache
-        id: python-image
-        uses: actions/cache@v5
-        with:
-          path: /tmp/python3-image.tar
-          key: python3-image-${{ steps.week.outputs.week }}
-          restore-keys: |
-            python3-image-
-
-      - name: Load or pull python:3
-        run: |
-          if [ "${{ steps.python-image.outputs.cache-hit }}" = "true" ]; then
-            docker load -i /tmp/python3-image.tar
-            exit 0
-          fi
-          pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
-          if pull python:3; then
-            docker save -o /tmp/python3-image.tar python:3
-          elif [ -f /tmp/python3-image.tar ]; then
-            # Docker Hub unreachable; fall back to last week's cached image
-            docker load -i /tmp/python3-image.tar
-          else
-            exit 1
-          fi
+      - name: Pre-pull Python image
+        run: docker pull python:3
 
       - name: Run go mod tidy
         run: go mod tidy
@@ -663,6 +608,15 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v4
+
+      - name: Pre-pull Python image
+        run: docker pull python:3
+        
+      - name: Pre-pull LocalStack image (if needed)
+        run: docker pull localstack/localstack:latest || true
+
       - name: Run go mod tidy
         run: go mod tidy
 
@@ -723,16 +677,14 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Configure Docker Hub mirror
-        run: |
-          echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v4
 
-      - name: Pre-pull images
-        run: |
-          pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
-          pull unitycatalog/unitycatalog:v0.4.0
-          pull python:3.11-slim
+      - name: Pre-pull Unity Catalog image
+        run: docker pull unitycatalog/unitycatalog:v0.4.0
+
+      - name: Pre-pull Python image for delta-rs writer
+        run: docker pull python:3.11-slim
 
       - name: Run go mod tidy
         run: go mod tidy

.github/workflows/s3tests.yml

@@ -45,7 +45,7 @@ jobs:
 
       - name: Fix S3 tests bucket creation conflicts
         run: |
-          python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
+          python3 test/s3/fix_s3_tests_bucket_conflicts.py
         env:
           S3_TESTS_PATH: s3-tests
 
@@ -388,7 +388,7 @@ jobs:
 
       - name: Fix S3 tests bucket creation conflicts
         run: |
-          python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
+          python3 test/s3/fix_s3_tests_bucket_conflicts.py
         env:
           S3_TESTS_PATH: s3-tests
 

.github/workflows/terraform_ci.yml

@@ -1,80 +0,0 @@
-name: "terraform: validate and test modules"
-
-on:
-  push:
-    branches: [ master ]
-    paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
-  pull_request:
-    branches: [ master ]
-    paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
-
-permissions:
-  contents: read
-
-jobs:
-  validate:
-    name: fmt, validate, plan-level tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: Set up OpenTofu
-        uses: opentofu/setup-opentofu@v2
-        with:
-          tofu_version: 1.12.1
-
-      - name: fmt check
-        working-directory: terraform
-        run: tofu fmt -recursive -check -diff
-
-      - name: validate core
-        working-directory: terraform/modules/core
-        run: |
-          tofu init -backend=false -input=false
-          tofu validate
-
-      - name: validate security
-        working-directory: terraform/modules/security
-        run: |
-          tofu init -backend=false -input=false
-          tofu validate
-
-      - name: plan-level tests (core)
-        working-directory: terraform/modules/core
-        run: tofu test
-
-      - name: validate examples
-        run: |
-          set -e
-          for ex in terraform/examples/*/; do
-            echo "== validate $ex =="
-            tofu -chdir="$ex" init -backend=false -input=false
-            tofu -chdir="$ex" validate
-          done
-
-  smoke:
-    name: local cluster smoke test (real weed)
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: Set up Go
-        uses: actions/setup-go@v6
-        with:
-          go-version-file: go.mod
-
-      - name: Build weed
-        run: go build -o "$RUNNER_TEMP/weed" ./weed
-
-      - name: Set up OpenTofu
-        uses: opentofu/setup-opentofu@v2
-        with:
-          tofu_version: 1.12.1
-
-      - name: Run local cluster harness
-        working-directory: terraform/test/local
-        run: WEED="$RUNNER_TEMP/weed" ./run_local_cluster.sh
-
-      - name: Run local mTLS cluster harness
-        working-directory: terraform/test/local-secure
-        run: WEED="$RUNNER_TEMP/weed" ./run_local_secure.sh

README.md

@@ -35,7 +35,6 @@ Your support will be really appreciated by me and other supporters!
 [![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com)
 [![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com)
 [![keepsec](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/keepsec.png)](https://www.keepsec.ca)
-[![zyner](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_zyner.png)](https://zyner.org)
 
 ---
 
@@ -655,8 +654,7 @@ Cluster Total: 3302.88 MiB/s, 550.51 obj/s over 43s.
 ## Enterprise ##
 
 For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition, 
-which has advanced features, including data recovery, self-healing storage, 
-customizable erasure coding, EC vacuum and repair, etc.
+which has a self-healing storage format with better data protection.
 
 [Back to TOC](#table-of-contents)
 

docker/Dockerfile.go_build

@@ -1,6 +1,4 @@
-# Pin the builder to the host arch and cross-compile the (CGO-free) Go binary,
-# so arm64/arm/386 targets skip QEMU emulation of the whole compile.
-FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder
+FROM golang:1.25-alpine AS builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/seaweedfs/
 ARG BRANCH=${BRANCH:-master}
@@ -14,12 +12,9 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs && \
       git checkout $BRANCH) || \
      (echo "ERROR: Branch/commit $BRANCH not found in repository" && \
       echo "Available branches:" && git branch -a && exit 1))
-ARG TARGETOS TARGETARCH TARGETVARIANT
 RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
   && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
-  && export GOOS=$TARGETOS GOARCH=$TARGETARCH \
-  && case "$TARGETARCH" in arm) export GOARM="${TARGETVARIANT#v}";; esac \
-  && CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}" -o /go/bin/weed .
+  && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
 
 # Rust volume server: use pre-built binary from CI when available (placed in
 # weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile

docker/Makefile

@@ -127,9 +127,6 @@ test_tarantool: tags = tarantool
 test_tarantool: build_tarantool_dev_env build
 	docker compose -f compose/test-tarantool-filer.yml -p seaweedfs up
 
-test_keycloak_s3: build
-	docker compose -f compose/test-keycloak-s3.yml -p seaweedfs up
-
 clean:
 	rm ./weed
 

docker/compose/test-keycloak-s3.yml

@@ -1,46 +0,0 @@
-services:
-  keycloak:
-    image: quay.io/keycloak/keycloak:26.0.7
-    command: ["start-dev", "--import-realm"]
-    environment:
-      KC_BOOTSTRAP_ADMIN_USERNAME: admin
-      KC_BOOTSTRAP_ADMIN_PASSWORD: admin
-      KC_HTTP_ENABLED: "true"
-      KC_HOSTNAME: "http://keycloak:8080"
-      KC_HOSTNAME_STRICT: "false"
-    ports:
-      - "8080:8080"
-    volumes:
-      - ../../test/s3/iam/seaweedfs-test-realm.json:/opt/keycloak/data/import/seaweedfs-test-realm.json:ro
-    healthcheck:
-      test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/8080"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-      start_period: 60s
-
-  s3:
-    image: chrislusf/seaweedfs:local
-    command:
-      - -v=9
-      - server
-      - -ip=s3
-      - -filer
-      - -master.volumeSizeLimitMB=16
-      - -volume
-      - -volume.max=0
-      - -volume.preStopSeconds=1
-      - -s3
-      - -s3.port=8333
-      - -s3.config=/etc/seaweedfs/s3_config.json
-      - -s3.iam.config=/etc/seaweedfs/iam_config.json
-    environment:
-      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
-    volumes:
-      - ../../test/s3/iam/test_config.json:/etc/seaweedfs/s3_config.json:ro
-      - ../../test/s3/iam/iam_config_docker.json:/etc/seaweedfs/iam_config.json:ro
-    ports:
-      - "8333:8333"
-    depends_on:
-      keycloak:
-        condition: service_healthy

go.mod

@@ -88,9 +88,9 @@ require (
 	go.etcd.io/etcd/client/v3 v3.6.10
 	go.mongodb.org/mongo-driver v1.17.9
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.46.0
+	gocloud.dev v0.45.0
 	gocloud.dev/pubsub/natspubsub v0.45.0
-	gocloud.dev/pubsub/rabbitpubsub v0.46.0
+	gocloud.dev/pubsub/rabbitpubsub v0.45.0
 	golang.org/x/crypto v0.52.0
 	golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
 	golang.org/x/image v0.39.0
@@ -102,7 +102,7 @@ require (
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.278.0
 	google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
-	google.golang.org/grpc v1.81.1
+	google.golang.org/grpc v1.80.0
 	google.golang.org/protobuf v1.36.11
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
@@ -117,18 +117,19 @@ require (
 	github.com/Jille/raft-grpc-transport v1.6.1
 	github.com/ThreeDotsLabs/watermill v1.5.1
 	github.com/a-h/templ v0.3.1001
-	github.com/apache/cassandra-gocql-driver/v2 v2.1.1
+	github.com/apache/cassandra-gocql-driver/v2 v2.1.0
 	github.com/apache/iceberg-go v0.5.0
 	github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
 	github.com/arangodb/go-driver v1.6.9
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.41.12
-	github.com/aws/aws-sdk-go-v2/config v1.32.21
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.20
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2
+	github.com/aws/aws-sdk-go-v2 v1.41.7
+	github.com/aws/aws-sdk-go-v2/config v1.32.14
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.14
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
 	github.com/cognusion/imaging v1.0.3
 	github.com/fluent/fluent-logger-golang v1.10.1
 	github.com/getsentry/sentry-go v0.44.1
+	github.com/go-git/go-billy/v5 v5.9.0
 	github.com/go-ldap/ldap/v3 v3.4.13
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
@@ -144,19 +145,21 @@ require (
 	github.com/rabbitmq/amqp091-go v1.11.0
 	github.com/rclone/rclone v1.74.1
 	github.com/rdleal/intervalst v1.5.0
-	github.com/redis/go-redis/v9 v9.20.0
+	github.com/redis/go-redis/v9 v9.19.0
 	github.com/schollz/progressbar/v3 v3.19.0
 	github.com/seaweedfs/go-fuse/v2 v2.9.3
 	github.com/shirou/gopsutil/v4 v4.26.3
 	github.com/tarantool/go-tarantool/v2 v2.4.2
 	github.com/testcontainers/testcontainers-go v0.40.0
 	github.com/tikv/client-go/v2 v2.0.7
+	github.com/willscott/go-nfs v0.0.4
+	github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.139.5
-	go.etcd.io/etcd/client/pkg/v3 v3.6.12
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2
+	go.etcd.io/etcd/client/pkg/v3 v3.6.11
 	go.uber.org/atomic v1.11.0
-	golang.org/x/sync v0.21.0
+	golang.org/x/sync v0.20.0
 	golang.org/x/tools/godoc v0.1.0-deprecated
 	google.golang.org/grpc/security/advancedtls v1.0.0
 )
@@ -179,7 +182,7 @@ require (
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
 	github.com/apache/thrift v0.23.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.1.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/bazelbuild/rules_go v0.46.0 // indirect
 	github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
@@ -208,7 +211,6 @@ require (
 	github.com/dromara/dongle v1.0.1 // indirect
 	github.com/gin-gonic/gin v1.11.0 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
-	github.com/go-git/go-billy/v5 v5.9.0 // indirect
 	github.com/goccy/go-yaml v1.18.0 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
@@ -231,7 +233,7 @@ require (
 	github.com/klauspost/asmfmt v1.3.2 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/lib/pq v1.12.0 // indirect
+	github.com/lib/pq v1.11.1 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
 	github.com/magiconair/properties v1.8.10 // indirect
@@ -255,6 +257,7 @@ require (
 	github.com/pquerna/otp v1.5.0 // indirect
 	github.com/pterm/pterm v0.12.82 // indirect
 	github.com/quic-go/qpack v0.6.0 // indirect
+	github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
 	github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
 	github.com/rclone/go-proton-api v1.0.2 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
@@ -274,10 +277,10 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/zeebo/xxh3 v1.1.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
 	go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.uber.org/mock v0.5.2 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
@@ -320,22 +323,23 @@ require (
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.26 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.26 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.26 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.27 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.26 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.31.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.43.0
-	github.com/aws/smithy-go v1.27.1
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
+	github.com/aws/smithy-go v1.25.1
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
@@ -346,7 +350,7 @@ require (
 	github.com/cloudinary/cloudinary-go/v2 v2.15.0 // indirect
 	github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
 	github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
-	github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
+	github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
 	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
 	github.com/creasty/defaults v1.8.0 // indirect
 	github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect
@@ -358,8 +362,8 @@ require (
 	github.com/elastic/gosigar v0.14.3 // indirect
 	github.com/emersion/go-message v0.18.2 // indirect
 	github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
-	github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
-	github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
+	github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
 	github.com/fatih/color v1.18.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/flynn/noise v1.1.0 // indirect
@@ -388,7 +392,7 @@ require (
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/gorilla/sessions v1.4.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -437,7 +441,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
 	github.com/philhofer/fwd v1.2.0 // indirect
-	github.com/pierrec/lz4/v4 v4.1.27
+	github.com/pierrec/lz4/v4 v4.1.26
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
 	github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
@@ -474,7 +478,7 @@ require (
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20260428144813-1c07baab7f7b // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 // indirect
 	github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
@@ -484,7 +488,7 @@ require (
 	go.etcd.io/bbolt v1.4.3 // indirect
 	go.etcd.io/etcd/api/v3 v3.6.10 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
 	go.opentelemetry.io/otel v1.43.0 // indirect

go.sum

@@ -573,6 +573,10 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 h1:tZh20RjgfMxKBxJiI
 github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4/go.mod h1:vGYAk36rhMVCfTP7v+RVruCR0zmPe6S+36KRpDCLySw=
 github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
 github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest/to v0.4.1 h1:CxNHBqdzTr7rLtdrtb5CMjJcDut+WNGCVv7OmS5+lTc=
+github.com/Azure/go-autorest/autorest/to v0.4.1/go.mod h1:EtaofgU4zmtvn1zT2ARsjRFdq9vXx0YWtmElwL+GZ9M=
 github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
 github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
@@ -690,8 +694,8 @@ github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmO
 github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 h1:r/gg2gzUXiXoy72VU3jnODh8l/5rL0aslnTAbtmai/U=
 github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87/go.mod h1:IJTMBTlHe7cDOhRh0ioGuEKBl5iTR6xPfl5BN4AgirU=
 github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
-github.com/apache/cassandra-gocql-driver/v2 v2.1.1 h1:DjmtmSRoUuIFswczA8LPBppRb8T9vxqNP5Txc/lQxhE=
-github.com/apache/cassandra-gocql-driver/v2 v2.1.1/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
+github.com/apache/cassandra-gocql-driver/v2 v2.1.0 h1:VEbbeJ2ift4deKMZ6Fs55Vs3fq/RrkjCcxCnqUxhwf8=
+github.com/apache/cassandra-gocql-driver/v2 v2.1.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
 github.com/apache/iceberg-go v0.5.0 h1:wQj4CK5YiXZcB+tj19gWG+Jf1I6MiORQ/StSL/E5gGQ=
 github.com/apache/iceberg-go v0.5.0/go.mod h1:F/rdP1yZmnO4mQ0Qew2HTGdc+ZV57cRfxbbq/uJm1eM=
 github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc=
@@ -711,50 +715,50 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck=
-github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94=
-github.com/aws/aws-sdk-go-v2/config v1.32.21 h1:1i1v5jWn6iNPl7zkH4VwIXQOb+OMqMRFNjdHQeZbFVo=
-github.com/aws/aws-sdk-go-v2/config v1.32.21/go.mod h1:7NXDJjk0e4QOMv3Y3ppwALx3ifdSQHzCACFtRX5J1hI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.20 h1:nIGz0+cQKwijxSsWNpggvPxjxFMtrAHNj16RYzGbu2Q=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.20/go.mod h1:/UQHYia3DSeilGdchXJDEVzJws15XuZSdnTlwbTBg0Q=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.26 h1:a1VPHz7eG7mVNHWHXUcCyC8rQDBEr4k+o4HJYbSoYZ0=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.26/go.mod h1:9G8LPblMwaodeQdS+IfDBgPqABDctJLpyZ/M6jflE6s=
+github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
+github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
+github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
+github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 h1:uMC4oL6G3MNhodo358QEqSDjrgvzV3TUQ58nyQSGq2E=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13/go.mod h1:Cer86AE2686DvVUe57LPve3jUBmbujuaonSX8pNzGgw=
-github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 h1:w5OoDiMN6x53ROmiIImGzmVcxXv2q1GXY+aKV4WAJYM=
-github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3/go.mod h1:dAhgYp776bX3LuWvnSCFwQEjNs6fuFg7YXIy5PXcP3Q=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.26 h1:dx1qc6V0exl5SK5/kwSc2IdOKCxLHoNbl/zXuYRwopw=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.26/go.mod h1:iDMGb+drKqPTqRjv1kWc5meMYzT6HsvPXxJV/Axq1t0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.26 h1:ZjsIjuahUd/OxnxgfeK6OZqrReRNru1BcPfFD95IoN0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.26/go.mod h1:57VbRA0K0Xxx5XUUTUrVxZ/gm4u8Bk3LBflfufwHtKQ=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.27 h1:i49S7g2smFWmlconi97Nn8dxb7jSQ0TeOFD20NX3M+o=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.27/go.mod h1:qiLes3uVQAK+cpb837HUEhypYKGBz2ZCDcDl5BYQtpM=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 h1:d5/908OJ4bXg8lyjeMPvXetEKqoDoLi5Owy1zNue3yg=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10/go.mod h1:a57l7Hwh+FWI+we50g5NPJHYUKeJKfXbc4w8SyXu8Ig=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.26 h1:oIRaBVJ2CxT7T61l3JkBCW/rnyh2TFZIRL71MvJTKRI=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.26/go.mod h1:1gCTZPMlqAUSFoFbkl35hKtTpE96WFl7ORvDXmSLpRc=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDbiit/GcBE2K4IUpMZymaA0kOz3xK978=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0=
-github.com/aws/aws-sdk-go-v2/service/signin v1.1.2 h1:qwU4CNor4G2gRW3Na24FWOCZSEvr967IAV3D+MXxdE0=
-github.com/aws/aws-sdk-go-v2/service/signin v1.1.2/go.mod h1:vwIj2P3URYf7ZtOQkO2iVbAaR8qFhqMv05w3aPPA5XA=
-github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 h1:p8WdWDh5AwSZdp19Haa3XMyPCICi9Z375a/Nu3IIEZY=
-github.com/aws/aws-sdk-go-v2/service/sns v1.39.14/go.mod h1:NKVY7DER6VXHkt2I/ycmHakALNboi3Rqwt4eEf/1Cnk=
-github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 h1:JP2wjWGmUp8lTCZb13Dv0Eciyc1jbO8pd0HZVMHFlrc=
-github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24/go.mod h1:Ql9ziDutk8ERAN9HMaYANCW3lop451ppebkxEJMLCTM=
-github.com/aws/aws-sdk-go-v2/service/sso v1.31.0 h1:cMDUWhi7vlmGFj6rldHofjnAQhLLBWwRvNRbtEoyZo4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.31.0/go.mod h1:VfGCm6HLUGwcouY6zjQJd81PoFsb8sfSmKd5QCg9XXc=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.3 h1:/pKy3mNES+ppFHf3g/XmI95+f3vdeXnkWYObByoYGAg=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.3/go.mod h1:0z2rhlVAjsYeBzV8fdT4jR23Nq+fYhoJNSJN+4SBlqM=
-github.com/aws/aws-sdk-go-v2/service/sts v1.43.0 h1:HN43uYvF8D2NtaPTeE4L3HMvXgd+RtqEqrea4uUztkM=
-github.com/aws/aws-sdk-go-v2/service/sts v1.43.0/go.mod h1:qzhDlfw1BRYyoNofevemNzrlRpUXa0BJamWx7klrR8w=
-github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8=
-github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
+github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
+github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
+github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
+github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
+github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
+github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
 github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
 github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
@@ -850,8 +854,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik=
-github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4=
+github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w=
+github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI=
 github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=
 github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc=
 github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
@@ -996,16 +1000,16 @@ github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJ
 github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI=
 github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA=
 github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU=
-github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ=
-github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A=
+github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g=
+github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98=
 github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
 github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo=
 github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w=
 github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss=
-github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds=
-github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0=
+github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4=
+github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA=
 github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw=
 github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA=
 github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
@@ -1320,8 +1324,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4=
 github.com/hamba/avro/v2 v2.31.0 h1:wv3nmua7lCEIwWsb6vqsTS3pXktTxcKg5eoyNu0VhrU=
 github.com/hamba/avro/v2 v2.31.0/go.mod h1:t6lJYAGE5Mswfn17zjtyQsssRQgnqO6TXLBCHHWRqrw=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -1507,8 +1511,8 @@ github.com/lanrat/extsort v1.4.2 h1:akbLIdo4PhNZtvjpaWnbXtGMmLtnGzXplkzfgl+XTTY=
 github.com/lanrat/extsort v1.4.2/go.mod h1:hceP6kxKPKebjN1RVrDBXMXXECbaI41Y94tt6MDazc4=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
-github.com/lib/pq v1.12.0 h1:mC1zeiNamwKBecjHarAr26c/+d8V5w/u4J0I/yASbJo=
-github.com/lib/pq v1.12.0/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
+github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI=
+github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
 github.com/linkedin/goavro/v2 v2.15.0 h1:pDj1UrjUOO62iXhgBiE7jQkpNIc5/tA5eZsgolMjgVI=
 github.com/linkedin/goavro/v2 v2.15.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
 github.com/linxGnu/grocksdb v1.10.8 h1:Nau01Hhm/0kaVTR6d4viwD6npYbnDvZAfzwJCLzKRYo=
@@ -1687,8 +1691,8 @@ github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2
 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
 github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk=
-github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
+github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY=
+github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
 github.com/pierrre/compare v1.0.2 h1:k4IUsHgh+dbcAOIWCfxVa/7G6STjADH2qmhomv+1quc=
 github.com/pierrre/compare v1.0.2/go.mod h1:8UvyRHH+9HS8Pczdd2z5x/wvv67krDwVxoOndaIIDVU=
 github.com/pierrre/geohash v1.0.0 h1:f/zfjdV4rVofTCz1FhP07T+EMQAvcMM2ioGZVt+zqjI=
@@ -1779,6 +1783,8 @@ github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SA
 github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
 github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8=
 github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
+github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
+github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
 github.com/rclone/Proton-API-Bridge v1.0.3 h1:Bs7RC4xCFSN0BPIYVda/BNxp0qo3NV0gB2VZqx2KIew=
 github.com/rclone/Proton-API-Bridge v1.0.3/go.mod h1:26RAest751Ofk+F/d8xtl4UyWXrZvMQwn39U8rm/WKM=
 github.com/rclone/go-proton-api v1.0.2 h1:cJtJUab0MGJ3C6q5kiEJs3pbyhSLnOKMyYOQehA0PBc=
@@ -1789,8 +1795,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
 github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
-github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0=
-github.com/redis/go-redis/v9 v9.20.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
+github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k=
+github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
 github.com/redis/rueidis v1.0.71 h1:pODtnAR5GAB7j4ekhldZ29HKOxe4Hph0GTDGk1ayEQY=
 github.com/redis/rueidis v1.0.71/go.mod h1:lfdcZzJ1oKGKL37vh9fO3ymwt+0TdjkkUCJxbgpmcgQ=
 github.com/redis/rueidis/rueidiscompat v1.0.71 h1:wNZ//kEjMZgBM0KCk7ncOX8KmAgROU2kDdDNpwheG4w=
@@ -2022,6 +2028,10 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
+github.com/willscott/go-nfs v0.0.4 h1:1vpOPAdECmoT2KmZ8u+ukO/jfvDjMEUNYhA2F1jGJtI=
+github.com/willscott/go-nfs v0.0.4/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg=
+github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
+github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
 github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
 github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
@@ -2054,14 +2064,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD
 github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
-github.com/ydb-platform/ydb-go-genproto v0.0.0-20260428144813-1c07baab7f7b h1:xeiobG1riqe6dTMZuTcOvwOY0BbFidSk3gRLyVeLfJA=
-github.com/ydb-platform/ydb-go-genproto v0.0.0-20260428144813-1c07baab7f7b/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 h1:avIdi8eGXjKbn1WLokNR1Ofnz1k8t7tJ88YQLD/iCi8=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaFdZ6zevvYe+TRxOR8ifa2s=
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.139.5 h1:mTih9wy1RbI4PplGlFn/CMnkgxm1HzD/nKSdnNMy4pE=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.139.5/go.mod h1:b9NEO6mgaiqsnOMkS003uS82XsKh6GL+ZTFfPqXWz+c=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2 h1:Bzra5hUQIWrxknQ0oV6HE5q/eZU1VhHMKSZ/XWpqMgM=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2/go.mod h1:VYUUkRJkKuQPkIpgtZJj6+58Fa2g8ccAqdmaaK6HP5k=
 github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
 github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
 github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
@@ -2101,8 +2111,8 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
 go.etcd.io/etcd/api/v3 v3.6.10 h1:jlwjtELjA8yi2VWpOFH+0w0lGr3K6mVDyn0RDB9aaAY=
 go.etcd.io/etcd/api/v3 v3.6.10/go.mod h1:pdV4VeFmvhdNjB4LWRkC8ReLyRBAxUOze3GarMhE2sk=
-go.etcd.io/etcd/client/pkg/v3 v3.6.12 h1:36zzB+pQOdHbhN+kH2iJz/K8bJn0ZLtLfPPO7jozTDo=
-go.etcd.io/etcd/client/pkg/v3 v3.6.12/go.mod h1:hh2+ZXtfLzs3o6mn92ntgNPBrTJJOvXqICM5g3L3DMY=
+go.etcd.io/etcd/client/pkg/v3 v3.6.11 h1:e41mp315Yn3QMGPmEzCyLsMINgJXTY/dX8kM++1csxU=
+go.etcd.io/etcd/client/pkg/v3 v3.6.11/go.mod h1:DysuMe/inqRyC/1tjRR6hReH/VV9Lufs27YKSKBWWJg=
 go.etcd.io/etcd/client/v3 v3.6.10 h1:J598zJ+C/ZPvImypmq5waj84+bovePrlZERHklf34y0=
 go.etcd.io/etcd/client/v3 v3.6.10/go.mod h1:iHhUDUcEwaKs1YFq3MgmI9U4zhTVasp/vgdVbFf1RS8=
 go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
@@ -2118,8 +2128,8 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ=
-go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8=
+go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE=
+go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc=
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0 h1:0tY123n7CdWMem7MOVdKOt0YfshufLCwfE5Bob+hQuM=
@@ -2128,14 +2138,14 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:Oyrsyzu
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
 go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
 go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 h1:MdKucPl/HbzckWWEisiNqMPhRrAOQX8r4jTuGr636gk=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0/go.mod h1:RolT8tWtfHcjajEH5wFIZ4Dgh5jpPdFXYV9pTAk/qjc=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0/go.mod h1:ChZSJbbfbl/DcRZNc9Gqh6DYGlfjw4PvO1pEOZH1ZsE=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA=
@@ -2153,8 +2163,8 @@ go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLh
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
-go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
-go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
+go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
+go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@@ -2178,12 +2188,12 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
 go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q=
-gocloud.dev v0.46.0/go.mod h1:ACQe+2qO+hEO+pdcvvsM+RB63r8TyGD1W3ESCLFyzvM=
+gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY=
+gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI=
 gocloud.dev/pubsub/natspubsub v0.45.0 h1:kfCupVejeynIQcS1GaIvkY3Nj/acLlM5yPWyxZN8wJ8=
 gocloud.dev/pubsub/natspubsub v0.45.0/go.mod h1:WU5SMDWuF7CCr2U8UpbrEONYYFOmvigCAvTpOOpUvYE=
-gocloud.dev/pubsub/rabbitpubsub v0.46.0 h1:uWJ8z/F8xYxoTlBafTweoKvX2V85cwtN+/+nyRRtjZ0=
-gocloud.dev/pubsub/rabbitpubsub v0.46.0/go.mod h1:Ef5XHxv17zV9B3U9xoYAqJ8Rutslxt5Emb5e1cScuXc=
+gocloud.dev/pubsub/rabbitpubsub v0.45.0 h1:hlejKjBFYQBUAj7ZsHKgW0JOER/3XUqrMFFxBOFUIOU=
+gocloud.dev/pubsub/rabbitpubsub v0.45.0/go.mod h1:Kqq6gppGY/RTrRQkKSHZq+/qbCufPEoQ0E1V3X389zM=
 golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
 golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -2395,8 +2405,8 @@ golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
-golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -2882,8 +2892,8 @@ google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsA
 google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
 google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
 google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
-google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ=
-google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=
+google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
+google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds=
 google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.33"
+appVersion: "4.29"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.33.0
+version: 4.29.0

k8s/charts/seaweedfs/README.md

@@ -384,4 +384,4 @@ helm install seaweedfs seaweedfs/seaweedfs \
 ## Enterprise
 
 For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition, 
-which has advanced features, including data recovery, self-healing storage, customizable erasure coding, EC vacuum and repair, etc.
+which has a self-healing storage format with better data protection.

k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json

@@ -3666,291 +3666,6 @@
       ],
       "title": "S3 Bucket Object Count",
       "type": "timeseries"
-    },
-    {
-      "collapsed": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_PROMETHEUS}"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 130
-      },
-      "id": 92,
-      "panels": [],
-      "title": "Filer Object Size Distribution",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_PROMETHEUS}"
-      },
-      "description": "Rate of new objects created, split by size range.",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "objects/s",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 25,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 4,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "normal"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "short",
-          "unitScale": true
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 131
-      },
-      "id": 93,
-      "links": [],
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "8.1.2",
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval]))",
-          "format": "time_series",
-          "hide": false,
-          "instant": false,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "< 1KB",
-          "refId": "A",
-          "step": 60
-        },
-        {
-          "exemplar": true,
-          "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
-          "format": "time_series",
-          "hide": false,
-          "instant": false,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "1KB - 100KB",
-          "refId": "B",
-          "step": 60
-        },
-        {
-          "exemplar": true,
-          "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
-          "format": "time_series",
-          "hide": false,
-          "instant": false,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "100KB - 1MB",
-          "refId": "C",
-          "step": 60
-        },
-        {
-          "exemplar": true,
-          "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
-          "format": "time_series",
-          "hide": false,
-          "instant": false,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "1MB - 100MB",
-          "refId": "D",
-          "step": 60
-        },
-        {
-          "exemplar": true,
-          "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
-          "format": "time_series",
-          "hide": false,
-          "instant": false,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "100MB - 1GB",
-          "refId": "E",
-          "step": 60
-        },
-        {
-          "exemplar": true,
-          "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
-          "format": "time_series",
-          "hide": false,
-          "instant": false,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "> 1GB",
-          "refId": "F",
-          "step": 60
-        }
-      ],
-      "title": "Filer Object Write Rate by Size Range",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_PROMETHEUS}"
-      },
-      "description": "Objects created per size range over the selected time window.",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 131
-      },
-      "id": 94,
-      "options": {
-        "displayMode": "gradient",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showUnfilled": true,
-        "valueMode": "color"
-      },
-      "pluginVersion": "8.1.2",
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range]))",
-          "format": "time_series",
-          "instant": true,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "< 1KB",
-          "refId": "A"
-        },
-        {
-          "exemplar": true,
-          "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
-          "format": "time_series",
-          "instant": true,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "1KB - 100KB",
-          "refId": "B"
-        },
-        {
-          "exemplar": true,
-          "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
-          "format": "time_series",
-          "instant": true,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "100KB - 1MB",
-          "refId": "C"
-        },
-        {
-          "exemplar": true,
-          "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
-          "format": "time_series",
-          "instant": true,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "1MB - 100MB",
-          "refId": "D"
-        },
-        {
-          "exemplar": true,
-          "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
-          "format": "time_series",
-          "instant": true,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "100MB - 1GB",
-          "refId": "E"
-        },
-        {
-          "exemplar": true,
-          "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
-          "format": "time_series",
-          "instant": true,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "> 1GB",
-          "refId": "F"
-        }
-      ],
-      "title": "Filer Object Size Distribution",
-      "type": "bargauge"
     }
   ],
   "refresh": "",

k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml

@@ -81,9 +81,8 @@ spec:
           imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
           env:
             {{- /* Determine default cluster alias and the corresponding env var keys to avoid conflicts */}}
-            {{- $mergedExtraEnvironmentVars := dict }}
-            {{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global.seaweedfs "component" .Values.allInOne "target" $mergedExtraEnvironmentVars) }}
-            {{- $clusterDefault := default "sw" (index $mergedExtraEnvironmentVars "WEED_CLUSTER_DEFAULT") }}
+            {{- $envMerged := merge (.Values.global.seaweedfs.extraEnvironmentVars | default dict) (.Values.allInOne.extraEnvironmentVars | default dict) }}
+            {{- $clusterDefault := default "sw" (index $envMerged "WEED_CLUSTER_DEFAULT") }}
             {{- $clusterUpper := upper $clusterDefault }}
             {{- $clusterMasterKey := printf "WEED_CLUSTER_%s_MASTER" $clusterUpper }}
             {{- $clusterFilerKey := printf "WEED_CLUSTER_%s_FILER" $clusterUpper }}
@@ -101,8 +100,8 @@ spec:
                   fieldPath: metadata.namespace
             - name: SEAWEEDFS_FULLNAME
               value: "{{ include "seaweedfs.fullname" . }}"
-            {{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
-            {{- $value := index $mergedExtraEnvironmentVars $key }}
+            {{- if .Values.allInOne.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.allInOne.extraEnvironmentVars }}
             {{- if and (ne $key $clusterMasterKey) (ne $key $clusterFilerKey) }}
             - name: {{ $key }}
               {{- if kindIs "string" $value }}
@@ -113,6 +112,20 @@ spec:
               {{- end }}
             {{- end }}
             {{- end }}
+            {{- end }}
+            {{- if .Values.global.seaweedfs.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.global.seaweedfs.extraEnvironmentVars }}
+            {{- if and (ne $key $clusterMasterKey) (ne $key $clusterFilerKey) }}
+            - name: {{ $key }}
+              {{- if kindIs "string" $value }}
+              value: {{ tpl $value $ | quote }}
+              {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 }}
+              {{- end }}
+            {{- end }}
+            {{- end }}
+            {{- end }}
             # Inject computed cluster endpoints for the default cluster
             - name: {{ $clusterMasterKey }}
               value: {{ include "seaweedfs.cluster.masterAddress" . | quote }}
@@ -259,9 +272,6 @@ spec:
               {{- $authMethods := .Values.allInOne.sftp.authMethods | default .Values.sftp.authMethods }}
               {{- if $authMethods }}
               -sftp.authMethods={{ $authMethods }} \
-              {{- if contains "certificate" $authMethods }}
-              -sftp.trustedUserCAKeysFile=/etc/sw/sftp_ca/ca_user.pub \
-              {{- end }}
               {{- end }}
               {{- $maxAuthTries := .Values.allInOne.sftp.maxAuthTries | default .Values.sftp.maxAuthTries }}
               {{- if $maxAuthTries }}
@@ -309,12 +319,6 @@ spec:
               name: config-users
               readOnly: true
             {{- end }}
-            {{- $aioAuthMethods := .Values.allInOne.sftp.authMethods | default .Values.sftp.authMethods }}
-            {{- if and $aioAuthMethods (contains "certificate" $aioAuthMethods) }}
-            - mountPath: /etc/sw/sftp_ca
-              name: config-sftp-ca
-              readOnly: true
-            {{- end }}
             {{- end }}
             {{- if .Values.filer.notificationConfig }}
             - name: notification-config
@@ -450,18 +454,6 @@ spec:
             defaultMode: 420
             secretName: {{ default (printf "%s-sftp-secret" (include "seaweedfs.fullname" .)) (or .Values.allInOne.sftp.existingConfigSecret .Values.sftp.existingConfigSecret) }}
         {{- end }}
-        {{- $aioAuthMethodsVol := .Values.allInOne.sftp.authMethods | default .Values.sftp.authMethods }}
-        {{- if and $aioAuthMethodsVol (contains "certificate" $aioAuthMethodsVol) }}
-        {{- $aioCASecret := or .Values.allInOne.sftp.existingCAKeysSecret .Values.sftp.existingCAKeysSecret }}
-        {{- $aioCAKeys := or .Values.allInOne.sftp.trustedUserCAKeys .Values.sftp.trustedUserCAKeys }}
-        {{- if and (not $aioCASecret) (not $aioCAKeys) }}
-        {{- fail "allInOne.sftp.authMethods includes \"certificate\" but neither trustedUserCAKeys nor existingCAKeysSecret is set" }}
-        {{- end }}
-        - name: config-sftp-ca
-          secret:
-            defaultMode: 420
-            secretName: {{ default (printf "%s-sftp-ca-secret" (include "seaweedfs.fullname" .)) $aioCASecret }}
-        {{- end }}
         {{- end }}
         {{- if .Values.filer.notificationConfig }}
         - name: notification-config

k8s/charts/seaweedfs/templates/sftp/sftp-ca-secret.yaml

@@ -1,40 +0,0 @@
-{{- include "seaweedfs.compat" . -}}
-{{- /*
-Render a chart-managed Secret carrying the SSH user CA public key(s) for
-certificate-based SFTP authentication, but only when:
-  - SFTP (standalone or all-in-one) is enabled,
-  - "certificate" is in the relevant authMethods list,
-  - the user provided inline CA keys via sftp.trustedUserCAKeys, and
-  - no existingCAKeysSecret is set (which would supersede this Secret).
-*/}}
-{{- $sftpEnabled := or .Values.sftp.enabled (and .Values.allInOne.enabled .Values.allInOne.sftp.enabled) -}}
-{{- $authMethods := .Values.sftp.authMethods -}}
-{{- if and .Values.allInOne.enabled .Values.allInOne.sftp.authMethods -}}
-{{- $authMethods = .Values.allInOne.sftp.authMethods -}}
-{{- end -}}
-{{- $certInMethods := and $authMethods (contains "certificate" $authMethods) -}}
-{{- $inlineCAKeys := .Values.sftp.trustedUserCAKeys -}}
-{{- if and .Values.allInOne.enabled .Values.allInOne.sftp.trustedUserCAKeys -}}
-{{- $inlineCAKeys = .Values.allInOne.sftp.trustedUserCAKeys -}}
-{{- end -}}
-{{- $existingSecret := .Values.sftp.existingCAKeysSecret -}}
-{{- if and .Values.allInOne.enabled .Values.allInOne.sftp.existingCAKeysSecret -}}
-{{- $existingSecret = .Values.allInOne.sftp.existingCAKeysSecret -}}
-{{- end -}}
-{{- if and $sftpEnabled $certInMethods $inlineCAKeys (not $existingSecret) }}
-apiVersion: v1
-kind: Secret
-type: Opaque
-metadata:
-  name: {{ include "seaweedfs.fullname" . }}-sftp-ca-secret
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/component: sftp
-stringData:
-  ca_user.pub: |
-{{ $inlineCAKeys | indent 4 }}
-{{- end }}

k8s/charts/seaweedfs/templates/sftp/sftp-deployment.yaml

@@ -135,9 +135,6 @@ spec:
               {{- end }}
               {{- if .Values.sftp.authMethods }}
               -authMethods={{ .Values.sftp.authMethods }} \
-              {{- if contains "certificate" .Values.sftp.authMethods }}
-              -trustedUserCAKeysFile=/etc/sw/sftp_ca/ca_user.pub \
-              {{- end }}
               {{- end }}
               {{- if .Values.sftp.maxAuthTries }}
               -maxAuthTries={{ .Values.sftp.maxAuthTries }} \
@@ -179,11 +176,6 @@ spec:
             - mountPath: /etc/sw/ssh
               name: config-ssh
               readOnly: true
-            {{- if and .Values.sftp.authMethods (contains "certificate" .Values.sftp.authMethods) }}
-            - mountPath: /etc/sw/sftp_ca
-              name: config-sftp-ca
-              readOnly: true
-            {{- end }}
             {{- if include "seaweedfs.securityConfigEnabled" . }}
             - name: security-config
               readOnly: true
@@ -264,19 +256,6 @@ spec:
             {{- else }}
             secretName: {{ include "seaweedfs.fullname" . }}-sftp-ssh-secret
             {{- end }}
-        {{- if and .Values.sftp.authMethods (contains "certificate" .Values.sftp.authMethods) }}
-        {{- if and (not .Values.sftp.existingCAKeysSecret) (not .Values.sftp.trustedUserCAKeys) }}
-        {{- fail "sftp.authMethods includes \"certificate\" but neither sftp.trustedUserCAKeys nor sftp.existingCAKeysSecret is set" }}
-        {{- end }}
-        - name: config-sftp-ca
-          secret:
-            defaultMode: 420
-            {{- if .Values.sftp.existingCAKeysSecret }}
-            secretName: {{ .Values.sftp.existingCAKeysSecret }}
-            {{- else }}
-            secretName: {{ include "seaweedfs.fullname" . }}-sftp-ca-secret
-            {{- end }}
-        {{- end }}
         {{- if eq .Values.sftp.logs.type "hostPath" }}
         - name: logs
           hostPath:

k8s/charts/seaweedfs/templates/shared/_helpers.tpl

@@ -333,13 +333,11 @@ Create the name of the service account to use
 {{- end -}}
 
 {{/* True when security.toml should be rendered and mounted. volumeWrite is
-     excluded unless its non-default expiration is configured. */}}
+     excluded since it defaults to true. */}}
 {{- define "seaweedfs.securityConfigEnabled" -}}
 {{- $sec := (.Values.global.seaweedfs).securityConfig | default dict -}}
 {{- $jwt := $sec.jwtSigning | default dict -}}
-{{- $expiresAfterSeconds := $jwt.expiresAfterSeconds | default dict -}}
-{{- $volumeWriteExpirationConfigured := and $jwt.volumeWrite (gt (int $expiresAfterSeconds.volumeWrite) 0) -}}
-{{- if or .Values.global.seaweedfs.enableSecurity $volumeWriteExpirationConfigured $jwt.volumeRead $jwt.filerWrite $jwt.filerRead -}}
+{{- if or .Values.global.seaweedfs.enableSecurity $jwt.volumeRead $jwt.filerWrite $jwt.filerRead -}}
 true
 {{- end -}}
 {{- end -}}

k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml

@@ -143,8 +143,6 @@ spec:
           {{- if kindIs "bool" .versioning }}
             {{- if .versioning }}
               {{- $bucketVersioning = "Enabled" }}
-            {{- else }}
-              {{- $bucketVersioning = "Suspended" }}
             {{- end }}
           {{- else if kindIs "string" .versioning }}
             {{- $versioningLower := lower .versioning }}

k8s/charts/seaweedfs/templates/shared/security-configmap.yaml

@@ -19,56 +19,40 @@ data:
   {{- $existing = lookup "v1" "ConfigMap" .Release.Namespace $legacyName }}
   {{- end }}
   {{- $securityConfig := fromToml (dig "data" "security.toml" "" $existing) }}
-  {{- $securityConfigValues := .Values.global.seaweedfs.securityConfig | default dict }}
-  {{- $jwtSigning := $securityConfigValues.jwtSigning | default dict }}
-  {{- $expiresAfterSeconds := $jwtSigning.expiresAfterSeconds | default dict }}
   security.toml: |-
     # this file is read by master, volume server, and filer
 
-    {{- if $jwtSigning.volumeWrite }}
+    {{- if .Values.global.seaweedfs.securityConfig.jwtSigning.volumeWrite }}
     # the jwt signing key is read by master and volume server
-    # the jwt defaults to expire after 10 seconds
+    # a jwt expires in 10 seconds
     [jwt.signing]
     key = "{{ dig "jwt" "signing" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
-    {{- if gt (int $expiresAfterSeconds.volumeWrite) 0 }}
-    expires_after_seconds = {{ int $expiresAfterSeconds.volumeWrite }}
-    {{- end }}
     {{- end }}
 
-    {{- if $jwtSigning.volumeRead }}
+    {{- if .Values.global.seaweedfs.securityConfig.jwtSigning.volumeRead }}
     # this jwt signing key is read by master and volume server, and it is used for read operations:
     # - the Master server generates the JWT, which can be used to read a certain file on a volume server
     # - the Volume server validates the JWT on reading
-    # the jwt defaults to expire after 60 seconds
     [jwt.signing.read]
     key = "{{ dig "jwt" "signing" "read" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
-    {{- if gt (int $expiresAfterSeconds.volumeRead) 0 }}
-    expires_after_seconds = {{ int $expiresAfterSeconds.volumeRead }}
-    {{- end }}
     {{- end }}
 
-    {{- if $jwtSigning.filerWrite }}
+    {{- if .Values.global.seaweedfs.securityConfig.jwtSigning.filerWrite }}
     # If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT:
     # - f.e. the S3 API Shim generates the JWT
     # - the Filer server validates the JWT on writing
-    # the jwt defaults to expire after 10 seconds
+    # the jwt defaults to expire after 10 seconds.
     [jwt.filer_signing]
     key = "{{ dig "jwt" "filer_signing" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
-    {{- if gt (int $expiresAfterSeconds.filerWrite) 0 }}
-    expires_after_seconds = {{ int $expiresAfterSeconds.filerWrite }}
-    {{- end }}
     {{- end }}
 
-    {{- if $jwtSigning.filerRead }}
+    {{- if .Values.global.seaweedfs.securityConfig.jwtSigning.filerRead }}
     # If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT:
     # - f.e. the S3 API Shim generates the JWT
     # - the Filer server validates the JWT on reading
-    # the jwt defaults to expire after 60 seconds
+    # the jwt defaults to expire after 10 seconds.
     [jwt.filer_signing.read]
     key = "{{ dig "jwt" "filer_signing" "read" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
-    {{- if gt (int $expiresAfterSeconds.filerRead) 0 }}
-    expires_after_seconds = {{ int $expiresAfterSeconds.filerRead }}
-    {{- end }}
     {{- end }}
 
     {{- if .Values.global.seaweedfs.enableSecurity }}

k8s/charts/seaweedfs/values.yaml

@@ -26,13 +26,6 @@ global:
         volumeRead: false
         filerWrite: false
         filerRead: false
-        # Positive values override SeaweedFS token lifetime defaults.
-        # Zero keeps the runtime defaults: 10s for writes and 60s for reads.
-        expiresAfterSeconds:
-          volumeWrite: 0
-          volumeRead: 0
-          filerWrite: 0
-          filerRead: 0
     # we will use this serviceAccountName for all ClusterRoles/ClusterRoleBindings
     serviceAccountName: "seaweedfs"
     serviceAccountAnnotations: {}
@@ -926,7 +919,7 @@ filer:
     # Buckets may be exposed publicly by setting `anonymousRead` to `true`
     # ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
     # objectLock enables S3 Object Lock (irreversible, forces versioning)
-    # versioning: Enabled or Suspended (or bool true/false)
+    # versioning: Enabled or Suspended (or true to enable)
     # createBuckets:
     #   - name: bucket-a
     #     anonymousRead: true
@@ -979,7 +972,7 @@ s3:
   # Buckets may be exposed publicly by setting `anonymousRead` to `true`
   # ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
   # objectLock enables S3 Object Lock (irreversible, forces versioning)
-  # versioning: Enabled or Suspended (or bool true/false)
+  # versioning: Enabled or Suspended (or true to enable)
   # createBuckets:
   #   - name: bucket-a
   #     anonymousRead: true
@@ -1145,7 +1138,7 @@ sftp:
   # SSH server configuration
   sshPrivateKey: "/etc/sw/seaweedfs_sftp_ssh_private_key"  # Path to the SSH private key file for host authentication
   hostKeysFolder: "/etc/sw/ssh"  # path to folder containing SSH private key files for host authentication
-  authMethods: "password,publickey"  # Comma-separated list of allowed auth methods: password, publickey, certificate
+  authMethods: "password,publickey"  # Comma-separated list of allowed auth methods: password, publickey, keyboard-interactive
   maxAuthTries: 6  # Maximum number of authentication attempts per connection
   bannerMessage: "SeaweedFS SFTP Server"  # Message displayed before authentication
   loginGraceTime: "2m"  # Timeout for authentication
@@ -1162,18 +1155,6 @@ sftp:
   # Set to the name of an existing kubernetes Secret with the list of ssh private keys for sftp
   existingSshConfigSecret: null
 
-  # SSH user-certificate authentication (CA-signed user certs). Mirrors
-  # OpenSSH `TrustedUserCAKeys` and MinIO `--sftp=trusted-user-ca-key`.
-  # Add "certificate" to `authMethods` to activate; when active, plain
-  # public keys are rejected on the public-key channel.
-  #
-  # Inline CA public keys in OpenSSH authorized_keys format (one per
-  # line). Ignored when `existingCAKeysSecret` is set.
-  trustedUserCAKeys: ""
-  # Set to the name of an existing kubernetes Secret carrying the CA
-  # public keys under data key `ca_user.pub`.
-  existingCAKeysSecret: null
-
   # Additional resources
   sidecars: []
   initContainers: ""
@@ -1533,7 +1514,7 @@ allInOne:
     # Buckets may be exposed publicly by setting `anonymousRead` to `true`
     # ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
     # objectLock enables S3 Object Lock (irreversible, forces versioning)
-    # versioning: Enabled or Suspended (or bool true/false)
+    # versioning: Enabled or Suspended (or true to enable)
     # createBuckets:
     #   - name: bucket-a
     #     anonymousRead: true
@@ -1564,10 +1545,6 @@ allInOne:
     existingConfigSecret: null
     # Set to the name of an existing kubernetes Secret with the SSH keys
     existingSshConfigSecret: null
-    # SSH user-certificate authentication. See sftp.trustedUserCAKeys above.
-    # (null on either field inherits from the top-level sftp.* setting.)
-    trustedUserCAKeys: null
-    existingCAKeysSecret: null
 
   # Service settings
   service:

other/java/client/src/main/proto/filer.proto

@@ -78,11 +78,6 @@ service SeaweedFiler {
     rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
     }
 
-    // List the metadata subscribers currently connected to this filer
-    // (FUSE mounts, S3, filer.sync, peer filers, ...).
-    rpc ListMetadataSubscribers (ListMetadataSubscribersRequest) returns (ListMetadataSubscribersResponse) {
-    }
-
     rpc KvGet (KvGetRequest) returns (KvGetResponse) {
     }
 
@@ -393,13 +388,7 @@ message PosixLockRequest {
     bool is_moved = 2;
     PosixLockOp op = 3;
     PosixLockRange lock = 4;
-    // locks carries the full set a mount holds on key for a KEEP_ALIVE
-    // re-assertion, so the current owner filer can rebuild its in-memory state
-    // after an ownership change or restart. lock.sid identifies the session.
     repeated PosixLockRange locks = 5;
-    // cooling_probe marks a dual-read a new owner sends to the previous owner
-    // during a ring change, so the previous owner answers from local state
-    // without itself cooling-off (no recursion).
     bool cooling_probe = 6;
 }
 
@@ -626,22 +615,6 @@ message SubscribeMetadataResponse {
     repeated SubscribeMetadataResponse events = 4; // batch of additional events (backlog catch-up)
     repeated LogFileChunkRef log_file_refs = 5;    // log file chunk refs for client direct-read
 }
-message ListMetadataSubscribersRequest {
-    repeated string client_types = 1; // optional filter by client type, e.g. "mount"; empty = all
-}
-message ListMetadataSubscribersResponse {
-    repeated MetadataSubscriber subscribers = 1;
-}
-message MetadataSubscriber {
-    string client_name = 1;     // "<type>@<address>"
-    string client_type = 2;     // e.g. "mount", "sw-vfs", "s3", "filer:<addr>"
-    string address = 3;         // client peer address
-    string path_prefix = 4;     // subscribed path prefix
-    int32 client_id = 5;
-    int32 client_epoch = 6;
-    int64 connected_at_ns = 7;
-    string filer_address = 8;   // the filer this subscriber is connected to
-}
 // A persisted log file that the client can read directly from volume servers.
 // The file format is: [4-byte size | protobuf LogEntry] repeated.
 // Each LogEntry.Data contains a marshaled SubscribeMetadataResponse.

seaweed-volume/proto/master.proto

@@ -147,11 +147,7 @@ message VolumeEcShardInformationMessage {
   repeated int64 shard_sizes = 7; // optimized: sizes for shards in order of set bits in ec_index_bits
   uint64 file_count = 8;   // total needles in the .ecx index (live + tombstoned)
   uint64 delete_count = 9; // node-local tombstones in the .ecj deletion journal
-  // encode-run identity (unix nanos) from the .vif EcShardConfig; lets the admin
-  // group shards by encode generation. Numbered 14 (not 10) to skip the
-  // enterprise fork's reserved 10-13.
-  int64 encode_ts_ns = 14;
-  // fields 15-19 reserved for future upstream open-source additions.
+  // fields 10-19 reserved for future upstream open-source additions.
   // fields 20+ are owned by the enterprise fork (e.g. data_shards/parity_shards)
   // and must not be used here without coordination.
 }

seaweed-volume/proto/volume_server.proto

@@ -450,11 +450,8 @@ message VolumeEcShardsDeleteRequest {
     uint32 volume_id = 1;
     string collection = 2;
     repeated uint32 shard_ids = 3;
-    bool full_teardown = 4; // pre-encode cleanup: wipe every EC artifact + generation for this volume, not just shard_ids
-    int64 encode_ts_ns = 5; // full_teardown generation fence: delete only a disk whose .vif generation is strictly OLDER than this; preserve same-or-newer, generation 0, and an unreadable .vif. 0 => wipe-all (shell pre-encode / pre-upgrade)
 }
 message VolumeEcShardsDeleteResponse {
-    bool full_teardown_done = 1; // set by a new server that performed full_teardown; absent from an old server lets the caller detect the silent no-op
 }
 
 message VolumeEcShardsMountRequest {
@@ -469,7 +466,6 @@ message VolumeEcShardsMountResponse {
 message VolumeEcShardsUnmountRequest {
     uint32 volume_id = 1;
     repeated uint32 shard_ids = 3;
-    int64 encode_ts_ns = 4; // generation fence: skip a disk whose mounted EC volume is this generation or newer (0 = unfenced, unmount all)
 }
 message VolumeEcShardsUnmountResponse {
 }
@@ -480,13 +476,10 @@ message VolumeEcShardReadRequest {
     int64 offset = 3;
     int64 size = 4;
     uint64 file_key = 5;
-    reserved 6;
-    int64 encode_ts_ns = 7; // caller's expected encode time; the server rejects a shard from a different encode run
 }
 message VolumeEcShardReadResponse {
     bytes data = 1;
     bool is_deleted = 2;
-    int64 encode_ts_ns = 3; // identity of the shard actually served; client rejects a mismatch (0 = pre-upgrade server)
 }
 
 message VolumeEcBlobDeleteRequest {
@@ -584,7 +577,6 @@ message VolumeInfo {
 message EcShardConfig {
     uint32 data_shards = 1;   // Number of data shards (e.g., 10)
     uint32 parity_shards = 2; // Number of parity shards (e.g., 4)
-    int64 encode_ts_ns = 3;   // encode time (unix nanos); a read served from a shard of a different encode run is rejected
 }
 message OldVersionVolumeInfo {
     repeated RemoteFile files = 1;

seaweed-volume/src/main.rs

@@ -545,12 +545,6 @@ async fn run(
                     whitelist.extend(sec.guard_white_list.iter().cloned());
                     let mut guard = state_reload.guard.write().unwrap();
                     guard.update_whitelist(&whitelist);
-                    guard.update_signing_keys(
-                        SigningKey(sec.jwt_signing_key),
-                        sec.jwt_signing_expires,
-                        SigningKey(sec.jwt_read_signing_key),
-                        sec.jwt_read_signing_expires,
-                    );
                 }
 
                 // Trigger heartbeat to report new volumes

seaweed-volume/src/metrics.rs

@@ -424,10 +424,6 @@ mod tests {
     #[tokio::test]
     async fn test_push_metrics_once() {
         register_metrics();
-        // A CounterVec with no children emits nothing, so create a labelset
-        // instead of depending on another test having touched the counter
-        // first (test order is nondeterministic).
-        REQUEST_COUNTER.with_label_values(&["GET", "200"]).inc();
 
         let captured = Arc::new(Mutex::new(None::<String>));
         let captured_clone = captured.clone();

seaweed-volume/src/security.rs

@@ -172,26 +172,6 @@ impl Guard {
         self.is_write_active = !is_empty_whitelist || !self.signing_key.is_empty();
     }
 
-    /// Refresh the JWT signing keys and their expirations in place so operators
-    /// can rotate keys via SIGHUP without restarting the process. Mirrors Go's
-    /// `Guard::UpdateSigningKeys`: it swaps the fields and recomputes
-    /// `is_write_active`.
-    pub fn update_signing_keys(
-        &mut self,
-        signing_key: SigningKey,
-        expires_after_sec: i64,
-        read_signing_key: SigningKey,
-        read_expires_after_sec: i64,
-    ) {
-        self.signing_key = signing_key;
-        self.expires_after_sec = expires_after_sec;
-        self.read_signing_key = read_signing_key;
-        self.read_expires_after_sec = read_expires_after_sec;
-
-        let is_empty_whitelist = self.whitelist_ips.is_empty() && self.whitelist_cidrs.is_empty();
-        self.is_write_active = !is_empty_whitelist || !self.signing_key.is_empty();
-    }
-
     /// Check if a remote IP is in the whitelist.
     /// Returns true if write security is inactive (no whitelist and no signing key),
     /// if the whitelist is empty, or if the IP matches.
@@ -483,43 +463,6 @@ mod tests {
         assert!(matches!(err, Err(JwtError::FileIdMismatch { .. })));
     }
 
-    #[test]
-    fn test_update_signing_keys_rotates() {
-        let mut guard = Guard::new(
-            &[],
-            SigningKey::from_string("old-write"),
-            10,
-            SigningKey::from_string("old-read"),
-            60,
-        );
-
-        guard.update_signing_keys(
-            SigningKey::from_string("new-write"),
-            11,
-            SigningKey::from_string("new-read"),
-            61,
-        );
-
-        let token = gen_jwt(&guard.signing_key, 60, "3,01637037d6").unwrap();
-        // Validates with the rotated key, not the old one.
-        assert!(decode_jwt(&guard.signing_key, &token).is_ok());
-        assert!(decode_jwt(&SigningKey::from_string("old-write"), &token).is_err());
-        assert_eq!(guard.expires_after_sec, 11);
-        assert_eq!(guard.read_expires_after_sec, 61);
-    }
-
-    #[test]
-    fn test_update_signing_keys_toggles_write_active() {
-        let mut guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
-        assert!(guard.check_jwt(None, true).is_ok());
-
-        guard.update_signing_keys(SigningKey::from_string("write"), 10, SigningKey(vec![]), 0);
-        assert!(guard.is_write_active);
-
-        guard.update_signing_keys(SigningKey(vec![]), 0, SigningKey(vec![]), 0);
-        assert!(!guard.is_write_active);
-    }
-
     #[test]
     fn test_extract_host() {
         assert_eq!(extract_host("192.168.1.1:8080"), "192.168.1.1");

seaweed-volume/src/server/grpc_server.rs

@@ -399,27 +399,6 @@ impl VolumeServer for VolumeGrpcService {
 
             if !is_ec_volume {
                 let mut store = self.state.store.write().unwrap();
-                // Recheck EC state under the write lock before mutating the .dat. The
-                // is_ec_volume snapshot was taken earlier under a separate read lock;
-                // ec.encode mounts EC shards (copied from the .dat) BEFORE deleting the
-                // originals, so the vid can be EC now while the .dat still exists. A
-                // delete_volume_needle here would tombstone that .dat, which the encode
-                // then removes — the delete is lost and the needle resurrected from the
-                // pre-tombstone shards. If the vid is now EC, return a retriable 503 so
-                // the filer requeues the delete onto the EC path.
-                if store.has_ec_volume(file_id.volume_id) {
-                    results.push(volume_server_pb::DeleteResult {
-                        file_id: fid_str.clone(),
-                        status: 503,
-                        error: format!(
-                            "volume {} became ec during delete, try again",
-                            file_id.volume_id
-                        ),
-                        size: 0,
-                        version: 0,
-                    });
-                    continue;
-                }
                 match store.delete_volume_needle(file_id.volume_id, &mut n) {
                     Ok(size) => {
                         if size.0 == 0 {
@@ -441,35 +420,13 @@ impl VolumeServer for VolumeGrpcService {
                         }
                     }
                     Err(e) => {
-                        // The volume vanished between the is_ec_volume snapshot and
-                        // this mutation. If an EC volume now occupies the vid (ec.encode
-                        // mounted EC then deleted the .dat under us), the delete belongs
-                        // on the EC journal, not here — return a retriable 503 with the
-                        // "try again" token so the filer requeues it and the retry hits
-                        // the EC path. A bare exact "not found" would be dropped
-                        // permanently by the filer chunk-GC.
-                        if matches!(e, crate::storage::volume::VolumeError::NotFound)
-                            && store.has_ec_volume(file_id.volume_id)
-                        {
-                            results.push(volume_server_pb::DeleteResult {
-                                file_id: fid_str.clone(),
-                                status: 503,
-                                error: format!(
-                                    "volume {} became ec during delete, try again",
-                                    file_id.volume_id
-                                ),
-                                size: 0,
-                                version: 0,
-                            });
-                        } else {
-                            results.push(volume_server_pb::DeleteResult {
-                                file_id: fid_str.clone(),
-                                status: 500,
-                                error: e.to_string(),
-                                size: 0,
-                                version: 0,
-                            });
-                        }
+                        results.push(volume_server_pb::DeleteResult {
+                            file_id: fid_str.clone(),
+                            status: 500,
+                            error: e.to_string(),
+                            size: 0,
+                            version: 0,
+                        });
                     }
                 }
             } else {
@@ -1080,15 +1037,21 @@ impl VolumeServer for VolumeGrpcService {
         let req = request.into_inner();
         let vid = VolumeId(req.volume_id);
 
-        // A pre-existing local replica is NOT deleted up front. Deleting before
-        // the source is confirmed reachable destroys a healthy copy on a
-        // transient source outage (and, on retry, can lose the volume
-        // entirely). The delete is deferred until read_volume_file_status below
-        // proves the source holds the volume; readability alone is the gate.
-        let had_existing_volume = {
+        // If volume already exists locally, delete it first
+        {
             let store = self.state.store.read().unwrap();
-            store.find_volume(vid).is_some()
-        };
+            if store.find_volume(vid).is_some() {
+                drop(store);
+                let mut store = self.state.store.write().unwrap();
+                // keep remote data: the inbound copy carries a .vif that may
+                // point at the same cloud-tier object the existing volume
+                // references.
+                store.delete_volume(vid, false, true).map_err(|e| {
+                    Status::internal(format!("failed to delete existing volume {}: {}", vid, e))
+                })?;
+                self.state.volume_state_notify.notify_one();
+            }
+        }
 
         // Parse source_data_node address: "ip:port.grpcPort" or "ip:port" (grpc = port + 10000)
         let source = &req.source_data_node;
@@ -1129,19 +1092,6 @@ impl VolumeServer for VolumeGrpcService {
             .map_err(|e| Status::internal(format!("read volume file status failed, {}", e)))?
             .into_inner();
 
-        // Source is reachable and holds the volume: only now is it safe to drop
-        // an existing local replica before overwriting its files.
-        if had_existing_volume {
-            let mut store = self.state.store.write().unwrap();
-            // keep remote data: the inbound copy carries a .vif that may point
-            // at the same cloud-tier object the existing volume references.
-            store.delete_volume(vid, false, true).map_err(|e| {
-                Status::internal(format!("failed to delete existing volume {}: {}", vid, e))
-            })?;
-            drop(store);
-            self.state.volume_state_notify.notify_one();
-        }
-
         let requested_disk_type = if !req.disk_type.is_empty() {
             DiskType::from_string(&req.disk_type)
         } else {
@@ -1173,12 +1123,9 @@ impl VolumeServer for VolumeGrpcService {
         let idx_base_name =
             crate::storage::volume::volume_file_name(&idx_base, &vol_info.collection, vid);
 
-        // Write a .note file to indicate copy in progress. A leftover note
-        // fails the volume load on restart, so a write failure must abort.
+        // Write a .note file to indicate copy in progress
         let note_path = format!("{}.note", data_base_name);
-        std::fs::write(&note_path, format!("copying from {}", source)).map_err(|e| {
-            Status::internal(format!("write .note for volume {}: {}", vid, e))
-        })?;
+        let _ = std::fs::write(&note_path, format!("copying from {}", source));
 
         let has_remote_dat = vol_info
             .volume_info
@@ -1257,7 +1204,7 @@ impl VolumeServer for VolumeGrpcService {
                     .await
                     .map_err(|e| Status::internal(e))?;
                     if dat_modified_ts_ns > 0 {
-                        let _ = set_file_mtime(&dat_path, dat_modified_ts_ns);
+                        set_file_mtime(&dat_path, dat_modified_ts_ns);
                     }
                 }
 
@@ -1282,7 +1229,7 @@ impl VolumeServer for VolumeGrpcService {
                 .await
                 .map_err(|e| Status::internal(e))?;
                 if idx_modified_ts_ns > 0 {
-                    let _ = set_file_mtime(&idx_path, idx_modified_ts_ns);
+                    set_file_mtime(&idx_path, idx_modified_ts_ns);
                 }
 
                 // Copy .vif file (ignore if not found on source)
@@ -1306,19 +1253,11 @@ impl VolumeServer for VolumeGrpcService {
                 .await
                 .map_err(|e| Status::internal(e))?;
                 if vif_modified_ts_ns > 0 {
-                    let _ = set_file_mtime(&vif_path, vif_modified_ts_ns);
+                    set_file_mtime(&vif_path, vif_modified_ts_ns);
                 }
 
-                // Remove the .note file. A leftover note fails the load on the
-                // next restart, so a removal failure must fail the copy.
-                if let Err(e) = std::fs::remove_file(&note_path) {
-                    if e.kind() != std::io::ErrorKind::NotFound {
-                        return Err(Status::internal(format!(
-                            "remove .note for volume {}: {}",
-                            vid, e
-                        )));
-                    }
-                }
+                // Remove the .note file
+                let _ = std::fs::remove_file(&note_path);
 
                 // Verify file sizes
                 if !has_remote_dat {
@@ -2194,12 +2133,6 @@ impl VolumeServer for VolumeGrpcService {
                 ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
                     data_shards: data_shards,
                     parity_shards: parity_shards,
-                    // This run's identity; the read path rejects a shard from a
-                    // different encode run.
-                    encode_ts_ns: std::time::SystemTime::now()
-                        .duration_since(std::time::UNIX_EPOCH)
-                        .unwrap_or_default()
-                        .as_nanos() as i64,
                 }),
                 ..Default::default()
             };
@@ -2660,74 +2593,12 @@ impl VolumeServer for VolumeGrpcService {
         self.state.check_maintenance()?;
         let req = request.into_inner();
         let vid = VolumeId(req.volume_id);
-
-        if req.full_teardown {
-            if req.encode_ts_ns == 0 {
-                // Blanket teardown (shell pre-encode cleanup / pre-upgrade caller): evict
-                // the volume and wipe every EC artifact for it on every disk, not just the
-                // listed shards, so a remote node retains no stale generation a fresh copy
-                // collides with. Echo the acknowledgement so the caller can tell a
-                // pre-upgrade server apart.
-                {
-                    let mut store = self.state.store.write().unwrap();
-                    let _ = store.remove_ec_volume(vid);
-                    for loc in &store.locations {
-                        loc.remove_ec_volume_files_full_teardown(&req.collection, vid)
-                            .map_err(|e| {
-                                Status::internal(format!(
-                                    "full teardown of ec volume {}: {}",
-                                    req.volume_id, e
-                                ))
-                            })?;
-                    }
-                }
-            } else {
-                // Generation-fenced teardown (stale-worker pre-distribute cleanup): wipe
-                // only a disk whose .vif generation is strictly OLDER than the request;
-                // preserve same-or-newer, generation 0 (recovered/pre-upgrade live volume),
-                // and an unreadable .vif, so a stale run never wipes a newer run's live
-                // shards. Unload and remove only the strictly-older disks, never node-wide.
-                let mut store = self.state.store.write().unwrap();
-                for disk_id in 0..store.locations.len() {
-                    let disk_gen = store.locations[disk_id].ec_generation_ts_ns(&req.collection, vid);
-                    let older = matches!(disk_gen, Some(g) if g > 0 && g < req.encode_ts_ns);
-                    if !older {
-                        tracing::info!(
-                            volume_id = vid.0,
-                            disk_id,
-                            ?disk_gen,
-                            req = req.encode_ts_ns,
-                            "ec full teardown preserved: generation not older than request"
-                        );
-                        continue;
-                    }
-                    store.locations[disk_id].remove_ec_volume(vid);
-                    store.locations[disk_id]
-                        .remove_ec_volume_files_full_teardown(&req.collection, vid)
-                        .map_err(|e| {
-                            Status::internal(format!(
-                                "fenced teardown of ec volume {}: {}",
-                                req.volume_id, e
-                            ))
-                        })?;
-                }
-            }
-            self.state.volume_state_notify.notify_one();
-            return Ok(Response::new(
-                volume_server_pb::VolumeEcShardsDeleteResponse {
-                    full_teardown_done: true,
-                },
-            ));
-        }
-
         let mut store = self.state.store.write().unwrap();
         store.delete_ec_shards(vid, &req.collection, &req.shard_ids);
         drop(store);
         self.state.volume_state_notify.notify_one();
         Ok(Response::new(
-            volume_server_pb::VolumeEcShardsDeleteResponse {
-                full_teardown_done: false,
-            },
+            volume_server_pb::VolumeEcShardsDeleteResponse {},
         ))
     }
 
@@ -2767,11 +2638,9 @@ impl VolumeServer for VolumeGrpcService {
         // Matches Go: for _, shardId := range req.ShardIds { err = vs.store.UnmountEcShards(...) }
         let mut store = self.state.store.write().unwrap();
         for &shard_id in &req.shard_ids {
-            store
-                .unmount_ec_shard(vid, shard_id, req.encode_ts_ns)
-                .map_err(|e| {
-                    Status::internal(format!("unmount {}.{}: {}", req.volume_id, shard_id, e))
-                })?;
+            store.unmount_ec_shard(vid, shard_id).map_err(|e| {
+                Status::internal(format!("unmount {}.{}: {}", req.volume_id, shard_id, e))
+            })?;
         }
         drop(store);
         self.state.volume_state_notify.notify_one();
@@ -2803,21 +2672,6 @@ impl VolumeServer for VolumeGrpcService {
                 ))
             })?;
 
-        // Reject a shard whose identity doesn't match the caller's index; the caller
-        // then recovers from parity. Lenient only when the caller has no identity
-        // (pre-upgrade reader): a known caller must not accept an unstamped holder,
-        // which would serve a stale shard from a different encode run.
-        if req.encode_ts_ns != 0 && req.encode_ts_ns != ec_vol.encode_ts_ns {
-            return Err(Status::failed_precondition(format!(
-                "ec shard {}.{} belongs to a different encode run",
-                req.volume_id, req.shard_id
-            )));
-        }
-
-        // Identity of the shard actually served, echoed on every response chunk so
-        // the client can reject a different encode run even from a pre-upgrade server.
-        let served_encode_ts_ns = ec_vol.encode_ts_ns;
-
         // Check if the requested needle is deleted (via .ecx index, matching Go)
         if req.file_key > 0 {
             let needle_id = NeedleId(req.file_key);
@@ -2828,7 +2682,6 @@ impl VolumeServer for VolumeGrpcService {
                 if size.is_deleted() {
                     let results = vec![Ok(volume_server_pb::VolumeEcShardReadResponse {
                         is_deleted: true,
-                        encode_ts_ns: served_encode_ts_ns,
                         ..Default::default()
                     })];
                     return Ok(Response::new(Box::pin(tokio_stream::iter(results))));
@@ -2877,7 +2730,6 @@ impl VolumeServer for VolumeGrpcService {
             results.push(Ok(volume_server_pb::VolumeEcShardReadResponse {
                 data: buf,
                 is_deleted: false,
-                encode_ts_ns: served_encode_ts_ns,
             }));
             if n < chunk_size {
                 break; // short read means EOF
@@ -3146,15 +2998,6 @@ impl VolumeServer for VolumeGrpcService {
             dat_path
         };
 
-        // Store the source .dat mtime, not the upload time, so a reload computes
-        // TTL from real data age (matches Go VolumeTierMoveDatToRemote).
-        let dat_modified_secs = std::fs::metadata(&dat_path)
-            .and_then(|m| m.modified())
-            .map_err(|e| Status::internal(format!("stat data file {}: {}", dat_path, e)))?
-            .duration_since(std::time::UNIX_EPOCH)
-            .map(|d| d.as_secs())
-            .unwrap_or(0);
-
         // Look up the S3 tier backend
         let backend = {
             let registry = self.state.s3_tier_registry.read().unwrap();
@@ -3207,13 +3050,18 @@ impl VolumeServer for VolumeGrpcService {
                 {
                     let mut store = state.store.write().unwrap();
                     if let Some((_, vol)) = store.find_volume_mut(vid) {
+                        let now_unix = std::time::SystemTime::now()
+                            .duration_since(std::time::UNIX_EPOCH)
+                            .unwrap_or_default()
+                            .as_secs();
+
                         vol.volume_info.files.push(volume_server_pb::RemoteFile {
                             backend_type: backend_type.clone(),
                             backend_id: backend_id.clone(),
                             key,
                             offset: 0,
                             file_size: size,
-                            modified_time: dat_modified_secs,
+                            modified_time: now_unix,
                             extension: ".dat".to_string(),
                         });
                         vol.refresh_remote_write_mode();
@@ -3264,7 +3112,7 @@ impl VolumeServer for VolumeGrpcService {
         let vid = VolumeId(req.volume_id);
 
         // Validate volume and get remote storage info
-        let (dat_path, storage_name, storage_key, remote_modified_secs) = {
+        let (dat_path, storage_name, storage_key) = {
             let store = self.state.store.read().unwrap();
             let (_, vol) = store
                 .find_volume(vid)
@@ -3294,14 +3142,7 @@ impl VolumeServer for VolumeGrpcService {
                 )));
             }
 
-            let remote_modified_secs = vol
-                .volume_info
-                .files
-                .first()
-                .map(|f| f.modified_time)
-                .unwrap_or(0);
-
-            (dat_path, storage_name, storage_key, remote_modified_secs)
+            (dat_path, storage_name, storage_key)
         };
 
         // Look up the S3 tier backend
@@ -3349,15 +3190,6 @@ impl VolumeServer for VolumeGrpcService {
                         ))
                     })?;
 
-                // Restore the .dat mtime so a reload computes TTL from real data age,
-                // not download time (matches Go VolumeTierMoveDatFromRemote).
-                if remote_modified_secs > 0 {
-                    let modified_ts_ns = (remote_modified_secs as i64).saturating_mul(1_000_000_000);
-                    if let Err(e) = set_file_mtime(&dat_path, modified_ts_ns) {
-                        tracing::warn!("volume {} restore data file {} modified time: {}", vid, dat_path, e);
-                    }
-                }
-
                 if !keep_remote {
                     // Delete remote file
                     backend.delete_file(&storage_key).await.map_err(|e| {
@@ -4229,16 +4061,17 @@ async fn ping_filer_target(
 use super::grpc_client::parse_grpc_address;
 
 /// Set the modification time of a file from nanoseconds since Unix epoch.
-fn set_file_mtime(path: &str, modified_ts_ns: i64) -> std::io::Result<()> {
+fn set_file_mtime(path: &str, modified_ts_ns: i64) {
     use std::time::{Duration, SystemTime};
     let ts = if modified_ts_ns >= 0 {
         SystemTime::UNIX_EPOCH + Duration::from_nanos(modified_ts_ns as u64)
     } else {
         SystemTime::UNIX_EPOCH
     };
-    let file = std::fs::File::open(path)?;
-    let ft = std::fs::FileTimes::new().set_accessed(ts).set_modified(ts);
-    file.set_times(ft)
+    if let Ok(file) = std::fs::File::open(path) {
+        let ft = std::fs::FileTimes::new().set_accessed(ts).set_modified(ts);
+        let _ = file.set_times(ft);
+    }
 }
 
 /// Copy a file from a remote volume server via CopyFile streaming RPC.

seaweed-volume/src/server/handlers.rs

@@ -23,27 +23,6 @@ use crate::pb::volume_server_pb;
 use crate::storage::needle::needle::Needle;
 use crate::storage::types::*;
 
-/// Slack added over the configured file-size limit when bounding the raw
-/// request body, to allow for multipart/form-data framing overhead. The exact
-/// per-file limit is still enforced on the parsed data after multipart parsing.
-const UPLOAD_BODY_OVERHEAD: usize = 16 * 1024 * 1024; // 16 MiB
-
-/// Upper bound on bytes we will materialize in memory for a single request when
-/// expanding stored content (gzip decompression or chunk-manifest assembly).
-/// Guards against gzip bombs and crafted/oversized manifest sizes OOM-killing
-/// the server; legitimate large objects are read via client-side chunk fetches.
-const MAX_EXPANSION_BYTES: u64 = 2 * 1024 * 1024 * 1024; // 2 GiB
-
-/// Why `maybe_decompress_gzip` failed, so callers can distinguish a recoverable
-/// "not valid gzip, use the raw bytes" from "the bomb cap was hit, reject it".
-#[derive(Debug)]
-enum GunzipError {
-    /// Input was not valid gzip / decode failed — callers may fall back to raw.
-    Decode,
-    /// Decompressed output would exceed `MAX_EXPANSION_BYTES` — must be rejected.
-    TooLarge,
-}
-
 // ============================================================================
 // Inflight Throttle Guard
 // ============================================================================
@@ -1473,16 +1452,12 @@ async fn get_or_head_handler_inner(
     if is_compressed {
         if needs_image_ops {
             // Always decompress for image operations (Go decompresses before resize/crop)
-            match maybe_decompress_gzip(&data) {
-                Ok(decompressed) => data = decompressed,
-                Err(GunzipError::TooLarge) => {
-                    return (
-                        StatusCode::PAYLOAD_TOO_LARGE,
-                        "compressed object exceeds decompression limit",
-                    )
-                        .into_response()
-                }
-                Err(GunzipError::Decode) => {} // not valid gzip; keep raw bytes
+            use flate2::read::GzDecoder;
+            use std::io::Read as _;
+            let mut decoder = GzDecoder::new(&data[..]);
+            let mut decompressed = Vec::new();
+            if decoder.read_to_end(&mut decompressed).is_ok() {
+                data = decompressed;
             }
         } else {
             let accept_encoding = headers
@@ -1499,16 +1474,12 @@ async fn get_or_head_handler_inner(
                 response_headers.insert(header::CONTENT_ENCODING, "gzip".parse().unwrap());
             } else {
                 // Decompress for client
-                match maybe_decompress_gzip(&data) {
-                    Ok(decompressed) => data = decompressed,
-                    Err(GunzipError::TooLarge) => {
-                        return (
-                            StatusCode::PAYLOAD_TOO_LARGE,
-                            "compressed object exceeds decompression limit",
-                        )
-                            .into_response()
-                    }
-                    Err(GunzipError::Decode) => {} // not valid gzip; keep raw bytes
+                use flate2::read::GzDecoder;
+                use std::io::Read as _;
+                let mut decoder = GzDecoder::new(&data[..]);
+                let mut decompressed = Vec::new();
+                if decoder.read_to_end(&mut decompressed).is_ok() {
+                    data = decompressed;
                 }
             }
         }
@@ -2163,30 +2134,15 @@ pub async fn post_handler(
         .and_then(|v| v.to_str().ok())
         .map(|s| s.to_string());
 
-    // Read body, bounded by the configured file-size limit so a single upload
-    // cannot buffer unbounded memory and OOM-kill the server (mirrors Go's
-    // io.LimitReader(r.Body, sizeLimit+1)). A margin covers multipart framing;
-    // the exact per-file limit is still enforced on the parsed data below.
-    let body_limit = if state.file_size_limit_bytes > 0 {
-        // try_from (not `as usize`) so a >usize::MAX limit on 32-bit caps at
-        // usize::MAX instead of silently truncating/wrapping to a tiny value.
-        usize::try_from(state.file_size_limit_bytes)
-            .unwrap_or(usize::MAX)
-            .saturating_add(UPLOAD_BODY_OVERHEAD)
-    } else {
-        usize::MAX
-    };
-    let body = match axum::body::to_bytes(request.into_body(), body_limit).await {
+    // Read body
+    let body = match axum::body::to_bytes(request.into_body(), usize::MAX).await {
         Ok(b) => b,
         Err(e) => {
-            // With a limit configured, an error here means the body exceeded it
-            // before we buffered the whole thing; report it like the size check.
-            let msg = if state.file_size_limit_bytes > 0 {
-                format!("file over the limited {} bytes", state.file_size_limit_bytes)
-            } else {
-                format!("read body: {}", e)
-            };
-            return json_error_with_query(StatusCode::BAD_REQUEST, msg, Some(&query));
+            return json_error_with_query(
+                StatusCode::BAD_REQUEST,
+                format!("read body: {}", e),
+                Some(&query),
+            )
         }
     };
 
@@ -2326,17 +2282,7 @@ pub async fn post_handler(
     };
 
     let uncompressed_data = if is_gzipped {
-        match maybe_decompress_gzip(&body_data_raw) {
-            Ok(d) => d,
-            Err(GunzipError::TooLarge) => {
-                return json_error_with_query(
-                    StatusCode::PAYLOAD_TOO_LARGE,
-                    "compressed object exceeds decompression limit",
-                    Some(&query),
-                );
-            }
-            Err(GunzipError::Decode) => body_data_raw.clone(),
-        }
+        maybe_decompress_gzip(&body_data_raw).unwrap_or_else(|| body_data_raw.clone())
     } else {
         body_data_raw.clone()
     };
@@ -2815,16 +2761,14 @@ pub async fn delete_handler(
     // If this is a chunk manifest, delete child chunks first
     if n.is_chunk_manifest() {
         let manifest_data = if n.is_compressed() {
-            match maybe_decompress_gzip(&n.data) {
-                Ok(d) => d,
-                Err(GunzipError::TooLarge) => {
-                    return json_error_with_query(
-                        StatusCode::PAYLOAD_TOO_LARGE,
-                        "compressed manifest exceeds decompression limit",
-                        Some(&del_query),
-                    );
-                }
-                Err(GunzipError::Decode) => n.data.clone(),
+            use flate2::read::GzDecoder;
+            use std::io::Read as _;
+            let mut decoder = GzDecoder::new(&n.data[..]);
+            let mut decompressed = Vec::new();
+            if decoder.read_to_end(&mut decompressed).is_ok() {
+                decompressed
+            } else {
+                n.data.clone()
             }
         } else {
             n.data.clone()
@@ -3180,19 +3124,14 @@ fn try_expand_chunk_manifest(
     last_modified_str: &Option<String>,
 ) -> Option<Response> {
     let data = if n.is_compressed() {
-        match maybe_decompress_gzip(&n.data) {
-            Ok(d) => d,
-            Err(GunzipError::TooLarge) => {
-                return Some(
-                    (
-                        StatusCode::PAYLOAD_TOO_LARGE,
-                        "compressed manifest exceeds decompression limit",
-                    )
-                        .into_response(),
-                )
-            }
-            Err(GunzipError::Decode) => return None,
+        use flate2::read::GzDecoder;
+        use std::io::Read as _;
+        let mut decoder = GzDecoder::new(&n.data[..]);
+        let mut decompressed = Vec::new();
+        if decoder.read_to_end(&mut decompressed).is_err() {
+            return None;
         }
+        decompressed
     } else {
         n.data.clone()
     };
@@ -3202,13 +3141,6 @@ fn try_expand_chunk_manifest(
         Err(_) => return None,
     };
 
-    // Guard the attacker-controlled manifest size before allocating: a negative
-    // value would wrap to a huge usize (capacity-overflow panic) and an oversized
-    // one would OOM-kill the server.
-    if manifest.size < 0 || manifest.size as u64 > MAX_EXPANSION_BYTES {
-        return None;
-    }
-
     // Read and concatenate all chunks
     let mut result = vec![0u8; manifest.size as usize];
     let store = state.store.read().unwrap();
@@ -3242,54 +3174,24 @@ fn try_expand_chunk_manifest(
                 )
             }
         }
-        // Validate the attacker-controlled chunk offset before indexing: a
-        // negative value would wrap to a huge usize, and an out-of-range one has
-        // nowhere to land.
-        if chunk.offset < 0 {
-            return Some(
-                (
-                    StatusCode::INTERNAL_SERVER_ERROR,
-                    format!("invalid negative chunk offset in {}", chunk.fid),
-                )
-                    .into_response(),
-            );
-        }
-        let offset = chunk.offset as usize;
-        if offset >= result.len() {
-            continue;
-        }
-        // Write the chunk into its window in `result`. Compressed chunks inflate
-        // directly into the destination slice (bounded by the remaining window),
-        // so a chunk never allocates a second large buffer — peak memory stays at
-        // ~manifest.size instead of doubling. Bytes past the window are dropped,
-        // matching the prior truncation behavior.
-        if chunk_needle.is_compressed() {
+        let chunk_data = if chunk_needle.is_compressed() {
             use flate2::read::GzDecoder;
             use std::io::Read as _;
             let mut decoder = GzDecoder::new(&chunk_needle.data[..]);
-            let window = &mut result[offset..];
-            let mut written = 0usize;
-            let mut decode_failed = false;
-            while written < window.len() {
-                match decoder.read(&mut window[written..]) {
-                    Ok(0) => break,
-                    Ok(n) => written += n,
-                    Err(_) => {
-                        decode_failed = true;
-                        break;
-                    }
-                }
-            }
-            // On any decode failure, drop the partial output and fall back to the
-            // chunk's raw bytes (truncated to the window), preserving prior behavior.
-            if decode_failed {
-                window[..written].fill(0);
-                let copy_len = chunk_needle.data.len().min(window.len());
-                window[..copy_len].copy_from_slice(&chunk_needle.data[..copy_len]);
+            let mut decompressed = Vec::new();
+            if decoder.read_to_end(&mut decompressed).is_ok() {
+                decompressed
+            } else {
+                chunk_needle.data.clone()
             }
         } else {
-            let copy_len = chunk_needle.data.len().min(result.len() - offset);
-            result[offset..offset + copy_len].copy_from_slice(&chunk_needle.data[..copy_len]);
+            chunk_needle.data.clone()
+        };
+        let offset = chunk.offset as usize;
+        let end = std::cmp::min(offset + chunk_data.len(), result.len());
+        let copy_len = end - offset;
+        if copy_len > 0 {
+            result[offset..offset + copy_len].copy_from_slice(&chunk_data[..copy_len]);
         }
     }
 
@@ -3662,22 +3564,13 @@ fn try_gzip_data(data: &[u8]) -> Option<Vec<u8>> {
     encoder.finish().ok()
 }
 
-fn maybe_decompress_gzip(data: &[u8]) -> Result<Vec<u8>, GunzipError> {
+fn maybe_decompress_gzip(data: &[u8]) -> Option<Vec<u8>> {
     use flate2::read::GzDecoder;
     use std::io::Read;
-    // Cap the output so a crafted, highly-compressible (gzip-bomb) needle cannot
-    // OOM the server when decompressed on read or upload. take(limit+1) lets us
-    // tell "exactly at the limit" apart from "over it", which we reject as
-    // TooLarge so callers fail the request instead of silently using raw bytes.
-    let mut decoder = GzDecoder::new(data).take(MAX_EXPANSION_BYTES + 1);
+    let mut decoder = GzDecoder::new(data);
     let mut decompressed = Vec::new();
-    decoder
-        .read_to_end(&mut decompressed)
-        .map_err(|_| GunzipError::Decode)?;
-    if decompressed.len() as u64 > MAX_EXPANSION_BYTES {
-        return Err(GunzipError::TooLarge);
-    }
-    Ok(decompressed)
+    decoder.read_to_end(&mut decompressed).ok()?;
+    Some(decompressed)
 }
 
 fn compute_md5_base64(data: &[u8]) -> String {
@@ -3919,11 +3812,7 @@ mod tests {
         let compressed = try_gzip_data(data).unwrap();
         let decompressed = maybe_decompress_gzip(&compressed).unwrap();
         assert_eq!(decompressed, data);
-        // Non-gzip input is reported as a decode error (callers fall back to raw).
-        assert!(matches!(
-            maybe_decompress_gzip(data),
-            Err(GunzipError::Decode)
-        ));
+        assert!(maybe_decompress_gzip(data).is_none());
     }
 
     #[test]

seaweed-volume/src/server/heartbeat.rs

@@ -314,7 +314,6 @@ fn collect_ec_shard_delta_messages(
                         disk_type: ec_vol.disk_type.to_string(),
                         expire_at_sec: ec_vol.expire_at_sec,
                         disk_id: disk_id as u32,
-                        encode_ts_ns: ec_vol.encode_ts_ns,
                         ..Default::default()
                     },
                 );

seaweed-volume/src/server/store_ec.rs

@@ -70,10 +70,6 @@ struct Snapshot {
     intervals: Vec<IntervalResult>,
     cached_locations: HashMap<ShardId, Vec<String>>,
     cache_refreshed_at: Option<Instant>,
-    /// This volume's encode identity, carried to peers on remote shard reads so a
-    /// shard from a different encode run is rejected rather than served at a
-    /// mismatched offset. 0 for a pre-feature volume (lenient).
-    encode_ts_ns: i64,
 }
 
 /// Top-level entry point. Returns `Ok(None)` for "not found" (matches
@@ -151,7 +147,6 @@ pub async fn read_ec_shard_needle_distributed(
                     &shard_locations,
                     snapshot.data_shards as usize,
                     snapshot.parity_shards as usize,
-                    snapshot.encode_ts_ns,
                 )
                 .await?;
                 assembled.push(buf);
@@ -264,7 +259,6 @@ fn snapshot_under_lock(
         intervals: interval_results,
         cached_locations,
         cache_refreshed_at,
-        encode_ts_ns: ecv.encode_ts_ns,
     }))
 }
 
@@ -389,7 +383,6 @@ async fn fetch_one_interval(
     shard_locations: &HashMap<ShardId, Vec<String>>,
     data_shards: usize,
     parity_shards: usize,
-    expected_encode_ts_ns: i64,
 ) -> io::Result<Vec<u8>> {
     // Direct peer read against the cached locations for this shard.
     if let Some(sources) = shard_locations.get(&shard_id) {
@@ -402,7 +395,6 @@ async fn fetch_one_interval(
                 shard_id,
                 shard_offset,
                 size,
-                expected_encode_ts_ns,
             )
             .await
             {
@@ -432,7 +424,6 @@ async fn fetch_one_interval(
         shard_locations,
         data_shards,
         parity_shards,
-        expected_encode_ts_ns,
     )
     .await
 }
@@ -445,7 +436,6 @@ async fn read_remote_ec_shard_interval(
     shard_id: ShardId,
     shard_offset: i64,
     size: usize,
-    expected_encode_ts_ns: i64,
 ) -> io::Result<Vec<u8>> {
     let mut last_err: Option<io::Error> = None;
     for src in sources {
@@ -457,7 +447,6 @@ async fn read_remote_ec_shard_interval(
             shard_id,
             shard_offset,
             size,
-            expected_encode_ts_ns,
         )
         .await
         {
@@ -481,7 +470,6 @@ async fn do_read_remote_ec_shard_interval(
     shard_id: ShardId,
     shard_offset: i64,
     size: usize,
-    expected_encode_ts_ns: i64,
 ) -> io::Result<Vec<u8>> {
     let grpc_addr =
         parse_grpc_address(source).map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
@@ -517,7 +505,6 @@ async fn do_read_remote_ec_shard_interval(
         offset: shard_offset,
         size: size as i64,
         file_key: needle_id.0,
-        encode_ts_ns: expected_encode_ts_ns,
     };
     let resp = client
         .volume_ec_shard_read(Request::new(req))
@@ -536,18 +523,6 @@ async fn do_read_remote_ec_shard_interval(
         .await
         .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("recv: {}", e)))?
     {
-        // Validate the served shard's identity client-side, so the guard holds even
-        // against a pre-upgrade server that ignored the request field (returns 0).
-        // A mismatch fails the read; the caller recovers from parity.
-        if expected_encode_ts_ns != 0 && msg.encode_ts_ns != expected_encode_ts_ns {
-            return Err(io::Error::new(
-                io::ErrorKind::Other,
-                format!(
-                    "ec shard {}.{} from {} belongs to a different encode run (want {} got {})",
-                    vid.0, shard_id, source, expected_encode_ts_ns, msg.encode_ts_ns
-                ),
-            ));
-        }
         if !msg.data.is_empty() {
             out.extend_from_slice(&msg.data);
         }
@@ -579,7 +554,6 @@ async fn recover_one_remote_ec_shard_interval(
     shard_locations: &HashMap<ShardId, Vec<String>>,
     data_shards: usize,
     parity_shards: usize,
-    expected_encode_ts_ns: i64,
 ) -> io::Result<Vec<u8>> {
     let total_shards = data_shards + parity_shards;
     let rs = ReedSolomon::new(data_shards, parity_shards).map_err(|e| {
@@ -640,7 +614,6 @@ async fn recover_one_remote_ec_shard_interval(
                 sid,
                 shard_offset,
                 size,
-                expected_encode_ts_ns,
             )
             .await;
             (sid, res)

seaweed-volume/src/storage/disk_location.rs

@@ -10,7 +10,7 @@ use std::io;
 use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
 use std::sync::Arc;
 
-use tracing::warn;
+use tracing::{info, warn};
 
 use crate::config::MinFreeSpace;
 use crate::storage::erasure_coding::ec_shard::{
@@ -19,11 +19,9 @@ use crate::storage::erasure_coding::ec_shard::{
 };
 use crate::storage::erasure_coding::ec_volume::EcVolume;
 use crate::storage::needle_map::NeedleMapKind;
-use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE};
+use crate::storage::super_block::ReplicaPlacement;
 use crate::storage::types::*;
-use crate::storage::volume::{
-    remove_volume_files, volume_file_name, VifVolumeInfo, Volume, VolumeError,
-};
+use crate::storage::volume::{remove_volume_files, volume_file_name, Volume, VolumeError};
 
 /// A single disk location managing volumes in one directory.
 pub struct DiskLocation {
@@ -106,12 +104,6 @@ impl DiskLocation {
             fs::create_dir_all(&self.idx_directory)?;
         }
 
-        // Recover any interrupted compaction commit before the volume scan. This
-        // must run here, not inside the .dat loop: that loop is keyed on .dat
-        // files and would miss the marker-only or already-renamed-.idx states a
-        // mid-commit crash can leave behind.
-        self.reconcile_compact_states();
-
         // Scan for .dat files
         let entries = fs::read_dir(&self.directory)?;
         let mut dat_files: Vec<(String, VolumeId)> = Vec::new();
@@ -131,9 +123,16 @@ impl DiskLocation {
             let volume_name = volume_file_name(&self.directory, &collection, vid);
             let idx_name = volume_file_name(&self.idx_directory, &collection, vid);
 
-            // Sweep a leftover empty `.dat` stub (a phantom from the pre-fix
-            // loader) before it loads as a phantom volume or blocks startup.
-            if remove_empty_ec_dat_stub(&volume_name, &idx_name, vid) {
+            // Check for incomplete volume (.note file means a VolumeCopy was interrupted)
+            let note_path = format!("{}.note", volume_name);
+            if std::path::Path::new(&note_path).exists() {
+                let note = fs::read_to_string(&note_path).unwrap_or_default();
+                warn!(
+                    volume_id = vid.0,
+                    "volume was not completed: {}, removing files", note
+                );
+                remove_volume_files(&volume_name);
+                remove_volume_files(&idx_name);
                 continue;
             }
 
@@ -157,36 +156,21 @@ impl DiskLocation {
                         volume_id = vid.0,
                         "EC volume validation failed, removing incomplete EC files"
                     );
-                    let _ = self.remove_ec_volume_files(&collection, vid);
+                    self.remove_ec_volume_files(&collection, vid);
                     // Fall through to load .dat file
                 }
             }
 
-            // Stale .cpd/.cpx temp files were already rolled forward or back by
-            // the reconcile_compact_states pre-pass above.
-
-            // Check for an incomplete volume (.note means a VolumeCopy was
-            // interrupted). This runs BELOW the empty-stub sweep and EC
-            // validation: when an .ecx for this vid coexists on the disk, the
-            // regular and EC volumes share <base>.vif, so removing the
-            // incomplete regular copy must keep the .vif (keep_vif=true) or it
-            // would strip the EC volume's info file.
-            let note_path = format!("{}.note", volume_name);
-            if std::path::Path::new(&note_path).exists() {
-                let note = fs::read_to_string(&note_path).unwrap_or_default();
-                warn!(
-                    volume_id = vid.0,
-                    "volume was not completed: {}, removing files", note
-                );
-                // Re-check .ecx now (not the pre-validation ecx_exists): the
-                // invalid-EC cleanup above may have removed it, in which case
-                // the .vif is no longer shared and must not be preserved.
-                let keep_vif = std::path::Path::new(&ecx_path).exists()
-                    || (self.idx_directory != self.directory
-                        && std::path::Path::new(&format!("{}.ecx", volume_name)).exists());
-                remove_volume_files(&volume_name, keep_vif);
-                remove_volume_files(&idx_name, keep_vif);
-                continue;
+            // Clean up stale compaction temp files
+            let cpd_path = format!("{}.cpd", volume_name);
+            let cpx_path = format!("{}.cpx", idx_name);
+            if std::path::Path::new(&cpd_path).exists() {
+                info!(volume_id = vid.0, "removing stale compaction file .cpd");
+                let _ = fs::remove_file(&cpd_path);
+            }
+            if std::path::Path::new(&cpx_path).exists() {
+                info!(volume_id = vid.0, "removing stale compaction file .cpx");
+                let _ = fs::remove_file(&cpx_path);
             }
 
             // Skip if already loaded (e.g., from a previous call)
@@ -194,26 +178,6 @@ impl DiskLocation {
                 continue;
             }
 
-            // Load existing data only; never create a phantom `.dat`. A lone
-            // `.vif`/`.idx` (e.g. an EC sidecar whose `.ecx` is on a sibling
-            // disk) would otherwise have Volume::new write an 8-byte stub that
-            // the sibling-.dat prune deletes real shards against. Remote-tiered
-            // volumes also have no local `.dat`, but their `.vif` points at
-            // remote files and must still load via the remote path.
-            // Plain existence check (not check_dat_file_exists): an empty
-            // .dat here is a fresh, needle-less volume that must still load.
-            // A stat error other than NotFound counts as present so a
-            // transient error doesn't skip a real volume.
-            let dat_path = format!("{}.dat", volume_name);
-            let dat_missing = matches!(fs::metadata(&dat_path),
-                Err(ref e) if e.kind() == io::ErrorKind::NotFound);
-            if dat_missing
-                && !vif_references_remote_file(&format!("{}.vif", volume_name))
-                && !vif_references_remote_file(&format!("{}.vif", idx_name))
-            {
-                continue;
-            }
-
             match Volume::new(
                 &self.directory,
                 &self.idx_directory,
@@ -250,78 +214,22 @@ impl DiskLocation {
         Ok(())
     }
 
-    /// Directory pre-pass that recovers interrupted compaction commits. Collects
-    /// every volume id that still has a .cpc commit marker or a leftover
-    /// .cpd/.cpx temp file across the data and idx directories, then rolls the
-    /// swap forward (marker present) or back (marker absent) per volume.
-    fn reconcile_compact_states(&self) {
-        let mut pending: HashSet<(String, VolumeId)> = HashSet::new();
-        let mut collect = |dir: &str| {
-            if let Ok(entries) = fs::read_dir(dir) {
-                for entry in entries.flatten() {
-                    let name = entry.file_name().into_string().unwrap_or_default();
-                    let stem = name
-                        .strip_suffix(".cpc")
-                        .or_else(|| name.strip_suffix(".cpd"))
-                        .or_else(|| name.strip_suffix(".cpx"));
-                    if let Some(stem) = stem {
-                        if let Some(key) = parse_collection_volume_id(stem) {
-                            pending.insert(key);
-                        }
-                    }
-                }
-            }
-        };
-        collect(&self.directory);
-        if self.idx_directory != self.directory {
-            collect(&self.idx_directory);
-        }
-
-        for (collection, vid) in pending {
-            // On a runtime reload (SIGHUP), an already-loaded volume may be
-            // mid-vacuum: its .cpd/.cpx are live, not crash leftovers, and
-            // rolling them back would clobber the in-flight compaction. Only
-            // reconcile vids not currently loaded; genuine startup recovery
-            // runs before any volume is loaded.
-            if self.volumes.contains_key(&vid) {
-                continue;
-            }
-            let v = Volume::new_unloaded(&self.directory, &self.idx_directory, &collection, vid);
-            if let Err(e) = v.reconcile_compact_state() {
-                warn!(volume_id = vid.0, error = %e, "reconcile interrupted compaction failed");
-            }
-        }
-    }
-
-    /// Reports whether the EC files for (collection, vid) on this disk may be
-    /// deleted to reclaim the local .dat. Returns false (delete) only when that
-    /// provably loses no data; every ambiguity returns true (keep), since the
-    /// shards may be the only copy. Mirrors Go's validateEcVolume.
+    /// Validate EC volume shards: all shards must be same size, and if .dat exists,
+    /// need at least DATA_SHARDS_COUNT shards with size matching expected.
     fn validate_ec_volume(&self, collection: &str, vid: VolumeId) -> bool {
         let base = volume_file_name(&self.directory, collection, vid);
         let dat_path = format!("{}.dat", base);
 
-        // Custom ratio comes from the volume's own .vif; the server holds no
-        // cluster EC config in memory.
-        let data_shards =
-            ec_data_shards_from_vif(&self.directory, &self.idx_directory, collection, vid);
-
-        // On-disk .dat: an empty <= superblock one is a stub; a transient stat
-        // error keeps the shards rather than deleting.
         let mut expected_shard_size: Option<i64> = None;
-        let dat_exists = match fs::metadata(&dat_path) {
-            Ok(meta) if meta.len() > SUPER_BLOCK_SIZE as u64 => {
-                expected_shard_size =
-                    Some(calculate_expected_shard_size(meta.len() as i64, data_shards));
-                true
+        let dat_exists = std::path::Path::new(&dat_path).exists();
+
+        if dat_exists {
+            if let Ok(meta) = fs::metadata(&dat_path) {
+                expected_shard_size = Some(calculate_expected_shard_size(meta.len() as i64));
+            } else {
+                return false;
             }
-            Ok(_) => false,
-            Err(e) if e.kind() == io::ErrorKind::NotFound => false,
-            Err(e) => {
-                warn!(volume_id = vid.0, error = %e, "cannot stat .dat; keeping EC shards");
-                return true;
-            }
-        };
+        }
 
         let mut shard_count = 0usize;
         let mut actual_shard_size: Option<i64> = None;
@@ -334,10 +242,14 @@ impl DiskLocation {
                     let size = meta.len() as i64;
                     if let Some(prev) = actual_shard_size {
                         if size != prev {
-                            // Inconsistent sizes signal corruption or mixed
-                            // generations; not trusted for deletion -> keep.
-                            warn!(volume_id = vid.0, shard = i, size, expected = prev, "EC shard size mismatch; keeping shards");
-                            return true;
+                            warn!(
+                                volume_id = vid.0,
+                                shard = i,
+                                size,
+                                expected = prev,
+                                "EC shard size mismatch"
+                            );
+                            return false;
                         }
                     } else {
                         actual_shard_size = Some(size);
@@ -345,30 +257,49 @@ impl DiskLocation {
                     shard_count += 1;
                 }
                 Err(e) if e.kind() != io::ErrorKind::NotFound => {
-                    warn!(volume_id = vid.0, shard = i, error = %e, "cannot stat EC shard; keeping shards");
-                    return true;
+                    warn!(
+                        volume_id = vid.0,
+                        shard = i,
+                        error = %e,
+                        "failed to stat EC shard"
+                    );
+                    return false;
                 }
                 _ => {} // not found or zero size — skip
             }
         }
 
-        if !dat_exists {
-            return true; // distributed EC; any shard count is valid
-        }
-
-        // Reclaim only when it loses no data. Shards smaller than this .dat's
-        // full encode are an interrupted encode whose .dat is the complete
-        // source -> reclaim. Shards >= expected (valid/distributing EC, or a
-        // stale/partial .dat beside larger real shards) may be the only copy -> keep.
-        if shard_count == 0 {
-            return false;
-        }
-        if let (Some(actual), Some(expected)) = (actual_shard_size, expected_shard_size) {
-            if actual < expected {
-                warn!(volume_id = vid.0, actual, expected, "shards smaller than the .dat's full encode; reclaiming the complete .dat");
-                return false;
+        // If .dat exists, validate shard size matches expected
+        if dat_exists {
+            if let (Some(actual), Some(expected)) = (actual_shard_size, expected_shard_size) {
+                if actual != expected {
+                    warn!(
+                        volume_id = vid.0,
+                        actual_shard_size = actual,
+                        expected_shard_size = expected,
+                        "EC shard size doesn't match .dat file"
+                    );
+                    return false;
+                }
             }
         }
+
+        // Distributed EC (no .dat): any shard count is valid
+        if !dat_exists {
+            return true;
+        }
+
+        // With .dat: need at least DATA_SHARDS_COUNT shards
+        if shard_count < DATA_SHARDS_COUNT {
+            warn!(
+                volume_id = vid.0,
+                shard_count,
+                required = DATA_SHARDS_COUNT,
+                "EC volume has .dat but too few shards"
+            );
+            return false;
+        }
+
         true
     }
 
@@ -377,80 +308,24 @@ impl DiskLocation {
     /// `store_ec_reconcile.rs` can call it to scrub partial EC artefacts
     /// when a healthy `.dat` for the same vid lives on a sibling disk
     /// (seaweedfs/seaweedfs#9478).
-    pub(crate) fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) -> io::Result<()> {
+    pub(crate) fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) {
         let base = volume_file_name(&self.directory, collection, vid);
         let idx_base = volume_file_name(&self.idx_directory, collection, vid);
         const MAX_SHARD_COUNT: usize = 32;
 
         // Remove index files from idx directory (.ecx, .ecj)
-        rm_if_present(format!("{}.ecx", idx_base))?;
-        rm_if_present(format!("{}.ecj", idx_base))?;
+        let _ = fs::remove_file(format!("{}.ecx", idx_base));
+        let _ = fs::remove_file(format!("{}.ecj", idx_base));
         // Also try data directory in case .ecx/.ecj were created before -dir.idx was configured
         if self.idx_directory != self.directory {
-            rm_if_present(format!("{}.ecx", base))?;
-            rm_if_present(format!("{}.ecj", base))?;
+            let _ = fs::remove_file(format!("{}.ecx", base));
+            let _ = fs::remove_file(format!("{}.ecj", base));
         }
 
         // Remove all EC shard files (.ec00 ~ .ec31)
         for i in 0..MAX_SHARD_COUNT {
-            rm_if_present(format!("{}.ec{:02}", base, i))?;
+            let _ = fs::remove_file(format!("{}.ec{:02}", base, i));
         }
-        Ok(())
-    }
-
-    /// Full-teardown variant: everything remove_ec_volume_files clears, PLUS the
-    /// normal-volume <base>.vif on a SHARD-ONLY node. An interrupted shard copy
-    /// (which installs shards + .ecx before .vif) could otherwise mount a fresh
-    /// generation under the prior run's identity / ratio / dat_file_size carried by
-    /// a stale .vif. Gated on .idx absence so a source-volume holder keeps its live
-    /// .vif. Reconcile/load-fallback call remove_ec_volume_files directly and
-    /// intentionally preserve it, mirroring Go's removeEcVolumeFiles (reconcile) vs
-    /// removeStaleEcArtifacts (teardown) split.
-    pub(crate) fn remove_ec_volume_files_full_teardown(
-        &self,
-        collection: &str,
-        vid: VolumeId,
-    ) -> io::Result<()> {
-        self.remove_ec_volume_files(collection, vid)?;
-        let base = volume_file_name(&self.directory, collection, vid);
-        let idx_base = volume_file_name(&self.idx_directory, collection, vid);
-        // try_exists, not exists(): a stat error on a PRESENT .idx must not read as
-        // "shard-only" and delete a live source volume's .vif. Only Ok(false) (genuine
-        // NotFound) clears it; on a stat error stay conservative and keep the sidecar.
-        if matches!(
-            std::path::Path::new(&format!("{}.idx", idx_base)).try_exists(),
-            Ok(false)
-        ) {
-            rm_if_present(format!("{}.vif", idx_base))?;
-            if self.idx_directory != self.directory
-                && matches!(
-                    std::path::Path::new(&format!("{}.idx", base)).try_exists(),
-                    Ok(false)
-                )
-            {
-                rm_if_present(format!("{}.vif", base))?;
-            }
-        }
-        Ok(())
-    }
-
-    /// EC encode generation recorded in this disk's .vif (data dir first, then idx
-    /// dir for the split-disk layout). None if no .vif was readable, so the fenced
-    /// teardown preserves it (fail-safe). A present .vif with no ec_shard_config
-    /// yields Some(0) (a recovered or pre-upgrade live volume), also preserved.
-    pub(crate) fn ec_generation_ts_ns(&self, collection: &str, vid: VolumeId) -> Option<i64> {
-        for dir in [&self.directory, &self.idx_directory] {
-            let vif = format!("{}.vif", volume_file_name(dir, collection, vid));
-            if let Ok(s) = fs::read_to_string(&vif) {
-                if let Ok(vi) = serde_json::from_str::<VifVolumeInfo>(&s) {
-                    return Some(vi.ec_shard_config.map(|c| c.encode_ts_ns).unwrap_or(0));
-                }
-            }
-            if self.directory == self.idx_directory {
-                break;
-            }
-        }
-        None
     }
 
     /// Find a volume by ID.
@@ -949,23 +824,34 @@ impl DiskLocation {
                 volume_id = vid.0,
                 "Incomplete or invalid EC volume: .dat exists but validation failed, cleaning up EC files",
             );
-            let _ = self.remove_ec_volume_files(collection, vid);
+            self.remove_ec_volume_files(collection, vid);
             return;
         }
 
         let shard_ids: Vec<u32> = shards.iter().map(|(_, sid)| *sid).collect();
         if let Err(e) = self.mount_ec_shards(vid, collection, &shard_ids, "") {
-            // A mount failure (corrupt/locked .ecx, EMFILE, transient I/O) is
-            // not proof the shards are disposable -- validate_ec_volume already
-            // decided they may be the only copy. Release partially-mounted
-            // shards (mirror-decrements the metric) but keep the files; never
-            // delete on a load error.
-            warn!(
-                volume_id = vid.0,
-                "Failed to load EC shards: {}; keeping files for retry",
-                e,
-            );
-            self.unmount_ec_shards(vid, &shard_ids);
+            // mount_ec_shards adds shards one at a time and increments
+            // the per-shard metric for each. If it fails halfway, plain
+            // ec_volumes.remove(vid) would leak metric increments for
+            // the shards that did mount. Drive cleanup through
+            // unmount_ec_shards which mirror-decrements the metric, then
+            // the empty EcVolume drops itself.
+            if dat_exists {
+                warn!(
+                    volume_id = vid.0,
+                    "Failed to load EC shards and .dat exists ({}), cleaning up EC files to use .dat",
+                    e,
+                );
+                self.unmount_ec_shards(vid, &shard_ids);
+                self.remove_ec_volume_files(collection, vid);
+            } else {
+                warn!(
+                    volume_id = vid.0,
+                    "Failed to load EC shards: {} (this may be normal for distributed EC volumes)",
+                    e,
+                );
+                self.unmount_ec_shards(vid, &shard_ids);
+            }
         }
     }
 
@@ -991,7 +877,7 @@ impl DiskLocation {
                 "Found {} EC shards without .ecx file (incomplete encoding interrupted before .ecx), cleaning up",
                 shards.len(),
             );
-            let _ = self.remove_ec_volume_files(collection, vid);
+            self.remove_ec_volume_files(collection, vid);
             return true;
         }
         false
@@ -1052,14 +938,14 @@ pub fn get_disk_stats(path: &str) -> (u64, u64) {
 /// Calculate expected EC shard size from .dat file size.
 /// Matches Go's `calculateExpectedShardSize`: large blocks (1GB * data_shards) first,
 /// then small blocks (1MB * data_shards) for the remainder.
-fn calculate_expected_shard_size(dat_file_size: i64, data_shards: usize) -> i64 {
-    let large_batch_size = ERASURE_CODING_LARGE_BLOCK_SIZE as i64 * data_shards as i64;
+fn calculate_expected_shard_size(dat_file_size: i64) -> i64 {
+    let large_batch_size = ERASURE_CODING_LARGE_BLOCK_SIZE as i64 * DATA_SHARDS_COUNT as i64;
     let num_large_batches = dat_file_size / large_batch_size;
     let mut shard_size = num_large_batches * ERASURE_CODING_LARGE_BLOCK_SIZE as i64;
     let remaining = dat_file_size - (num_large_batches * large_batch_size);
 
     if remaining > 0 {
-        let small_batch_size = ERASURE_CODING_SMALL_BLOCK_SIZE as i64 * data_shards as i64;
+        let small_batch_size = ERASURE_CODING_SMALL_BLOCK_SIZE as i64 * DATA_SHARDS_COUNT as i64;
         // Ceiling division
         let num_small_batches = (remaining + small_batch_size - 1) / small_batch_size;
         shard_size += num_small_batches * ERASURE_CODING_SMALL_BLOCK_SIZE as i64;
@@ -1068,39 +954,6 @@ fn calculate_expected_shard_size(dat_file_size: i64, data_shards: usize) -> i64
     shard_size
 }
 
-/// Resolve the EC data-shard count from the volume's own `.vif` (the volume
-/// server never holds the cluster EC config in memory), checking the data dir
-/// then the idx dir. Falls back to the default ratio when no EC `.vif` is found.
-/// Remove a file, treating a missing file as success but propagating real errors
-/// (permissions, disk full). Mirrors Go's removeFileIfExists so a teardown failure
-/// surfaces instead of silently leaving stale artifacts.
-fn rm_if_present(path: String) -> io::Result<()> {
-    match fs::remove_file(&path) {
-        Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e),
-        _ => Ok(()),
-    }
-}
-
-fn ec_data_shards_from_vif(directory: &str, idx_directory: &str, collection: &str, vid: VolumeId) -> usize {
-    for dir in [directory, idx_directory] {
-        let vif = format!("{}.vif", volume_file_name(dir, collection, vid));
-        if let Some(ds) = fs::read_to_string(&vif)
-            .ok()
-            .and_then(|s| serde_json::from_str::<VifVolumeInfo>(&s).ok())
-            .and_then(|vi| vi.ec_shard_config)
-            .map(|c| c.data_shards as usize)
-        {
-            if ds > 0 {
-                return ds;
-            }
-        }
-        if directory == idx_directory {
-            break;
-        }
-    }
-    DATA_SHARDS_COUNT
-}
-
 /// Parse a volume filename like "collection_42.dat" or "42.dat" into (collection, VolumeId).
 /// Parse a `<collection>_<vid>` or `<vid>` base name into its parts.
 /// Mirrors `parseCollectionVolumeId` in
@@ -1154,77 +1007,31 @@ fn parse_ec_shard_extension(ext: &str) -> Option<u32> {
     Some(id)
 }
 
-/// Robust check that a `.dat` with actual data exists. An empty `.dat`
-/// (<= a superblock, zero needles) is a leftover stub, not an encode source,
-/// and counts as absent so it never justifies deleting shards. Any unexpected
-/// stat error (permission, I/O) is treated as "exists" so we don't misclassify
-/// local EC as distributed EC. Mirrors `checkDatFileExists` in Go.
+/// Robust `.dat` existence check: any unexpected stat error (permission,
+/// I/O) is treated as "exists" so we don't misclassify local EC as
+/// distributed EC. Mirrors `checkDatFileExists` in Go.
 fn check_dat_file_exists(path: &str) -> bool {
     match fs::metadata(path) {
-        Ok(meta) => meta.len() > SUPER_BLOCK_SIZE as u64,
+        Ok(_) => true,
         Err(e) if e.kind() == io::ErrorKind::NotFound => false,
         Err(_) => true,
     }
 }
 
-/// True when a `.vif` references remote-tier files: a remote-only volume
-/// that has no local `.dat` but must still load via the remote path,
-/// rather than be skipped as a lone EC sidecar.
-fn vif_references_remote_file(vif_path: &str) -> bool {
-    fs::read_to_string(vif_path)
-        .ok()
-        .and_then(|s| serde_json::from_str::<VifVolumeInfo>(&s).ok())
-        .map(|vif| !vif.files.is_empty())
-        .unwrap_or(false)
-}
-
-/// True when a `.vif` records an EC shard config, i.e. the volume was
-/// erasure-coded. Such a volume keeps no local `.dat`, so an empty `.dat`
-/// alongside it is a leftover stub safe to remove.
-fn vif_is_ec_volume(vif_path: &str) -> bool {
-    fs::read_to_string(vif_path)
-        .ok()
-        .and_then(|s| serde_json::from_str::<VifVolumeInfo>(&s).ok())
-        .map(|vif| vif.ec_shard_config.is_some())
-        .unwrap_or(false)
-}
-
-/// Remove a leftover empty EC `.dat` stub and return whether one was swept.
-///
-/// A stub is an empty `.dat` (<= a superblock, i.e. zero needles) whose `.vif`
-/// records an EC shard config. An EC volume keeps no local `.dat`, so the stub
-/// holds no data — its shards live on other servers. Such stubs (phantoms from
-/// the pre-fix loader) otherwise load as phantom empty volumes, and a same-vid
-/// stub on two disks blocks startup via the duplicate-vid check. The `.dat` and
-/// its empty `.idx` are removed; non-EC empty `.dat` files are left alone. The
-/// `.vif` is looked up in both the data and idx directories (which differ only
-/// when `-dir.idx` is configured), each read at most once.
-fn remove_empty_ec_dat_stub(volume_name: &str, idx_name: &str, vid: VolumeId) -> bool {
-    let dat_path = format!("{}.dat", volume_name);
-    match fs::metadata(&dat_path) {
-        Ok(meta) if meta.len() <= SUPER_BLOCK_SIZE as u64 => {}
-        _ => return false,
-    }
-
-    let data_vif = format!("{}.vif", volume_name);
-    let idx_vif = format!("{}.vif", idx_name);
-    let is_ec = vif_is_ec_volume(&data_vif) || (idx_vif != data_vif && vif_is_ec_volume(&idx_vif));
-    if !is_ec {
-        return false;
-    }
-
-    warn!(volume_id = vid.0, "removing leftover empty .dat stub for EC volume");
-    let _ = fs::remove_file(&dat_path);
-    let _ = fs::remove_file(format!("{}.idx", idx_name));
-    true
-}
-
 fn parse_volume_filename(filename: &str) -> Option<(String, VolumeId)> {
     let stem = filename
         .strip_suffix(".dat")
         .or_else(|| filename.strip_suffix(".vif"))
         .or_else(|| filename.strip_suffix(".idx"))?;
-    parse_collection_volume_id(stem)
+    if let Some(pos) = stem.rfind('_') {
+        let collection = &stem[..pos];
+        let id_str = &stem[pos + 1..];
+        let id: u32 = id_str.parse().ok()?;
+        Some((collection.to_string(), VolumeId(id)))
+    } else {
+        let id: u32 = stem.parse().ok()?;
+        Some((String::new(), VolumeId(id)))
+    }
 }
 
 // ============================================================================
@@ -1236,138 +1043,6 @@ mod tests {
     use super::*;
     use tempfile::TempDir;
 
-    /// When `-dir.idx` is configured the EC `.vif` may live in the idx
-    /// directory; the sweep must look there too, not only the data dir.
-    #[test]
-    fn test_remove_empty_ec_dat_stub_finds_vif_in_idx_dir() {
-        let tmp = TempDir::new().unwrap();
-        let data = tmp.path().join("data");
-        let idx = tmp.path().join("idx");
-        std::fs::create_dir_all(&data).unwrap();
-        std::fs::create_dir_all(&idx).unwrap();
-        let vbase = format!("{}/warp-cal_42", data.to_str().unwrap());
-        let ibase = format!("{}/warp-cal_42", idx.to_str().unwrap());
-        std::fs::write(format!("{}.dat", vbase), vec![0u8; 8]).unwrap();
-
-        let vif = VifVolumeInfo {
-            version: 3,
-            ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
-                data_shards: 10,
-                parity_shards: 4,
-                ..Default::default()
-            }),
-            ..Default::default()
-        };
-        std::fs::write(format!("{}.vif", ibase), serde_json::to_string(&vif).unwrap()).unwrap();
-
-        assert!(
-            remove_empty_ec_dat_stub(&vbase, &ibase, VolumeId(42)),
-            "EC .vif in the idx dir should be found and the stub removed",
-        );
-        assert!(!std::path::Path::new(&format!("{}.dat", vbase)).exists());
-    }
-
-    // The headline geometry: a stale/partial .dat (e.g. an interrupted decode)
-    // smaller than the volume's real source sits next to the full-size shards
-    // that are the only copy. validate_ec_volume must keep the shards.
-    #[test]
-    fn test_validate_ec_volume_partial_dat_next_to_full_shards_keeps() {
-        let tmp = TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-        let loc = DiskLocation::new(dir, dir, 10, DiskType::HardDrive, MinFreeSpace::Percent(1.0), Vec::new()).unwrap();
-        let base = volume_file_name(dir, "", VolumeId(70));
-        let ds = crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
-        let full = calculate_expected_shard_size(30 * 1024 * 1024, ds);
-        for i in 0..ds {
-            std::fs::File::create(format!("{}.ec{:02}", base, i)).unwrap().set_len(full as u64).unwrap();
-        }
-        // Partial .dat: bigger than a superblock so it is not swept as a stub,
-        // but smaller than what these shards encode.
-        std::fs::File::create(format!("{}.dat", base)).unwrap().set_len(5 * 1024 * 1024).unwrap();
-        assert!(
-            loc.validate_ec_volume("", VolumeId(70)),
-            "full-size shards beside a smaller (stale/partial) .dat must be kept",
-        );
-    }
-
-    // The legitimate cleanup: a full source .dat next to shards SMALLER than it
-    // would encode (an interrupted encode). The .dat is the complete source, so
-    // reclaiming it loses no data.
-    #[test]
-    fn test_validate_ec_volume_interrupted_encode_reclaims() {
-        let tmp = TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-        let loc = DiskLocation::new(dir, dir, 10, DiskType::HardDrive, MinFreeSpace::Percent(1.0), Vec::new()).unwrap();
-        let base = volume_file_name(dir, "", VolumeId(71));
-        let ds = crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
-        let dat_size = 30 * 1024 * 1024i64;
-        std::fs::File::create(format!("{}.dat", base)).unwrap().set_len(dat_size as u64).unwrap();
-        let partial = calculate_expected_shard_size(dat_size, ds) / 3;
-        assert!(partial > 0);
-        for i in 0..ds {
-            std::fs::File::create(format!("{}.ec{:02}", base, i)).unwrap().set_len(partial as u64).unwrap();
-        }
-        assert!(
-            !loc.validate_ec_volume("", VolumeId(71)),
-            "shards smaller than the full source .dat should be reclaimable",
-        );
-    }
-
-    // The per-disk generation reader behind the fenced EC teardown: a present .vif
-    // yields its generation (or 0 with no EC config), a missing .vif is unreadable
-    // (preserved, fail-safe), and the idx dir is a fallback for the split-disk layout.
-    #[test]
-    fn test_ec_generation_ts_ns_reads_and_fences() {
-        let tmp = TempDir::new().unwrap();
-        let data = tmp.path().join("data");
-        let idx = tmp.path().join("idx");
-        std::fs::create_dir_all(&data).unwrap();
-        std::fs::create_dir_all(&idx).unwrap();
-        let loc = DiskLocation::new(
-            data.to_str().unwrap(),
-            idx.to_str().unwrap(),
-            10,
-            DiskType::HardDrive,
-            MinFreeSpace::Percent(1.0),
-            Vec::new(),
-        )
-        .unwrap();
-        let vid = VolumeId(88);
-        let dbase = volume_file_name(data.to_str().unwrap(), "", vid);
-        let ibase = volume_file_name(idx.to_str().unwrap(), "", vid);
-
-        // No .vif anywhere -> unreadable (the fenced teardown preserves on None).
-        assert_eq!(loc.ec_generation_ts_ns("", vid), None);
-
-        // .vif in the data dir carries its generation.
-        let with_gen = VifVolumeInfo {
-            version: 3,
-            ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
-                data_shards: 10,
-                parity_shards: 4,
-                encode_ts_ns: 4242,
-                ..Default::default()
-            }),
-            ..Default::default()
-        };
-        std::fs::write(format!("{}.vif", dbase), serde_json::to_string(&with_gen).unwrap()).unwrap();
-        assert_eq!(loc.ec_generation_ts_ns("", vid), Some(4242));
-
-        // A .vif with no EC config reads as generation 0 (recovered/pre-upgrade live volume).
-        std::fs::remove_file(format!("{}.vif", dbase)).unwrap();
-        let no_cfg = VifVolumeInfo {
-            version: 3,
-            ..Default::default()
-        };
-        std::fs::write(format!("{}.vif", dbase), serde_json::to_string(&no_cfg).unwrap()).unwrap();
-        assert_eq!(loc.ec_generation_ts_ns("", vid), Some(0));
-
-        // idx-dir fallback: only the idx dir holds the .vif.
-        std::fs::remove_file(format!("{}.vif", dbase)).unwrap();
-        std::fs::write(format!("{}.vif", ibase), serde_json::to_string(&with_gen).unwrap()).unwrap();
-        assert_eq!(loc.ec_generation_ts_ns("", vid), Some(4242));
-    }
-
     #[test]
     fn test_parse_volume_filename() {
         assert_eq!(

seaweed-volume/src/storage/erasure_coding/ec_decoder.rs

@@ -11,7 +11,7 @@ use crate::storage::idx;
 use crate::storage::needle::needle::get_actual_size;
 use crate::storage::super_block::SUPER_BLOCK_SIZE;
 use crate::storage::types::*;
-use crate::storage::volume::{fsync_dir, volume_file_name};
+use crate::storage::volume::volume_file_name;
 
 /// Calculate .dat file size from the max offset entry in .ecx.
 /// Reads the volume version from the first EC shard (.ec00) superblock,
@@ -123,81 +123,60 @@ pub fn write_dat_file_from_shards_with_dirs(
     }
     let base = volume_file_name(dat_dir, collection, volume_id);
     let dat_path = format!("{}.dat", base);
-    // Write to a temp file and atomically rename into place, so a crash
-    // mid-write never leaves a partial .dat at the final name beside the
-    // source shards.
-    let tmp_path = format!("{}.tmp", dat_path);
 
-    let write_result = (|| -> io::Result<()> {
-        // Open data shards from their individual home dirs.
-        let mut shards: Vec<EcVolumeShard> = (0..data_shards as u8)
-            .map(|i| EcVolumeShard::new(&shard_dirs[i as usize], collection, volume_id, i))
-            .collect();
+    // Open data shards from their individual home dirs.
+    let mut shards: Vec<EcVolumeShard> = (0..data_shards as u8)
+        .map(|i| EcVolumeShard::new(&shard_dirs[i as usize], collection, volume_id, i))
+        .collect();
 
-        for shard in &mut shards {
-            shard.open()?;
-        }
-
-        let mut dat_file = File::create(&tmp_path)?;
-        let mut remaining = dat_file_size;
-        let large_block_size = ERASURE_CODING_LARGE_BLOCK_SIZE;
-        let small_block_size = ERASURE_CODING_SMALL_BLOCK_SIZE;
-        let large_row_size = (large_block_size * data_shards) as i64;
-
-        let mut shard_offset: u64 = 0;
-
-        // Read large blocks
-        while remaining >= large_row_size {
-            for i in 0..data_shards {
-                let mut buf = vec![0u8; large_block_size];
-                shards[i].read_at(&mut buf, shard_offset)?;
-                let to_write = large_block_size.min(remaining as usize);
-                dat_file.write_all(&buf[..to_write])?;
-                remaining -= to_write as i64;
-                if remaining <= 0 {
-                    break;
-                }
-            }
-            shard_offset += large_block_size as u64;
-        }
-
-        // Read small blocks
-        while remaining > 0 {
-            for i in 0..data_shards {
-                let mut buf = vec![0u8; small_block_size];
-                shards[i].read_at(&mut buf, shard_offset)?;
-                let to_write = small_block_size.min(remaining as usize);
-                dat_file.write_all(&buf[..to_write])?;
-                remaining -= to_write as i64;
-                if remaining <= 0 {
-                    break;
-                }
-            }
-            shard_offset += small_block_size as u64;
-        }
-
-        for shard in &mut shards {
-            shard.close();
-        }
-
-        // fsync, rename, then fsync the dir so the decoded .dat is durable and
-        // atomically published before the caller deletes the source shards.
-        dat_file.sync_all()?;
-        drop(dat_file);
-        // Windows rename does not replace an existing file on every version;
-        // remove the destination first, matching the compaction commit path.
-        #[cfg(windows)]
-        {
-            let _ = std::fs::remove_file(&dat_path);
-        }
-        std::fs::rename(&tmp_path, &dat_path)?;
-        fsync_dir(&dat_path)?;
-        Ok(())
-    })();
-    if write_result.is_err() {
-        let _ = std::fs::remove_file(&tmp_path);
+    for shard in &mut shards {
+        shard.open()?;
     }
-    write_result
+
+    let mut dat_file = File::create(&dat_path)?;
+    let mut remaining = dat_file_size;
+    let large_block_size = ERASURE_CODING_LARGE_BLOCK_SIZE;
+    let small_block_size = ERASURE_CODING_SMALL_BLOCK_SIZE;
+    let large_row_size = (large_block_size * data_shards) as i64;
+
+    let mut shard_offset: u64 = 0;
+
+    // Read large blocks
+    while remaining >= large_row_size {
+        for i in 0..data_shards {
+            let mut buf = vec![0u8; large_block_size];
+            shards[i].read_at(&mut buf, shard_offset)?;
+            let to_write = large_block_size.min(remaining as usize);
+            dat_file.write_all(&buf[..to_write])?;
+            remaining -= to_write as i64;
+            if remaining <= 0 {
+                break;
+            }
+        }
+        shard_offset += large_block_size as u64;
+    }
+
+    // Read small blocks
+    while remaining > 0 {
+        for i in 0..data_shards {
+            let mut buf = vec![0u8; small_block_size];
+            shards[i].read_at(&mut buf, shard_offset)?;
+            let to_write = small_block_size.min(remaining as usize);
+            dat_file.write_all(&buf[..to_write])?;
+            remaining -= to_write as i64;
+            if remaining <= 0 {
+                break;
+            }
+        }
+        shard_offset += small_block_size as u64;
+    }
+
+    for shard in &mut shards {
+        shard.close();
+    }
+
+    dat_file.sync_all()?;
+    Ok(())
 }
 
 /// Write .idx file from .ecx index + .ecj deletion journal.
@@ -213,59 +192,34 @@ pub fn write_idx_file_from_ec_index(
     let ecx_path = format!("{}.ecx", base);
     let ecj_path = format!("{}.ecj", base);
     let idx_path = format!("{}.idx", base);
-    // Write to a temp file and atomically rename into place, so a crash
-    // mid-write never leaves a partial .idx at the final name beside the
-    // source shards.
-    let tmp_path = format!("{}.tmp", idx_path);
 
-    let write_result = (|| -> io::Result<()> {
-        // Copy .ecx to the temp .idx
-        std::fs::copy(&ecx_path, &tmp_path)?;
+    // Copy .ecx to .idx
+    std::fs::copy(&ecx_path, &idx_path)?;
 
-        // Append deletions from .ecj as tombstones. Read the journal directly
-        // and treat only NotFound as "no journal": Path::exists would also
-        // swallow a permission/IO error and silently skip deletions, which
-        // would resurrect deleted needles as live.
-        let mut idx_file = std::fs::OpenOptions::new()
-            .write(true)
-            .append(true)
-            .open(&tmp_path)?;
-        match std::fs::read(&ecj_path) {
-            Ok(ecj_data) => {
-                let count = ecj_data.len() / NEEDLE_ID_SIZE;
-                for i in 0..count {
-                    let start = i * NEEDLE_ID_SIZE;
-                    let needle_id = NeedleId::from_bytes(&ecj_data[start..start + NEEDLE_ID_SIZE]);
-                    idx::write_index_entry(
-                        &mut idx_file,
-                        needle_id,
-                        Offset::default(),
-                        TOMBSTONE_FILE_SIZE,
-                    )?;
-                }
+    // Append deletions from .ecj as tombstones
+    if std::path::Path::new(&ecj_path).exists() {
+        let ecj_data = std::fs::read(&ecj_path)?;
+        if !ecj_data.is_empty() {
+            let mut idx_file = std::fs::OpenOptions::new()
+                .write(true)
+                .append(true)
+                .open(&idx_path)?;
+
+            let count = ecj_data.len() / NEEDLE_ID_SIZE;
+            for i in 0..count {
+                let start = i * NEEDLE_ID_SIZE;
+                let needle_id = NeedleId::from_bytes(&ecj_data[start..start + NEEDLE_ID_SIZE]);
+                idx::write_index_entry(
+                    &mut idx_file,
+                    needle_id,
+                    Offset::default(),
+                    TOMBSTONE_FILE_SIZE,
+                )?;
             }
-            Err(e) if e.kind() == io::ErrorKind::NotFound => {}
-            Err(e) => return Err(e),
         }
-
-        // fsync, rename, then fsync the dir so the decoded .idx is durable and
-        // atomically published before the caller deletes the source shards.
-        idx_file.sync_all()?;
-        drop(idx_file);
-        // Windows rename does not replace an existing file on every version;
-        // remove the destination first, matching the compaction commit path.
-        #[cfg(windows)]
-        {
-            let _ = std::fs::remove_file(&idx_path);
-        }
-        std::fs::rename(&tmp_path, &idx_path)?;
-        fsync_dir(&idx_path)?;
-        Ok(())
-    })();
-    if write_result.is_err() {
-        let _ = std::fs::remove_file(&tmp_path);
     }
-    write_result
+
+    Ok(())
 }
 
 #[cfg(test)]
@@ -334,10 +288,6 @@ mod tests {
             .unwrap();
         write_idx_file_from_ec_index(dir, "", VolumeId(1)).unwrap();
 
-        // Atomic publish must rename the temp files away, never leaving them behind.
-        assert!(!std::path::Path::new(&format!("{}/1.dat.tmp", dir)).exists());
-        assert!(!std::path::Path::new(&format!("{}/1.idx.tmp", dir)).exists());
-
         // Verify reconstructed .dat matches original
         let reconstructed_dat = std::fs::read(format!("{}/1.dat", dir)).unwrap();
         assert_eq!(
@@ -369,16 +319,4 @@ mod tests {
             assert_eq!(&n.data, expected_data, "needle {} data should match", id);
         }
     }
-
-    #[test]
-    fn test_decode_missing_shard_leaves_no_dat() {
-        let tmp = TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-        // No shard files exist, so de-striping must fail and publish nothing:
-        // neither the final .dat nor a partial .dat.tmp may remain.
-        let res = write_dat_file_from_shards(dir, "", VolumeId(7), 100, 10);
-        assert!(res.is_err());
-        assert!(!std::path::Path::new(&format!("{}/7.dat", dir)).exists());
-        assert!(!std::path::Path::new(&format!("{}/7.dat.tmp", dir)).exists());
-    }
 }

seaweed-volume/src/storage/erasure_coding/ec_encoder.rs

@@ -136,22 +136,11 @@ pub fn rebuild_ec_files(
         // Allocate buffers for all shards. Option<Vec<u8>> is required by rs.reconstruct()
         let mut buffers: Vec<Option<Vec<u8>>> = vec![None; total_shards];
 
-        // Read available shards. A short read means a truncated/corrupt input
-        // shard; treat it as an error rather than reconstructing over a
-        // zero-padded tail and publishing the result as restored redundancy.
+        // Read available shards
         for (i, shard) in shards.iter().enumerate() {
             if !missing_shard_ids.contains(&(i as u32)) {
                 let mut buf = vec![0u8; to_process];
-                let n = shard.read_at(&mut buf, offset)?;
-                if n != to_process {
-                    return Err(io::Error::new(
-                        io::ErrorKind::UnexpectedEof,
-                        format!(
-                            "ec rebuild short read shard {} at {}: got {} want {}",
-                            i, offset, n, to_process
-                        ),
-                    ));
-                }
+                shard.read_at(&mut buf, offset)?;
                 buffers[i] = Some(buf);
             }
         }
@@ -666,81 +655,6 @@ mod tests {
         assert!(std::path::Path::new(&ecx_path).exists());
     }
 
-    // encode_sample_volume writes a small volume and EC-encodes it, returning
-    // the dir path so a test can drop/truncate shards and rebuild.
-    fn encode_sample_volume(tmp: &TempDir) -> String {
-        let dir = tmp.path().to_str().unwrap().to_string();
-        let mut v = Volume::new(
-            &dir,
-            &dir,
-            "",
-            VolumeId(1),
-            NeedleMapKind::InMemory,
-            None,
-            None,
-            0,
-            Version::current(),
-        )
-        .unwrap();
-        for i in 1..=20 {
-            let data = format!("test data for needle {} padded with bytes", i).repeat(64);
-            let mut n = Needle {
-                id: NeedleId(i),
-                cookie: Cookie(i as u32),
-                data: data.as_bytes().to_vec(),
-                data_size: data.len() as u32,
-                ..Needle::default()
-            };
-            v.write_needle(&mut n, true).unwrap();
-        }
-        v.sync_to_disk().unwrap();
-        v.close();
-        write_ec_files(&dir, &dir, "", VolumeId(1), 10, 4).unwrap();
-        dir
-    }
-
-    // A truncated/corrupt input shard must abort rebuild_ec_files with an error
-    // rather than reconstructing over a zero-padded tail and publishing a
-    // truncated shard as restored redundancy.
-    #[test]
-    fn test_rebuild_ec_files_short_read_input_errors() {
-        let tmp = TempDir::new().unwrap();
-        let dir = encode_sample_volume(&tmp);
-
-        // Truncate a present input shard to half its size.
-        let victim = format!("{}/1.ec03", dir);
-        let full = std::fs::metadata(&victim).unwrap().len();
-        assert!(full > 0, "encoded shard should be non-empty");
-        let f = std::fs::OpenOptions::new().write(true).open(&victim).unwrap();
-        f.set_len(full / 2).unwrap();
-        drop(f);
-
-        // Rebuild a different (genuinely missing) shard.
-        std::fs::remove_file(format!("{}/1.ec07", dir)).unwrap();
-        let res = rebuild_ec_files(&dir, "", VolumeId(1), &[7], 10, 4);
-        assert!(res.is_err(), "truncated input shard must abort the rebuild");
-    }
-
-    // Happy path: dropping a shard and rebuilding it from the rest must succeed
-    // and recreate the shard file (the short-read guard must not false-positive).
-    #[test]
-    fn test_rebuild_ec_files_happy_path() {
-        let tmp = TempDir::new().unwrap();
-        let dir = encode_sample_volume(&tmp);
-
-        let dropped = format!("{}/1.ec07", dir);
-        std::fs::remove_file(&dropped).unwrap();
-        rebuild_ec_files(&dir, "", VolumeId(1), &[7], 10, 4).unwrap();
-        assert!(
-            std::path::Path::new(&dropped).exists(),
-            "rebuilt shard .ec07 should exist"
-        );
-        assert!(
-            std::fs::metadata(&dropped).unwrap().len() > 0,
-            "rebuilt shard should be non-empty"
-        );
-    }
-
     #[test]
     fn test_reed_solomon_basic() {
         let data_shards = 10;

seaweed-volume/src/storage/erasure_coding/ec_volume.rs

@@ -57,10 +57,6 @@ pub struct EcVolume {
     pub shard_locations_refresh_time: std::sync::Mutex<Option<std::time::Instant>>,
     /// EC volume expiration time (unix epoch seconds), set during EC encode from TTL.
     pub expire_at_sec: u64,
-    /// Encode-run identity (unix nanos) loaded from the .vif EcShardConfig. A read
-    /// served from a shard of a different encode run is rejected (server- and
-    /// client-side); 0 for a pre-feature volume, which is treated leniently.
-    pub encode_ts_ns: i64,
 }
 
 /// Locate the `.vif` for a (collection, vid) by preferring the data dir
@@ -141,7 +137,7 @@ impl EcVolume {
         // for shard-size math and by the Store-level prune in
         // `store_ec_reconcile.rs` to verify a sibling-disk .dat is
         // plausibly the encoding source (#9478).
-        let (expire_at_sec, vif_version, vif_dat_file_size, encode_ts_ns) = {
+        let (expire_at_sec, vif_version, vif_dat_file_size) = {
             let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
             if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
                 if let Ok(vif_info) =
@@ -152,21 +148,12 @@ impl EcVolume {
                     } else {
                         Version::current()
                     };
-                    let cfg_encode_ts_ns = vif_info
-                        .ec_shard_config
-                        .as_ref()
-                        .map_or(0, |c| c.encode_ts_ns);
-                    (
-                        vif_info.expire_at_sec,
-                        ver,
-                        vif_info.dat_file_size,
-                        cfg_encode_ts_ns,
-                    )
+                    (vif_info.expire_at_sec, ver, vif_info.dat_file_size)
                 } else {
-                    (0, Version::current(), 0, 0)
+                    (0, Version::current(), 0)
                 }
             } else {
-                (0, Version::current(), 0, 0)
+                (0, Version::current(), 0)
             }
         };
 
@@ -190,7 +177,6 @@ impl EcVolume {
             shard_locations: std::sync::RwLock::new(HashMap::new()),
             shard_locations_refresh_time: std::sync::Mutex::new(None),
             expire_at_sec,
-            encode_ts_ns,
         };
 
         // Open .ecx file (sorted index) in read/write mode for in-place deletion marking.
@@ -448,7 +434,6 @@ impl EcVolume {
             disk_id,
             file_count,
             delete_count,
-            encode_ts_ns: self.encode_ts_ns,
             ..Default::default()
         }]
     }
@@ -620,19 +605,7 @@ impl EcVolume {
                 })?;
 
             let mut buf = vec![0u8; interval.size as usize];
-            let n = shard.read_at(&mut buf, shard_offset as u64)?;
-            if n != buf.len() {
-                return Err(io::Error::new(
-                    io::ErrorKind::UnexpectedEof,
-                    format!(
-                        "short read on ec shard {}: read {} of {} bytes for needle {}",
-                        shard_id,
-                        n,
-                        buf.len(),
-                        needle_id
-                    ),
-                ));
-            }
+            shard.read_at(&mut buf, shard_offset as u64)?;
             bytes.extend_from_slice(&buf);
         }
 
@@ -1270,7 +1243,6 @@ mod tests {
             ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
                 data_shards: 6,
                 parity_shards: 3,
-                ..Default::default()
             }),
             ..Default::default()
         };
@@ -1296,7 +1268,6 @@ mod tests {
             ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
                 data_shards: 10,
                 parity_shards: 10,
-                ..Default::default()
             }),
             ..Default::default()
         };

seaweed-volume/src/storage/store.rs

@@ -369,16 +369,6 @@ impl Store {
             let vif_path = format!("{}.vif", base);
             if std::path::Path::new(&dat_path).exists() || std::path::Path::new(&vif_path).exists()
             {
-                // A persisting .note means the copy that produced these files
-                // never completed; mounting it would expose a truncated volume.
-                // Fail the mount so the caller (VolumeCopy) treats it as an error.
-                let note_path = format!("{}.note", base);
-                if std::path::Path::new(&note_path).exists() {
-                    return Err(VolumeError::Io(io::Error::new(
-                        io::ErrorKind::Other,
-                        format!("volume {} copy incomplete: .note still present", vid),
-                    )));
-                }
                 return loc.create_volume(
                     vid,
                     collection,
@@ -608,16 +598,8 @@ impl Store {
                     as i32;
                 let mut max_count = vol_count + ec_equivalent;
 
-                // One slot per full volume that fits in the unclaimed space.
-                // A "- 1" here used to zero the count when the disk had room for
-                // exactly one volume (free between 1x and 2x the limit), stranding
-                // auto-sized disks at max_volume_count 0 with no writable volume.
-                if unclaimed > 0 {
-                    max_count += (unclaimed as u64 / volume_size_limit) as i32;
-                }
-                // An auto-sized disk with free space always hosts at least one volume.
-                if max_count < 1 {
-                    max_count = 1;
+                if unclaimed > volume_size_limit as i64 {
+                    max_count += (unclaimed as u64 / volume_size_limit) as i32 - 1;
                 }
 
                 loc.max_volume_count.store(max_count, Ordering::Relaxed);
@@ -716,37 +698,17 @@ impl Store {
 
     /// Unmount a single EC shard, searching all locations.
     /// Matches Go's Store.UnmountEcShards which unmounts one shard at a time.
-    pub fn unmount_ec_shard(
-        &mut self,
-        vid: VolumeId,
-        shard_id: u32,
-        req_encode_ts_ns: i64,
-    ) -> Result<(), VolumeError> {
+    pub fn unmount_ec_shard(&mut self, vid: VolumeId, shard_id: u32) -> Result<(), VolumeError> {
         // Walk all locations rather than stopping at the first with the
         // vid — split-disk reconciled volumes can have the same vid on
         // multiple disks, with the target shard on any of them.
         for disk_id in 0..self.locations.len() {
-            let ec_vol = self.locations[disk_id].find_ec_volume(vid);
-            let has_shard = ec_vol.is_some_and(|ec_vol| ec_vol.has_shard(shard_id as u8));
+            let has_shard = self.locations[disk_id]
+                .find_ec_volume(vid)
+                .is_some_and(|ec_vol| ec_vol.has_shard(shard_id as u8));
             if !has_shard {
                 continue;
             }
-            // Generation fence: when the caller carries a generation (stale-worker
-            // cleanup), only unmount a strictly-older generation; preserve a disk
-            // whose generation is same-or-newer, 0, or unknown, so a stale run cannot
-            // unmount a newer run's live shards. req 0 (legacy/shell) unmounts all.
-            let disk_gen = ec_vol.map_or(0, |v| v.encode_ts_ns);
-            if req_encode_ts_ns > 0 && !(disk_gen > 0 && disk_gen < req_encode_ts_ns) {
-                tracing::info!(
-                    volume_id = vid.0,
-                    shard_id,
-                    disk_id,
-                    disk_gen,
-                    req = req_encode_ts_ns,
-                    "UnmountEcShards skipped: generation not older than request"
-                );
-                continue;
-            }
             tracing::info!(
                 volume_id = vid.0,
                 shard_id,
@@ -1500,47 +1462,6 @@ mod tests {
         assert!(with_preallocate > without_preallocate);
     }
 
-    #[test]
-    fn test_maybe_adjust_volume_max_no_dead_zone() {
-        // With auto max (original_max_volume_count == 0) and a large volume size
-        // limit, a disk with room for at least one volume must not report 0
-        // slots. It once did so for disks sized between 1x and 2x the limit,
-        // leaving no writable volume and timing out every assign.
-        let tmp = TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-
-        let (_, free) = crate::storage::disk_location::get_disk_stats(dir);
-        if free < 4 {
-            return; // cannot determine free disk space
-        }
-
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        store
-            .add_location(
-                dir,
-                dir,
-                0, // auto
-                DiskType::HardDrive,
-                MinFreeSpace::Percent(1.0),
-                Vec::new(),
-            )
-            .unwrap();
-
-        // free disk is 1.5x the limit: room for one full volume, less than two.
-        let volume_size_limit = free * 2 / 3;
-        store
-            .volume_size_limit
-            .store(volume_size_limit, Ordering::Relaxed);
-
-        store.maybe_adjust_volume_max();
-
-        let max = store.locations[0].max_volume_count.load(Ordering::Relaxed);
-        assert!(
-            max >= 1,
-            "auto-sized disk with room for one volume reported max_volume_count={max}; want >= 1"
-        );
-    }
-
     #[test]
     fn test_find_free_location_predicate_prefers_more_capacity_and_skips_low_disk() {
         let tmp1 = TempDir::new().unwrap();

seaweed-volume/src/storage/store_ec_mirror.rs

@@ -327,7 +327,6 @@ mod tests {
             ec_shard_config: Some(VifEcShardConfig {
                 data_shards,
                 parity_shards,
-                ..Default::default()
             }),
             ..Default::default()
         };

seaweed-volume/src/storage/store_ec_reconcile.rs

@@ -23,6 +23,7 @@ use tracing::{info, warn};
 use crate::storage::disk_location::{is_ec_shard_extension, parse_collection_volume_id_pub};
 use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
 use crate::storage::store::Store;
+use crate::storage::super_block::SUPER_BLOCK_SIZE;
 use crate::storage::types::VolumeId;
 
 pub(crate) fn ec_local_ecx_path(dir: &str, collection: &str, vid: VolumeId) -> String {
@@ -202,11 +203,14 @@ impl Store {
     /// this server) also fall through unchanged because the `.dat`
     /// index never matches.
     ///
-    /// The sibling `.dat` must be a credible encoding source before we
-    /// delete anything: at least the size `.vif` recorded at encode time,
-    /// or — when unknown (0) — more than a bare superblock so an empty
-    /// 8-byte stub can't pass. A truncated `.dat` leaves the partial EC
-    /// alone; those shards may still reconstruct from other servers.
+    /// Before deleting any EC files we also check that the sibling
+    /// `.dat` is plausibly the encoding source: at least
+    /// `SUPER_BLOCK_SIZE` bytes long, and — when the EC's `.vif`
+    /// recorded a non-zero source size in `dat_file_size` — at least
+    /// that many bytes. A zero-byte shell or a truncated `.dat` does
+    /// not justify wiping the partial EC, because that EC shard may
+    /// still combine usefully with shards on other servers in a
+    /// recoverable distributed-EC layout.
     ///
     /// We don't have to push anything to a deleted-shards channel
     /// here: the Rust heartbeat path in `server/heartbeat.rs` diffs
@@ -236,15 +240,8 @@ impl Store {
         let mut victims: Vec<Victim> = Vec::new();
         for (loc_idx, loc) in self.locations.iter().enumerate() {
             for (vid, ev) in loc.ec_volumes() {
-                // Use the volume's own ratio, not the OSS default, so a full
-                // custom-ratio data set (e.g. 9 of a 9+3) is not mistaken for a leftover.
-                let data_shards = if ev.data_shards > 0 {
-                    ev.data_shards as usize
-                } else {
-                    DATA_SHARDS_COUNT
-                };
                 let shard_count = ev.shard_count();
-                if shard_count >= data_shards {
+                if shard_count >= DATA_SHARDS_COUNT {
                     continue;
                 }
                 let key = EcKey {
@@ -260,10 +257,16 @@ impl Store {
                     // per-disk pass; don't second-guess it here.
                     continue;
                 }
-                // Delete only against a byte-exact committed source: the sibling
-                // .dat must equal the size .vif recorded at encode time. An
-                // unknown (0) or mismatched size cannot prove the .dat holds this data.
-                if ev.dat_file_size <= 0 || owner.size != ev.dat_file_size as u64 {
+                // Decide whether the sibling .dat is credible. Prefer
+                // the size baked into .vif at encode time; fall back
+                // to "at least a superblock" for old EC volumes whose
+                // .vif predates the field.
+                let required = if ev.dat_file_size > 0 {
+                    ev.dat_file_size as u64
+                } else {
+                    SUPER_BLOCK_SIZE as u64
+                };
+                if owner.size < required {
                     warn!(
                         volume_id = vid.0,
                         collection = %ev.collection,
@@ -271,30 +274,8 @@ impl Store {
                         shard_count,
                         sibling_dir = %self.locations[owner.location].directory,
                         sibling_dat_size = owner.size,
-                        recorded = ev.dat_file_size,
-                        "sibling .dat does not byte-exactly match the recorded EC source size; leaving partial EC in place",
-                    );
-                    continue;
-                }
-                // Never prune when the shards are recoverable node-wide (a set
-                // split across sibling disks summing to >= data_shards); they
-                // may be sole copies of a distributed volume.
-                let mut node_wide_bits = ev.shard_bits().0;
-                for other in &self.locations {
-                    if let Some(other_ev) = other.find_ec_volume(*vid) {
-                        if other_ev.collection == ev.collection {
-                            node_wide_bits |= other_ev.shard_bits().0;
-                        }
-                    }
-                }
-                let node_wide = node_wide_bits.count_ones() as usize;
-                if node_wide >= data_shards {
-                    warn!(
-                        volume_id = vid.0,
-                        collection = %ev.collection,
-                        node_wide,
-                        data_shards,
-                        "shards present node-wide are independently recoverable; leaving EC in place despite a sibling .dat",
+                        required,
+                        "sibling .dat is smaller than the EC source size; leaving partial EC in place so distributed reconstruction is still possible (issue 9478)",
                     );
                     continue;
                 }
@@ -332,7 +313,7 @@ impl Store {
             // Also sweep any unmounted shard files (.ec00 .. .ec31)
             // that the per-disk loader skipped — destroy() only walks
             // the in-memory shards, but the disk may still hold others.
-            let _ = loc.remove_ec_volume_files(&v.collection, v.vid);
+            loc.remove_ec_volume_files(&v.collection, v.vid);
         }
     }
 
@@ -478,7 +459,6 @@ mod tests {
             ec_shard_config: Some(VifEcShardConfig {
                 data_shards,
                 parity_shards,
-                ..Default::default()
             }),
             ..Default::default()
         };
@@ -489,287 +469,6 @@ mod tests {
         .unwrap();
     }
 
-    fn write_ec_vif(dir: &str, collection: &str, vid: u32) {
-        let vif = VifVolumeInfo {
-            version: 3,
-            ec_shard_config: Some(VifEcShardConfig {
-                data_shards: 10,
-                parity_shards: 4,
-                ..Default::default()
-            }),
-            ..Default::default()
-        };
-        std::fs::write(
-            format!("{}/{}_{}.vif", dir, collection, vid),
-            serde_json::to_string(&vif).unwrap(),
-        )
-        .unwrap();
-    }
-
-    /// An empty `.dat` (<= a superblock, i.e. zero needles) for an EC volume
-    /// is a leftover stub from the pre-fix loader. It must be swept on startup,
-    /// not loaded as a phantom empty volume. With the same vid's stub on two
-    /// disks this also unblocks startup, which previously failed the
-    /// duplicate-vid check (the volume6 incident).
-    #[test]
-    fn test_empty_ec_dat_stub_removed_and_unblocks_startup() {
-        let tmp = TempDir::new().unwrap();
-        let d0 = tmp.path().join("data0");
-        let d1 = tmp.path().join("data1");
-        std::fs::create_dir_all(&d0).unwrap();
-        std::fs::create_dir_all(&d1).unwrap();
-        let coll = "warp-cal";
-        let vid = 41u32;
-
-        // A real (loadable) but empty superblock: without the sweep both disks
-        // load it as vid 41 and add_location fails the duplicate-vid check.
-        let stub = crate::storage::super_block::SuperBlock {
-            version: crate::storage::types::Version::current(),
-            ..Default::default()
-        }
-        .to_bytes();
-        for d in [&d0, &d1] {
-            let dir = d.to_str().unwrap();
-            std::fs::write(format!("{}/{}_{}.dat", dir, coll, vid), &stub).unwrap();
-            write_ec_vif(dir, coll, vid);
-        }
-
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        for d in [&d0, &d1] {
-            store
-                .add_location(
-                    d.to_str().unwrap(),
-                    d.to_str().unwrap(),
-                    100,
-                    DiskType::HardDrive,
-                    MinFreeSpace::Percent(0.0),
-                    Vec::new(),
-                )
-                .expect("a same-vid empty stub on two disks must not block startup");
-        }
-
-        let loaded: usize = store.locations.iter().map(|l| l.volume_ids().len()).sum();
-        assert_eq!(loaded, 0, "empty EC stub was loaded as a phantom volume");
-        for d in [&d0, &d1] {
-            assert!(
-                !std::path::Path::new(&format!("{}/{}_{}.dat", d.to_str().unwrap(), coll, vid))
-                    .exists(),
-                "empty .dat stub was not removed",
-            );
-        }
-    }
-
-    /// Safety: an empty `.dat` for a NON-EC volume (no EC `.vif`) is left
-    /// alone — only EC stubs are swept, so freshly-allocated empty volumes
-    /// survive.
-    #[test]
-    fn test_keeps_empty_dat_for_non_ec_volume() {
-        let tmp = TempDir::new().unwrap();
-        let dir = tmp.path().join("data0");
-        std::fs::create_dir_all(&dir).unwrap();
-        let dat = format!("{}/7.dat", dir.to_str().unwrap());
-        std::fs::write(&dat, vec![0u8; 8]).unwrap();
-
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        store
-            .add_location(
-                dir.to_str().unwrap(),
-                dir.to_str().unwrap(),
-                100,
-                DiskType::HardDrive,
-                MinFreeSpace::Percent(0.0),
-                Vec::new(),
-            )
-            .unwrap();
-
-        assert!(
-            std::path::Path::new(&dat).exists(),
-            "a non-EC empty .dat must not be swept",
-        );
-    }
-
-    /// Regression: a lone `.vif` whose `.ecx` is on a sibling disk must not
-    /// make the loader create a phantom `.dat`, nor the sibling-.dat prune
-    /// delete the real shards on the sibling.
-    #[test]
-    fn test_lone_vif_does_not_create_phantom_dat_or_delete_shards() {
-        let tmp = TempDir::new().unwrap();
-        let d0 = tmp.path().join("data0");
-        let d1 = tmp.path().join("data1");
-        std::fs::create_dir_all(&d0).unwrap();
-        std::fs::create_dir_all(&d1).unwrap();
-        let coll = "warp-loadtest";
-        let vid = 57u32;
-
-        // d0: a self-contained but partial (2 < 10) EC volume.
-        write_shard(d0.to_str().unwrap(), coll, vid, 2);
-        write_shard(d0.to_str().unwrap(), coll, vid, 4);
-        write_index_files(d0.to_str().unwrap(), coll, vid, 10, 4);
-
-        // d1: ONLY the mirrored `.vif` — no `.ecx`, no shard, no `.dat`.
-        std::fs::copy(
-            d0.join(format!("{}_{}.vif", coll, vid)),
-            d1.join(format!("{}_{}.vif", coll, vid)),
-        )
-        .unwrap();
-
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        for d in [&d0, &d1] {
-            store
-                .add_location(
-                    d.to_str().unwrap(),
-                    d.to_str().unwrap(),
-                    100,
-                    DiskType::HardDrive,
-                    MinFreeSpace::Percent(0.0),
-                    Vec::new(),
-                )
-                .unwrap();
-        }
-
-        let phantom = d1.join(format!("{}_{}.dat", coll, vid));
-        assert!(
-            !phantom.exists(),
-            "loader created a phantom .dat from a lone .vif"
-        );
-        let total: usize = store.locations.iter().map(|l| l.ec_shard_count()).sum();
-        assert_eq!(total, 2, "real EC shards were deleted (total={})", total);
-    }
-
-    /// A disk holding a few local shards of a healthy distributed EC volume
-    /// plus a leftover empty `.dat` stub: the stub must be swept before EC
-    /// validation can mistake the volume for an interrupted local encode
-    /// (fewer than data_shards local shards) and delete the only copies of
-    /// those shards.
-    #[test]
-    fn test_empty_dat_stub_next_to_ecx_does_not_delete_shards() {
-        let tmp = TempDir::new().unwrap();
-        let d0 = tmp.path().join("data0");
-        std::fs::create_dir_all(&d0).unwrap();
-        let dir = d0.to_str().unwrap();
-        let coll = "warp-rec";
-        let vid = 87u32;
-
-        write_shard(dir, coll, vid, 0);
-        write_shard(dir, coll, vid, 5);
-        write_index_files(dir, coll, vid, 10, 4);
-        // Zero-byte stub .dat: the phantom left by the pre-fix loader.
-        std::fs::write(d0.join(format!("{}_{}.dat", coll, vid)), b"").unwrap();
-
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        store
-            .add_location(
-                dir,
-                dir,
-                100,
-                DiskType::HardDrive,
-                MinFreeSpace::Percent(0.0),
-                Vec::new(),
-            )
-            .unwrap();
-
-        assert!(
-            !d0.join(format!("{}_{}.dat", coll, vid)).exists(),
-            "empty .dat stub was not swept"
-        );
-        assert_eq!(
-            store.locations[0].ec_shard_count(),
-            2,
-            "EC shards were deleted on the stub's account"
-        );
-        assert_eq!(
-            store.locations[0].volume_ids().len(),
-            0,
-            "stub was loaded as a phantom volume"
-        );
-    }
-
-    /// Same shard-holding disk but the stub has no `.vif` at all, so the
-    /// sweep has no EC evidence and must leave it. The empty `.dat` still
-    /// must not count as an encode source: the shards survive and load as
-    /// a distributed EC volume.
-    #[test]
-    fn test_empty_dat_without_vif_does_not_delete_shards() {
-        let tmp = TempDir::new().unwrap();
-        let d0 = tmp.path().join("data0");
-        std::fs::create_dir_all(&d0).unwrap();
-        let dir = d0.to_str().unwrap();
-        let coll = "warp-rec";
-        let vid = 88u32;
-
-        write_shard(dir, coll, vid, 1);
-        write_shard(dir, coll, vid, 7);
-        write_index_files(dir, coll, vid, 10, 4);
-        std::fs::remove_file(d0.join(format!("{}_{}.vif", coll, vid))).unwrap();
-        std::fs::write(d0.join(format!("{}_{}.dat", coll, vid)), b"").unwrap();
-
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        store
-            .add_location(
-                dir,
-                dir,
-                100,
-                DiskType::HardDrive,
-                MinFreeSpace::Percent(0.0),
-                Vec::new(),
-            )
-            .unwrap();
-
-        assert_eq!(
-            store.locations[0].ec_shard_count(),
-            2,
-            "EC shards were deleted on the empty .dat's account"
-        );
-        assert_eq!(
-            store.locations[0].volume_ids().len(),
-            0,
-            "empty .dat was loaded as a phantom volume"
-        );
-    }
-
-    /// Regression for the prune credibility gate: when the EC `.vif`
-    /// records no source size (`dat_file_size == 0`), an empty 8-byte
-    /// sibling `.dat` stub must NOT justify deleting partial EC shards.
-    #[test]
-    fn test_prune_refuses_unverifiable_8byte_dat() {
-        let tmp = TempDir::new().unwrap();
-        let d0 = tmp.path().join("data0");
-        let d1 = tmp.path().join("data1");
-        std::fs::create_dir_all(&d0).unwrap();
-        std::fs::create_dir_all(&d1).unwrap();
-        let coll = "warp-loadtest";
-        let vid = 99u32;
-
-        // d0: partial EC (2 shards); its `.vif` records no source size.
-        write_shard(d0.to_str().unwrap(), coll, vid, 0);
-        write_shard(d0.to_str().unwrap(), coll, vid, 1);
-        write_index_files(d0.to_str().unwrap(), coll, vid, 10, 4);
-
-        // d1: a real but empty 8-byte (superblock-sized) `.dat` stub.
-        std::fs::write(d1.join(format!("{}_{}.dat", coll, vid)), vec![0u8; 8]).unwrap();
-
-        let mut store = Store::new(NeedleMapKind::InMemory);
-        for d in [&d0, &d1] {
-            store
-                .add_location(
-                    d.to_str().unwrap(),
-                    d.to_str().unwrap(),
-                    100,
-                    DiskType::HardDrive,
-                    MinFreeSpace::Percent(0.0),
-                    Vec::new(),
-                )
-                .unwrap();
-        }
-
-        let total: usize = store.locations.iter().map(|l| l.ec_shard_count()).sum();
-        assert_eq!(
-            total, 2,
-            "prune deleted shards against an unverifiable 8-byte .dat (total={})",
-            total
-        );
-    }
-
     /// Reproduces the orphan-shard layout from issue #9212. Shards live
     /// on dir0; the .ecx / .ecj / .vif live on dir1. Without
     /// reconciliation, dir0's shards are silently dropped at startup.
@@ -1044,7 +743,7 @@ mod tests {
         let (mut store, _tmp) = build_split_disk_store(7003);
         let vid = VolumeId(7003);
 
-        store.unmount_ec_shard(vid, 1, 0).unwrap();
+        store.unmount_ec_shard(vid, 1).unwrap();
         assert_eq!(store.find_ec_shard_location(vid, 1), None);
         // Other shards untouched.
         assert_eq!(store.find_ec_shard_location(vid, 0), Some(0));
@@ -1267,39 +966,20 @@ mod tests {
         std::fs::create_dir_all(&dat_dir).unwrap();
         std::fs::create_dir_all(&ec_dir).unwrap();
 
-        // Real collection so the loader's volume_file_name-based `.ecx`
-        // lookup matches the helpers (empty collection emits `_122.*` vs the
-        // expected `122.*`).
-        let collection = "pics";
+        let collection = "";
         let vid = 122u32;
 
-        // Disk A (sdd): a .dat whose size must byte-exactly match the EC
-        // source size recorded in the sibling .vif for the prune to treat it
-        // as the committed source.
-        let dat_path = dat_dir.join(format!("{}_{}.dat", collection, vid));
+        // Disk A (sdd): a non-trivial .dat plus the regular-volume
+        // sidecars. The .dat content doesn't matter for the prune —
+        // load_existing_volumes refuses to mount it because the
+        // superblock is absent — but its file name has to be present
+        // so index_dat_owners records this disk as the .dat owner.
+        let dat_path = dat_dir.join(format!("{}_{}.dat", collection, vid).trim_start_matches('_'));
         std::fs::write(&dat_path, vec![0u8; 1024]).unwrap();
 
         // Disk B (sdf): partial EC — one shard, plus .ecx / .ecj / .vif.
         write_shard(ec_dir.to_str().unwrap(), collection, vid, 1);
         write_index_files(ec_dir.to_str().unwrap(), collection, vid, 10, 4);
-        // Record the encode-time source size in the EC .vif so the prune's
-        // byte-exact credibility gate recognizes the 1024-byte sibling .dat as
-        // the source (a real encoded volume records this).
-        std::fs::write(
-            ec_dir.join(format!("{}_{}.vif", collection, vid)),
-            serde_json::to_string(&VifVolumeInfo {
-                version: 3,
-                dat_file_size: 1024,
-                ec_shard_config: Some(VifEcShardConfig {
-                    data_shards: 10,
-                    parity_shards: 4,
-                    ..Default::default()
-                }),
-                ..Default::default()
-            })
-            .unwrap(),
-        )
-        .unwrap();
 
         let mut store = Store::new(NeedleMapKind::InMemory);
         store
@@ -1398,7 +1078,6 @@ mod tests {
             ec_shard_config: Some(VifEcShardConfig {
                 data_shards: 10,
                 parity_shards: 4,
-                ..Default::default()
             }),
             ..Default::default()
         };

seaweed-volume/src/storage/volume.rs

@@ -189,10 +189,6 @@ pub struct VifEcShardConfig {
     pub data_shards: u32,
     #[serde(default, rename = "parityShards")]
     pub parity_shards: u32,
-    /// Encode time (unix nanos). Shards and .ecx of one encode run share it,
-    /// so a read served from a different run's shard is rejected.
-    #[serde(default, rename = "encodeTsNs", with = "string_or_i64")]
-    pub encode_ts_ns: i64,
 }
 
 /// Serde-compatible representation of OldVersionVolumeInfo for legacy .vif JSON deserialization.
@@ -283,7 +279,6 @@ impl VifVolumeInfo {
             ec_shard_config: pb.ec_shard_config.as_ref().map(|c| VifEcShardConfig {
                 data_shards: c.data_shards,
                 parity_shards: c.parity_shards,
-                encode_ts_ns: c.encode_ts_ns,
             }),
         }
     }
@@ -314,7 +309,6 @@ impl VifVolumeInfo {
                 crate::pb::volume_server_pb::EcShardConfig {
                     data_shards: c.data_shards,
                     parity_shards: c.parity_shards,
-                    encode_ts_ns: c.encode_ts_ns,
                 }
             }),
         }
@@ -588,36 +582,6 @@ impl Volume {
         Ok(v)
     }
 
-    /// Build a minimal, unloaded Volume holding only the identity/path fields
-    /// reconcile_compact_state needs. Used by the disk-location load pre-pass to
-    /// recover an interrupted commit before the volume itself is loaded.
-    pub fn new_unloaded(dirname: &str, dir_idx: &str, collection: &str, id: VolumeId) -> Self {
-        Volume {
-            id,
-            dir: dirname.to_string(),
-            dir_idx: dir_idx.to_string(),
-            collection: collection.to_string(),
-            dat_file: None,
-            remote_dat_file: None,
-            nm: None,
-            needle_map_kind: NeedleMapKind::InMemory,
-            data_file_access_control: Arc::new(DataFileAccessControl::default()),
-            super_block: SuperBlock::default(),
-            no_write_or_delete: false,
-            no_write_can_delete: false,
-            location_disk_space_low: Arc::new(AtomicBool::new(false)),
-            last_modified_ts_seconds: 0,
-            last_append_at_ns: 0,
-            last_compact_index_offset: 0,
-            last_compact_revision: 0,
-            is_compacting: false,
-            compaction_byte_per_second: 0,
-            last_io_error: Mutex::new(None),
-            volume_info: PbVolumeInfo::default(),
-            has_remote_file: false,
-        }
-    }
-
     /// Returns true if the volume is currently being compacted.
     pub fn is_compacting(&self) -> bool {
         self.is_compacting
@@ -2927,14 +2891,28 @@ impl Volume {
         self.dat_file = None;
         self.remote_dat_file = None;
 
-        // makeup_diff has fsynced the .cpd/.cpx contents. Persist a durable .cpc
-        // commit marker BEFORE the renames so the two-rename swap is atomic
-        // across a crash: a marker on disk means the swap is decided and
-        // reconcile rolls forward; no marker means roll back. Without it, a
-        // crash between the two renames leaves a stale .idx that a later vacuum
-        // compacts to empty.
-        self.write_compact_commit_marker()?;
-        self.apply_compact_swap()?;
+        let cpd_path = self.file_name(".cpd");
+        let cpx_path = self.file_name(".cpx");
+        let dat_path = self.file_name(".dat");
+        let idx_path = self.file_name(".idx");
+
+        // Check that compact files exist
+        if !Path::new(&cpd_path).exists() || !Path::new(&cpx_path).exists() {
+            return Err(VolumeError::Io(io::Error::new(
+                io::ErrorKind::NotFound,
+                "compact files (.cpd/.cpx) not found",
+            )));
+        }
+
+        // Swap files: .cpd → .dat, .cpx → .idx
+        fs::rename(&cpd_path, &dat_path)?;
+        fs::rename(&cpx_path, &idx_path)?;
+
+        // Remove any leveldb/redb index files (rebuilt from .idx on reload)
+        let ldb_path = self.file_name(".ldb");
+        let _ = fs::remove_dir_all(&ldb_path);
+        let rdb_path = self.file_name(".rdb");
+        let _ = fs::remove_file(&rdb_path);
 
         // Reload
         self.load(true, false, 0, self.version())?;
@@ -2942,165 +2920,30 @@ impl Volume {
         Ok(())
     }
 
-    /// Write and fsync the .cpc marker, then fsync the directory so the marker
-    /// survives a crash before apply_compact_swap.
-    fn write_compact_commit_marker(&self) -> Result<(), VolumeError> {
-        let marker_path = self.file_name(".cpc");
-        let f = OpenOptions::new()
-            .write(true)
-            .create(true)
-            .truncate(true)
-            .open(&marker_path)?;
-        f.sync_all()?;
-        drop(f);
-        fsync_dir(&marker_path)?;
-        Ok(())
-    }
-
-    /// Perform the durable two-rename compaction commit. Idempotent: run both at
-    /// the tail of commit and by reconcile rolling forward after a crash. It
-    /// requires BOTH .cpd/.cpx to be present, so a stale or duplicate commit
-    /// returns an error without deleting the live .dat/.idx.
-    fn apply_compact_swap(&self) -> Result<(), VolumeError> {
-        // Normal commit: both temp files must be present, or a stale/duplicate
-        // commit could clobber the live .dat/.idx.
-        if !Path::new(&self.file_name(".cpd")).exists()
-            || !Path::new(&self.file_name(".cpx")).exists()
-        {
-            return Err(VolumeError::Io(io::Error::new(
-                io::ErrorKind::NotFound,
-                "compact files (.cpd/.cpx) not found",
-            )));
-        }
-        self.finish_compact_swap()
-    }
-
-    /// Renames whichever compaction temp file is still present (.cpd->.dat,
-    /// .cpx->.idx), fsyncs, drops the stale .ldb/.rdb, then clears the .cpc
-    /// marker. Tolerates a partial state: a crash after the .dat rename but
-    /// before the .idx rename leaves only .cpx, which must still be applied --
-    /// not abandoned, which would pair a fresh .dat with a stale .idx. Existence
-    /// is checked robustly so a transient error never silently skips the swap.
-    fn finish_compact_swap(&self) -> Result<(), VolumeError> {
-        let exists = |p: String| match fs::metadata(&p) {
-            Ok(_) => true,
-            Err(e) if e.kind() == io::ErrorKind::NotFound => false,
-            Err(_) => true,
-        };
-        let dat_path = self.file_name(".dat");
-        let idx_path = self.file_name(".idx");
-        let cpd_exists = exists(self.file_name(".cpd"));
-        let cpx_exists = exists(self.file_name(".cpx"));
-
-        if cpd_exists {
-            #[cfg(windows)]
-            {
-                let _ = fs::remove_file(&dat_path);
-            }
-            fs::rename(self.file_name(".cpd"), &dat_path)?;
-        }
-        if cpx_exists {
-            #[cfg(windows)]
-            {
-                let _ = fs::remove_file(&idx_path);
-            }
-            fs::rename(self.file_name(".cpx"), &idx_path)?;
-        }
-        if cpd_exists || cpx_exists {
-            fsync_dir(&dat_path)?;
-            if self.dir != self.dir_idx {
-                fsync_dir(&idx_path)?;
-            }
-            let _ = fs::remove_dir_all(self.file_name(".ldb"));
-            let _ = fs::remove_file(self.file_name(".rdb"));
-        }
-
-        // Clear the marker last and fsync the dir so a restart does not re-run a
-        // completed swap.
-        let marker_path = self.file_name(".cpc");
-        match fs::remove_file(&marker_path) {
-            Ok(()) => {}
-            Err(e) if e.kind() == io::ErrorKind::NotFound => {}
-            Err(e) => return Err(e.into()),
-        }
-        fsync_dir(&marker_path)?;
-        Ok(())
-    }
-
-    /// Recover an interrupted compaction commit on load. When the .cpc marker is
-    /// present the swap was decided, so roll FORWARD by finishing the renames;
-    /// when it is absent any leftover .cpd/.cpx are an abandoned generation, so
-    /// roll BACK by deleting them (and any stale .ldb beside a healthy .idx).
-    pub fn reconcile_compact_state(&self) -> Result<(), VolumeError> {
-        let cpc_path = self.file_name(".cpc");
-        if Path::new(&cpc_path).exists() {
-            // Marker present: the swap was decided. Finish whichever rename is
-            // still pending -- a crash may have completed only the .dat rename,
-            // so the lone remaining .cpx must still be applied, not abandoned.
-            // If neither temp file remains, finish_compact_swap clears the marker.
-            tracing::info!(
-                volume_id = self.id.0,
-                "rolling forward interrupted compaction commit"
-            );
-            return self.finish_compact_swap();
-        }
-
-        // No marker: roll back any orphan compaction temp files.
-        let mut rolled_back = false;
-        for ext in &[".cpd", ".cpx"] {
-            let p = self.file_name(ext);
-            if Path::new(&p).exists() {
-                tracing::info!(
-                    volume_id = self.id.0,
-                    "rolling back orphan compaction file {}",
-                    ext
-                );
-                match fs::remove_file(&p) {
-                    Ok(()) => {}
-                    Err(e) if e.kind() == io::ErrorKind::NotFound => {}
-                    Err(e) => return Err(e.into()),
-                }
-                rolled_back = true;
-            }
-        }
-        if rolled_back {
-            let _ = fs::remove_dir_all(self.file_name(".ldb"));
-            let _ = fs::remove_file(self.file_name(".rdb"));
-        }
-        Ok(())
-    }
-
     /// Clean up leftover compaction files (.cpd, .cpx).
     pub fn cleanup_compact(&self) -> Result<(), VolumeError> {
-        // Refuse to unlink .cpd/.cpx while a .cpc marker exists: those temp files
-        // are the only inputs reconcile can roll forward to, so removing them
-        // mid-commit would strand a decided swap.
-        if Path::new(&self.file_name(".cpc")).exists() {
-            return Err(VolumeError::Io(io::Error::new(
-                io::ErrorKind::Other,
-                format!(
-                    "volume {}: refusing cleanup while commit marker present",
-                    self.id
-                ),
-            )));
-        }
-
         let cpd_path = self.file_name(".cpd");
         let cpx_path = self.file_name(".cpx");
         let cpldb_path = self.file_name(".cpldb");
-        let cpc_path = self.file_name(".cpc");
 
         let e1 = fs::remove_file(&cpd_path);
         let e2 = fs::remove_file(&cpx_path);
         let e3 = fs::remove_dir_all(&cpldb_path);
-        let e4 = fs::remove_file(&cpc_path);
 
         // Ignore NotFound errors
-        for e in [e1, e2, e3, e4] {
-            if let Err(e) = e {
-                if e.kind() != io::ErrorKind::NotFound {
-                    return Err(e.into());
-                }
+        if let Err(e) = e1 {
+            if e.kind() != io::ErrorKind::NotFound {
+                return Err(e.into());
+            }
+        }
+        if let Err(e) = e2 {
+            if e.kind() != io::ErrorKind::NotFound {
+                return Err(e.into());
+            }
+        }
+        if let Err(e) = e3 {
+            if e.kind() != io::ErrorKind::NotFound {
+                return Err(e.into());
             }
         }
 
@@ -3228,15 +3071,11 @@ impl Volume {
                 let bytes = fake_del_needle.write_bytes(version);
                 dst_dat.write_all(&bytes)?;
 
-                // Record the tombstone's real .dat offset, like the normal delete path,
-                // so a deletion left at the .dat tail stays visible to the integrity
-                // check on reload. Offset 0 hid the trailing tombstone and falsely
-                // flipped the volume read-only.
                 let mut idx_entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
                 crate::storage::types::idx_entry_to_bytes(
                     &mut idx_entry_buf,
                     key,
-                    Offset::from_actual_offset(dat_offset as i64),
+                    Offset::from_actual_offset(0),
                     Size(crate::storage::types::TOMBSTONE_FILE_SIZE.into()),
                 );
                 dst_idx.write_all(&idx_entry_buf)?;
@@ -3318,34 +3157,12 @@ impl Volume {
             }
         }
 
-        // A regular volume and an EC volume for the same id share <base>.vif.
-        // When EC artefacts coexist on this disk (e.g. shards distributed onto
-        // a source replica before it is deleted), keep the .vif so removing the
-        // regular volume does not strip the EC volume's info file.
-        let keep_vif = self.shares_vif_with_ec_volume();
         self.close();
-        remove_volume_files(&self.data_file_name(), keep_vif);
-        remove_volume_files(&self.index_file_name(), keep_vif);
+        remove_volume_files(&self.data_file_name());
+        remove_volume_files(&self.index_file_name());
         Ok(())
     }
 
-    /// Reports whether an EC volume for this id has a sealed .ecx on the same
-    /// disk, in which case its .vif is the same file as the regular volume's
-    /// and must outlive the regular volume's deletion. Mirrors the on-disk
-    /// portion of Go's Volume.sharesVifWithEcVolume / HasEcxFileOnDisk.
-    fn shares_vif_with_ec_volume(&self) -> bool {
-        let has_ecx = |base: &str| -> bool {
-            fs::metadata(format!("{}.ecx", base))
-                .map(|m| !m.is_dir() && m.len() > 0)
-                .unwrap_or(false)
-        };
-        if has_ecx(&volume_file_name(&self.dir_idx, &self.collection, self.id)) {
-            return true;
-        }
-        self.dir != self.dir_idx
-            && has_ecx(&volume_file_name(&self.dir, &self.collection, self.id))
-    }
-
     /// Check if an I/O error is EIO (errno 5) and record it for health monitoring.
     /// On success (None), clears any previously recorded EIO error.
     /// Matches Go's `checkReadWriteError` in volume_write.go.
@@ -3414,35 +3231,10 @@ fn get_append_at_ns(last: u64) -> u64 {
 
 /// Remove all files associated with a volume.
 /// .dat/.idx removals log at info level so destructive calls are traceable.
-/// fsync the parent directory of `path` so a rename/create/unlink inside it is
-/// durable, propagating a sync failure so the commit path can abort rather than
-/// proceed with an undurable rename or marker. A path with no openable parent is
-/// tolerated; directory fsync is unsupported on Windows, so it is a no-op there
-/// (matching the Go fsyncDir helper, which ignores that error).
-pub(crate) fn fsync_dir(path: &str) -> io::Result<()> {
-    #[cfg(windows)]
-    {
-        let _ = path;
-        Ok(())
-    }
-    #[cfg(not(windows))]
-    {
-        if let Some(parent) = Path::new(path).parent() {
-            if let Ok(d) = File::open(parent) {
-                return d.sync_all();
-            }
-        }
-        Ok(())
-    }
-}
-
-pub(crate) fn remove_volume_files(base: &str, keep_vif: bool) {
+pub(crate) fn remove_volume_files(base: &str) {
     for ext in &[
-        ".dat", ".idx", ".vif", ".sdx", ".cpd", ".cpx", ".cpc", ".note", ".rdb",
+        ".dat", ".idx", ".vif", ".sdx", ".cpd", ".cpx", ".note", ".rdb",
     ] {
-        if *ext == ".vif" && keep_vif {
-            continue;
-        }
         let path = format!("{}{}", base, ext);
         let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
         let existed = fs::remove_file(&path).is_ok();
@@ -4948,184 +4740,4 @@ mod tests {
             ".vif removed from data dir"
         );
     }
-
-    /// When an EC volume for the same id has a sealed .ecx on the same disk, the
-    /// .vif is shared with it and must survive the regular volume's deletion.
-    #[test]
-    fn test_destroy_keeps_vif_when_ec_coexists() {
-        let tmp = TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-
-        let mut v = make_test_volume(dir);
-        let mut n = Needle {
-            id: NeedleId(1),
-            cookie: Cookie(1),
-            data: b"test".to_vec(),
-            data_size: 4,
-            ..Needle::default()
-        };
-        v.write_needle(&mut n, true).unwrap();
-
-        let vif_path = format!("{}/1.vif", dir);
-        std::fs::write(&vif_path, r#"{"version":3}"#).unwrap();
-        // A sealed .ecx marks a coexisting EC volume for the same id.
-        let ecx_path = format!("{}/1.ecx", dir);
-        std::fs::write(&ecx_path, b"ec-index").unwrap();
-
-        v.destroy(false, false).unwrap();
-
-        let dat_path = format!("{}/1.dat", dir);
-        let idx_path = format!("{}/1.idx", dir);
-        assert!(!std::path::Path::new(&dat_path).exists(), ".dat removed");
-        assert!(!std::path::Path::new(&idx_path).exists(), ".idx removed");
-        assert!(
-            std::path::Path::new(&vif_path).exists(),
-            ".vif kept: shared with the coexisting EC volume"
-        );
-        assert!(
-            std::path::Path::new(&ecx_path).exists(),
-            ".ecx is an EC sidecar, never touched here"
-        );
-    }
-
-    /// A crash after the .idx/.dat were consumed but before the .cpx->.idx
-    /// rename committed leaves only .cpd + .cpx + .cpc. reconcile_compact_state
-    /// must roll the swap FORWARD and produce a loadable, writable volume
-    /// holding the compacted needle count.
-    #[test]
-    fn test_reconcile_roll_forward_marker_only() {
-        let tmp = TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-        let mut v = make_test_volume(dir);
-
-        for i in 1..=6u64 {
-            let mut n = Needle {
-                id: NeedleId(i),
-                cookie: Cookie(i as u32),
-                data: format!("data-{}", i).into_bytes(),
-                data_size: format!("data-{}", i).len() as u32,
-                ..Needle::default()
-            };
-            v.write_needle(&mut n, true).unwrap();
-        }
-        for id in [2u64, 5u64] {
-            let mut del = Needle {
-                id: NeedleId(id),
-                cookie: Cookie(id as u32),
-                ..Needle::default()
-            };
-            v.delete_needle(&mut del).unwrap();
-        }
-        let expected_live = v.file_count() - v.deleted_count();
-
-        v.compact_by_index(0, 0, |_| true).unwrap();
-        v.close();
-
-        let cpd = v.file_name(".cpd");
-        let cpx = v.file_name(".cpx");
-        let cpc = v.file_name(".cpc");
-        let dat = v.file_name(".dat");
-        let idx = v.file_name(".idx");
-        assert!(Path::new(&cpd).exists());
-        assert!(Path::new(&cpx).exists());
-
-        // Simulate the mid-commit crash: original .dat/.idx are gone and only
-        // the compacted temp files plus the commit marker survive.
-        fs::remove_file(&dat).unwrap();
-        fs::remove_file(&idx).unwrap();
-        let _ = fs::remove_dir_all(v.file_name(".ldb"));
-        fs::write(&cpc, b"").unwrap();
-
-        v.reconcile_compact_state().unwrap();
-
-        assert!(!Path::new(&cpd).exists(), ".cpd consumed by roll-forward");
-        assert!(!Path::new(&cpx).exists(), ".cpx consumed by roll-forward");
-        assert!(!Path::new(&cpc).exists(), ".cpc cleared after swap");
-        assert!(Path::new(&dat).exists(), ".dat restored by roll-forward");
-        assert!(Path::new(&idx).exists(), ".idx restored by roll-forward");
-
-        // The rolled-forward files must load as a writable volume with the
-        // compacted needle count.
-        let reloaded = make_test_volume(dir);
-        assert!(!reloaded.is_read_only(), "rolled-forward volume read-only");
-        assert_eq!(
-            reloaded.file_count(),
-            expected_live,
-            "rolled-forward volume needle count"
-        );
-    }
-
-    /// With no .cpc marker, leftover .cpd/.cpx are an abandoned generation;
-    /// reconcile must roll BACK by deleting them and leave the live .dat/.idx.
-    #[test]
-    fn test_reconcile_roll_back_no_marker() {
-        let tmp = TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-        let mut v = make_test_volume(dir);
-
-        for i in 1..=4u64 {
-            let mut n = Needle {
-                id: NeedleId(i),
-                cookie: Cookie(i as u32),
-                data: format!("data-{}", i).into_bytes(),
-                data_size: format!("data-{}", i).len() as u32,
-                ..Needle::default()
-            };
-            v.write_needle(&mut n, true).unwrap();
-        }
-        v.compact_by_index(0, 0, |_| true).unwrap();
-
-        let cpd = v.file_name(".cpd");
-        let cpx = v.file_name(".cpx");
-        let dat = v.file_name(".dat");
-        let idx = v.file_name(".idx");
-        assert!(Path::new(&cpd).exists());
-        assert!(Path::new(&cpx).exists());
-        // No .cpc marker exists.
-
-        v.reconcile_compact_state().unwrap();
-
-        assert!(!Path::new(&cpd).exists(), ".cpd rolled back");
-        assert!(!Path::new(&cpx).exists(), ".cpx rolled back");
-        assert!(Path::new(&dat).exists(), "live .dat kept");
-        assert!(Path::new(&idx).exists(), "live .idx kept");
-    }
-
-    /// apply_compact_swap must abort without touching the live .dat/.idx when
-    /// the .cpd/.cpx temp files are missing (a stale or duplicate commit).
-    #[test]
-    fn test_apply_compact_swap_missing_temp_files_preserves_live() {
-        let tmp = TempDir::new().unwrap();
-        let dir = tmp.path().to_str().unwrap();
-        let mut v = make_test_volume(dir);
-
-        for i in 1..=3u64 {
-            let mut n = Needle {
-                id: NeedleId(i),
-                cookie: Cookie(i as u32),
-                data: format!("data-{}", i).into_bytes(),
-                data_size: format!("data-{}", i).len() as u32,
-                ..Needle::default()
-            };
-            v.write_needle(&mut n, true).unwrap();
-        }
-        v.sync_to_disk().unwrap();
-
-        let dat = v.file_name(".dat");
-        let idx = v.file_name(".idx");
-        let dat_len_before = fs::metadata(&dat).unwrap().len();
-
-        // No .cpd/.cpx present: the swap must refuse.
-        assert!(
-            v.apply_compact_swap().is_err(),
-            "swap should error when .cpd/.cpx are missing"
-        );
-        assert!(Path::new(&dat).exists(), "live .dat preserved");
-        assert!(Path::new(&idx).exists(), "live .idx preserved");
-        assert_eq!(
-            fs::metadata(&dat).unwrap().len(),
-            dat_len_before,
-            "live .dat unchanged"
-        );
-    }
 }

seaweedfs-rdma-sidecar/rdma-engine/Cargo.lock

@@ -161,29 +161,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "bindgen"
-version = "0.69.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
-dependencies = [
- "bitflags 2.9.2",
- "cexpr",
- "clang-sys",
- "itertools",
- "lazy_static",
- "lazycell",
- "log",
- "prettyplease",
- "proc-macro2",
- "quote",
- "regex",
- "rustc-hash",
- "shlex",
- "syn",
- "which",
-]
-
 [[package]]
 name = "bit-set"
 version = "0.8.0"
@@ -253,15 +230,6 @@ dependencies = [
  "shlex",
 ]
 
-[[package]]
-name = "cexpr"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
-dependencies = [
- "nom",
-]
-
 [[package]]
 name = "cfg-if"
 version = "1.0.1"
@@ -280,7 +248,7 @@ dependencies = [
  "num-traits",
  "serde",
  "wasm-bindgen",
- "windows-link 0.1.3",
+ "windows-link",
 ]
 
 [[package]]
@@ -310,17 +278,6 @@ dependencies = [
  "half",
 ]
 
-[[package]]
-name = "clang-sys"
-version = "1.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
-dependencies = [
- "glob",
- "libc",
- "libloading",
-]
-
 [[package]]
 name = "clap"
 version = "4.5.45"
@@ -573,12 +530,6 @@ version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
 
-[[package]]
-name = "glob"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
-
 [[package]]
 name = "half"
 version = "2.6.0"
@@ -610,15 +561,6 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
 
-[[package]]
-name = "home"
-version = "0.5.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d"
-dependencies = [
- "windows-sys 0.61.2",
-]
-
 [[package]]
 name = "iana-time-zone"
 version = "0.1.63"
@@ -713,12 +655,6 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
-[[package]]
-name = "lazycell"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
-
 [[package]]
 name = "libc"
 version = "0.2.175"
@@ -741,12 +677,6 @@ version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
 
-[[package]]
-name = "linux-raw-sys"
-version = "0.4.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.9.4"
@@ -793,15 +723,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "memoffset"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -989,12 +910,6 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
 
-[[package]]
-name = "pkg-config"
-version = "0.3.33"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
-
 [[package]]
 name = "plotters"
 version = "0.3.7"
@@ -1032,16 +947,6 @@ dependencies = [
  "zerocopy",
 ]
 
-[[package]]
-name = "prettyplease"
-version = "0.2.37"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
-dependencies = [
- "proc-macro2",
- "syn",
-]
-
 [[package]]
 name = "proc-macro2"
 version = "1.0.100"
@@ -1168,7 +1073,6 @@ dependencies = [
  "nix",
  "parking_lot",
  "proptest",
- "rdma-sys",
  "rmp-serde",
  "serde",
  "tempfile",
@@ -1180,17 +1084,6 @@ dependencies = [
  "uuid",
 ]
 
-[[package]]
-name = "rdma-sys"
-version = "0.3.0"
-dependencies = [
- "bindgen",
- "libc",
- "memoffset",
- "paste",
- "pkg-config",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.5.17"
@@ -1278,25 +1171,6 @@ version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace"
 
-[[package]]
-name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
-[[package]]
-name = "rustix"
-version = "0.38.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
-dependencies = [
- "bitflags 2.9.2",
- "errno",
- "libc",
- "linux-raw-sys 0.4.15",
- "windows-sys 0.59.0",
-]
-
 [[package]]
 name = "rustix"
 version = "1.0.8"
@@ -1306,7 +1180,7 @@ dependencies = [
  "bitflags 2.9.2",
  "errno",
  "libc",
- "linux-raw-sys 0.9.4",
+ "linux-raw-sys",
  "windows-sys 0.60.2",
 ]
 
@@ -1464,7 +1338,7 @@ dependencies = [
  "fastrand",
  "getrandom 0.3.3",
  "once_cell",
- "rustix 1.0.8",
+ "rustix",
  "windows-sys 0.59.0",
 ]
 
@@ -1797,18 +1671,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "which"
-version = "4.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
-dependencies = [
- "either",
- "home",
- "once_cell",
- "rustix 0.38.44",
-]
-
 [[package]]
 name = "winapi-util"
 version = "0.1.9"
@@ -1826,7 +1688,7 @@ checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
 dependencies = [
  "windows-implement",
  "windows-interface",
- "windows-link 0.1.3",
+ "windows-link",
  "windows-result",
  "windows-strings",
 ]
@@ -1859,19 +1721,13 @@ version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
 
-[[package]]
-name = "windows-link"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
-
 [[package]]
 name = "windows-result"
 version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
 dependencies = [
- "windows-link 0.1.3",
+ "windows-link",
 ]
 
 [[package]]
@@ -1880,7 +1736,7 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
 dependencies = [
- "windows-link 0.1.3",
+ "windows-link",
 ]
 
 [[package]]
@@ -1910,15 +1766,6 @@ dependencies = [
  "windows-targets 0.53.3",
 ]
 
-[[package]]
-name = "windows-sys"
-version = "0.61.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
-dependencies = [
- "windows-link 0.2.1",
-]
-
 [[package]]
 name = "windows-targets"
 version = "0.52.6"
@@ -1941,7 +1788,7 @@ version = "0.53.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91"
 dependencies = [
- "windows-link 0.1.3",
+ "windows-link",
  "windows_aarch64_gnullvm 0.53.0",
  "windows_aarch64_msvc 0.53.0",
  "windows_i686_gnu 0.53.0",

seaweedfs-rdma-sidecar/rdma-engine/Cargo.toml

@@ -10,24 +10,6 @@ license = "Apache-2.0"
 name = "rdma-engine-server"
 path = "src/main.rs"
 
-# Phase-1 proof: a real IBV_WR_RDMA_READ over SoftRoCE loopback (requires Linux +
-# libibverbs/librdmacm). Build with: cargo build --features real-rdma --bin rdma-loopback
-[[bin]]
-name = "rdma-loopback"
-path = "src/bin/rdma_loopback.rs"
-required-features = ["real-rdma"]
-
-# Standalone cross-process RDMA target + client (real file data over rxe).
-[[bin]]
-name = "rdma-target"
-path = "src/bin/rdma_target.rs"
-required-features = ["real-rdma"]
-
-[[bin]]
-name = "rdma-client"
-path = "src/bin/rdma_client.rs"
-required-features = ["real-rdma"]
-
 [lib]
 name = "rdma_engine"
 path = "src/lib.rs"
@@ -66,10 +48,6 @@ parking_lot = "0.12"  # Fast mutexes
 nix = { version = "0.27", features = ["mman"] }  # Unix domain sockets and system calls
 async-trait = "0.1"  # Async traits
 
-# Real RDMA via libibverbs + librdmacm (raw FFI). Optional, Linux-only; pulled in
-# by the `real-rdma` feature. Used by src/rdma_real.rs (RealRdmaContext).
-rdma-sys = { version = "0.3", optional = true }
-
 # Configuration
 clap = { version = "4.0", features = ["derive"] }
 config = "0.13"
@@ -83,7 +61,6 @@ tempfile = "3.0"
 default = ["mock-ucx"]
 mock-ucx = []
 real-ucx = []  # UCX integration for production RDMA
-real-rdma = ["dep:rdma-sys"]  # real libibverbs/rdma_cm datapath (SoftRoCE/real NIC)
 
 [profile.release]
 opt-level = 3
@@ -95,6 +72,3 @@ panic = "abort"
 
 [package.metadata.docs.rs]
 features = ["real-rdma"]
-
-[patch.crates-io]
-rdma-sys = { path = "vendor/rdma-sys" }

seaweedfs-rdma-sidecar/rdma-engine/REAL_RDMA.md

@@ -1,58 +0,0 @@
-# Real RDMA datapath (`real-rdma` feature) — proof of plumbing
-
-The engine's default backend is a **mock** (`src/rdma.rs` `MockRdmaContext`): its
-`post_read` fabricates `byte = i % 256` and never touches a wire (the Go sidecar
-in `../pkg/rdma/client.go` even discards the engine's buffer and re-fakes the
-same pattern). This module replaces that with a **real** libibverbs / `rdma_cm`
-datapath, validated over SoftRoCE (rxe).
-
-## What's here
-
-- `src/rdma_real.rs`
-  - `RealRdmaContext` — the real client backend: `connect` (via `rdma_cm` /
-    `rdma_create_ep`), `register_memory`, `post_read` (a genuine
-    `IBV_WR_RDMA_READ` via `rdma_post_read`), `poll_completion`, `device_info`.
-  - `RealResponder` — a throwaway loopback target: registers a buffer for
-    `REMOTE_READ` (`rdma_reg_read`) and hands the client its `{addr, rkey, len}`.
-- `src/bin/rdma_loopback.rs` — the proof: a responder registers a buffer with a
-  verifiable pattern (a `REALRDMA` magic header — deliberately **not** the mock's
-  `i%256`), the client RDMA-READs it, and asserts the bytes match.
-
-Connection setup and QP state transitions use `rdma_cm` (much simpler than a
-manual GID/QPN/LID handshake for RoCEv2 addressing).
-
-## Build & run (Linux + SoftRoCE only)
-
-Requires `libibverbs-dev librdmacm-dev ibverbs-providers ibverbs-utils
-rdma-core clang libclang-dev`, and a SoftRoCE link:
-
-```sh
-sudo modprobe rdma_rxe rdma_ucm
-sudo rdma link add rxe0 type rxe netdev eth0     # eth0 = your primary netdev
-ibv_rc_pingpong -d rxe0 -g 1                      # sanity-check the fabric
-
-cargo build --features real-rdma --bin rdma-loopback
-sudo ./target/debug/rdma-loopback <rxe_ip> 7471 1048576   # rxe_ip = eth0's IP
-# => REAL-RDMA-READ-SUCCESS, dest head "REALRDMA", bytes match
-```
-
-(rxe on a low-vCPU VM is a **correctness** proof, not a representative benchmark.)
-
-## Why `vendor/rdma-sys`
-
-`rdma-sys 0.3.0` (crates.io) pins **bindgen 0.59.2**, which cannot parse the
-`rdma-core 56` headers (`ib_uverbs_flow_action_esp_encap_union_… is not a valid
-Ident`). `vendor/rdma-sys` is rdma-sys 0.3.0 with bindgen bumped to 0.69.4 (and
-the two methods removed in modern bindgen — `rustfmt_bindings` / `size_t_is_usize`
-— dropped from its `build.rs`); `Cargo.toml` redirects to it via
-`[patch.crates-io]`. This is a temporary workaround — upstreaming a bindgen bump
-to rdma-sys, or moving to a maintained bindings crate, is the cleaner long-term
-fix.
-
-## Not done (the real RDMA *win* is multi-week)
-
-This is Phase-1 proof-of-plumbing only. A demonstrable zero-copy read of real
-volume data still needs: a needle-backed RDMA **target** in the volume server
-(none exists today), a control-plane handshake to publish `{remote_addr, rkey}`,
-a breaking IPC-contract change (Rust + Go, in lockstep), mount wiring, and then
-the zero-copy READ path. See the scoping notes for the phased plan.

seaweedfs-rdma-sidecar/rdma-engine/src/bin/rdma_client.rs

@@ -1,58 +0,0 @@
-//! Standalone RDMA-READ client: connect to an `rdma-target`, RDMA-READ its whole
-//! registered buffer into a fresh local buffer (separate process / address
-//! space), and write it out. md5 of the output vs the target's source file
-//! proves a real cross-process RDMA read of real data.
-//!
-//!   cargo run --features real-rdma --bin rdma-client -- <target_ip> <port> <out_file> [offset] [size]
-//!
-//! With [offset]/[size] it RDMA-READs only that byte range (the real read API
-//! shape: file_id + offset + size); without them it reads the whole MR.
-
-use std::fs;
-
-use rdma_engine::rdma_real::RealRdmaContext;
-
-fn main() {
-    let args: Vec<String> = std::env::args().collect();
-    if args.len() < 4 {
-        eprintln!("usage: rdma-client <target_ip> <port> <out_file> [offset] [size]");
-        std::process::exit(2);
-    }
-    let ip = args[1].clone();
-    let port: u16 = args[2].parse().expect("port");
-    let out = &args[3];
-    let offset: u64 = args.get(4).and_then(|s| s.parse().ok()).unwrap_or(0);
-
-    let (ctx, raddr, rkey, rlen) = match RealRdmaContext::connect(&ip, port) {
-        Ok(v) => v,
-        Err(e) => {
-            eprintln!("client connect failed: {e}");
-            std::process::exit(1);
-        }
-    };
-    // Default to the rest of the MR from `offset`.
-    let size: usize = args
-        .get(5)
-        .and_then(|s| s.parse().ok())
-        .unwrap_or(rlen - offset.min(rlen as u64) as usize);
-    eprintln!(
-        "client: connected to {ip}:{port}, remote MR len={rlen} rkey=0x{rkey:x}; reading [{offset}, {})",
-        offset as usize + size
-    );
-    assert!(offset as usize + size <= rlen, "range exceeds remote MR");
-
-    let mut dest = vec![0u8; size];
-    let (_mi, mr) = ctx.register_memory(&mut dest).expect("register dest");
-    // RDMA-READ the sub-range: remote source is the MR base + offset.
-    ctx.post_read(&mut dest, mr, raddr + offset, rkey, 1)
-        .expect("post_read");
-    let wc = ctx.poll_completion().expect("poll completion");
-    ctx.finish();
-    eprintln!(
-        "client: RDMA READ status={:?} byte_len={}",
-        wc.status, wc.byte_len
-    );
-
-    fs::write(out, &dest).expect("write output");
-    eprintln!("client: wrote {} bytes to {out}", dest.len());
-}

seaweedfs-rdma-sidecar/rdma-engine/src/bin/rdma_loopback.rs

@@ -1,95 +0,0 @@
-//! Phase-1 proof: a real `IBV_WR_RDMA_READ` over SoftRoCE (rxe) loopback.
-//!
-//! Spawns a [`RealResponder`] that registers a buffer holding a verifiable
-//! pattern (with a `REALRDMA` magic header — deliberately NOT the mock's `i%256`),
-//! then a [`RealRdmaContext`] client connects, RDMA-READs the buffer, and asserts
-//! the bytes match. Proves the engine moves real bytes over a genuine RDMA verb.
-//!
-//!   rmmod-free run (after SoftRoCE is up):
-//!     cargo run --features real-rdma --bin rdma-loopback -- <rxe_ip> [port] [len]
-//!
-//! `<rxe_ip>` must be an IP on the netdev backing the rxe link (e.g. eth0's IP),
-//! not 127.0.0.1 (rxe is bound to that netdev, not loopback).
-
-use std::sync::mpsc;
-use std::time::Instant;
-
-use rdma_engine::rdma_real::{RealRdmaContext, RealResponder};
-
-fn main() {
-    let args: Vec<String> = std::env::args().collect();
-    let ip = args.get(1).cloned().unwrap_or_else(|| "127.0.0.1".into());
-    let port: u16 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(7471);
-    let len: usize = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1 << 20);
-
-    // A verifiable pattern, distinct from the mock's i%256, with a magic header.
-    let mut src = vec![0u8; len];
-    for (i, b) in src.iter_mut().enumerate() {
-        *b = (i.wrapping_mul(131).wrapping_add(17) & 0xff) as u8;
-    }
-    let magic = b"REALRDMA";
-    src[..magic.len()].copy_from_slice(magic);
-    let expected = src.clone();
-
-    let (ready_tx, ready_rx) = mpsc::channel::<()>();
-    let sip = ip.clone();
-    let server = std::thread::spawn(move || {
-        let mut data = src;
-        if let Err(e) = RealResponder::serve_once(&sip, port, &mut data, move || {
-            let _ = ready_tx.send(());
-        }) {
-            eprintln!("responder error: {e}");
-        }
-    });
-    // Wait until the responder is listening before connecting.
-    let _ = ready_rx.recv();
-
-    let t0 = Instant::now();
-    let (ctx, raddr, rkey, rlen) = match RealRdmaContext::connect(&ip, port) {
-        Ok(v) => v,
-        Err(e) => {
-            eprintln!("client connect failed: {e}");
-            let _ = server.join();
-            std::process::exit(1);
-        }
-    };
-    println!(
-        "connected: device={:?} remote MR addr=0x{raddr:x} rkey=0x{rkey:x} len={rlen}",
-        ctx.device_info().name
-    );
-
-    let mut dest = vec![0u8; rlen];
-    let (_mrinfo, mr) = ctx.register_memory(&mut dest).expect("register dest");
-    ctx.post_read(&mut dest, mr, raddr, rkey, 1).expect("post_read");
-    let wc = ctx.poll_completion().expect("poll completion");
-    let dt = t0.elapsed();
-    ctx.finish();
-    let _ = server.join();
-
-    let head: String = dest[..16.min(dest.len())]
-        .iter()
-        .map(|b| format!("{b:02x}"))
-        .collect::<Vec<_>>()
-        .join(" ");
-    println!(
-        "RDMA READ done: status={:?} byte_len={}  dest head: {head}  (\"{}\")",
-        wc.status,
-        wc.byte_len,
-        String::from_utf8_lossy(&dest[..magic.len().min(dest.len())])
-    );
-
-    if dest == expected {
-        let mibps = (rlen as f64 / (1024.0 * 1024.0)) / dt.as_secs_f64();
-        println!("VERIFY OK: {rlen} bytes match the responder's registered buffer");
-        println!("({mibps:.1} MiB/s incl. connection setup; rxe on a 2-vCPU VM is a correctness proof, not a benchmark)");
-        println!("REAL-RDMA-READ-SUCCESS");
-    } else {
-        let mismatch = dest
-            .iter()
-            .zip(expected.iter())
-            .position(|(a, b)| a != b)
-            .unwrap_or(0);
-        eprintln!("VERIFY FAILED: first mismatch at byte {mismatch}");
-        std::process::exit(1);
-    }
-}

seaweedfs-rdma-sidecar/rdma-engine/src/bin/rdma_target.rs

@@ -1,38 +0,0 @@
-//! Standalone RDMA-READ target: load a file, register it for `REMOTE_READ`, and
-//! serve one client over SoftRoCE (rxe) — a real, separate-process precursor to
-//! a volume-side RDMA target. Pairs with `rdma-client`.
-//!
-//!   cargo run --features real-rdma --bin rdma-target -- <bind_ip> <port> <file>
-//!
-//! `<bind_ip>` must be an IP on the rxe-backing netdev (e.g. eth0's IP).
-
-use std::fs;
-
-use rdma_engine::rdma_real::RealResponder;
-
-fn main() {
-    let args: Vec<String> = std::env::args().collect();
-    if args.len() < 4 {
-        eprintln!("usage: rdma-target <bind_ip> <port> <file>");
-        std::process::exit(2);
-    }
-    let ip = args[1].clone();
-    let port: u16 = args[2].parse().expect("port");
-    let file = &args[3];
-
-    let mut data = fs::read(file).expect("read source file");
-    eprintln!(
-        "target: serving {} bytes of {file} on {ip}:{port} (REMOTE_READ)",
-        data.len()
-    );
-
-    match RealResponder::serve_once(&ip, port, &mut data, || {
-        eprintln!("target: listening; waiting for a client");
-    }) {
-        Ok(()) => eprintln!("target: served one client; done"),
-        Err(e) => {
-            eprintln!("target error: {e}");
-            std::process::exit(1);
-        }
-    }
-}

seaweedfs-rdma-sidecar/rdma-engine/src/lib.rs

@@ -27,8 +27,6 @@ use anyhow::Result;
 
 pub mod ucx;
 pub mod rdma;
-#[cfg(feature = "real-rdma")]
-pub mod rdma_real; // real libibverbs/rdma_cm RDMA datapath (SoftRoCE/real NIC)
 pub mod ipc;
 pub mod session;
 pub mod memory;

seaweedfs-rdma-sidecar/rdma-engine/src/rdma_real.rs

@@ -1,360 +0,0 @@
-//! Real RDMA datapath over libibverbs + librdmacm (`rdma_cm`).
-//!
-//! Phase-1 proof-of-plumbing: a genuine `IBV_WR_RDMA_READ` (via the librdmacm
-//! convenience verb `rdma_post_read`) that pulls *real* bytes out of a remote
-//! peer's registered memory region over SoftRoCE (rxe) — replacing the mock
-//! engine that fabricated `byte = i % 256` and never touched a wire.
-//!
-//! [`RealRdmaContext`] is the client (the engine's real backend: connect /
-//! register_memory / post_read / poll_completion / device_info); [`RealResponder`]
-//! is a throwaway loopback target that registers a buffer for `REMOTE_READ` and
-//! hands the client its `{addr, rkey, len}`. Connection setup and QP state
-//! transitions are handled by `rdma_cm` (much simpler than a manual GID/QPN/LID
-//! handshake for RoCEv2). Linux + libibverbs/librdmacm only (`real-rdma` feature).
-
-#![allow(clippy::missing_safety_doc)]
-
-use std::ffi::{CStr, CString};
-use std::os::raw::{c_int, c_void};
-use std::ptr::null_mut;
-
-use rdma_sys::*;
-
-use crate::error::{RdmaError, RdmaResult};
-use crate::rdma::{CompletionStatus, MemoryRegion, RdmaDeviceInfo, RdmaOp, WorkCompletion};
-
-/// Metadata the responder hands the client so it can RDMA-READ the source MR.
-#[repr(C)]
-#[derive(Clone, Copy, Default)]
-struct RemoteInfo {
-    addr: u64,
-    rkey: u32,
-    len: u32,
-}
-
-const META_LEN: usize = std::mem::size_of::<RemoteInfo>();
-
-fn op_failed(op: &str) -> RdmaError {
-    RdmaError::OperationFailed {
-        operation: format!("{op}: {}", std::io::Error::last_os_error()),
-        status: -1,
-    }
-}
-
-unsafe fn qp_init_attr() -> ibv_qp_init_attr {
-    let mut a: ibv_qp_init_attr = std::mem::zeroed();
-    a.cap.max_send_wr = 8;
-    a.cap.max_recv_wr = 8;
-    a.cap.max_send_sge = 1;
-    a.cap.max_recv_sge = 1;
-    a.sq_sig_all = 1;
-    a
-}
-
-// Allow at least one outstanding RDMA READ in each direction.
-unsafe fn conn_param() -> rdma_conn_param {
-    let mut cp: rdma_conn_param = std::mem::zeroed();
-    cp.responder_resources = 1;
-    cp.initiator_depth = 1;
-    cp.retry_count = 7;
-    cp.rnr_retry_count = 7;
-    cp
-}
-
-unsafe fn query_device(id: *mut rdma_cm_id) -> RdmaDeviceInfo {
-    let ctx = (*id).verbs;
-    let mut attr: ibv_device_attr = std::mem::zeroed();
-    let mut name = String::from("unknown");
-    if !ctx.is_null() {
-        ibv_query_device(ctx, &mut attr);
-        let dev = (*ctx).device;
-        if !dev.is_null() {
-            name = CStr::from_ptr((*dev).name.as_ptr())
-                .to_string_lossy()
-                .into_owned();
-        }
-    }
-    RdmaDeviceInfo {
-        name,
-        vendor_id: attr.vendor_id,
-        vendor_part_id: attr.vendor_part_id,
-        hw_ver: attr.hw_ver,
-        max_mr: attr.max_mr as u32,
-        max_qp: attr.max_qp as u32,
-        max_cq: attr.max_cq as u32,
-        max_mr_size: attr.max_mr_size,
-        port_gid: String::new(),
-        port_lid: 0,
-    }
-}
-
-/// Busy-poll a librdmacm completion helper (`rdma_get_send_comp` /
-/// `rdma_get_recv_comp`) until it returns a completion or an error.
-unsafe fn wait_comp(
-    f: unsafe fn(*mut rdma_cm_id, *mut ibv_wc) -> c_int,
-    id: *mut rdma_cm_id,
-    what: &str,
-) -> RdmaResult<ibv_wc> {
-    let mut wc: ibv_wc = std::mem::zeroed();
-    loop {
-        let n = f(id, &mut wc);
-        if n < 0 {
-            return Err(op_failed(what));
-        }
-        if n > 0 {
-            break;
-        }
-    }
-    if wc.status != ibv_wc_status::IBV_WC_SUCCESS {
-        return Err(RdmaError::OperationFailed {
-            operation: format!("{what}: wc status {}", wc.status),
-            status: wc.status as i32,
-        });
-    }
-    Ok(wc)
-}
-
-/// The real RDMA client backend.
-pub struct RealRdmaContext {
-    id: *mut rdma_cm_id,
-    device: RdmaDeviceInfo,
-}
-
-impl RealRdmaContext {
-    /// Connect to a responder at `ip:port` and receive the `{addr, rkey, len}`
-    /// of its RDMA-READable source memory region.
-    pub fn connect(ip: &str, port: u16) -> RdmaResult<(Self, u64, u32, usize)> {
-        unsafe {
-            let node = CString::new(ip).map_err(|_| op_failed("bad ip"))?;
-            let svc = CString::new(port.to_string()).unwrap();
-            let mut hints: rdma_addrinfo = std::mem::zeroed();
-            hints.ai_port_space = rdma_port_space::RDMA_PS_TCP as c_int;
-            let mut res: *mut rdma_addrinfo = null_mut();
-            if rdma_getaddrinfo(node.as_ptr(), svc.as_ptr(), &hints, &mut res) != 0 {
-                return Err(op_failed("rdma_getaddrinfo(client)"));
-            }
-            let mut attr = qp_init_attr();
-            let mut id: *mut rdma_cm_id = null_mut();
-            let r = rdma_create_ep(&mut id, res, null_mut(), &mut attr);
-            rdma_freeaddrinfo(res);
-            if r != 0 {
-                return Err(op_failed("rdma_create_ep(client)"));
-            }
-
-            // Post a receive for the 16-byte RemoteInfo before connecting.
-            let mut meta = RemoteInfo::default();
-            let meta_ptr = (&mut meta as *mut RemoteInfo).cast::<c_void>();
-            let meta_mr = rdma_reg_msgs(id, meta_ptr, META_LEN);
-            if meta_mr.is_null() {
-                rdma_destroy_ep(id);
-                return Err(op_failed("rdma_reg_msgs(meta)"));
-            }
-            if rdma_post_recv(id, null_mut(), meta_ptr, META_LEN, meta_mr) != 0 {
-                rdma_destroy_ep(id);
-                return Err(op_failed("rdma_post_recv(meta)"));
-            }
-
-            let mut cp = conn_param();
-            if rdma_connect(id, &mut cp) != 0 {
-                rdma_destroy_ep(id);
-                return Err(op_failed("rdma_connect"));
-            }
-            wait_comp(rdma_get_recv_comp, id, "recv(meta)")?;
-            rdma_dereg_mr(meta_mr);
-
-            let device = query_device(id);
-            Ok((
-                RealRdmaContext { id, device },
-                meta.addr,
-                meta.rkey,
-                meta.len as usize,
-            ))
-        }
-    }
-
-    /// Register a local buffer (LOCAL_WRITE) as the destination of an RDMA READ.
-    /// Returns the engine's `MemoryRegion` plus the raw `ibv_mr` needed to post.
-    pub fn register_memory(&self, buf: &mut [u8]) -> RdmaResult<(MemoryRegion, *mut ibv_mr)> {
-        unsafe {
-            let mr = rdma_reg_msgs(self.id, buf.as_mut_ptr().cast(), buf.len());
-            if mr.is_null() {
-                return Err(op_failed("rdma_reg_msgs(dest)"));
-            }
-            Ok((
-                MemoryRegion {
-                    addr: buf.as_ptr() as u64,
-                    rkey: (*mr).rkey,
-                    lkey: (*mr).lkey,
-                    size: buf.len(),
-                    registered: true,
-                },
-                mr,
-            ))
-        }
-    }
-
-    /// Post a genuine `IBV_WR_RDMA_READ`: pull `buf.len()` bytes from the remote
-    /// `{remote_addr, rkey}` into the local buffer described by `mr`.
-    pub fn post_read(
-        &self,
-        buf: &mut [u8],
-        mr: *mut ibv_mr,
-        remote_addr: u64,
-        rkey: u32,
-        wr_id: u64,
-    ) -> RdmaResult<()> {
-        unsafe {
-            let flags = ibv_send_flags::IBV_SEND_SIGNALED.0 as c_int;
-            if rdma_post_read(
-                self.id,
-                wr_id as *mut c_void,
-                buf.as_mut_ptr().cast(),
-                buf.len(),
-                mr,
-                flags,
-                remote_addr,
-                rkey,
-            ) != 0
-            {
-                return Err(op_failed("rdma_post_read"));
-            }
-            Ok(())
-        }
-    }
-
-    /// Wait for the RDMA READ completion (it lands on the send queue).
-    pub fn poll_completion(&self) -> RdmaResult<WorkCompletion> {
-        unsafe {
-            let wc = wait_comp(rdma_get_send_comp, self.id, "send_comp(read)")?;
-            Ok(WorkCompletion {
-                wr_id: wc.wr_id,
-                status: CompletionStatus::from(wc.status as u32),
-                opcode: RdmaOp::Read,
-                byte_len: wc.byte_len,
-                imm_data: None,
-            })
-        }
-    }
-
-    pub fn device_info(&self) -> &RdmaDeviceInfo {
-        &self.device
-    }
-
-    /// Signal the responder we are done (1-byte send) so it can tear down, then
-    /// disconnect.
-    pub fn finish(&self) {
-        unsafe {
-            let mut done = [1u8; 1];
-            let mr = rdma_reg_msgs(self.id, done.as_mut_ptr().cast(), 1);
-            if !mr.is_null() {
-                if rdma_post_send(self.id, null_mut(), done.as_mut_ptr().cast(), 1, mr, 0) == 0 {
-                    let _ = wait_comp(rdma_get_send_comp, self.id, "send_comp(done)");
-                }
-                rdma_dereg_mr(mr);
-            }
-            rdma_disconnect(self.id);
-        }
-    }
-}
-
-impl Drop for RealRdmaContext {
-    fn drop(&mut self) {
-        unsafe {
-            if !self.id.is_null() {
-                rdma_destroy_ep(self.id);
-            }
-        }
-    }
-}
-
-/// A throwaway loopback RDMA-READ target: registers `data` for `REMOTE_READ`,
-/// accepts one client, hands it `{addr, rkey, len}`, and serves until the client
-/// signals done. Blocking — run on a thread; `data` must outlive the call so the
-/// client's reads see real bytes.
-pub struct RealResponder;
-
-impl RealResponder {
-    pub fn serve_once(
-        ip: &str,
-        port: u16,
-        data: &mut [u8],
-        ready: impl FnOnce(),
-    ) -> RdmaResult<()> {
-        unsafe {
-            let node = CString::new(ip).map_err(|_| op_failed("bad ip"))?;
-            let svc = CString::new(port.to_string()).unwrap();
-            let mut hints: rdma_addrinfo = std::mem::zeroed();
-            hints.ai_flags = RAI_PASSIVE as c_int;
-            hints.ai_port_space = rdma_port_space::RDMA_PS_TCP as c_int;
-            let mut res: *mut rdma_addrinfo = null_mut();
-            if rdma_getaddrinfo(node.as_ptr(), svc.as_ptr(), &hints, &mut res) != 0 {
-                return Err(op_failed("rdma_getaddrinfo(server)"));
-            }
-            let mut attr = qp_init_attr();
-            let mut listen_id: *mut rdma_cm_id = null_mut();
-            let r = rdma_create_ep(&mut listen_id, res, null_mut(), &mut attr);
-            rdma_freeaddrinfo(res);
-            if r != 0 {
-                return Err(op_failed("rdma_create_ep(server)"));
-            }
-            if rdma_listen(listen_id, 1) != 0 {
-                rdma_destroy_ep(listen_id);
-                return Err(op_failed("rdma_listen"));
-            }
-            ready(); // the listener is up; let the client connect
-
-            let mut id: *mut rdma_cm_id = null_mut();
-            if rdma_get_request(listen_id, &mut id) != 0 {
-                rdma_destroy_ep(listen_id);
-                return Err(op_failed("rdma_get_request"));
-            }
-
-            // Expose the source buffer for REMOTE_READ.
-            let src_mr = rdma_reg_read(id, data.as_mut_ptr().cast(), data.len());
-            if src_mr.is_null() {
-                rdma_destroy_ep(id);
-                rdma_destroy_ep(listen_id);
-                return Err(op_failed("rdma_reg_read"));
-            }
-
-            let mut meta = RemoteInfo {
-                addr: data.as_ptr() as u64,
-                rkey: (*src_mr).rkey,
-                len: data.len() as u32,
-            };
-            let meta_ptr = (&mut meta as *mut RemoteInfo).cast::<c_void>();
-            let meta_mr = rdma_reg_msgs(id, meta_ptr, META_LEN);
-            let mut done = [0u8; 1];
-            let done_mr = rdma_reg_msgs(id, done.as_mut_ptr().cast(), 1);
-            if meta_mr.is_null() || done_mr.is_null() {
-                return Err(op_failed("rdma_reg_msgs(server)"));
-            }
-            // Post the done-signal receive before accepting.
-            if rdma_post_recv(id, null_mut(), done.as_mut_ptr().cast(), 1, done_mr) != 0 {
-                return Err(op_failed("rdma_post_recv(done)"));
-            }
-
-            let mut cp = conn_param();
-            if rdma_accept(id, &mut cp) != 0 {
-                return Err(op_failed("rdma_accept"));
-            }
-
-            // Hand the client the source MR coordinates.
-            if rdma_post_send(id, null_mut(), meta_ptr, META_LEN, meta_mr, 0) != 0 {
-                return Err(op_failed("rdma_post_send(meta)"));
-            }
-            wait_comp(rdma_get_send_comp, id, "send_comp(meta)")?;
-
-            // Keep src_mr/data alive until the client finishes its read.
-            wait_comp(rdma_get_recv_comp, id, "recv_comp(done)")?;
-
-            rdma_disconnect(id);
-            rdma_dereg_mr(src_mr);
-            rdma_dereg_mr(meta_mr);
-            rdma_dereg_mr(done_mr);
-            rdma_destroy_ep(id);
-            rdma_destroy_ep(listen_id);
-            Ok(())
-        }
-    }
-}

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.cargo-ok

@@ -1 +0,0 @@
-{"v":1}

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.cargo_vcs_info.json

@@ -1,6 +0,0 @@
-{
-  "git": {
-    "sha1": "ad39d0cba38a5ffe57c3596cfca7a4476c41b61c"
-  },
-  "path_in_vcs": ""
-}

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.github/workflows/ci.yml

@@ -1,54 +0,0 @@
-name: RDMA-SYS-CI
-on:
-  pull_request:
-    branches: [master]
-  push:
-    branches: [master]
-  schedule: [cron: "0 */24 * * *"]
-
-env:
-  CI_RUST_TOOLCHAIN: 1.61.0
-
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-      - name: Git Sumbodule Update
-        run: |
-          git submodule update --remote --recursive
-      - name: Install dependencies
-        run: |
-          # Install libibverbs-dev as dependency automatically
-          sudo apt-get install -y librdmacm-dev
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: ${{ env.CI_RUST_TOOLCHAIN }}
-          override: true
-      - name: Test
-        run: |
-          ./rdma-env-setup/scripts/setup.sh
-          ./scripts/run.sh
-      - name: Setup tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
-
-  fmt:
-    name: Fmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: ${{ env.CI_RUST_TOOLCHAIN }}
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.gitignore

@@ -1,10 +0,0 @@
-# Generated by Cargo
-# will have compiled files and executables
-/target/
-
-# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
-# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
-
-# These are backup files generated by rustfmt
-**/*.rs.bk

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.gitmodules

@@ -1,4 +0,0 @@
-[submodule "rdma-env-setup"]
-	path = rdma-env-setup
-	url = https://github.com/datenlord/rdma-env-setup.git
-	branch = main

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/Cargo.lock

@@ -1,382 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "aho-corasick"
-version = "0.7.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "ansi_term"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
-dependencies = [
- "winapi",
-]
-
-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi",
- "libc",
- "winapi",
-]
-
-[[package]]
-name = "autocfg"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
-
-[[package]]
-name = "bindgen"
-version = "0.59.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8"
-dependencies = [
- "bitflags",
- "cexpr",
- "clang-sys",
- "clap",
- "env_logger",
- "lazy_static",
- "lazycell",
- "log",
- "peeking_take_while",
- "proc-macro2",
- "quote",
- "regex",
- "rustc-hash",
- "shlex",
- "which",
-]
-
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
-[[package]]
-name = "cexpr"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
-dependencies = [
- "nom",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "clang-sys"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3"
-dependencies = [
- "glob",
- "libc",
- "libloading",
-]
-
-[[package]]
-name = "clap"
-version = "2.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
-dependencies = [
- "ansi_term",
- "atty",
- "bitflags",
- "strsim",
- "textwrap",
- "unicode-width",
- "vec_map",
-]
-
-[[package]]
-name = "either"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
-
-[[package]]
-name = "env_logger"
-version = "0.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7"
-dependencies = [
- "atty",
- "humantime",
- "log",
- "regex",
- "termcolor",
-]
-
-[[package]]
-name = "glob"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
-
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "humantime"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
-
-[[package]]
-name = "lazy_static"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
-
-[[package]]
-name = "lazycell"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
-
-[[package]]
-name = "libc"
-version = "0.2.139"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
-
-[[package]]
-name = "libloading"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f"
-dependencies = [
- "cfg-if",
- "winapi",
-]
-
-[[package]]
-name = "log"
-version = "0.4.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "memchr"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
-
-[[package]]
-name = "memoffset"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "minimal-lexical"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
-
-[[package]]
-name = "nom"
-version = "7.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
-dependencies = [
- "memchr",
- "minimal-lexical",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
-
-[[package]]
-name = "paste"
-version = "1.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
-
-[[package]]
-name = "peeking_take_while"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
-
-[[package]]
-name = "pkg-config"
-version = "0.3.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.49"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "rdma-sys"
-version = "0.3.0"
-dependencies = [
- "bindgen",
- "libc",
- "memoffset",
- "paste",
- "pkg-config",
-]
-
-[[package]]
-name = "regex"
-version = "1.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-syntax"
-version = "0.6.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
-
-[[package]]
-name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
-[[package]]
-name = "shlex"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
-
-[[package]]
-name = "strsim"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
-
-[[package]]
-name = "termcolor"
-version = "1.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
-dependencies = [
- "winapi-util",
-]
-
-[[package]]
-name = "textwrap"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
-dependencies = [
- "unicode-width",
-]
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
-
-[[package]]
-name = "unicode-width"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
-
-[[package]]
-name = "vec_map"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
-
-[[package]]
-name = "which"
-version = "4.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b"
-dependencies = [
- "either",
- "libc",
- "once_cell",
-]
-
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-util"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
-dependencies = [
- "winapi",
-]
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/Cargo.toml

@@ -1,35 +0,0 @@
-# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
-#
-# When uploading crates to the registry Cargo will automatically
-# "normalize" Cargo.toml files for maximal compatibility
-# with all versions of Cargo and also rewrite `path` dependencies
-# to registry (e.g., crates.io) dependencies.
-#
-# If you are reading this file be aware that the original Cargo.toml
-# will likely look very different (and much more reasonable).
-# See Cargo.toml.orig for the original contents.
-
-[package]
-edition = "2021"
-name = "rdma-sys"
-version = "0.3.0"
-authors = ["Pu Wang <nicolas.weeks@gmail.com>"]
-description = "Rdma ibverbs lib Rust binding"
-license-file = "LICENSE"
-repository = "https://github.com/datenlord/rdma-sys"
-resolver = "2"
-
-[dependencies.libc]
-version = "0.2"
-
-[dependencies.memoffset]
-version = "0.6"
-
-[dependencies.paste]
-version = "1.0"
-
-[build-dependencies.bindgen]
-version = "0.69.4"
-
-[build-dependencies.pkg-config]
-version = "0.3"

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/Cargo.toml.orig

@@ -1,19 +0,0 @@
-[package]
-name = "rdma-sys"
-version = "0.3.0"
-authors = ["Pu Wang <nicolas.weeks@gmail.com>"]
-edition = "2021"
-description = "Rdma ibverbs lib Rust binding"
-license-file = "LICENSE"
-repository = "https://github.com/datenlord/rdma-sys"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-libc = "0.2"
-memoffset = "0.6"
-paste = "1.0"
-# intrusive-collections = "0.9"
-
-[build-dependencies]
-bindgen = "0.59.2"
-pkg-config = "0.3"

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2021 datenlord
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/README.md

@@ -1,3 +0,0 @@
-# Rdma ibverbs lib Rust binding
-
-This lib is the ibverbs low-level Rust binding. As inline function and nested structure are not handled properly in the automatic bind generator, we deal with them case by case manually.

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/build.rs

@@ -1,161 +0,0 @@
-use std::env;
-use std::path::Path;
-
-fn link_rdma_core(lib_name: &str, pkg_name: &str, version: &str, include_paths: &mut Vec<String>) {
-    let result: _ = pkg_config::Config::new()
-        .atleast_version(version)
-        .statik(false)
-        .probe(lib_name);
-
-    let lib = result.unwrap_or_else(|_| panic!("please install {pkg_name} {version})"));
-    println!("found {pkg_name} {}", lib.version);
-
-    for path in lib.include_paths {
-        let path = path.to_str().expect("non-utf8 path");
-        include_paths.push(path.to_owned());
-    }
-}
-
-fn main() {
-    let mut include_paths: Vec<String> = Vec::new();
-
-    {
-        let lib_name = "libibverbs";
-        let pkg_name = "libibverbs-dev";
-        let version = "1.8.28";
-        link_rdma_core(lib_name, pkg_name, version, &mut include_paths);
-    }
-
-    {
-        let lib_name = "librdmacm";
-        let pkg_name = "librdmacm-dev";
-        let version = "1.2.28";
-        link_rdma_core(lib_name, pkg_name, version, &mut include_paths);
-    }
-
-    {
-        include_paths.sort_unstable();
-        include_paths.dedup_by(|x, first| x == first);
-        include_paths.push("/usr/include".into());
-        println!("include paths: {:?}", include_paths);
-    }
-
-    let include_args = include_paths.iter().map(|p| format!("-I{}", p));
-
-    let bindings = bindgen::Builder::default()
-        .clang_args(include_args)
-        .header("src/bindings.h")
-        .allowlist_function("ibv_.*")
-        .allowlist_type("ibv_.*")
-        .allowlist_function("rdma_.*")
-        .allowlist_type("rdma_.*")
-        .allowlist_type("verbs_.*")
-        .allowlist_type("ib_uverbs_access_flags")
-        //.allowlist_type("verbs_devices_ops")
-        //.allowlist_var("verbs_provider_.*")
-        .blocklist_type("in6_addr")
-        .opaque_type("pthread_.*")
-        .blocklist_type("sockaddr.*")
-        .blocklist_type("timespec")
-        .blocklist_type("ibv_ah_attr")
-        .blocklist_type("ibv_async_event")
-        .blocklist_type("ibv_flow_spec")
-        .blocklist_type("ibv_gid")
-        .blocklist_type("ibv_global_route")
-        .blocklist_type("ibv_mw_bind_info")
-        .blocklist_type("ibv_ops_wr")
-        .blocklist_type("ibv_send_wr")
-        .blocklist_type("ibv_wc")
-        .blocklist_type("rdma_addr")
-        .blocklist_type("rdma_cm_event")
-        .blocklist_type("rdma_ib_addr")
-        .blocklist_type("rdma_ud_param")
-        // Following ENUM will used with bitwise-or
-        // including flags, mask, caps, bits, fields, size
-        .bitfield_enum("ibv_device_cap_flags")
-        .bitfield_enum("ibv_odp_transport_cap_bits")
-        .bitfield_enum("ibv_odp_general_caps")
-        .bitfield_enum("ibv_rx_hash_function_flags")
-        .bitfield_enum("ibv_rx_hash_fields")
-        .bitfield_enum("ibv_raw_packet_caps")
-        .bitfield_enum("ibv_tm_cap_flags")
-        .bitfield_enum("ibv_pci_atomic_op_size")
-        .bitfield_enum("ibv_port_cap_flags")
-        .bitfield_enum("ibv_port_cap_flags2")
-        .bitfield_enum("ibv_create_cq_wc_flags")
-        .bitfield_enum("ibv_wc_flags")
-        .bitfield_enum("ibv_access_flags")
-        .bitfield_enum("ibv_xrcd_init_attr_mask")
-        .bitfield_enum("ibv_rereg_mr_flags")
-        .bitfield_enum("ibv_srq_attr_mask")
-        .bitfield_enum("ibv_srq_init_attr_mask") // TODO: need to be bitfield?
-        .bitfield_enum("ibv_wq_init_attr_mask")
-        .bitfield_enum("ibv_wq_flags")
-        .bitfield_enum("ibv_wq_attr_mask")
-        .bitfield_enum("ibv_ind_table_init_attr_mask")
-        .bitfield_enum("ibv_qp_init_attr_mask") // TODO: need to be bitfield?
-        .bitfield_enum("ibv_qp_create_flags")
-        .bitfield_enum("ibv_qp_create_send_ops_flags")
-        .bitfield_enum("ibv_qp_open_attr_mask")
-        .bitfield_enum("ibv_qp_attr_mask")
-        .bitfield_enum("ibv_send_flags")
-        .bitfield_enum("ibv_ops_flags")
-        .bitfield_enum("ibv_cq_attr_mask")
-        .bitfield_enum("ibv_flow_flags")
-        .bitfield_enum("ibv_flow_action_esp_mask")
-        .bitfield_enum("ibv_cq_init_attr_mask")
-        .bitfield_enum("ibv_create_cq_attr_flags")
-        .bitfield_enum("ibv_parent_domain_init_attr_mask")
-        .bitfield_enum("ibv_read_counters_flags")
-        .bitfield_enum("ibv_values_mask")
-        .bitfield_enum("ib_uverbs_access_flags")
-        .bitfield_enum("rdma_cm_join_mc_attr_mask")
-        .bitfield_enum("rdma_cm_mc_join_flags")
-        // Following ENUM will be const in a sub-mod
-        .constified_enum_module("ibv_node_type")
-        .constified_enum_module("ibv_transport_type")
-        .constified_enum_module("ibv_atomic_cap")
-        .constified_enum_module("ibv_mtu")
-        .constified_enum_module("ibv_port_state")
-        .constified_enum_module("ibv_wc_status")
-        .constified_enum_module("ibv_wc_opcode")
-        .constified_enum_module("ibv_mw_type")
-        .constified_enum_module("ibv_rate")
-        .constified_enum_module("ibv_srq_type")
-        .constified_enum_module("ibv_wq_type")
-        .constified_enum_module("ibv_wq_state")
-        .constified_enum_module("ibv_qp_type")
-        .constified_enum_module("ibv_qp_state")
-        .constified_enum_module("ibv_mig_state")
-        .constified_enum_module("ibv_wr_opcode")
-        .constified_enum_module("ibv_ops_wr_opcode")
-        .constified_enum_module("ibv_flow_attr_type")
-        .constified_enum_module("ibv_flow_spec_type")
-        .constified_enum_module("ibv_counter_description")
-        .constified_enum_module("ibv_rereg_mr_err_code")
-        .constified_enum_module("ib_uverbs_advise_mr_advice")
-        .constified_enum_module("rdma_cm_event_type")
-        .constified_enum_module("rdma_driver_id")
-        .constified_enum_module("rdma_port_space")
-        .rustified_enum("ibv_event_type")
-        // unions with non-`Copy` fields other than `ManuallyDrop<T>` are unstable
-        // for example: `pub eth: ibv_flow_spec_eth`
-        // note: see issue #55149 <https://github.com/rust-lang/rust/issues/55149> for more information
-        .derive_copy(true)
-        .derive_debug(false)
-        .derive_default(false)
-        .generate_comments(false)
-        //.generate_inline_functions(true)
-        //.default_macro_constant_type(bindgen::MacroTypeVariation::Unsigned)
-        .prepend_enum_name(false)
-        .disable_untagged_union()
-        .generate()
-        .expect("Unable to generate bindings");
-
-    let out_dir = env::var_os("OUT_DIR").unwrap();
-    let dest_path = Path::new(&out_dir).join("bindings.rs");
-
-    bindings
-        .write_to_file(dest_path)
-        .expect("Could not write bindings");
-}

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/examples/client.rs

@@ -1,168 +0,0 @@
-//! This demo shows how to establish a connection between server and client
-//! and send msg to the other end.
-//!
-//! You can try this example by running:
-//!
-//!     cargo run --example server
-//!
-//! And then start client in another terminal by running:
-//!
-//!     cargo run --example client <server_ip> <port>
-//!
-//! The default port is 7471.
-
-use rdma_sys::*;
-use std::{env, process::exit, ptr::null_mut};
-
-fn run(ip: &str, port: &str) -> i32 {
-    let mut send_msg = vec![1_u8; 16];
-    let mut recv_msg = vec![0_u8; 16];
-    let mut hints = unsafe { std::mem::zeroed::<rdma_addrinfo>() };
-    let mut res: *mut rdma_addrinfo = null_mut();
-
-    hints.ai_port_space = rdma_port_space::RDMA_PS_TCP as i32;
-    let mut ret =
-        unsafe { rdma_getaddrinfo(ip.as_ptr().cast(), port.as_ptr().cast(), &hints, &mut res) };
-
-    if ret != 0 {
-        println!("rdma_getaddrinfo");
-        return ret;
-    }
-
-    let mut attr = unsafe { std::mem::zeroed::<ibv_qp_init_attr>() };
-    let mut id: *mut rdma_cm_id = null_mut();
-    attr.cap.max_send_wr = 1;
-    attr.cap.max_recv_wr = 1;
-    attr.cap.max_send_sge = 1;
-    attr.cap.max_recv_sge = 1;
-    attr.cap.max_inline_data = 16;
-    attr.qp_context = id.cast();
-    attr.sq_sig_all = 1;
-    ret = unsafe { rdma_create_ep(&mut id, res, null_mut(), &mut attr) };
-    // Check to see if we got inline data allowed or not
-    let mut send_flags = 0_u32;
-    if attr.cap.max_inline_data >= 16 {
-        send_flags = ibv_send_flags::IBV_SEND_INLINE.0;
-    } else {
-        println!("rdma_client: device doesn't support IBV_SEND_INLINE, using sge sends");
-    }
-
-    if ret != 0 {
-        println!("rdma_create_ep");
-        unsafe {
-            rdma_freeaddrinfo(res);
-        }
-        return ret;
-    }
-
-    let mr = unsafe { rdma_reg_msgs(id, recv_msg.as_mut_ptr().cast(), 16) };
-    if mr.is_null() {
-        println!("rdma_reg_msgs for recv_msg");
-        unsafe {
-            rdma_destroy_ep(id);
-        }
-        return -1;
-    }
-
-    let mut send_mr = null_mut();
-    if (send_flags & ibv_send_flags::IBV_SEND_INLINE.0) as u32 == 0 {
-        println!("flags {:?}", send_flags);
-        send_mr = unsafe { rdma_reg_msgs(id, send_msg.as_mut_ptr().cast(), 16) };
-        if send_mr.is_null() {
-            println!("rdma_reg_msgs for send_msg");
-            unsafe {
-                rdma_dereg_mr(mr);
-            }
-            return -1;
-        }
-    }
-
-    ret = unsafe { rdma_post_recv(id, null_mut(), recv_msg.as_mut_ptr().cast(), 16, mr) };
-    if ret != 0 {
-        println!("rdma_post_recv");
-        if (send_flags & ibv_send_flags::IBV_SEND_INLINE.0) as u32 == 0 {
-            unsafe { rdma_dereg_mr(send_mr) };
-        }
-        return ret;
-    }
-
-    ret = unsafe { rdma_connect(id, null_mut()) };
-    if ret != 0 {
-        println!("rdma_connect");
-        unsafe {
-            rdma_disconnect(id);
-        }
-        return ret;
-    }
-
-    ret = unsafe {
-        rdma_post_send(
-            id,
-            null_mut(),
-            send_msg.as_mut_ptr().cast(),
-            16,
-            send_mr,
-            send_flags.try_into().unwrap(),
-        )
-    };
-    if ret != 0 {
-        println!("rdma_post_send");
-        unsafe {
-            rdma_disconnect(id);
-        }
-        return ret;
-    }
-
-    let mut wc = unsafe { std::mem::zeroed::<ibv_wc>() };
-    while ret == 0 {
-        ret = unsafe { rdma_get_send_comp(id, &mut wc) };
-    }
-    if ret < 0 {
-        println!("rdma_get_send_comp");
-        unsafe {
-            rdma_disconnect(id);
-        }
-        return ret;
-    }
-
-    ret = 0;
-    while ret == 0 {
-        ret = unsafe { rdma_get_recv_comp(id, &mut wc) };
-    }
-    println!("rdma_client: recv msg : {:?}", recv_msg);
-    if ret < 0 {
-        println!("rdma_get_recv_comp");
-    } else {
-        ret = 0;
-    }
-
-    ret
-}
-
-fn main() {
-    println!("rdma_client: start");
-    let args: Vec<String> = env::args().collect();
-    if args.len() != 3 {
-        println!("usage : cargo run --example client <server_ip> <port>");
-        println!("input : {:?}", args);
-        exit(-1);
-    }
-    let ip = args.get(1).unwrap().as_str();
-    let port = args.get(2).unwrap().as_str();
-
-    let ret = run(ip, port);
-
-    if ret != 0 {
-        println!(
-            "rdma_client: ret error {:?}",
-            std::io::Error::from_raw_os_error(-ret)
-        );
-        if ret == -1 {
-            println!(
-                "rdma_client: last os error {:?}",
-                std::io::Error::last_os_error()
-            );
-        }
-    }
-    println!("rdma_client: end");
-}

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/examples/server.rs

@@ -1,201 +0,0 @@
-//! This demo shows how to establish a connection between server and client
-//! and send msg to the other end.
-//!
-//! You can try this example by running:
-//!
-//!     cargo run --example server
-//!
-//! And then start client in another terminal by running:
-//!
-//!     cargo run --example client <server_ip> <port>
-//!
-//! The default port is 7471.
-
-use rdma_sys::*;
-use std::ptr::null_mut;
-
-static SERVER: &str = "0.0.0.0\0";
-static PORT: &str = "7471\0";
-
-fn run() -> i32 {
-    let mut send_msg = vec![1_u8; 16];
-    let mut recv_msg = vec![0_u8; 16];
-    let mut hints = unsafe { std::mem::zeroed::<rdma_addrinfo>() };
-    let mut res: *mut rdma_addrinfo = null_mut();
-    hints.ai_flags = RAI_PASSIVE.try_into().unwrap();
-    hints.ai_port_space = rdma_port_space::RDMA_PS_TCP.try_into().unwrap();
-    let mut ret = unsafe {
-        rdma_getaddrinfo(
-            SERVER.as_ptr().cast(),
-            PORT.as_ptr().cast(),
-            &hints,
-            &mut res,
-        )
-    };
-
-    if ret != 0 {
-        println!("rdma_getaddrinfo");
-        return ret;
-    }
-
-    let mut listen_id = null_mut();
-    let mut id = null_mut();
-
-    let mut init_attr = unsafe { std::mem::zeroed::<ibv_qp_init_attr>() };
-    init_attr.cap.max_send_wr = 1;
-    init_attr.cap.max_recv_wr = 1;
-    init_attr.cap.max_send_sge = 1;
-    init_attr.cap.max_recv_sge = 1;
-    init_attr.cap.max_inline_data = 16;
-    init_attr.sq_sig_all = 1;
-    ret = unsafe { rdma_create_ep(&mut listen_id, res, null_mut(), &mut init_attr) };
-    // Check to see if we got inline data allowed or not
-    if ret != 0 {
-        println!("rdma_create_ep");
-        unsafe {
-            rdma_freeaddrinfo(res);
-        }
-        return ret;
-    }
-    ret = unsafe { rdma_listen(listen_id, 0) };
-    if ret != 0 {
-        println!("rdma_listen");
-        unsafe {
-            rdma_destroy_ep(listen_id);
-        }
-        return ret;
-    }
-
-    ret = unsafe { rdma_get_request(listen_id, &mut id) };
-    if ret != 0 {
-        println!("rdma_get_request");
-        unsafe {
-            rdma_destroy_ep(listen_id);
-        }
-        return ret;
-    }
-
-    let mut qp_attr = unsafe { std::mem::zeroed::<ibv_qp_attr>() };
-    ret = unsafe {
-        ibv_query_qp(
-            (*id).qp,
-            &mut qp_attr,
-            ibv_qp_attr_mask::IBV_QP_CAP.0.try_into().unwrap(),
-            &mut init_attr,
-        )
-    };
-
-    if ret != 0 {
-        println!("ibv_query_qp");
-        unsafe {
-            rdma_destroy_ep(id);
-        }
-        return ret;
-    }
-
-    let mut send_flags = 0_u32;
-    if init_attr.cap.max_inline_data >= 16 {
-        send_flags = ibv_send_flags::IBV_SEND_INLINE.0;
-    } else {
-        println!("rdma_server: device doesn't support IBV_SEND_INLINE, using sge sends");
-    }
-
-    let recv_mr = unsafe { rdma_reg_msgs(id, recv_msg.as_mut_ptr().cast(), 16) };
-    if recv_mr.is_null() {
-        ret = -1;
-        println!("rdma_reg_msgs for recv_msg");
-        unsafe {
-            rdma_dereg_mr(recv_mr);
-        }
-        return ret;
-    }
-
-    let mut send_mr = null_mut();
-    if (send_flags & ibv_send_flags::IBV_SEND_INLINE.0) == 0 {
-        send_mr = unsafe { rdma_reg_msgs(id, send_msg.as_mut_ptr().cast(), 16) };
-        if send_mr.is_null() {
-            ret = -1;
-            println!("rdma_reg_msgs for send_msg");
-            unsafe {
-                rdma_dereg_mr(recv_mr);
-            }
-            return ret;
-        }
-    }
-    ret = unsafe { rdma_post_recv(id, null_mut(), recv_msg.as_mut_ptr().cast(), 16, recv_mr) };
-
-    if ret != 0 {
-        println!("rdma_post_recv");
-        unsafe {
-            rdma_dereg_mr(recv_mr);
-        }
-        return ret;
-    }
-
-    ret = unsafe { rdma_accept(id, null_mut()) };
-    if ret != 0 {
-        println!("rdma_accept");
-        if (send_flags & ibv_send_flags::IBV_SEND_INLINE.0) == 0 {
-            unsafe { rdma_dereg_mr(send_mr) };
-        }
-        return ret;
-    }
-
-    let mut wc = unsafe { std::mem::zeroed::<ibv_wc>() };
-    while ret == 0 {
-        ret = unsafe { rdma_get_recv_comp(id, &mut wc) };
-    }
-    if ret < 0 {
-        println!("rdma_get_recv_comp");
-        unsafe {
-            rdma_disconnect(id);
-        }
-        return ret;
-    }
-    println!("rdma_server: recv msg : {:?}", recv_msg);
-    ret = unsafe {
-        rdma_post_send(
-            id,
-            null_mut(),
-            send_msg.as_mut_ptr().cast(),
-            16,
-            send_mr,
-            send_flags.try_into().unwrap(),
-        )
-    };
-    if ret != 0 {
-        println!("rdma_post_send");
-        unsafe {
-            rdma_disconnect(id);
-        }
-        return ret;
-    }
-
-    while ret == 0 {
-        ret = unsafe { rdma_get_send_comp(id, &mut wc) };
-    }
-    if ret < 0 {
-        println!("rdma_get_send_comp");
-    } else {
-        ret = 0;
-    }
-    ret
-}
-
-fn main() {
-    println!("rdma_server: start");
-    let ret = run();
-    if ret != 0 {
-        println!(
-            "rdma_server: ret error {:?}",
-            std::io::Error::from_raw_os_error(-ret)
-        );
-        if ret == -1 {
-            println!(
-                "rdma_server: last os error {:?}",
-                std::io::Error::last_os_error()
-            );
-        }
-    }
-    println!("rdma_server: end");
-}

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/scripts/run.sh

@@ -1,30 +0,0 @@
-#! /bin/sh
-
-RXE_DEV=rxe_eth0
-
-# Remove existing devices if any
-sudo rdma link delete $RXE_DEV
-
-set -o errexit
-set -o nounset
-set -o xtrace
-
-if [ `ifconfig -s | grep -c '^e'` -eq 0 ]; then
-    echo "no eth device"
-    exit 1
-elif [ `ifconfig -s | grep -c '^e'` -gt 1 ]; then
-    echo "multiple eth devices, select the first one"
-    ifconfig -s | grep '^e'
-fi
-
-ETH_DEV=`ifconfig -s | grep '^e' | cut -d ' ' -f 1 | head -n 1`
-ETH_IP=`ifconfig $ETH_DEV | grep inet | grep -v inet6 | awk '{print $2}' | tr -d "addr:"`
-CM_PORT=7471
-# Setup soft-roce device
-sudo rdma link add $RXE_DEV type rxe netdev $ETH_DEV
-rdma link | grep $RXE_DEV
-
-cargo test --all
-cargo run --example server &
-sleep 1 &&
-cargo run --example client $ETH_IP $CM_PORT

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/bindings.h

@@ -1,3 +0,0 @@
-#include <infiniband/verbs.h>
-#include <rdma/rdma_cma.h>
-#include <rdma/rdma_verbs.h>

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/lib.rs

@@ -1,16 +0,0 @@
-#![deny(warnings)]
-#![allow(non_snake_case, non_camel_case_types, non_upper_case_globals)]
-#![allow(deref_nullptr)] // TODO(fxbug.dev/74605): Remove once bindgen is fixed.
-#![allow(clippy::missing_safety_doc, clippy::too_many_arguments)]
-
-use libc::*;
-
-include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
-
-mod opcode;
-mod types;
-mod verbs;
-
-pub use self::opcode::*;
-pub use self::types::*;
-pub use self::verbs::*;

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/opcode.rs

@@ -1,103 +0,0 @@
-pub mod ibv_opcode {
-    use paste::paste;
-    pub type Type = std::os::raw::c_int;
-    macro_rules! concat_ibv_opcode {
-        ($transport: expr, $op : expr ) => {
-            paste! {
-                pub const [<IBV_OPCODE_ $transport _ $op>]: Type = [<IBV_OPCODE_ $transport>] + [<IBV_OPCODE_ $op>] ;
-            }
-        };
-    }
-
-    /* transport types -- just used to define real constants */
-    pub const IBV_OPCODE_RC: Type = 0x00;
-    pub const IBV_OPCODE_UC: Type = 0x20;
-    pub const IBV_OPCODE_RD: Type = 0x40;
-    pub const IBV_OPCODE_UD: Type = 0x60;
-    /* operations -- just used to define real constants */
-    pub const IBV_OPCODE_SEND_FIRST: Type = 0x00;
-    pub const IBV_OPCODE_SEND_MIDDLE: Type = 0x01;
-    pub const IBV_OPCODE_SEND_LAST: Type = 0x02;
-    pub const IBV_OPCODE_SEND_LAST_WITH_IMMEDIATE: Type = 0x03;
-    pub const IBV_OPCODE_SEND_ONLY: Type = 0x04;
-    pub const IBV_OPCODE_SEND_ONLY_WITH_IMMEDIATE: Type = 0x05;
-    pub const IBV_OPCODE_RDMA_WRITE_FIRST: Type = 0x06;
-    pub const IBV_OPCODE_RDMA_WRITE_MIDDLE: Type = 0x07;
-    pub const IBV_OPCODE_RDMA_WRITE_LAST: Type = 0x08;
-    pub const IBV_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE: Type = 0x09;
-    pub const IBV_OPCODE_RDMA_WRITE_ONLY: Type = 0x0a;
-    pub const IBV_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE: Type = 0x0b;
-    pub const IBV_OPCODE_RDMA_READ_REQUEST: Type = 0x0c;
-    pub const IBV_OPCODE_RDMA_READ_RESPONSE_FIRST: Type = 0x0d;
-    pub const IBV_OPCODE_RDMA_READ_RESPONSE_MIDDLE: Type = 0x0e;
-    pub const IBV_OPCODE_RDMA_READ_RESPONSE_LAST: Type = 0x0f;
-    pub const IBV_OPCODE_RDMA_READ_RESPONSE_ONLY: Type = 0x10;
-    pub const IBV_OPCODE_ACKNOWLEDGE: Type = 0x11;
-    pub const IBV_OPCODE_ATOMIC_ACKNOWLEDGE: Type = 0x12;
-    pub const IBV_OPCODE_COMPARE_SWAP: Type = 0x13;
-    pub const IBV_OPCODE_FETCH_ADD: Type = 0x14;
-
-    /* RC */
-    concat_ibv_opcode!(RC, SEND_FIRST);
-    concat_ibv_opcode!(RC, SEND_MIDDLE);
-    concat_ibv_opcode!(RC, SEND_LAST);
-    concat_ibv_opcode!(RC, SEND_LAST_WITH_IMMEDIATE);
-    concat_ibv_opcode!(RC, SEND_ONLY);
-    concat_ibv_opcode!(RC, SEND_ONLY_WITH_IMMEDIATE);
-    concat_ibv_opcode!(RC, RDMA_WRITE_FIRST);
-    concat_ibv_opcode!(RC, RDMA_WRITE_MIDDLE);
-    concat_ibv_opcode!(RC, RDMA_WRITE_LAST);
-    concat_ibv_opcode!(RC, RDMA_WRITE_LAST_WITH_IMMEDIATE);
-    concat_ibv_opcode!(RC, RDMA_WRITE_ONLY);
-    concat_ibv_opcode!(RC, RDMA_WRITE_ONLY_WITH_IMMEDIATE);
-    concat_ibv_opcode!(RC, RDMA_READ_REQUEST);
-    concat_ibv_opcode!(RC, RDMA_READ_RESPONSE_FIRST);
-    concat_ibv_opcode!(RC, RDMA_READ_RESPONSE_MIDDLE);
-    concat_ibv_opcode!(RC, RDMA_READ_RESPONSE_LAST);
-    concat_ibv_opcode!(RC, RDMA_READ_RESPONSE_ONLY);
-    concat_ibv_opcode!(RC, ACKNOWLEDGE);
-    concat_ibv_opcode!(RC, ATOMIC_ACKNOWLEDGE);
-    concat_ibv_opcode!(RC, COMPARE_SWAP);
-    concat_ibv_opcode!(RC, FETCH_ADD);
-
-    /* UC */
-    concat_ibv_opcode!(UC, SEND_FIRST);
-    concat_ibv_opcode!(UC, SEND_MIDDLE);
-    concat_ibv_opcode!(UC, SEND_LAST);
-    concat_ibv_opcode!(UC, SEND_LAST_WITH_IMMEDIATE);
-    concat_ibv_opcode!(UC, SEND_ONLY);
-    concat_ibv_opcode!(UC, SEND_ONLY_WITH_IMMEDIATE);
-    concat_ibv_opcode!(UC, RDMA_WRITE_FIRST);
-    concat_ibv_opcode!(UC, RDMA_WRITE_MIDDLE);
-    concat_ibv_opcode!(UC, RDMA_WRITE_LAST);
-    concat_ibv_opcode!(UC, RDMA_WRITE_LAST_WITH_IMMEDIATE);
-    concat_ibv_opcode!(UC, RDMA_WRITE_ONLY);
-    concat_ibv_opcode!(UC, RDMA_WRITE_ONLY_WITH_IMMEDIATE);
-
-    /* RD */
-    concat_ibv_opcode!(RD, SEND_FIRST);
-    concat_ibv_opcode!(RD, SEND_MIDDLE);
-    concat_ibv_opcode!(RD, SEND_LAST);
-    concat_ibv_opcode!(RD, SEND_LAST_WITH_IMMEDIATE);
-    concat_ibv_opcode!(RD, SEND_ONLY);
-    concat_ibv_opcode!(RD, SEND_ONLY_WITH_IMMEDIATE);
-    concat_ibv_opcode!(RD, RDMA_WRITE_FIRST);
-    concat_ibv_opcode!(RD, RDMA_WRITE_MIDDLE);
-    concat_ibv_opcode!(RD, RDMA_WRITE_LAST);
-    concat_ibv_opcode!(RD, RDMA_WRITE_LAST_WITH_IMMEDIATE);
-    concat_ibv_opcode!(RD, RDMA_WRITE_ONLY);
-    concat_ibv_opcode!(RD, RDMA_WRITE_ONLY_WITH_IMMEDIATE);
-    concat_ibv_opcode!(RD, RDMA_READ_REQUEST);
-    concat_ibv_opcode!(RD, RDMA_READ_RESPONSE_FIRST);
-    concat_ibv_opcode!(RD, RDMA_READ_RESPONSE_MIDDLE);
-    concat_ibv_opcode!(RD, RDMA_READ_RESPONSE_LAST);
-    concat_ibv_opcode!(RD, RDMA_READ_RESPONSE_ONLY);
-    concat_ibv_opcode!(RD, ACKNOWLEDGE);
-    concat_ibv_opcode!(RD, ATOMIC_ACKNOWLEDGE);
-    concat_ibv_opcode!(RD, COMPARE_SWAP);
-    concat_ibv_opcode!(RD, FETCH_ADD);
-
-    /* UD */
-    concat_ibv_opcode!(UD, SEND_ONLY);
-    concat_ibv_opcode!(UD, SEND_ONLY_WITH_IMMEDIATE);
-}

seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/types.rs

@@ -1,299 +0,0 @@
-use crate::*;
-
-/// This file defines the types directly or indirectly involving union,
-/// in that BindGen cannot handle union very well, so mannually define them.
-
-/// Struct types involve union in <infiniband/verbs.h>
-
-// ibv_gid related union and struct types
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct ibv_gid_global_t {
-    pub subnet_prefix: __be64,
-    pub interface_id: __be64,
-}
-
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub union ibv_gid {
-    pub raw: [u8; 16],
-    pub global: ibv_gid_global_t,
-}
-
-// ibv_async_event related union and struct type
-#[repr(C)]
-pub union ibv_async_event_element_t {
-    pub cq: *mut ibv_cq,
-    pub qp: *mut ibv_qp,
-    pub srq: *mut ibv_srq,
-    pub wq: *mut ibv_wq,
-    pub port_num: c_int,
-}
-
-#[repr(C)]
-pub struct ibv_async_event {
-    pub element: ibv_async_event_element_t,
-    pub event_type: ibv_event_type,
-}
-
-// ibv_wc related union and struct types
-#[repr(C)]
-pub union imm_data_invalidated_rkey_union_t {
-    /// When (wc_flags & IBV_WC_WITH_IMM): Immediate data in network byte order.
-    pub imm_data: __be32,
-    /// When (wc_flags & IBV_WC_WITH_INV): Stores the invalidated rkey.
-    pub invalidated_rkey: u32,
-}
-
-#[repr(C)]
-pub struct ibv_wc {
-    pub wr_id: u64,
-    pub status: ibv_wc_status::Type,
-    pub opcode: ibv_wc_opcode::Type,
-    pub vendor_err: u32,
-    pub byte_len: u32,
-    pub imm_data_invalidated_rkey_union: imm_data_invalidated_rkey_union_t,
-    pub qp_num: u32,
-    pub src_qp: u32,
-    pub wc_flags: c_uint,
-    pub pkey_index: u16,
-    pub slid: u16,
-    pub sl: u8,
-    pub dlid_path_bits: u8,
-}
-
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct ibv_global_route {
-    pub dgid: ibv_gid,
-    pub flow_label: u32,
-    pub sgid_index: u8,
-    pub hop_limit: u8,
-    pub traffic_class: u8,
-}
-
-// ibv_send_wr related union and struct types
-#[repr(C)]
-#[derive(Copy, Clone)]
-pub struct ibv_mw_bind_info {
-    pub mr: *mut ibv_mr,
-    pub addr: u64,
-    pub length: u64,
-    pub mw_access_flags: ::std::os::raw::c_uint,
-}
-
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct rdma_t {
-    pub remote_addr: u64,
-    pub rkey: u32,
-}
-
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct atomic_t {
-    pub remote_addr: u64,
-    pub compare_add: u64,
-    pub swap: u64,
-    pub rkey: u32,
-}
-
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct ud_t {
-    pub ah: *mut ibv_ah,
-    pub remote_qpn: u32,
-    pub remote_qkey: u32,
-}
-
-#[repr(C)]
-pub union wr_t {
-    pub rdma: rdma_t,
-    pub atomic: atomic_t,
-    pub ud: ud_t,
-}
-
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct xrc_t {
-    pub remote_srqn: u32,
-}
-
-#[repr(C)]
-pub union qp_type_t {
-    pub xrc: xrc_t,
-}
-
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct bind_mw_t {
-    pub mw: *mut ibv_mw,
-    pub rkey: u32,
-    pub bind_info: ibv_mw_bind_info,
-}
-
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct tso_t {
-    pub hdr: *mut c_void,
-    pub hdr_sz: u16,
-    pub mss: u16,
-}
-
-#[repr(C)]
-pub union bind_mw_tso_union_t {
-    pub bind_mw: bind_mw_t,
-    pub tso: tso_t,
-}
-
-#[repr(C)]
-pub struct ibv_send_wr {
-    pub wr_id: u64,
-    pub next: *mut Self,
-    pub sg_list: *mut ibv_sge,
-    pub num_sge: c_int,
-    pub opcode: ibv_wr_opcode::Type,
-    pub send_flags: c_uint,
-    /// When opcode is *_WITH_IMM: Immediate data in network byte order.
-    /// When opcode is *_INV: Stores the rkey to invalidate
-    pub imm_data_invalidated_rkey_union: imm_data_invalidated_rkey_union_t,
-    pub wr: wr_t,
-    pub qp_type: qp_type_t,
-    pub bind_mw_tso_union: bind_mw_tso_union_t,
-}
-
-#[repr(C)]
-#[derive(Copy, Clone)]
-pub struct add_t {
-    pub recv_wr_id: u64,
-    pub sg_list: *mut ibv_sge,
-    pub num_sge: c_int,
-    pub tag: u64,
-    pub mask: u64,
-}
-
-#[repr(C)]
-#[derive(Copy, Clone)]
-pub struct tm_t {
-    pub unexpected_cnt: u32,
-    pub handle: u32,
-    pub add: add_t,
-}
-
-#[repr(C)]
-pub struct ibv_ops_wr {
-    wr_id: u64,
-    next: *mut Self,
-    opcode: ibv_ops_wr_opcode::Type,
-    flags: c_int,
-    tm: tm_t,
-}
-
-// ibv_flow_spec related union and struct types
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct hdr_t {
-    pub type_: ibv_flow_spec_type::Type,
-    pub size: u16,
-}
-
-#[repr(C)]
-pub union ibv_flow_spec_union_t {
-    pub hdr: hdr_t,
-    pub eth: ibv_flow_spec_eth,
-    pub ipv4: ibv_flow_spec_ipv4,
-    pub tcp_udp: ibv_flow_spec_tcp_udp,
-    pub ipv4_ext: ibv_flow_spec_ipv4_ext,
-    pub ipv6: ibv_flow_spec_ipv6,
-    pub esp: ibv_flow_spec_esp,
-    pub tunnel: ibv_flow_spec_tunnel,
-    pub gre: ibv_flow_spec_gre,
-    pub mpls: ibv_flow_spec_mpls,
-    pub flow_tag: ibv_flow_spec_action_tag,
-    pub drop: ibv_flow_spec_action_drop,
-    pub handle: ibv_flow_spec_action_handle,
-    pub flow_count: ibv_flow_spec_counter_action,
-}
-
-#[repr(C)]
-pub struct ibv_flow_spec {
-    pub ibv_flow_spec_union: ibv_flow_spec_union_t,
-}
-
-/// Struct types involve union in <rdma/rdma_cma.h>
-
-// rdma_addr related union and struct types
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct rdma_ib_addr {
-    pub sgid: ibv_gid,
-    pub dgid: ibv_gid,
-    pub pkey: __be16,
-}
-
-#[repr(C)]
-pub union src_addr_union_t {
-    pub src_addr: libc::sockaddr,
-    pub src_sin: libc::sockaddr_in,
-    pub src_sin6: libc::sockaddr_in6,
-    pub src_storage: libc::sockaddr_storage,
-}
-
-#[repr(C)]
-pub union dst_addr_union_t {
-    pub dst_addr: libc::sockaddr,
-    pub dst_sin: libc::sockaddr_in,
-    pub dst_sin6: libc::sockaddr_in6,
-    pub dst_storage: libc::sockaddr_storage,
-}
-
-#[repr(C)]
-pub union addr_union_t {
-    pub ibaddr: rdma_ib_addr,
-}
-
-#[repr(C)]
-pub struct rdma_addr {
-    pub src_addr_union: src_addr_union_t,
-    pub dst_addr_union: dst_addr_union_t,
-    pub addr: addr_union_t,
-}
-
-/// rdma_cm_event related union and struct types
-
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct ibv_ah_attr {
-    pub grh: ibv_global_route,
-    pub dlid: u16,
-    pub sl: u8,
-    pub src_path_bits: u8,
-    pub static_rate: u8,
-    pub is_global: u8,
-    pub port_num: u8,
-}
-
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct rdma_ud_param {
-    pub private_data: *const ::std::os::raw::c_void,
-    pub private_data_len: u8,
-    pub ah_attr: ibv_ah_attr,
-    pub qp_num: u32,
-    pub qkey: u32,
-}
-
-#[repr(C)]
-pub union param_t {
-    pub conn: rdma_conn_param,
-    pub ud: rdma_ud_param,
-}
-
-#[repr(C)]
-pub struct rdma_cm_event {
-    pub id: *mut rdma_cm_id,
-    pub listen_id: *mut rdma_cm_id,
-    pub event: rdma_cm_event_type::Type,
-    pub status: c_int,
-    pub param: param_t,
-}

terraform/.gitignore

@@ -1,15 +0,0 @@
-# Terraform / OpenTofu working files
-.terraform/
-.terraform.lock.hcl
-*.tfstate
-*.tfstate.*
-crash.log
-crash.*.log
-*.tfvars
-!*.tfvars.example
-override.tf
-override.tf.json
-*_override.tf
-*_override.tf.json
-.terraformrc
-terraform.rc

terraform/README.md

@@ -1,128 +0,0 @@
-# SeaweedFS on Terraform
-
-> **Experimental.** This Terraform support is an early scaffold under active
-> development. Interfaces (variables, outputs, module layout) may change without
-> notice, and not every tier is implemented yet. Not recommended for production
-> without your own review and testing.
-
-Self-contained Terraform/OpenTofu modules to deploy SeaweedFS on cloud VMs
-running the `weed` binary directly under systemd. No Helm and no Kubernetes
-required.
-
-What works today is verified end-to-end against a real `weed` cluster (see
-"Test it locally" below).
-
-## Layout
-
-```text
-terraform/
-  modules/
-    core/        cloud-agnostic renderer (ZERO cloud resources): turns an
-                 address map + config into per-node weed argv, systemd units,
-                 config files, disk-mount + secret-fetch scripts, and cloud-init.
-                 Both the cloud wrappers and the local harnesses consume `nodes`.
-    security/    cloud-agnostic CA + per-component mTLS certs (distinct CNs) +
-                 JWT signing keys (tls/random providers). Emits a `core_security`
-                 object ready for the core, plus PEMs for secret-store delivery.
-    aws/         thin AWS wrapper: reserves stable ENIs (fixed private IPs)
-                 first, feeds them to core, creates instances + protected EBS
-                 data disks + SG; with enable_security it generates certs/JWT,
-                 stores them in SSM SecureString, grants an instance role, and
-                 renders a boot fetch-secrets.sh. Keyed for_each throughout.
-  examples/
-    aws-ha-distributed/   3 masters + 3 volumes (1/AZ) + 2 filers + 1 S3,
-                          secure-by-default (mTLS via SSM-delivered certs)
-    aws-all-in-one/       single `weed server` instance via core directly
-  test/
-    local/         render a cluster with core and run it as real weed processes
-                   on 127.0.0.1 (no cloud, no docker), then assert it works
-    local-secure/  generate certs/JWT with the security module, run a real
-                   mTLS cluster, and assert it forms + enforces JWT auth
-```
-
-## Design in one paragraph
-
-The chart is the structural reference, not a dependency. A cloud-agnostic
-**core** renders everything portable; thin per-cloud wrappers provision infra.
-Addressing is an **input** to the core (wrapper reserves static IPs first), so
-the wrapper -> core dependency is one-way with no apply-time cycle. Stateful
-tiers (master/volume/filer) are keyed `for_each` maps, never `count`, so a
-middle node can be replaced without reindexing its peers or reattaching the
-wrong disk. Flag names are verified against the real `weed` binary
-(notably: volume uses `-mserver` for the master list; gRPC is `-port.grpc`,
-auto = http+10000; `minFreeSpacePercent` is a string).
-
-## Test it locally
-
-Requires `tofu` (or `terraform`), `jq`, `curl`, and a `weed` binary. Renders a
-3-master + volume + filer + S3 cluster from the core module and runs it as real
-processes, asserting quorum, volume registration, and filer/S3 round-trips:
-
-```bash
-cd terraform/test/local
-WEED=/path/to/weed ./run_local_cluster.sh
-# => 7 passed, 0 failed
-```
-
-The harness uses a high port range (29333/28080/28888/28333) so it does not
-collide with a SeaweedFS cluster already running on the machine, and aborts if a
-required port is taken. `KEEP=1 ./run_local_cluster.sh` leaves the cluster up.
-
-### mTLS end-to-end
-
-```bash
-cd terraform/test/local-secure
-WEED=/path/to/weed ./run_local_secure.sh
-# => generates a CA + component certs + JWT, renders security.toml, runs a real
-#    mTLS cluster, asserts master/volume/filer form over mutual TLS and that the
-#    filer enforces JWT signing (unsigned writes get 401). 5 passed.
-```
-
-## Plan-level tests (no cloud)
-
-```bash
-cd terraform/modules/core && tofu test
-# => 11 passed: peers list, -mserver vs -master, metrics gating,
-#    security.toml conditions, all-in-one inheritance, ...
-```
-
-## Validate the cloud wrappers
-
-```bash
-cd terraform/examples/aws-ha-distributed && tofu init && tofu validate
-```
-
-`apply` needs AWS credentials, a VPC, subnets, and an AMI with `weed` installed
-(bake with Packer, or install at boot).
-
-## Status
-
-Implemented and verified:
-- Tiers: master / volume / filer / s3 / all-in-one rendered by the core.
-- Disk mount: cloud-init runs a `mount-disks.sh` that auto-discovers (or takes
-  explicit candidate devices), `blkid`-guards `mkfs`, mounts, and persists to
-  `/etc/fstab` by UUID. The AWS wrapper wires the protected EBS disk to `/data`.
-- mTLS + JWT: the `security` module generates the CA + per-component certs
-  (distinct CNs) + JWT keys; the AWS wrapper stores them in SSM SecureString,
-  grants an instance role, and the core renders a boot fetch script so secrets
-  stay OUT of user_data. Proven end-to-end by `test/local-secure`.
-
-Not yet implemented:
-- sftp / admin / worker tiers (core does not render them yet).
-- GCP/Azure wrappers, the data-plane provider, and the K8s-native module.
-- CA in Vault PKI (currently TF-generated, so the CA key lives in state). KMS
-  decrypt in the IAM policy is scoped by
-  `ViaService` but uses `Resource="*"`; tighten to the SSM CMK in production.
-
-## Gotchas
-
-- This `weed` build starts an Iceberg REST catalog on `:8181` by default; set
-  `s3.iceberg_port = 0` to disable, or a port to relocate it.
-- Disk auto-discovery assumes a single attached data disk per node. For multiple
-  data disks, pass explicit `devices` candidates in `disk_mounts`.
-
-## Security note
-
-`tofu output`/state can contain secrets. Generate secrets outside Terraform
-(cloud secret manager / Vault) and fetch them at boot; never commit secret
-material.

terraform/examples/aws-all-in-one/main.tf

@@ -1,133 +0,0 @@
-# SeaweedFS all-in-one on a single AWS instance.
-#
-# Demonstrates the layered design directly: call the cloud-agnostic core to
-# render cloud-init for a `weed server` node, then attach it to one instance
-# with a protected data disk. No wrapper module needed for the simple case.
-#
-#   tofu init && tofu validate
-#   tofu apply   # requires AWS credentials + a weed AMI
-
-terraform {
-  required_version = ">= 1.3.0"
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = ">= 5.40"
-    }
-  }
-}
-
-provider "aws" {
-  region = var.region
-}
-
-variable "region" {
-  type    = string
-  default = "us-east-1"
-}
-variable "vpc_id" {
-  type = string
-}
-variable "subnet_id" {
-  type = string
-}
-variable "availability_zone" {
-  type    = string
-  default = "us-east-1a"
-}
-variable "ami_id" {
-  type = string
-}
-variable "private_ip" {
-  type    = string
-  default = "10.0.1.50"
-}
-
-module "core" {
-  source      = "../../modules/core"
-  weed_binary = "/usr/bin/weed"
-
-  # disable the distributed tiers; run a single all-in-one process
-  master = { enabled = false }
-  volume = { enabled = false }
-  filer  = { enabled = false }
-
-  all_in_one = {
-    enabled = true
-    nodes   = { a0 = { address = var.private_ip, data_dir = "/data" } }
-    s3      = { enabled = true }
-  }
-
-  s3_identities = [{
-    name       = "anonymous"
-    access_key = ""
-    secret_key = ""
-    actions    = ["Read", "List"]
-  }]
-}
-
-resource "aws_network_interface" "aio" {
-  subnet_id       = var.subnet_id
-  private_ips     = [var.private_ip]
-  security_groups = [aws_security_group.aio.id]
-  tags            = { Name = "seaweedfs-all-in-one" }
-}
-
-resource "aws_security_group" "aio" {
-  name_prefix = "seaweedfs-aio-"
-  vpc_id      = var.vpc_id
-  lifecycle {
-    create_before_destroy = true
-  }
-}
-
-resource "aws_vpc_security_group_ingress_rule" "s3" {
-  security_group_id = aws_security_group.aio.id
-  cidr_ipv4         = "10.0.0.0/8"
-  ip_protocol       = "tcp"
-  from_port         = 8333
-  to_port           = 8333
-}
-
-resource "aws_vpc_security_group_egress_rule" "all" {
-  security_group_id = aws_security_group.aio.id
-  cidr_ipv4         = "0.0.0.0/0"
-  ip_protocol       = "-1"
-}
-
-resource "aws_instance" "aio" {
-  ami           = var.ami_id
-  instance_type = "m5.large"
-  user_data     = module.core.cloud_init_by_node["all-in-one-a0"]
-
-  network_interface {
-    network_interface_id = aws_network_interface.aio.id
-    device_index         = 0
-  }
-  metadata_options {
-    http_tokens = "required"
-  }
-  tags = { Name = "seaweedfs-all-in-one", Role = "all-in-one" }
-}
-
-resource "aws_ebs_volume" "data" {
-  availability_zone = var.availability_zone
-  size              = 200
-  type              = "gp3"
-  encrypted         = true
-  tags              = { Name = "seaweedfs-all-in-one-data" }
-  lifecycle {
-    prevent_destroy = true
-  }
-}
-
-resource "aws_volume_attachment" "data" {
-  device_name                    = "/dev/xvdf"
-  volume_id                      = aws_ebs_volume.data.id
-  instance_id                    = aws_instance.aio.id
-  stop_instance_before_detaching = true
-}
-
-output "instance_id" {
-  value = aws_instance.aio.id
-}

terraform/examples/aws-ha-distributed/main.tf

@@ -1,117 +0,0 @@
-# SeaweedFS HA on AWS: 3-master quorum + 3 volume servers (one per AZ) + 2
-# filers (leveldb2-replicated HA) + 1 standalone S3 gateway.
-#
-#   tofu init && tofu validate
-#   tofu apply   # requires AWS credentials, a VPC, subnets, and a weed AMI
-#
-# This is a scaffold: it provisions instances, protected EBS data disks, and the
-# security group. Mounting the EBS disk at /data and secret-store cert delivery
-# are documented follow-ups (see terraform/README.md).
-
-terraform {
-  required_version = ">= 1.3.0"
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = ">= 5.40"
-    }
-  }
-}
-
-provider "aws" {
-  region = var.region
-}
-
-variable "region" {
-  type    = string
-  default = "us-east-1"
-}
-
-variable "vpc_id" {
-  type = string
-}
-
-variable "ami_id" {
-  description = "AMI with the weed binary at /usr/bin/weed."
-  type        = string
-}
-
-# subnet per AZ
-variable "subnet_a" { type = string }
-variable "subnet_b" { type = string }
-variable "subnet_c" { type = string }
-
-# AZ defaults assume region us-east-1. When you change `region`, override az_a/az_b/az_c
-# (and the matching subnets) with AZs that belong to that region, or the volume/filer
-# EBS volumes will fail to create in a mismatched AZ.
-variable "az_a" {
-  type    = string
-  default = "us-east-1a"
-}
-variable "az_b" {
-  type    = string
-  default = "us-east-1b"
-}
-variable "az_c" {
-  type    = string
-  default = "us-east-1c"
-}
-
-variable "client_ingress_cidrs" {
-  type    = list(string)
-  default = ["10.0.0.0/8"]
-}
-
-variable "ssh_ingress_cidrs" {
-  type    = list(string)
-  default = []
-}
-
-module "seaweedfs" {
-  source = "../../modules/aws"
-
-  name   = "seaweedfs"
-  vpc_id = var.vpc_id
-  ami_id = var.ami_id
-
-  # secure-by-default flagship: mTLS (certs + JWT generated by the security
-  # submodule, delivered via SSM and fetched at boot) plus monitoring.
-  enable_security    = true
-  monitoring_enabled = true
-
-  masters = {
-    m0 = { subnet_id = var.subnet_a, private_ip = "10.0.1.10" }
-    m1 = { subnet_id = var.subnet_b, private_ip = "10.0.2.10" }
-    m2 = { subnet_id = var.subnet_c, private_ip = "10.0.3.10" }
-  }
-
-  volumes = {
-    v0 = { subnet_id = var.subnet_a, availability_zone = var.az_a, private_ip = "10.0.1.20", rack = var.az_a, data_center = var.region, data_volume_size_gb = 500 }
-    v1 = { subnet_id = var.subnet_b, availability_zone = var.az_b, private_ip = "10.0.2.20", rack = var.az_b, data_center = var.region, data_volume_size_gb = 500 }
-    v2 = { subnet_id = var.subnet_c, availability_zone = var.az_c, private_ip = "10.0.3.20", rack = var.az_c, data_center = var.region, data_volume_size_gb = 500 }
-  }
-
-  filers = {
-    f0 = { subnet_id = var.subnet_a, availability_zone = var.az_a, private_ip = "10.0.1.30" }
-    f1 = { subnet_id = var.subnet_b, availability_zone = var.az_b, private_ip = "10.0.2.30" }
-  }
-
-  s3_nodes = {
-    s0 = { subnet_id = var.subnet_a, private_ip = "10.0.1.40" }
-  }
-
-  client_ingress_cidrs = var.client_ingress_cidrs
-  ssh_ingress_cidrs    = var.ssh_ingress_cidrs
-}
-
-output "master_peers" {
-  value = module.seaweedfs.master_peers
-}
-
-output "instance_ids" {
-  value = module.seaweedfs.instance_ids
-}
-
-output "security_group_id" {
-  value = module.seaweedfs.security_group_id
-}

terraform/modules/aws/main.tf

@@ -1,205 +0,0 @@
-# ---- step 1: reserve stable addressing (ENIs with fixed private IPs) -------
-# These exist BEFORE the core renders config, so the address map is known and
-# the core <- wrapper dependency is one-way (no apply-time cycle).
-
-resource "aws_network_interface" "master" {
-  for_each        = var.masters
-  subnet_id       = each.value.subnet_id
-  private_ips     = [each.value.private_ip]
-  security_groups = [aws_security_group.cluster.id]
-  tags            = merge(var.tags, { Name = "${var.name}-master-${each.key}", Role = "master" })
-}
-
-resource "aws_network_interface" "volume" {
-  for_each        = var.volumes
-  subnet_id       = each.value.subnet_id
-  private_ips     = [each.value.private_ip]
-  security_groups = [aws_security_group.cluster.id]
-  tags            = merge(var.tags, { Name = "${var.name}-volume-${each.key}", Role = "volume" })
-}
-
-resource "aws_network_interface" "filer" {
-  for_each        = var.filers
-  subnet_id       = each.value.subnet_id
-  private_ips     = [each.value.private_ip]
-  security_groups = [aws_security_group.cluster.id]
-  tags            = merge(var.tags, { Name = "${var.name}-filer-${each.key}", Role = "filer" })
-}
-
-resource "aws_network_interface" "s3" {
-  for_each        = var.s3_nodes
-  subnet_id       = each.value.subnet_id
-  private_ips     = [each.value.private_ip]
-  security_groups = [aws_security_group.cluster.id]
-  tags            = merge(var.tags, { Name = "${var.name}-s3-${each.key}", Role = "s3" })
-}
-
-# ---- step 2: render config with the cloud-agnostic core --------------------
-module "core" {
-  source = "../core"
-
-  weed_binary        = var.weed_binary
-  monitoring_enabled = var.monitoring_enabled
-  enable_security    = var.enable_security
-  security           = local.core_security
-
-  # Keep secrets out of cloud-init user_data when security is on; deliver them
-  # from SSM via the boot fetch script instead.
-  render_secret_files = !var.enable_security
-  boot_fetch_script   = local.fetch_script
-
-  master = {
-    nodes = { for k, v in var.masters : k => { address = v.private_ip } }
-  }
-
-  volume = {
-    enabled = length(var.volumes) > 0
-    nodes = { for k, v in var.volumes : k => {
-      address     = v.private_ip
-      rack        = v.rack
-      data_center = v.data_center
-      data_dirs   = [{ path = "/data", max_volumes = 0 }]
-      # auto-discover the single attached data disk and mount it at /data
-      disk_mounts = [{ mountpoint = "/data", fstype = "xfs" }]
-    } }
-  }
-
-  filer = {
-    enabled = length(var.filers) > 0
-    nodes = { for k, v in var.filers : k => {
-      address     = v.private_ip
-      data_dir    = "/data/filerldb2"
-      disk_mounts = [{ mountpoint = "/data", fstype = "xfs" }]
-    } }
-    s3 = var.embedded_s3
-  }
-
-  s3 = {
-    enabled = length(var.s3_nodes) > 0
-    nodes   = { for k, v in var.s3_nodes : k => { address = v.private_ip } }
-  }
-
-  s3_identities = var.s3_identities
-}
-
-# ---- step 3: instances (keyed for_each; user_data = rendered cloud-init) ----
-resource "aws_instance" "master" {
-  for_each                = var.masters
-  ami                     = var.ami_id
-  instance_type           = each.value.instance_type
-  key_name                = var.key_name
-  user_data               = module.core.cloud_init_by_node["master-${each.key}"]
-  iam_instance_profile    = var.enable_security ? aws_iam_instance_profile.node[0].name : null
-  disable_api_termination = var.termination_protection
-
-  network_interface {
-    network_interface_id = aws_network_interface.master[each.key].id
-    device_index         = 0
-  }
-  metadata_options {
-    http_tokens = "required" # IMDSv2
-  }
-  tags = merge(var.tags, { Name = "${var.name}-master-${each.key}", Role = "master" })
-}
-
-resource "aws_instance" "volume" {
-  for_each                = var.volumes
-  ami                     = var.ami_id
-  instance_type           = each.value.instance_type
-  key_name                = var.key_name
-  user_data               = module.core.cloud_init_by_node["volume-${each.key}"]
-  iam_instance_profile    = var.enable_security ? aws_iam_instance_profile.node[0].name : null
-  disable_api_termination = var.termination_protection
-
-  network_interface {
-    network_interface_id = aws_network_interface.volume[each.key].id
-    device_index         = 0
-  }
-  metadata_options {
-    http_tokens = "required"
-  }
-  tags = merge(var.tags, { Name = "${var.name}-volume-${each.key}", Role = "volume" })
-}
-
-resource "aws_instance" "filer" {
-  for_each                = var.filers
-  ami                     = var.ami_id
-  instance_type           = each.value.instance_type
-  key_name                = var.key_name
-  user_data               = module.core.cloud_init_by_node["filer-${each.key}"]
-  iam_instance_profile    = var.enable_security ? aws_iam_instance_profile.node[0].name : null
-  disable_api_termination = var.termination_protection
-
-  network_interface {
-    network_interface_id = aws_network_interface.filer[each.key].id
-    device_index         = 0
-  }
-  metadata_options {
-    http_tokens = "required"
-  }
-  tags = merge(var.tags, { Name = "${var.name}-filer-${each.key}", Role = "filer" })
-}
-
-resource "aws_instance" "s3" {
-  for_each             = var.s3_nodes
-  ami                  = var.ami_id
-  instance_type        = each.value.instance_type
-  key_name             = var.key_name
-  user_data            = module.core.cloud_init_by_node["s3-${each.key}"]
-  iam_instance_profile = var.enable_security ? aws_iam_instance_profile.node[0].name : null
-
-  network_interface {
-    network_interface_id = aws_network_interface.s3[each.key].id
-    device_index         = 0
-  }
-  metadata_options {
-    http_tokens = "required"
-  }
-  tags = merge(var.tags, { Name = "${var.name}-s3-${each.key}", Role = "s3" })
-}
-
-# ---- step 4: protected data disks (decoupled; survive instance replacement) -
-# The core's mount-disks.sh (wired via disk_mounts above) mkfs+mounts these at
-# /data before the weed unit starts.
-resource "aws_ebs_volume" "volume_data" {
-  for_each          = var.volumes
-  availability_zone = each.value.availability_zone
-  size              = each.value.data_volume_size_gb
-  type              = each.value.data_volume_type
-  iops              = each.value.data_volume_iops
-  encrypted         = true
-  tags              = merge(var.tags, { Name = "${var.name}-volume-${each.key}-data", Role = "volume" })
-
-  lifecycle {
-    prevent_destroy = true # break-glass: remove to allow destroy
-  }
-}
-
-resource "aws_volume_attachment" "volume_data" {
-  for_each                       = var.volumes
-  device_name                    = "/dev/xvdf"
-  volume_id                      = aws_ebs_volume.volume_data[each.key].id
-  instance_id                    = aws_instance.volume[each.key].id
-  stop_instance_before_detaching = true
-}
-
-resource "aws_ebs_volume" "filer_data" {
-  for_each          = var.filers
-  availability_zone = each.value.availability_zone
-  size              = each.value.data_volume_size_gb
-  type              = "gp3"
-  encrypted         = true
-  tags              = merge(var.tags, { Name = "${var.name}-filer-${each.key}-data", Role = "filer" })
-
-  lifecycle {
-    prevent_destroy = true
-  }
-}
-
-resource "aws_volume_attachment" "filer_data" {
-  for_each                       = var.filers
-  device_name                    = "/dev/xvdf"
-  volume_id                      = aws_ebs_volume.filer_data[each.key].id
-  instance_id                    = aws_instance.filer[each.key].id
-  stop_instance_before_detaching = true
-}

terraform/modules/aws/outputs.tf

@@ -1,34 +0,0 @@
-output "security_group_id" {
-  description = "Cluster security group id."
-  value       = aws_security_group.cluster.id
-}
-
-output "master_private_ips" {
-  description = "Master private IPs keyed by node id."
-  value       = { for k, v in var.masters : k => v.private_ip }
-}
-
-output "master_peers" {
-  description = "Master peer list passed to the weed processes."
-  value       = module.core.master_peers
-}
-
-output "filer_private_ips" {
-  description = "Filer private IPs keyed by node id."
-  value       = { for k, v in var.filers : k => v.private_ip }
-}
-
-output "s3_private_ips" {
-  description = "Standalone S3 gateway private IPs keyed by node id."
-  value       = { for k, v in var.s3_nodes : k => v.private_ip }
-}
-
-output "instance_ids" {
-  description = "All instance ids keyed by role-node."
-  value = merge(
-    { for k, v in aws_instance.master : "master-${k}" => v.id },
-    { for k, v in aws_instance.volume : "volume-${k}" => v.id },
-    { for k, v in aws_instance.filer : "filer-${k}" => v.id },
-    { for k, v in aws_instance.s3 : "s3-${k}" => v.id },
-  )
-}

terraform/modules/aws/security.tf

@@ -1,136 +0,0 @@
-# =============================================================================
-# mTLS + JWT generation and secret delivery (active when enable_security=true).
-#
-# Generates the CA + per-component certs + JWT keys with the security submodule,
-# stores them (plus the rendered security.toml / S3 config) in SSM Parameter
-# Store as SecureString, and renders a boot fetch-secrets.sh that pulls them via
-# the instance role. This keeps secrets OUT of cloud-init user_data / IMDS.
-# Secrets still live in Terraform state (TF-generated); prefer Vault PKI for the
-# CA in production.
-# =============================================================================
-
-locals {
-  all_private_ips = concat(
-    [for k, v in var.masters : v.private_ip],
-    [for k, v in var.volumes : v.private_ip],
-    [for k, v in var.filers : v.private_ip],
-    [for k, v in var.s3_nodes : v.private_ip],
-  )
-
-  ssm_prefix        = "/seaweedfs/${var.name}"
-  deliver_s3_config = length(var.s3_nodes) > 0 || var.embedded_s3.enabled
-
-  # security.toml is identical across nodes; take it from the first master.
-  first_master       = var.enable_security ? sort(keys(var.masters))[0] : ""
-  security_toml_path = "/etc/seaweedfs/security.toml"
-  s3_config_path     = "/etc/seaweedfs/s3_config.json"
-
-  # Boot fetch entries: cert files + security.toml + (optional) S3 config.
-  fetch_entries = var.enable_security ? concat(
-    [for m in module.security[0].secret_manifest : { param = "${local.ssm_prefix}/certs/${m.key}", path = m.path, mode = m.mode }],
-    [{ param = "${local.ssm_prefix}/security.toml", path = local.security_toml_path, mode = "0600" }],
-    local.deliver_s3_config ? [{ param = "${local.ssm_prefix}/s3_config.json", path = local.s3_config_path, mode = "0600" }] : [],
-  ) : []
-
-  fetch_script = var.enable_security ? templatefile("${path.module}/templates/fetch-secrets.sh.tftpl", {
-    run_as_user = "seaweedfs"
-    entries     = local.fetch_entries
-  }) : ""
-
-  core_security = var.enable_security ? module.security[0].core_security : var.security
-}
-
-module "security" {
-  count           = var.enable_security ? 1 : 0
-  source          = "../security"
-  internal_domain = var.internal_domain
-  cert_dir        = var.cert_dir
-  ip_sans         = local.all_private_ips
-}
-
-# ---- SSM SecureString parameters -------------------------------------------
-resource "aws_ssm_parameter" "cert" {
-  for_each = var.enable_security ? { for m in module.security[0].secret_manifest : m.key => m } : {}
-  name     = "${local.ssm_prefix}/certs/${each.key}"
-  type     = "SecureString"
-  value    = module.security[0].secret_contents[each.key]
-  tags     = var.tags
-}
-
-resource "aws_ssm_parameter" "security_toml" {
-  count = var.enable_security ? 1 : 0
-  name  = "${local.ssm_prefix}/security.toml"
-  type  = "SecureString"
-  value = module.core.secret_files_by_node["master-${local.first_master}"][local.security_toml_path]
-  tags  = var.tags
-}
-
-resource "aws_ssm_parameter" "s3_config" {
-  count = var.enable_security && local.deliver_s3_config ? 1 : 0
-  name  = "${local.ssm_prefix}/s3_config.json"
-  type  = "SecureString"
-  # identical content across s3/filer nodes; render once via the security module's
-  # JSON is not available here, so read it back from any node that carries it.
-  value = local.s3_config_source
-  tags  = var.tags
-}
-
-locals {
-  # Pull the rendered S3 identity JSON from whichever node carries it.
-  s3_config_source = !local.deliver_s3_config ? "" : (
-    length(var.s3_nodes) > 0 ?
-    module.core.secret_files_by_node["s3-${sort(keys(var.s3_nodes))[0]}"][local.s3_config_path] :
-    (length(var.filers) > 0 ? module.core.secret_files_by_node["filer-${sort(keys(var.filers))[0]}"][local.s3_config_path] : "")
-  )
-}
-
-# ---- IAM: instance role allowed to read the SSM params ----------------------
-data "aws_iam_policy_document" "assume" {
-  count = var.enable_security ? 1 : 0
-  statement {
-    actions = ["sts:AssumeRole"]
-    principals {
-      type        = "Service"
-      identifiers = ["ec2.amazonaws.com"]
-    }
-  }
-}
-
-data "aws_iam_policy_document" "ssm_read" {
-  count = var.enable_security ? 1 : 0
-  statement {
-    sid       = "ReadSeaweedfsParams"
-    actions   = ["ssm:GetParameter", "ssm:GetParameters"]
-    resources = ["arn:aws:ssm:*:*:parameter${local.ssm_prefix}/*"]
-  }
-  statement {
-    sid       = "DecryptSecureStrings"
-    actions   = ["kms:Decrypt"]
-    resources = [var.kms_key_arn] # defaults to "*"; set a CMK ARN in production
-    condition {
-      test     = "StringEquals"
-      variable = "kms:ViaService"
-      values   = ["ssm.*.amazonaws.com"]
-    }
-  }
-}
-
-resource "aws_iam_role" "node" {
-  count              = var.enable_security ? 1 : 0
-  name_prefix        = "${var.name}-node-"
-  assume_role_policy = data.aws_iam_policy_document.assume[0].json
-  tags               = var.tags
-}
-
-resource "aws_iam_role_policy" "ssm_read" {
-  count  = var.enable_security ? 1 : 0
-  name   = "ssm-read"
-  role   = aws_iam_role.node[0].id
-  policy = data.aws_iam_policy_document.ssm_read[0].json
-}
-
-resource "aws_iam_instance_profile" "node" {
-  count       = var.enable_security ? 1 : 0
-  name_prefix = "${var.name}-node-"
-  role        = aws_iam_role.node[0].name
-}

terraform/modules/aws/security_group.tf

@@ -1,62 +0,0 @@
-# Cluster security group + the SeaweedFS port matrix.
-# Intra-cluster: all TCP allowed within the SG (master 9333/19333, volume
-# 8080/18080, filer 8888/18888, admin 33646, ...). Client-facing: only S3 (8333)
-# and filer (8888). Metrics ports (9327) are never opened to clients.
-
-resource "aws_security_group" "cluster" {
-  name_prefix = "${var.name}-cluster-"
-  description = "SeaweedFS cluster"
-  vpc_id      = var.vpc_id
-  tags        = merge(var.tags, { Name = "${var.name}-cluster" })
-
-  lifecycle {
-    create_before_destroy = true
-  }
-}
-
-# All intra-cluster TCP via SG self-reference.
-resource "aws_vpc_security_group_ingress_rule" "intra_cluster" {
-  security_group_id            = aws_security_group.cluster.id
-  referenced_security_group_id = aws_security_group.cluster.id
-  ip_protocol                  = "tcp"
-  from_port                    = 0
-  to_port                      = 65535
-  description                  = "intra-cluster (http + gRPC for all roles)"
-}
-
-resource "aws_vpc_security_group_ingress_rule" "s3" {
-  for_each          = toset(var.client_ingress_cidrs)
-  security_group_id = aws_security_group.cluster.id
-  cidr_ipv4         = each.value
-  ip_protocol       = "tcp"
-  from_port         = 8333
-  to_port           = 8333
-  description       = "S3 gateway"
-}
-
-resource "aws_vpc_security_group_ingress_rule" "filer" {
-  for_each          = toset(var.client_ingress_cidrs)
-  security_group_id = aws_security_group.cluster.id
-  cidr_ipv4         = each.value
-  ip_protocol       = "tcp"
-  from_port         = 8888
-  to_port           = 8888
-  description       = "filer HTTP"
-}
-
-resource "aws_vpc_security_group_ingress_rule" "ssh" {
-  for_each          = toset(var.ssh_ingress_cidrs)
-  security_group_id = aws_security_group.cluster.id
-  cidr_ipv4         = each.value
-  ip_protocol       = "tcp"
-  from_port         = 22
-  to_port           = 22
-  description       = "SSH"
-}
-
-resource "aws_vpc_security_group_egress_rule" "all" {
-  security_group_id = aws_security_group.cluster.id
-  cidr_ipv4         = "0.0.0.0/0"
-  ip_protocol       = "-1"
-  description       = "all egress"
-}

terraform/modules/aws/templates/fetch-secrets.sh.tftpl

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-# Rendered by terraform-aws-seaweedfs. Pulls certs + secret config from SSM
-# Parameter Store (SecureString) at boot using the instance role. Region is
-# auto-resolved by the AWS CLI from IMDS, so no region is baked in. Requires
-# the AWS CLI and a working instance profile (see security.tf).
-set -euo pipefail
-
-run_as_user="${run_as_user}"
-
-fetch() {
-  local param="$1" dest="$2" mode="$3"
-  mkdir -p "$(dirname "$dest")"
-  aws ssm get-parameter --with-decryption --name "$param" \
-    --query 'Parameter.Value' --output text > "$dest"
-  chmod "$mode" "$dest"
-  chown "$run_as_user:$run_as_user" "$dest" 2>/dev/null || true
-}
-
-%{ for e in entries ~}
-fetch "${e.param}" "${e.path}" "${e.mode}"
-%{ endfor ~}

terraform/modules/aws/variables.tf

@@ -1,171 +0,0 @@
-# =============================================================================
-# terraform-aws-seaweedfs - thin AWS wrapper around the cloud-agnostic core.
-#
-# Reserves stable per-node addressing (ENIs with fixed private IPs) FIRST, feeds
-# that address map to the core to render cloud-init, then creates instances and
-# protected EBS data disks. All tiers are keyed for_each (never count), so a
-# middle node can be replaced without reindexing its peers/disks.
-# =============================================================================
-
-variable "name" {
-  description = "Name prefix for all resources."
-  type        = string
-  default     = "seaweedfs"
-}
-
-variable "vpc_id" {
-  description = "VPC to deploy into."
-  type        = string
-}
-
-variable "ami_id" {
-  description = "AMI with the weed binary installed (e.g. baked with Packer)."
-  type        = string
-}
-
-variable "weed_binary" {
-  description = "Path to weed on the AMI."
-  type        = string
-  default     = "/usr/bin/weed"
-}
-
-variable "key_name" {
-  description = "EC2 key pair for SSH (optional)."
-  type        = string
-  default     = null
-}
-
-variable "monitoring_enabled" {
-  description = "Bind metrics ports and pass them through to the core."
-  type        = bool
-  default     = false
-}
-
-variable "enable_security" {
-  description = "Enable mTLS (security.toml). Cert material must be supplied via the security variable / secret store."
-  type        = bool
-  default     = false
-}
-
-variable "security" {
-  description = "Passed through to the core security variable when enable_security=false. When enable_security=true this is ignored and certs/JWT are generated by the security submodule."
-  type        = any
-  default     = {}
-}
-
-variable "internal_domain" {
-  description = "Internal domain for generated component cert CNs / peer-auth wildcard (enable_security=true)."
-  type        = string
-  default     = "seaweedfs.internal"
-}
-
-variable "cert_dir" {
-  description = "On-host directory where certs are fetched/placed."
-  type        = string
-  default     = "/usr/local/share/ca-certificates"
-}
-
-variable "kms_key_arn" {
-  description = "KMS key ARN the instance role may use to decrypt SSM SecureStrings. Defaults to \"*\" (the SSM default key); set to a specific CMK ARN in production."
-  type        = string
-  default     = "*"
-}
-
-variable "termination_protection" {
-  description = "Set disable_api_termination on stateful instances (masters, volumes)."
-  type        = bool
-  default     = true
-}
-
-# ---- stateful tiers: keyed node maps ---------------------------------------
-variable "masters" {
-  description = "Master nodes keyed by stable id (m0/m1/m2). Quorum size must be odd."
-  type = map(object({
-    subnet_id     = string
-    private_ip    = string
-    instance_type = optional(string, "t3.medium")
-  }))
-  validation {
-    condition     = length(var.masters) % 2 == 1
-    error_message = "Master quorum must be an odd number of nodes (1, 3, 5)."
-  }
-}
-
-variable "volumes" {
-  description = "Volume nodes keyed by stable id. Each owns one protected EBS data disk."
-  type = map(object({
-    subnet_id           = string
-    availability_zone   = string
-    private_ip          = string
-    instance_type       = optional(string, "m5.large")
-    rack                = optional(string)
-    data_center         = optional(string)
-    data_volume_size_gb = optional(number, 100)
-    data_volume_type    = optional(string, "gp3")
-    data_volume_iops    = optional(number, null)
-  }))
-  default = {}
-}
-
-variable "filers" {
-  description = "Filer nodes keyed by stable id. Each owns a protected EBS disk for its leveldb2 store."
-  type = map(object({
-    subnet_id           = string
-    availability_zone   = string
-    private_ip          = string
-    instance_type       = optional(string, "t3.medium")
-    data_volume_size_gb = optional(number, 50)
-  }))
-  default = {}
-}
-
-variable "s3_nodes" {
-  description = "Standalone S3 gateway nodes keyed by stable id (stateless)."
-  type = map(object({
-    subnet_id     = string
-    private_ip    = string
-    instance_type = optional(string, "t3.medium")
-  }))
-  default = {}
-}
-
-variable "embedded_s3" {
-  description = "Embed the S3 gateway in the filer instead of standalone nodes."
-  type = object({
-    enabled     = optional(bool, false)
-    port        = optional(number, 8333)
-    domain_name = optional(string, "")
-  })
-  default = {}
-}
-
-variable "s3_identities" {
-  description = "Non-admin S3 identities for the gateway config (admin key goes via EnvironmentFile)."
-  type = list(object({
-    name       = string
-    access_key = string
-    secret_key = string
-    actions    = list(string)
-  }))
-  default   = []
-  sensitive = true
-}
-
-# ---- networking ------------------------------------------------------------
-variable "client_ingress_cidrs" {
-  description = "CIDRs allowed to reach the client-facing S3 (8333) and filer (8888) ports."
-  type        = list(string)
-  default     = []
-}
-
-variable "ssh_ingress_cidrs" {
-  description = "CIDRs allowed SSH (22)."
-  type        = list(string)
-  default     = []
-}
-
-variable "tags" {
-  description = "Extra tags applied to all resources."
-  type        = map(string)
-  default     = {}
-}

terraform/modules/aws/versions.tf

@@ -1,12 +0,0 @@
-terraform {
-  required_version = ">= 1.3.0"
-  required_providers {
-    aws = {
-      source = "hashicorp/aws"
-      # >= 5.40 for the modern aws_vpc_security_group_*_rule resources.
-      # Provider >= 6.0 is recommended for the EBS-attachment no-replace behavior;
-      # bump the floor here once you standardize on it.
-      version = ">= 5.40"
-    }
-  }
-}

terraform/modules/core/main.tf

@@ -1,373 +0,0 @@
-# =============================================================================
-# SeaweedFS Terraform core - pure renderer (zero cloud resources).
-#
-# Computes, per node: the verified `weed` argv, the full systemd ExecStart,
-# config files, environment, and rendered systemd unit + cloud-init.
-# =============================================================================
-
-locals {
-  # ---- shared security.toml (rendered when mTLS on OR any JWT key set) ----
-  jwt_any = anytrue([
-    var.security.jwt_signing_key != "",
-    var.security.jwt_signing_read_key != "",
-    var.security.jwt_filer_signing_key != "",
-    var.security.jwt_filer_signing_read_key != "",
-  ])
-  render_security = var.enable_security || local.jwt_any
-  security_toml = local.render_security ? templatefile("${path.module}/templates/security.toml.tftpl", {
-    enable_security            = var.enable_security
-    cert_dir                   = var.security.cert_dir
-    allowed_wildcard_domain    = var.security.allowed_wildcard_domain
-    allowed_common_names       = var.security.allowed_common_names
-    jwt_signing_key            = var.security.jwt_signing_key
-    jwt_signing_read_key       = var.security.jwt_signing_read_key
-    jwt_filer_signing_key      = var.security.jwt_filer_signing_key
-    jwt_filer_signing_read_key = var.security.jwt_filer_signing_read_key
-  }) : ""
-  security_files = local.render_security ? { "/etc/seaweedfs/security.toml" = local.security_toml } : {}
-
-  # ---- S3 identity JSON (non-admin identities only) ----
-  # Empty access_key => anonymous identity (no credentials block), per SeaweedFS
-  # convention where the identity literally named "anonymous" grants anon access.
-  s3_config_json = jsonencode({
-    identities = [for id in var.s3_identities : {
-      name        = id.name
-      credentials = id.access_key != "" ? [{ accessKey = id.access_key, secretKey = id.secret_key }] : []
-      actions     = id.actions
-    }]
-  })
-
-  # ---- global flags (before the subcommand) ----
-  global_flags = compact([
-    format("-v=%d", var.log_level),
-    var.log_to_stderr ? "-logtostderr=true" : "",
-  ])
-
-  # ---- master peers (sorted by key for determinism) ----
-  master_keys = sort(keys(var.master.nodes))
-  master_peer_list = [for k in local.master_keys :
-    format("%s:%d", var.master.nodes[k].address, coalesce(var.master.nodes[k].port, var.master.port))
-  ]
-  master_peers = join(",", local.master_peer_list)
-
-  # ---- first filer (default s3 target + cluster env) ----
-  filer_keys  = sort(keys(var.filer.nodes))
-  first_filer = length(local.filer_keys) > 0 ? format("%s:%d", var.filer.nodes[local.filer_keys[0]].address, coalesce(var.filer.nodes[local.filer_keys[0]].port, var.filer.port)) : ""
-
-  # ---- cluster discovery env (WEED_CLUSTER_*) ----
-  cluster_env = merge(
-    { WEED_CLUSTER_DEFAULT = var.cluster_name },
-    local.master_peers != "" ? { "WEED_CLUSTER_${replace(upper(var.cluster_name), "/[^A-Z0-9_]/", "_")}_MASTER" = local.master_peers } : {},
-    local.first_filer != "" ? { "WEED_CLUSTER_${replace(upper(var.cluster_name), "/[^A-Z0-9_]/", "_")}_FILER" = local.first_filer } : {},
-  )
-
-  metrics_flags_master = var.monitoring_enabled ? compact([
-    format("-metricsPort=%d", var.master.metrics_port),
-    var.master.metrics_ip != "" ? format("-metricsIp=%s", var.master.metrics_ip) : "",
-  ]) : []
-  metrics_flags_volume = var.monitoring_enabled ? compact([
-    format("-metricsPort=%d", var.volume.metrics_port),
-    var.volume.metrics_ip != "" ? format("-metricsIp=%s", var.volume.metrics_ip) : "",
-  ]) : []
-  metrics_flags_filer = var.monitoring_enabled ? compact([
-    format("-metricsPort=%d", var.filer.metrics_port),
-    var.filer.metrics_ip != "" ? format("-metricsIp=%s", var.filer.metrics_ip) : "",
-  ]) : []
-  metrics_flags_s3 = var.monitoring_enabled ? [format("-metricsPort=%d", var.s3.metrics_port)] : []
-
-  # ===========================================================================
-  # MASTER nodes
-  # ===========================================================================
-  master_nodes = var.master.enabled ? {
-    for k in local.master_keys : "master-${k}" => {
-      role        = "master"
-      name        = "master-${k}"
-      address     = var.master.nodes[k].address
-      disk_mounts = var.master.nodes[k].disk_mounts
-      data_dir    = coalesce(var.master.nodes[k].data_dir, var.master.data_dir)
-      data_dirs   = [coalesce(var.master.nodes[k].data_dir, var.master.data_dir)]
-      ports = {
-        http    = coalesce(var.master.nodes[k].port, var.master.port)
-        metrics = var.master.metrics_port
-      }
-      env          = local.cluster_env
-      config_files = local.security_files
-      argv = concat(
-        local.global_flags,
-        ["master"],
-        [
-          format("-port=%d", coalesce(var.master.nodes[k].port, var.master.port)),
-          format("-mdir=%s", coalesce(var.master.nodes[k].data_dir, var.master.data_dir)),
-          format("-ip=%s", var.master.nodes[k].address),
-          format("-ip.bind=%s", var.master.ip_bind),
-          format("-peers=%s", local.master_peers),
-          format("-defaultReplication=%s", var.master.default_replication),
-          format("-volumeSizeLimitMB=%d", var.master.volume_size_limit_mb),
-          format("-electionTimeout=%s", var.master.election_timeout),
-          format("-heartbeatInterval=%s", var.master.heartbeat_interval),
-        ],
-        var.master.nodes[k].grpc_port != null ? [format("-port.grpc=%d", var.master.nodes[k].grpc_port)] : (
-        var.master.grpc_port != null ? [format("-port.grpc=%d", var.master.grpc_port)] : []),
-        var.master.volume_preallocate ? ["-volumePreallocate"] : [],
-        var.master.garbage_threshold != null ? [format("-garbageThreshold=%s", tostring(var.master.garbage_threshold))] : [],
-        var.master.raft_hashicorp ? ["-raftHashicorp"] : [],
-        var.master.resume_state ? ["-resumeState"] : [],
-        var.master.disable_http ? ["-disableHttp"] : [],
-        var.master.white_list != "" ? [format("-whiteList=%s", var.master.white_list)] : [],
-        local.metrics_flags_master,
-        var.master.extra_args,
-      )
-    }
-  } : {}
-
-  # ===========================================================================
-  # VOLUME nodes
-  # ===========================================================================
-  volume_keys = sort(keys(var.volume.nodes))
-  volume_nodes = var.volume.enabled ? {
-    for k in local.volume_keys : "volume-${k}" => {
-      role        = "volume"
-      name        = "volume-${k}"
-      address     = var.volume.nodes[k].address
-      disk_mounts = var.volume.nodes[k].disk_mounts
-      data_dirs   = [for d in(var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs) : d.path]
-      data_dir    = (var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs)[0].path
-      ports = {
-        http    = coalesce(var.volume.nodes[k].port, var.volume.port)
-        metrics = var.volume.metrics_port
-      }
-      env          = local.cluster_env
-      config_files = local.security_files
-      argv = concat(
-        local.global_flags,
-        ["volume"],
-        [
-          format("-port=%d", coalesce(var.volume.nodes[k].port, var.volume.port)),
-          format("-dir=%s", join(",", [for d in(var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs) : d.path])),
-          format("-max=%s", join(",", [for d in(var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs) : tostring(d.max_volumes)])),
-          format("-ip=%s", var.volume.nodes[k].address),
-          format("-ip.bind=%s", var.volume.ip_bind),
-          format("-mserver=%s", local.master_peers),
-          format("-readMode=%s", var.volume.read_mode),
-          format("-compactionMBps=%d", var.volume.compaction_mbps),
-          format("-minFreeSpacePercent=%s", var.volume.min_free_space_percent),
-          format("-index=%s", var.volume.index),
-        ],
-        var.volume.nodes[k].grpc_port != null ? [format("-port.grpc=%d", var.volume.nodes[k].grpc_port)] : [],
-        (var.volume.nodes[k].idx_dir != null && var.volume.nodes[k].idx_dir != "") ? [format("-dir.idx=%s", var.volume.nodes[k].idx_dir)] : (var.volume.idx_dir != "" ? [format("-dir.idx=%s", var.volume.idx_dir)] : []),
-        var.volume.nodes[k].data_center != null ? [format("-dataCenter=%s", var.volume.nodes[k].data_center)] : [],
-        var.volume.nodes[k].rack != null ? [format("-rack=%s", var.volume.nodes[k].rack)] : [],
-        var.volume.nodes[k].public_url != null ? [format("-publicUrl=%s", var.volume.nodes[k].public_url)] : [],
-        var.volume.file_size_limit_mb != null ? [format("-fileSizeLimitMB=%d", var.volume.file_size_limit_mb)] : [],
-        var.volume.images_fix_orientation ? ["-images.fix.orientation"] : [],
-        var.volume.white_list != "" ? [format("-whiteList=%s", var.volume.white_list)] : [],
-        local.metrics_flags_volume,
-        var.volume.extra_args,
-      )
-    }
-  } : {}
-
-  # ===========================================================================
-  # FILER nodes (+ optional embedded S3)
-  # ===========================================================================
-  filer_s3_files = var.filer.s3.enabled ? { (var.filer.s3.config_path) = local.s3_config_json } : {}
-  filer_nodes = var.filer.enabled ? {
-    for k in local.filer_keys : "filer-${k}" => {
-      role        = "filer"
-      name        = "filer-${k}"
-      address     = var.filer.nodes[k].address
-      disk_mounts = var.filer.nodes[k].disk_mounts
-      data_dir    = coalesce(var.filer.nodes[k].data_dir, var.filer.data_dir)
-      data_dirs   = [coalesce(var.filer.nodes[k].data_dir, var.filer.data_dir)]
-      ports = {
-        http    = coalesce(var.filer.nodes[k].port, var.filer.port)
-        metrics = var.filer.metrics_port
-      }
-      env          = merge(local.cluster_env, var.filer.extra_env)
-      config_files = merge(local.security_files, local.filer_s3_files)
-      argv = concat(
-        local.global_flags,
-        ["filer"],
-        [
-          format("-port=%d", coalesce(var.filer.nodes[k].port, var.filer.port)),
-          format("-ip=%s", var.filer.nodes[k].address),
-          format("-ip.bind=%s", var.filer.ip_bind),
-          format("-master=%s", local.master_peers),
-          format("-defaultReplicaPlacement=%s", var.filer.default_replica_placement),
-          format("-dirListLimit=%d", var.filer.dir_list_limit),
-          format("-defaultStoreDir=%s", coalesce(var.filer.nodes[k].data_dir, var.filer.data_dir)),
-        ],
-        var.filer.nodes[k].grpc_port != null ? [format("-port.grpc=%d", var.filer.nodes[k].grpc_port)] : [],
-        var.filer.max_mb != null ? [format("-maxMB=%d", var.filer.max_mb)] : [],
-        var.filer.disable_dir_listing ? ["-disableDirListing"] : [],
-        var.filer.disable_http ? ["-disableHttp"] : [],
-        var.filer.encrypt_volume_data ? ["-encryptVolumeData"] : [],
-        var.filer.rack != "" ? [format("-rack=%s", var.filer.rack)] : [],
-        var.filer.data_center != "" ? [format("-dataCenter=%s", var.filer.data_center)] : [],
-        var.filer.filer_group != "" ? [format("-filerGroup=%s", var.filer.filer_group)] : [],
-        var.filer.s3.enabled ? [
-          "-s3",
-          format("-s3.port=%d", var.filer.s3.port),
-          format("-s3.config=%s", var.filer.s3.config_path),
-        ] : [],
-        (var.filer.s3.enabled && var.filer.s3.https_port > 0) ? [format("-s3.port.https=%d", var.filer.s3.https_port)] : [],
-        (var.filer.s3.enabled && var.filer.s3.domain_name != "") ? [format("-s3.domainName=%s", var.filer.s3.domain_name)] : [],
-        local.metrics_flags_filer,
-        var.filer.extra_args,
-      )
-    }
-  } : {}
-
-  # ===========================================================================
-  # S3 standalone nodes
-  # ===========================================================================
-  s3_keys      = sort(keys(var.s3.nodes))
-  s3_target    = var.s3.filer_address != "" ? var.s3.filer_address : local.first_filer
-  s3_std_files = merge(local.security_files, { (var.s3.config_path) = local.s3_config_json })
-  s3_nodes = var.s3.enabled ? {
-    for k in local.s3_keys : "s3-${k}" => {
-      role        = "s3"
-      name        = "s3-${k}"
-      address     = var.s3.nodes[k].address
-      disk_mounts = []
-      data_dir    = ""
-      data_dirs   = []
-      ports = {
-        http    = coalesce(var.s3.nodes[k].port, var.s3.port)
-        metrics = var.s3.metrics_port
-      }
-      env          = local.cluster_env
-      config_files = local.s3_std_files
-      argv = concat(
-        local.global_flags,
-        ["s3"],
-        [
-          format("-port=%d", coalesce(var.s3.nodes[k].port, var.s3.port)),
-          format("-ip.bind=%s", var.s3.ip_bind),
-          format("-filer=%s", local.s3_target),
-          format("-config=%s", var.s3.config_path),
-        ],
-        var.s3.https_port > 0 ? [format("-port.https=%d", var.s3.https_port)] : [],
-        var.s3.iceberg_port != null ? [format("-port.iceberg=%d", var.s3.iceberg_port)] : [],
-        var.s3.domain_name != "" ? [format("-domainName=%s", var.s3.domain_name)] : [],
-        var.s3.audit_log_config_path != "" ? [format("-auditLogConfig=%s", var.s3.audit_log_config_path)] : [],
-        var.s3.cert_file != "" ? [format("-cert.file=%s", var.s3.cert_file)] : [],
-        var.s3.key_file != "" ? [format("-key.file=%s", var.s3.key_file)] : [],
-        var.s3.cacert_file != "" ? [format("-cacert.file=%s", var.s3.cacert_file)] : [],
-        var.s3.verify_client_cert ? ["-tlsVerifyClientCert"] : [],
-        local.metrics_flags_s3,
-        var.s3.extra_args,
-      )
-    }
-  } : {}
-
-  # ===========================================================================
-  # ALL-IN-ONE nodes (weed server)
-  # ===========================================================================
-  aio_keys     = sort(keys(var.all_in_one.nodes))
-  aio_s3_files = var.all_in_one.s3.enabled ? { (var.all_in_one.s3.config_path) = local.s3_config_json } : {}
-  aio_nodes = var.all_in_one.enabled ? {
-    for k in local.aio_keys : "all-in-one-${k}" => {
-      role        = "server"
-      name        = "all-in-one-${k}"
-      address     = var.all_in_one.nodes[k].address
-      disk_mounts = var.all_in_one.nodes[k].disk_mounts
-      data_dir    = coalesce(var.all_in_one.nodes[k].data_dir, var.all_in_one.data_dir)
-      data_dirs   = [coalesce(var.all_in_one.nodes[k].data_dir, var.all_in_one.data_dir)]
-      ports = {
-        http    = var.all_in_one.master_port
-        metrics = var.all_in_one.metrics_port
-      }
-      env          = local.cluster_env
-      config_files = merge(local.security_files, local.aio_s3_files)
-      argv = concat(
-        local.global_flags,
-        ["server"],
-        [
-          format("-dir=%s", coalesce(var.all_in_one.nodes[k].data_dir, var.all_in_one.data_dir)),
-          format("-ip=%s", var.all_in_one.nodes[k].address),
-          format("-ip.bind=%s", var.all_in_one.ip_bind),
-          "-master",
-          format("-master.port=%d", var.all_in_one.master_port),
-          format("-master.peers=%s:%d", var.all_in_one.nodes[k].address, var.all_in_one.master_port),
-          format("-master.defaultReplication=%s", var.all_in_one.default_replication),
-          format("-master.volumeSizeLimitMB=%d", var.all_in_one.volume_size_limit_mb),
-          "-volume",
-          format("-volume.port=%d", var.all_in_one.volume_port),
-          "-filer",
-          format("-filer.port=%d", var.all_in_one.filer_port),
-          format("-idleTimeout=%d", var.all_in_one.idle_timeout),
-        ],
-        var.all_in_one.disable_http ? [format("-disableHttp=%t", var.all_in_one.disable_http)] : [],
-        var.all_in_one.s3.enabled ? [
-          "-s3",
-          format("-s3.port=%d", var.all_in_one.s3.port),
-          format("-s3.config=%s", var.all_in_one.s3.config_path),
-        ] : [],
-        (var.all_in_one.s3.enabled && var.all_in_one.s3.domain_name != "") ? [format("-s3.domainName=%s", var.all_in_one.s3.domain_name)] : [],
-        var.monitoring_enabled ? [format("-metricsPort=%d", var.all_in_one.metrics_port)] : [],
-        var.all_in_one.extra_args,
-      )
-    }
-  } : {}
-
-  # ===========================================================================
-  # Merge + render systemd unit and cloud-init per node
-  # ===========================================================================
-  nodes_base = merge(local.master_nodes, local.volume_nodes, local.filer_nodes, local.s3_nodes, local.aio_nodes)
-
-  # Disk-mount script per node (empty unless the node declares disk_mounts).
-  mount_scripts = {
-    for name, n in local.nodes_base : name => length(n.disk_mounts) > 0 ? templatefile("${path.module}/templates/mount-disks.sh.tftpl", {
-      run_as_user = var.hardening.run_as_user
-      mounts      = n.disk_mounts
-    }) : ""
-  }
-
-  nodes = {
-    for name, n in local.nodes_base : name => merge(n, {
-      exec_start = "${var.weed_binary} ${join(" ", n.argv)}"
-      # secret_files: the secret-bearing config a wrapper should deliver from a
-      # secret store when render_secret_files=false (else they go in cloud-init).
-      secret_files = n.config_files
-      mount_script = local.mount_scripts[name]
-      file_modes   = { for p in keys(n.config_files) : p => "0600" }
-      systemd_unit = templatefile("${path.module}/templates/weed.service.tftpl", {
-        role                = n.role
-        name                = n.name
-        run_as_user         = var.hardening.run_as_user
-        exec_start          = "${var.weed_binary} ${join(" ", n.argv)}"
-        no_new_privileges   = var.hardening.no_new_privileges
-        protect_system      = var.hardening.protect_system
-        cap_drop_all        = var.hardening.cap_drop_all
-        read_write_paths    = n.data_dirs
-        requires_mounts_for = n.data_dirs
-        env_file            = var.env_file
-        environment         = n.env
-      })
-      cloud_init = templatefile("${path.module}/templates/cloud-init.yaml.tftpl", {
-        role         = n.role
-        name         = n.name
-        run_as_user  = var.hardening.run_as_user
-        config_files = var.render_secret_files ? n.config_files : {}
-        file_modes   = { for p in keys(n.config_files) : p => "0600" }
-        pre_runcmd   = [for p in n.data_dirs : "install -d -o ${var.hardening.run_as_user} -g ${var.hardening.run_as_user} ${p}"]
-        mount_script = local.mount_scripts[name]
-        fetch_script = var.boot_fetch_script
-        systemd_unit = templatefile("${path.module}/templates/weed.service.tftpl", {
-          role                = n.role
-          name                = n.name
-          run_as_user         = var.hardening.run_as_user
-          exec_start          = "${var.weed_binary} ${join(" ", n.argv)}"
-          no_new_privileges   = var.hardening.no_new_privileges
-          protect_system      = var.hardening.protect_system
-          cap_drop_all        = var.hardening.cap_drop_all
-          read_write_paths    = n.data_dirs
-          requires_mounts_for = n.data_dirs
-          env_file            = var.env_file
-          environment         = n.env
-        })
-      })
-    })
-  }
-}

terraform/modules/core/outputs.tf

@@ -1,32 +0,0 @@
-output "nodes" {
-  description = "Per-node rendered artifacts keyed by node name. Each value: role, name, address, ports, data_dir(s), argv (list, after the weed binary), exec_start, env, config_files, systemd_unit, cloud_init."
-  value       = local.nodes
-  sensitive   = true # config_files/env may carry JWT keys or S3 identities
-}
-
-output "master_peers" {
-  description = "Comma-separated master peer list (address:port), as passed to -peers / -mserver / -master."
-  value       = local.master_peers
-}
-
-output "first_filer" {
-  description = "address:port of the lexically-first filer node (default S3 target)."
-  value       = local.first_filer
-}
-
-output "node_names" {
-  description = "List of rendered node names (non-sensitive)."
-  value       = sort(keys(local.nodes))
-}
-
-output "cloud_init_by_node" {
-  description = "Map of node name -> rendered cloud-init user_data (for the per-cloud wrappers)."
-  value       = { for name, n in local.nodes : name => n.cloud_init }
-  sensitive   = true
-}
-
-output "secret_files_by_node" {
-  description = "Map of node name -> {path => content} of secret-bearing files (security.toml, S3 identity JSON). Wrappers push these to a secret store and fetch them at boot when render_secret_files=false."
-  value       = { for name, n in local.nodes : name => n.secret_files }
-  sensitive   = true
-}

terraform/modules/core/templates/cloud-init.yaml.tftpl

@@ -1,44 +0,0 @@
-#cloud-config
-# Rendered by the SeaweedFS Terraform core module for node ${name} (role ${role}).
-# When render_secret_files=false, secret-bearing files are NOT here; they are
-# fetched at boot by fetch-secrets.sh from the cloud secret store.
-users:
-  - name: ${run_as_user}
-    system: true
-    shell: /usr/sbin/nologin
-write_files:
-%{ for path, content in config_files ~}
-  - path: ${path}
-    owner: "${run_as_user}:${run_as_user}"
-    permissions: '${lookup(file_modes, path, "0644")}'
-    content: |
-      ${indent(6, content)}
-%{ endfor ~}
-%{ if mount_script != "" ~}
-  - path: /opt/seaweedfs/mount-disks.sh
-    permissions: '0755'
-    content: |
-      ${indent(6, mount_script)}
-%{ endif ~}
-%{ if fetch_script != "" ~}
-  - path: /opt/seaweedfs/fetch-secrets.sh
-    permissions: '0755'
-    content: |
-      ${indent(6, fetch_script)}
-%{ endif ~}
-  - path: /etc/systemd/system/weed-${role}.service
-    permissions: '0644'
-    content: |
-      ${indent(6, systemd_unit)}
-runcmd:
-%{ if mount_script != "" ~}
-  - /opt/seaweedfs/mount-disks.sh
-%{ endif ~}
-%{ if fetch_script != "" ~}
-  - /opt/seaweedfs/fetch-secrets.sh
-%{ endif ~}
-%{ for cmd in pre_runcmd ~}
-  - ${cmd}
-%{ endfor ~}
-  - systemctl daemon-reload
-  - systemctl enable --now weed-${role}.service

terraform/modules/core/templates/mount-disks.sh.tftpl

@@ -1,73 +0,0 @@
-#!/usr/bin/env bash
-# Rendered by the SeaweedFS Terraform core module. Formats (only if blank) and
-# mounts each data disk, persisting by UUID to /etc/fstab. Safe to re-run.
-set -euo pipefail
-
-run_as_user="${run_as_user}"
-
-# Base block device backing "/" so auto-discovery never touches the root disk.
-root_src="$(findmnt -no SOURCE / 2>/dev/null || true)"
-root_base="$(lsblk -no pkname "$root_src" 2>/dev/null || true)"
-[ -n "$root_base" ] && root_base="/dev/$root_base"
-
-is_mounted() { findmnt -rno TARGET "$1" >/dev/null 2>&1; }
-
-# Pick the first unmounted, non-root whole disk (single-data-disk node).
-discover_dev() {
-  local d base
-  for d in /dev/nvme*n1 /dev/xvd[b-z] /dev/sd[b-z] /dev/vd[b-z]; do
-    [ -b "$d" ] || continue
-    [ "$d" = "$root_src" ] && continue
-    [ "$d" = "$root_base" ] && continue
-    base="/dev/$(lsblk -no pkname "$d" 2>/dev/null || true)"
-    [ "$base" = "$root_base" ] && continue
-    is_mounted "$d" && continue
-    echo "$d"; return 0
-  done
-  return 1
-}
-
-# Wait up to ~120s for one of the explicit candidate devices to appear.
-wait_dev() {
-  local i d
-  for i in $(seq 1 60); do
-    for d in "$@"; do [ -b "$d" ] && { echo "$d"; return 0; }; done
-    sleep 2
-  done
-  return 1
-}
-
-mount_one() {
-  local mountpoint="$1" fstype="$2"; shift 2
-  local dev=""
-  if [ "$#" -gt 0 ]; then
-    dev="$(wait_dev "$@" || true)"
-  fi
-  if [ -z "$dev" ]; then
-    dev="$(discover_dev || true)"
-  fi
-  if [ -z "$dev" ]; then
-    echo "mount-disks: no device found for $mountpoint" >&2
-    return 1
-  fi
-  if ! blkid "$dev" >/dev/null 2>&1; then
-    echo "mount-disks: formatting $dev as $fstype"
-    "mkfs.$fstype" -q "$dev"
-  fi
-  mkdir -p "$mountpoint"
-  local uuid; uuid="$(blkid -s UUID -o value "$dev")"
-  if [ -z "$uuid" ]; then
-    echo "mount-disks: failed to read a filesystem UUID for $dev" >&2
-    return 1
-  fi
-  if ! grep -q "$uuid" /etc/fstab; then
-    printf 'UUID=%s %s %s defaults,nofail 0 2\n' "$uuid" "$mountpoint" "$fstype" >> /etc/fstab
-  fi
-  is_mounted "$mountpoint" || mount "$mountpoint"
-  chown -R "$run_as_user:$run_as_user" "$mountpoint"
-  echo "mount-disks: $dev mounted at $mountpoint"
-}
-
-%{ for m in mounts ~}
-mount_one "${m.mountpoint}" "${m.fstype}"${length(m.devices) > 0 ? " ${join(" ", m.devices)}" : ""}
-%{ endfor ~}

terraform/modules/core/templates/security.toml.tftpl

@@ -1,53 +0,0 @@
-# SeaweedFS security.toml rendered by the Terraform core module.
-# Rendered when enable_security OR any non-default JWT signing option is set.
-# Presence of this file does NOT imply mTLS: the [grpc] TLS block is emitted
-# only under enable_security; a JWT-only config carries [jwt.*] and no TLS.
-
-%{ if enable_security ~}
-[grpc]
-ca = "${cert_dir}/ca/tls.crt"
-# Peer-identity authorization: without an allow-list, mTLS collapses to
-# "trusted the CA" with no authZ. Set allowed_wildcard_domain / commonNames.
-%{ if allowed_wildcard_domain != "" ~}
-allowed_wildcard_domain = "${allowed_wildcard_domain}"
-%{ endif ~}
-
-[grpc.master]
-cert = "${cert_dir}/master/tls.crt"
-key  = "${cert_dir}/master/tls.key"
-%{ if allowed_common_names != "" ~}
-allowed_commonNames = "${allowed_common_names}"
-%{ endif ~}
-
-[grpc.volume]
-cert = "${cert_dir}/volume/tls.crt"
-key  = "${cert_dir}/volume/tls.key"
-
-[grpc.filer]
-cert = "${cert_dir}/filer/tls.crt"
-key  = "${cert_dir}/filer/tls.key"
-
-[grpc.client]
-cert = "${cert_dir}/client/tls.crt"
-key  = "${cert_dir}/client/tls.key"
-%{ endif ~}
-%{ if jwt_signing_key != "" ~}
-
-[jwt.signing]
-key = "${jwt_signing_key}"
-%{ endif ~}
-%{ if jwt_signing_read_key != "" ~}
-
-[jwt.signing.read]
-key = "${jwt_signing_read_key}"
-%{ endif ~}
-%{ if jwt_filer_signing_key != "" ~}
-
-[jwt.filer_signing]
-key = "${jwt_filer_signing_key}"
-%{ endif ~}
-%{ if jwt_filer_signing_read_key != "" ~}
-
-[jwt.filer_signing.read]
-key = "${jwt_filer_signing_read_key}"
-%{ endif ~}

terraform/modules/core/templates/weed.service.tftpl

@@ -1,39 +0,0 @@
-[Unit]
-Description=SeaweedFS ${role} (${name})
-Documentation=https://github.com/seaweedfs/seaweedfs/wiki
-After=network-online.target
-Wants=network-online.target
-%{ for path in requires_mounts_for ~}
-RequiresMountsFor=${path}
-%{ endfor ~}
-
-[Service]
-Type=simple
-User=${run_as_user}
-Group=${run_as_user}
-ExecStart=${exec_start}
-Restart=always
-RestartSec=5
-LimitNOFILE=1048576
-TimeoutStopSec=60
-# Hardening (OpenShift SCC analogs; relax via the hardening variable)
-NoNewPrivileges=${no_new_privileges}
-%{ if protect_system ~}
-ProtectSystem=strict
-ProtectHome=true
-PrivateTmp=true
-%{ for path in read_write_paths ~}
-ReadWritePaths=${path}
-%{ endfor ~}
-%{ endif ~}
-%{ if cap_drop_all ~}
-CapabilityBoundingSet=
-AmbientCapabilities=
-%{ endif ~}
-EnvironmentFile=-${env_file}
-%{ for k, v in environment ~}
-Environment="${k}=${v}"
-%{ endfor ~}
-
-[Install]
-WantedBy=multi-user.target

terraform/modules/core/tests/render.tftest.hcl

@@ -1,234 +0,0 @@
-# Plan-level tests for the core renderer. No cloud credentials required.
-#   cd terraform/modules/core && tofu test
-
-variables {
-  weed_binary = "/usr/bin/weed"
-}
-
-# ---------------------------------------------------------------------------
-run "master_peers_and_count" {
-  command = plan
-  variables {
-    master = {
-      nodes = {
-        m0 = { address = "10.0.0.10" }
-        m1 = { address = "10.0.0.11" }
-        m2 = { address = "10.0.0.12" }
-      }
-    }
-    volume = { enabled = false }
-    filer  = { enabled = false }
-  }
-  assert {
-    condition     = output.master_peers == "10.0.0.10:9333,10.0.0.11:9333,10.0.0.12:9333"
-    error_message = "master -peers list is wrong"
-  }
-  assert {
-    condition     = length(output.node_names) == 3
-    error_message = "expected exactly 3 master nodes"
-  }
-}
-
-# ---------------------------------------------------------------------------
-run "volume_uses_mserver_not_master" {
-  command = plan
-  variables {
-    master = { nodes = { m0 = { address = "10.0.0.10" } } }
-    volume = { nodes = { v0 = { address = "10.0.0.20", rack = "rack-a", data_center = "dc1" } } }
-    filer  = { enabled = false }
-  }
-  assert {
-    condition     = contains(output.nodes["volume-v0"].argv, "-mserver=10.0.0.10:9333")
-    error_message = "volume must pass the master list via -mserver (verified flag), not -master"
-  }
-  assert {
-    condition     = !contains(output.nodes["volume-v0"].argv, "-master=10.0.0.10:9333")
-    error_message = "volume should not use -master for the master list"
-  }
-  assert {
-    condition     = contains(output.nodes["volume-v0"].argv, "-rack=rack-a") && contains(output.nodes["volume-v0"].argv, "-dataCenter=dc1")
-    error_message = "per-node rack/dataCenter must render"
-  }
-}
-
-# ---------------------------------------------------------------------------
-run "filer_uses_master_and_store_dir" {
-  command = plan
-  variables {
-    master = { nodes = { m0 = { address = "10.0.0.10" } } }
-    volume = { enabled = false }
-    filer  = { nodes = { f0 = { address = "10.0.0.30", data_dir = "/srv/filer" } } }
-  }
-  assert {
-    condition     = contains(output.nodes["filer-f0"].argv, "-master=10.0.0.10:9333")
-    error_message = "filer must use -master for the master list"
-  }
-  assert {
-    condition     = contains(output.nodes["filer-f0"].argv, "-defaultStoreDir=/srv/filer")
-    error_message = "filer leveldb2 store dir must render via -defaultStoreDir"
-  }
-}
-
-# ---------------------------------------------------------------------------
-run "metrics_gated_off_by_default" {
-  command = plan
-  variables {
-    master = { nodes = { m0 = { address = "10.0.0.10" } } }
-    volume = { enabled = false }
-    filer  = { enabled = false }
-  }
-  assert {
-    condition     = length([for a in output.nodes["master-m0"].argv : a if startswith(a, "-metricsPort")]) == 0
-    error_message = "metrics port must NOT be bound when monitoring is disabled"
-  }
-}
-
-run "metrics_on_when_enabled" {
-  command = plan
-  variables {
-    monitoring_enabled = true
-    master             = { nodes = { m0 = { address = "10.0.0.10" } }, metrics_port = 9327 }
-    volume             = { enabled = false }
-    filer              = { enabled = false }
-  }
-  assert {
-    condition     = contains(output.nodes["master-m0"].argv, "-metricsPort=9327")
-    error_message = "metrics port must bind when monitoring is enabled"
-  }
-}
-
-# ---------------------------------------------------------------------------
-run "security_toml_off_by_default" {
-  command = plan
-  variables {
-    master = { nodes = { m0 = { address = "10.0.0.10" } } }
-    volume = { enabled = false }
-    filer  = { enabled = false }
-  }
-  assert {
-    condition     = length(keys(output.nodes["master-m0"].config_files)) == 0
-    error_message = "security.toml must not render when security is off and no JWT keys set"
-  }
-}
-
-run "security_toml_on_with_mtls" {
-  command = plan
-  variables {
-    enable_security = true
-    security = {
-      allowed_wildcard_domain = ".seaweedfs.internal"
-      jwt_signing_key         = "test-signing-key-not-a-real-secret-0123456789"
-    }
-    master = { nodes = { m0 = { address = "10.0.0.10" } } }
-    volume = { enabled = false }
-    filer  = { enabled = false }
-  }
-  assert {
-    condition     = contains(keys(output.nodes["master-m0"].config_files), "/etc/seaweedfs/security.toml")
-    error_message = "security.toml must render when enable_security is true"
-  }
-}
-
-# ---------------------------------------------------------------------------
-run "all_in_one_renders_server" {
-  command = plan
-  variables {
-    master = { enabled = false }
-    volume = { enabled = false }
-    filer  = { enabled = false }
-    all_in_one = {
-      enabled = true
-      nodes   = { a0 = { address = "10.0.0.40" } }
-      s3      = { enabled = true }
-    }
-  }
-  assert {
-    condition     = contains(output.nodes["all-in-one-a0"].argv, "server")
-    error_message = "all-in-one must invoke the weed server subcommand"
-  }
-  assert {
-    condition     = contains(output.nodes["all-in-one-a0"].argv, "-s3") && contains(output.nodes["all-in-one-a0"].argv, "-s3.port=8333")
-    error_message = "all-in-one S3 must render when enabled"
-  }
-}
-
-# ---------------------------------------------------------------------------
-run "security_toml_mtls_only_no_jwt" {
-  command = plan
-  variables {
-    enable_security = true
-    master          = { nodes = { m0 = { address = "10.0.0.10" } } }
-    volume          = { enabled = false }
-    filer           = { enabled = false }
-  }
-  assert {
-    condition     = contains(keys(output.nodes["master-m0"].config_files), "/etc/seaweedfs/security.toml")
-    error_message = "security.toml must render for mTLS even without JWT keys"
-  }
-  assert {
-    condition     = !strcontains(output.nodes["master-m0"].config_files["/etc/seaweedfs/security.toml"], "[jwt.signing]")
-    error_message = "the jwt.signing block must be omitted when no signing key is set"
-  }
-}
-
-run "mount_fetch_and_secret_delivery" {
-  command = plan
-  variables {
-    enable_security     = true
-    render_secret_files = false
-    boot_fetch_script   = "#!/usr/bin/env bash\necho fetch\n"
-    security = {
-      jwt_signing_key = "test-signing-key-not-a-real-secret-0123456789"
-    }
-    master = { nodes = { m0 = { address = "10.0.0.10", disk_mounts = [{ mountpoint = "/data", fstype = "xfs" }] } } }
-    volume = { enabled = false }
-    filer  = { enabled = false }
-  }
-  assert {
-    condition     = strcontains(output.nodes["master-m0"].cloud_init, "/opt/seaweedfs/mount-disks.sh")
-    error_message = "cloud-init must write+run the mount script when disk_mounts is set"
-  }
-  assert {
-    condition     = strcontains(output.nodes["master-m0"].cloud_init, "/opt/seaweedfs/fetch-secrets.sh")
-    error_message = "cloud-init must write+run the fetch script when boot_fetch_script is set"
-  }
-  assert {
-    condition     = !strcontains(output.nodes["master-m0"].cloud_init, "[jwt.signing]")
-    error_message = "security.toml must NOT be inlined in cloud-init when render_secret_files=false"
-  }
-  assert {
-    condition     = contains(keys(output.nodes["master-m0"].secret_files), "/etc/seaweedfs/security.toml")
-    error_message = "secret_files must expose security.toml for secret-store delivery"
-  }
-}
-
-run "mount_script_absent_without_disks" {
-  command = plan
-  variables {
-    master = { nodes = { m0 = { address = "10.0.0.10" } } }
-    volume = { enabled = false }
-    filer  = { enabled = false }
-  }
-  assert {
-    condition     = !strcontains(output.nodes["master-m0"].cloud_init, "mount-disks.sh")
-    error_message = "no mount script should render when the node declares no disk_mounts"
-  }
-}
-
-run "s3_standalone_targets_filer" {
-  command = plan
-  variables {
-    master = { nodes = { m0 = { address = "10.0.0.10" } } }
-    volume = { enabled = false }
-    filer  = { nodes = { f0 = { address = "10.0.0.30" } } }
-    s3     = { enabled = true, nodes = { s0 = { address = "10.0.0.50" } } }
-  }
-  assert {
-    condition     = contains(output.nodes["s3-s0"].argv, "-filer=10.0.0.30:8888")
-    error_message = "standalone S3 must point -filer at the first filer node"
-  }
-  assert {
-    condition     = contains(output.nodes["s3-s0"].argv, "-config=/etc/seaweedfs/s3_config.json")
-    error_message = "standalone S3 must reference the rendered config path"
-  }
-}