4.19

* filer: bootstrap pre-existing metadata when a new filer joins a cluster

When a filer connects to a peer for the first time (no stored sync
offset), it now does a full BFS traversal of the peer's metadata via
TraverseBfsMetadata before starting the incremental change stream.
This ensures filer2 sees all data that existed before it started,
fixing the issue where only post-startup changes were synced.

Closes #8961

* filer: upsert during bootstrap and persist offset immediately

- Use upsert (insert, then update on conflict) during metadata
  traversal so the bootstrap doesn't fail on the root directory
  or after a partial previous attempt.
- Persist the sync offset right after a successful traversal so
  a retry doesn't redo the full BFS.

* filer: address review feedback on metadata bootstrap

- Use peer-side max Mtime as the streaming cursor instead of local
  time.Now() to avoid missing events due to clock skew between filers.
  traversePeerMetadata now returns the high-water Mtime (nanoseconds)
  observed during BFS traversal.

- Compare Mtime before overwriting during bootstrap: if a local entry
  is newer than the peer's version, skip the update instead of
  clobbering it.

- Only trigger full BFS traversal on ErrKvNotFound (key genuinely
  missing). Transient KvGet errors (connection issues, etc.) are now
  propagated instead of silently falling through to a full re-sync.
  Changed readOffset to use %w so errors.Is works through the chain.

* filer: address review findings on bootstrap sync

- Use wall-clock time with safety margin for stream cursor instead of
  entry Mtime. Mtime is file modification time (can be arbitrary),
  while the metadata stream uses TsNs (event log time). Using
  time.Now() minus 1 minute before traversal ensures no events are
  missed even with clock skew, matching the proven filer.meta.backup
  pattern.

- Pass ExcludedPrefixes=[SystemLogDir] to TraverseBfsMetadata so
  the server prunes internal log entries server-side instead of
  transferring them over the network only to be filtered client-side.

- Fail fast if updateOffset fails after bootstrap. If we can't
  persist the offset, bail out rather than proceeding and potentially
  losing the expensive BFS work on the next retry.

.github/workflows/container_dev.yml

@@ -9,17 +9,96 @@ permissions:
 
 jobs:
 
+  # ── Pre-build Rust volume server binaries natively ──────────────────
+  build-rust-binaries:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-musl
+            arch: amd64
+          - target: aarch64-unknown-linux-musl
+            arch: arm64
+            cross: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Install musl tools (amd64)
+        if: ${{ !matrix.cross }}
+        run: sudo apt-get install -y musl-tools
+
+      - name: Install cross-compilation tools (arm64)
+        if: matrix.cross
+        run: |
+          sudo apt-get install -y gcc-aarch64-linux-gnu
+          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
+          # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
+          echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-docker-dev-${{ matrix.target }}-
+
+      - name: Build normal variant
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }} --no-default-features
+          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-volume-${{ matrix.arch }}
+          path: weed-volume-normal-${{ matrix.arch }}
+
   build-dev-containers:
+    needs: [build-rust-binaries]
     runs-on: [ubuntu-latest]
 
     steps:
-      -
-        name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
-      -
-        name: Docker meta
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Download pre-built Rust binaries
+        uses: actions/download-artifact@v8
+        with:
+          pattern: rust-volume-*
+          merge-multiple: true
+          path: ./rust-bins
+
+      - name: Place Rust binaries in Docker context
+        run: |
+          mkdir -p docker/weed-volume-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-volume-normal-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
+              echo "Placed pre-built Rust binary for ${arch}"
+            fi
+          done
+          ls -la docker/weed-volume-prebuilt/
+
+      - name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v3
+        uses: docker/metadata-action@v6
         with:
           images: |
             chrislusf/seaweedfs
@@ -30,40 +109,40 @@ jobs:
             org.opencontainers.image.title=seaweedfs
             org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
             org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
-      -
-        name: Create BuildKit config
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Create BuildKit config
         run: |
           cat > /tmp/buildkitd.toml <<EOF
           [registry."docker.io"]
             mirrors = ["https://mirror.gcr.io"]
           EOF
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
         with:
           buildkitd-flags: "--debug"
           buildkitd-config: /tmp/buildkitd.toml
-      -
-        name: Login to Docker Hub
+
+      - name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
+
+      - name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
+        uses: docker/login-action@v4.1.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
+
+      - name: Build
+        uses: docker/build-push-action@v7
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_foundationdb_version.yml

@@ -133,7 +133,7 @@ jobs:
 
       - name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/container_latest.yml

@@ -58,8 +58,80 @@ jobs:
             echo "publish=true" >> "$GITHUB_OUTPUT"
           fi
 
+  # ── Pre-build Rust volume server binaries natively ──────────────────
+  build-rust-binaries:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-musl
+            arch: amd64
+          - target: aarch64-unknown-linux-musl
+            arch: arm64
+            cross: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
+
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Install musl tools (amd64)
+        if: ${{ !matrix.cross }}
+        run: sudo apt-get install -y musl-tools
+
+      - name: Install cross-compilation tools (arm64)
+        if: matrix.cross
+        run: |
+          sudo apt-get install -y gcc-aarch64-linux-gnu
+          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
+          # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
+          echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-docker-${{ matrix.target }}-
+
+      - name: Build large-disk variant
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }}
+          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
+
+      - name: Build normal variant
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }} --no-default-features
+          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-volume-${{ matrix.arch }}
+          path: |
+            weed-volume-large-disk-${{ matrix.arch }}
+            weed-volume-normal-${{ matrix.arch }}
+
   build:
-    needs: [setup]
+    needs: [setup, build-rust-binaries]
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -94,11 +166,32 @@ jobs:
           if [ "${{ matrix.variant }}" == "large_disk" ]; then
             echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
             echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
+            echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
           else
             echo "tag_suffix=" >> $GITHUB_OUTPUT
             echo "build_args=" >> $GITHUB_OUTPUT
+            echo "rust_variant=normal" >> $GITHUB_OUTPUT
           fi
 
+      - name: Download pre-built Rust binaries
+        uses: actions/download-artifact@v8
+        with:
+          pattern: rust-volume-*
+          merge-multiple: true
+          path: ./rust-bins
+
+      - name: Place Rust binaries in Docker context
+        run: |
+          mkdir -p docker/weed-volume-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
+              echo "Placed pre-built Rust binary for ${arch}"
+            fi
+          done
+          ls -la docker/weed-volume-prebuilt/
+
       - name: Docker meta
         id: docker_meta
         uses: docker/metadata-action@v6
@@ -127,13 +220,13 @@ jobs:
           buildkitd-config: /tmp/buildkitd.toml
       - name: Login to Docker Hub
         if: needs.setup.outputs.publish == 'true'
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Login to GHCR
         if: needs.setup.outputs.publish == 'true'
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
@@ -166,7 +259,7 @@ jobs:
 
   trivy-scan:
     runs-on: ubuntu-latest
-    needs: [setup, build]
+    needs: [setup, build, build-rust-binaries]
     strategy:
       matrix:
         variant: ${{ fromJSON(needs.setup.outputs.variants) }}
@@ -181,7 +274,7 @@ jobs:
           fi
       - name: Login to GHCR
         if: needs.setup.outputs.publish == 'true'
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
@@ -191,6 +284,29 @@ jobs:
         uses: actions/checkout@v6
         with:
           ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
+      - name: Download pre-built Rust binaries for local scan
+        if: needs.setup.outputs.publish != 'true'
+        uses: actions/download-artifact@v8
+        with:
+          pattern: rust-volume-*
+          merge-multiple: true
+          path: ./rust-bins
+      - name: Place Rust binaries in Docker context for local scan
+        if: needs.setup.outputs.publish != 'true'
+        run: |
+          rust_variant="normal"
+          if [ "${{ matrix.variant }}" == "large_disk" ]; then
+            rust_variant="large-disk"
+          fi
+          mkdir -p docker/weed-volume-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-volume-${rust_variant}-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
+              echo "Placed pre-built Rust binary for ${arch}"
+            fi
+          done
+          ls -la docker/weed-volume-prebuilt/
       - name: Create BuildKit config for local scan build
         if: needs.setup.outputs.publish != 'true'
         run: |
@@ -313,12 +429,12 @@ jobs:
             ghcr.io/chrislusf/seaweedfs
           tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
       - name: Login to Docker Hub
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Login to GHCR
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}

.github/workflows/container_release_foundationdb.yml

@@ -42,7 +42,7 @@ jobs:
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/container_release_unified.yml

@@ -74,6 +74,8 @@ jobs:
         run: |
           sudo apt-get install -y gcc-aarch64-linux-gnu
           echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
+          # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
+          echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
 
       - name: Cache cargo registry and target
         uses: actions/cache@v5
@@ -103,7 +105,7 @@ jobs:
           cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: rust-volume-${{ matrix.arch }}
           path: |
@@ -166,7 +168,7 @@ jobs:
 
       - name: Download pre-built Rust binaries
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           pattern: rust-volume-*
           merge-multiple: true
@@ -234,14 +236,14 @@ jobs:
           
       - name: Login to Docker Hub
         if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           
       - name: Login to GHCR
         if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
@@ -297,14 +299,14 @@ jobs:
     steps:
       - name: Login to Docker Hub
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           
       - name: Login to GHCR
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}

.github/workflows/container_rocksdb_version.yml

@@ -88,7 +88,7 @@ jobs:
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
 
       - name: Login to Docker Hub
-        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/rust_binaries_release.yml

@@ -99,7 +99,7 @@ jobs:
 
       - name: Upload artifacts
         if: ${{ !startsWith(github.ref, 'refs/tags/') }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: rust-volume-${{ matrix.asset_suffix }}
           path: |
@@ -176,7 +176,7 @@ jobs:
 
       - name: Upload artifacts
         if: ${{ !startsWith(github.ref, 'refs/tags/') }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: rust-volume-${{ matrix.asset_suffix }}
           path: |
@@ -245,7 +245,7 @@ jobs:
 
       - name: Upload artifacts
         if: ${{ !startsWith(github.ref, 'refs/tags/') }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: rust-volume-windows_amd64
           path: |

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/google/btree v1.1.3
 	github.com/google/uuid v1.6.0
 	github.com/google/wire v0.7.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.17.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.19.0 // indirect
 	github.com/gorilla/mux v1.8.1
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -86,7 +86,7 @@ require (
 	github.com/xdg-go/scram v1.2.0
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	go.etcd.io/etcd/client/v3 v3.6.9
+	go.etcd.io/etcd/client/v3 v3.6.10
 	go.mongodb.org/mongo-driver v1.17.9
 	go.opencensus.io v0.24.0 // indirect
 	gocloud.dev v0.45.0
@@ -95,22 +95,21 @@ require (
 	golang.org/x/crypto v0.49.0
 	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
 	golang.org/x/image v0.38.0
-	golang.org/x/net v0.51.0
+	golang.org/x/net v0.52.0
 	golang.org/x/oauth2 v0.36.0
 	golang.org/x/sys v0.42.0
 	golang.org/x/text v0.35.0 // indirect
 	golang.org/x/tools v0.42.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
-	google.golang.org/api v0.267.0
-	google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
+	google.golang.org/api v0.274.0
+	google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 // indirect
 	google.golang.org/grpc v1.79.3
 	google.golang.org/protobuf v1.36.11
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
-	modernc.org/mathutil v1.7.1
+	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.11.0 // indirect
 	modernc.org/sqlite v1.46.1
-	modernc.org/strutil v1.2.1
 )
 
 require (
@@ -126,8 +125,8 @@ require (
 	github.com/armon/go-metrics v0.4.1
 	github.com/aws/aws-sdk-go-v2 v1.41.5
 	github.com/aws/aws-sdk-go-v2/config v1.32.13
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.13
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.14
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
 	github.com/getsentry/sentry-go v0.44.1
@@ -136,10 +135,9 @@ require (
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
 	github.com/hashicorp/raft v1.7.3
 	github.com/hashicorp/raft-boltdb/v2 v2.3.1
-	github.com/hashicorp/vault/api v1.22.0
+	github.com/hashicorp/vault/api v1.23.0
 	github.com/jhump/protoreflect v1.18.0
 	github.com/linkedin/goavro/v2 v2.15.0
-	github.com/mattn/go-sqlite3 v1.14.34
 	github.com/minio/crc64nvme v1.1.1
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/parquet-go/parquet-go v0.28.0
@@ -157,7 +155,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
-	go.etcd.io/etcd/client/pkg/v3 v3.6.9
+	go.etcd.io/etcd/client/pkg/v3 v3.6.10
 	go.uber.org/atomic v1.11.0
 	golang.org/x/sync v0.20.0
 	golang.org/x/tools/godoc v0.1.0-deprecated
@@ -291,7 +289,7 @@ require (
 
 require (
 	cel.dev/expr v0.25.1 // indirect
-	cloud.google.com/go/auth v0.18.1 // indirect
+	cloud.google.com/go/auth v0.18.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	cloud.google.com/go/iam v1.5.3 // indirect
@@ -323,21 +321,21 @@ require (
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
 	github.com/aws/smithy-go v1.24.2
 	github.com/boltdb/bolt v1.3.1 // indirect
@@ -387,7 +385,7 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
 	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/gorilla/sessions v1.4.0
@@ -486,22 +484,22 @@ require (
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
 	go.etcd.io/bbolt v1.4.3 // indirect
-	go.etcd.io/etcd/api/v3 v3.6.9 // indirect
+	go.etcd.io/etcd/api/v3 v3.6.10 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
-	go.opentelemetry.io/otel v1.40.0 // indirect
-	go.opentelemetry.io/otel/metric v1.40.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.40.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
-	go.opentelemetry.io/otel/trace v1.40.0 // indirect
+	go.opentelemetry.io/otel v1.42.0 // indirect
+	go.opentelemetry.io/otel/metric v1.42.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.42.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect
+	go.opentelemetry.io/otel/trace v1.42.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
 	golang.org/x/term v0.41.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -94,8 +94,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo
 cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
 cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
 cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
-cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
-cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
+cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
+cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
@@ -310,8 +310,8 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6
 cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
 cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
 cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
-cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
-cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
+cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA=
+cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak=
 cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
 cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
 cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
@@ -716,12 +716,12 @@ github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
 github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
 github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
 github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
 github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
@@ -732,28 +732,28 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgq
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 h1:foqo/ocQ7WqKwy3FojGtZQJo0FR4vto9qnz9VaumbCo=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
 github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
@@ -1284,8 +1284,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
 github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
 github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
 github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
-github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
-github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
+github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
+github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
@@ -1296,8 +1296,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK
 github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
 github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
 github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
-github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
-github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
+github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
+github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
 github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
 github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
@@ -1383,8 +1383,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKc
 github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
 github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc=
 github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE=
-github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0=
-github.com/hashicorp/vault/api v1.22.0/go.mod h1:IUZA2cDvr4Ok3+NtK2Oq/r+lJeXkeCrHRmqdyWfpmGM=
+github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A=
+github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -2101,12 +2101,12 @@ go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
 go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
 go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
-go.etcd.io/etcd/api/v3 v3.6.9 h1:UA7iKfEW1AzgihcBSGXci2kDGQiokSq41F9HMCI/RTI=
-go.etcd.io/etcd/api/v3 v3.6.9/go.mod h1:csEk/qTfxKL36NqJdU15Tgtl65A8dyEY2BYo7PRsIwk=
-go.etcd.io/etcd/client/pkg/v3 v3.6.9 h1:T8nuk8Lz64C+Hzb0coBFLMSlVSQZBpAtFk46swdM1DA=
-go.etcd.io/etcd/client/pkg/v3 v3.6.9/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
-go.etcd.io/etcd/client/v3 v3.6.9 h1:3X555hQXmhRr27O37wls53g68CpUiPOiHXrZfz2Al+o=
-go.etcd.io/etcd/client/v3 v3.6.9/go.mod h1:KO7H1HLYh1qaljuVZJQwBFk1lRce6pJzt+C81GEnrlM=
+go.etcd.io/etcd/api/v3 v3.6.10 h1:jlwjtELjA8yi2VWpOFH+0w0lGr3K6mVDyn0RDB9aaAY=
+go.etcd.io/etcd/api/v3 v3.6.10/go.mod h1:pdV4VeFmvhdNjB4LWRkC8ReLyRBAxUOze3GarMhE2sk=
+go.etcd.io/etcd/client/pkg/v3 v3.6.10 h1:tBT7podcPhuVbCVkAEzx8bC5I+aqxfLwBN8/As1arrA=
+go.etcd.io/etcd/client/pkg/v3 v3.6.10/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
+go.etcd.io/etcd/client/v3 v3.6.10 h1:J598zJ+C/ZPvImypmq5waj84+bovePrlZERHklf34y0=
+go.etcd.io/etcd/client/v3 v3.6.10/go.mod h1:iHhUDUcEwaKs1YFq3MgmI9U4zhTVasp/vgdVbFf1RS8=
 go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
 go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -2128,8 +2128,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0/go.mod h1:CosX/aS4eHnG9D7nESYpV753l4j9q5j3SL/PUYd2lR8=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
-go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
-go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
+go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
+go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8=
@@ -2144,14 +2144,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
 go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
 go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
-go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
-go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
-go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
-go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
-go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
-go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
-go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
-go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
+go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
+go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
+go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
+go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
+go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
+go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
+go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
+go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
@@ -2344,8 +2344,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
-golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2552,8 +2552,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
-golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -2703,8 +2703,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
 google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
 google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
 google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
-google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
-google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
+google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
+google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -2838,12 +2838,12 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
 google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
 google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
 google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
-google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
-google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
-google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
-google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
+google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
+google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
+google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.18"
+appVersion: "4.19"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.18.0
+version: 4.19.0

seaweed-volume/Cargo.toml

@@ -82,7 +82,7 @@ memmap2 = "0.9"
 uuid = { version = "1", features = ["v4"] }
 
 # HTTP client (for proxying, remote fetch)
-reqwest = { version = "0.12", features = ["rustls-tls", "stream", "multipart", "json"] }
+reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream", "multipart", "json"] }
 
 # Content hashing
 md-5 = "0.10"

test/plugin_workers/volume_fixtures.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 )
 
@@ -21,7 +22,12 @@ func WriteTestVolumeFiles(t *testing.T, baseDir string, volumeID uint32, datSize
 	datPath := filepath.Join(baseDir, volumeFilename(volumeID, ".dat"))
 	idxPath := filepath.Join(baseDir, volumeFilename(volumeID, ".idx"))
 
-	data := make([]byte, datSize)
+	// The idx entry records the needle's data size. The actual on-disk size
+	// includes header + checksum + timestamp (GetActualSize). The .dat must
+	// be large enough to hold the full needle.
+	needleDataSize := types.Size(datSize)
+	actualSize := needle.GetActualSize(needleDataSize, needle.Version3)
+	data := make([]byte, actualSize)
 	rng := rand.New(rand.NewSource(99))
 	_, _ = rng.Read(data)
 	if err := os.WriteFile(datPath, data, 0644); err != nil {
@@ -35,7 +41,7 @@ func WriteTestVolumeFiles(t *testing.T, baseDir string, volumeID uint32, datSize
 
 	types.NeedleIdToBytes(entry[:idEnd], types.NeedleId(1))
 	types.OffsetToBytes(entry[idEnd:offsetEnd], types.ToOffset(0))
-	types.SizeToBytes(entry[offsetEnd:sizeEnd], types.Size(datSize))
+	types.SizeToBytes(entry[offsetEnd:sizeEnd], needleDataSize)
 
 	if err := os.WriteFile(idxPath, entry, 0644); err != nil {
 		t.Fatalf("write idx file: %v", err)

test/s3/spark/issue_8285_repro_test.go

@@ -70,7 +70,8 @@ print("WRITE_COUNT=" + str(count))
 		"issue-8285/output/_temporary/0/",
 		"issue-8285/output/_temporary/0/_temporary/",
 	}
-	lingering := waitForObjectsToDisappear(t, env, "test", temporaryCandidates, 35*time.Second)
+	// Empty folder cleanup has a 2m default delay + 30s processor interval
+	lingering := waitForObjectsToDisappear(t, env, "test", temporaryCandidates, 3*time.Minute)
 	if len(lingering) > 0 {
 		t.Fatalf("issue #8285 regression detected: lingering temporary directories: %v", lingering)
 	}

test/s3tables/catalog_spark/spark_operations_test.go

@@ -277,3 +277,80 @@ print(f"Count at snapshot: {count}")
 
 	t.Logf(">>> Time travel test passed")
 }
+
+// TestSparkMultiLevelNamespace tests that multi-level namespaces produce correct
+// S3 paths (dot-separated) so that Spark can read back the data it writes.
+// Regression test for https://github.com/seaweedfs/seaweedfs/issues/8959
+func TestSparkMultiLevelNamespace(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env, _, _ := setupSparkTestEnv(t)
+
+	// Use a two-level namespace like "analytics.daily"
+	nsLevel1 := "analytics_" + randomString(4)
+	nsLevel2 := "daily_" + randomString(4)
+	multiNs := fmt.Sprintf("%s.%s", nsLevel1, nsLevel2)
+	tableName := "events_" + randomString(4)
+
+	// Create multi-level namespace
+	t.Logf(">>> Creating multi-level namespace: %s", multiNs)
+	createNsSQL := fmt.Sprintf(`
+spark.sql("CREATE NAMESPACE iceberg.%s")
+print("Namespace created")
+`, multiNs)
+	output := runSparkPySQL(t, env.sparkContainer, createNsSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Namespace created") {
+		t.Fatalf("multi-level namespace creation failed, output: %s", output)
+	}
+
+	// Create table under multi-level namespace
+	t.Logf(">>> Creating table under multi-level namespace")
+	createTableSQL := fmt.Sprintf(`
+spark.sql("""
+CREATE TABLE iceberg.%s.%s (
+    id INT,
+    event STRING,
+    ts TIMESTAMP
+)
+USING iceberg
+""")
+print("Table created")
+`, multiNs, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, createTableSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Table created") {
+		t.Fatalf("table creation under multi-level namespace failed, output: %s", output)
+	}
+
+	// Insert data
+	t.Logf(">>> Inserting data into multi-level namespace table")
+	insertSQL := fmt.Sprintf(`
+spark.sql("""
+INSERT INTO iceberg.%s.%s VALUES
+    (1, 'click', TIMESTAMP '2025-01-01 00:00:00'),
+    (2, 'view',  TIMESTAMP '2025-01-01 01:00:00'),
+    (3, 'click', TIMESTAMP '2025-01-02 00:00:00')
+""")
+print("Data inserted")
+`, multiNs, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, insertSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Data inserted") {
+		t.Fatalf("data insertion failed, output: %s", output)
+	}
+
+	// Query data back — this is the key test: if the namespace path separator
+	// was wrong (\x1F instead of "."), Spark would not find the data files.
+	t.Logf(">>> Querying data from multi-level namespace table")
+	querySQL := fmt.Sprintf(`
+result = spark.sql("SELECT COUNT(*) as count FROM iceberg.%s.%s")
+count = result.collect()[0]['count']
+print(f"Row count: {count}")
+`, multiNs, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, querySQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Row count: 3") {
+		t.Errorf("expected row count 3 from multi-level namespace table, got output: %s", output)
+	}
+
+	t.Logf(">>> Multi-level namespace test passed")
+}

test/s3tables/catalog_trino/trino_catalog_test.go

@@ -82,6 +82,101 @@ func TestTrinoIcebergCatalog(t *testing.T) {
 	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SHOW TABLES FROM iceberg.%s", schemaName))
 }
 
+// TestTrinoMultiLevelNamespace tests that multi-level namespaces (dot-separated)
+// produce correct S3 paths so Trino can read back data it writes.
+// Regression test for https://github.com/seaweedfs/seaweedfs/issues/8959
+func TestTrinoMultiLevelNamespace(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	if !env.dockerAvailable {
+		t.Skip("Docker not available, skipping Trino integration test")
+	}
+
+	t.Logf(">>> Starting SeaweedFS...")
+	env.StartSeaweedFS(t)
+
+	tableBucket := "iceberg-tables"
+	createTableBucket(t, env, tableBucket)
+
+	configDir := env.writeTrinoConfig(t, tableBucket, withNestedNamespace())
+	env.startTrinoContainer(t, configDir)
+	waitForTrino(t, env.trinoContainer, 60*time.Second)
+
+	// Use a two-level namespace: "analytics.daily"
+	nsLevel1 := "analytics_" + randomString(4)
+	nsLevel2 := "daily_" + randomString(4)
+	flatNs := fmt.Sprintf("%s.%s", nsLevel1, nsLevel2)
+	// Trino uses double-quoted schema names for multi-level namespaces
+	multiNs := fmt.Sprintf(`"%s"`, flatNs)
+	tableName := "events_" + randomString(4)
+
+	// Create multi-level namespace (schema)
+	t.Logf(">>> Creating multi-level schema: %s", flatNs)
+	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", multiNs))
+
+	// Verify the schema shows up
+	output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
+	if !strings.Contains(output, flatNs) {
+		t.Fatalf("Expected schema %s in output:\n%s", flatNs, output)
+	}
+
+	// Create table with explicit location to avoid non-empty location conflict.
+	// The location uses the dot-separated namespace — if #8959 regresses
+	// (unit separator instead of dot), data would be written to the wrong path.
+	tableLocation := fmt.Sprintf("s3://%s/%s/%s_%s", tableBucket, flatNs, tableName, randomString(6))
+	t.Logf(">>> Creating table at location: %s", tableLocation)
+	createSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS iceberg.%s.%s (
+		id INTEGER,
+		event VARCHAR,
+		ts TIMESTAMP(6)
+	) WITH (
+		format = 'PARQUET',
+		location = '%s'
+	)`, multiNs, tableName, tableLocation)
+	runTrinoSQLAllowExists(t, env.trinoContainer, createSQL)
+
+	// Insert data
+	t.Logf(">>> Inserting data into multi-level namespace table")
+	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`
+		INSERT INTO iceberg.%s.%s VALUES
+			(1, 'click', TIMESTAMP '2025-01-01 00:00:00'),
+			(2, 'view',  TIMESTAMP '2025-01-01 01:00:00'),
+			(3, 'click', TIMESTAMP '2025-01-02 00:00:00')
+	`, multiNs, tableName))
+
+	// Query data back — if the namespace path separator were wrong (\x1F
+	// instead of "."), the metadata location would point to a non-existent
+	// S3 path and this query would fail.
+	t.Logf(">>> Querying data from multi-level namespace table")
+	countOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
+		"SELECT count(*) FROM iceberg.%s.%s", multiNs, tableName))
+	rowCount := mustParseCSVInt64(t, countOutput)
+	if rowCount != 3 {
+		t.Fatalf("expected row count 3, got %d", rowCount)
+	}
+
+	// Verify the S3 file path contains the dot-separated namespace, not \x1F.
+	filesOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
+		`SELECT file_path FROM iceberg.%s."%s$files" LIMIT 1`, multiNs, tableName))
+	filePath := strings.TrimSpace(filesOutput)
+	if filePath == "" {
+		t.Fatalf("expected at least one data file, got empty output")
+	}
+	if !strings.Contains(filePath, flatNs+"/") {
+		t.Errorf("expected file path to contain dot-separated namespace %q, got: %s", flatNs, filePath)
+	}
+	if strings.Contains(filePath, "\x1F") {
+		t.Errorf("file path contains unit separator (\\x1F), expected dot separator: %s", filePath)
+	}
+
+	t.Logf(">>> Trino multi-level namespace test passed")
+}
+
 func NewTestEnvironment(t *testing.T) *TestEnvironment {
 	t.Helper()
 
@@ -332,18 +427,32 @@ func testIcebergRestAPI(t *testing.T, env *TestEnvironment) {
 	}
 }
 
-func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket string) string {
+func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket string, opts ...func(*trinoConfigOptions)) string {
 	t.Helper()
 
-	configDir := filepath.Join(env.dataDir, "trino")
+	o := trinoConfigOptions{}
+	for _, fn := range opts {
+		fn(&o)
+	}
+
+	dirName := "trino"
+	if o.nestedNamespace {
+		dirName = "trino-nested"
+	}
+	configDir := filepath.Join(env.dataDir, dirName)
 	if err := os.MkdirAll(configDir, 0755); err != nil {
 		t.Fatalf("Failed to create Trino config dir: %v", err)
 	}
 
+	nestedLine := ""
+	if o.nestedNamespace {
+		nestedLine = "\niceberg.rest-catalog.nested-namespace-enabled=true"
+	}
+
 	config := fmt.Sprintf(`connector.name=iceberg
 iceberg.catalog.type=rest
 iceberg.rest-catalog.uri=http://host.docker.internal:%d
-iceberg.rest-catalog.warehouse=s3://%s
+iceberg.rest-catalog.warehouse=s3://%s%s
 iceberg.file-format=PARQUET
 iceberg.unique-table-location=true
 
@@ -358,7 +467,7 @@ s3.region=us-west-2
 
 # REST catalog authentication
 iceberg.rest-catalog.security=SIGV4
-`, env.icebergPort, warehouseBucket, env.s3Port, env.accessKey, env.secretKey)
+`, env.icebergPort, warehouseBucket, nestedLine, env.s3Port, env.accessKey, env.secretKey)
 
 	if err := os.WriteFile(filepath.Join(configDir, "iceberg.properties"), []byte(config), 0644); err != nil {
 		t.Fatalf("Failed to write Trino config: %v", err)
@@ -367,6 +476,14 @@ iceberg.rest-catalog.security=SIGV4
 	return configDir
 }
 
+type trinoConfigOptions struct {
+	nestedNamespace bool
+}
+
+func withNestedNamespace() func(*trinoConfigOptions) {
+	return func(o *trinoConfigOptions) { o.nestedNamespace = true }
+}
+
 func (env *TestEnvironment) startTrinoContainer(t *testing.T, configDir string) {
 	t.Helper()
 

test/volume_server/framework/volume_fixture.go

@@ -80,6 +80,42 @@ func EnableMaintenanceMode(t testing.TB, ctx context.Context, client volume_serv
 	}
 }
 
+// CorruptDatFile truncates a volume's .dat file to just the superblock (8 bytes)
+// so that needle reads fail during a full scrub due to data file size mismatch.
+func CorruptDatFile(t testing.TB, baseDir string, volumeID uint32) {
+	t.Helper()
+	datPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.dat", volumeID))
+	// Truncate to superblock size only, removing all needle data.
+	if err := os.Truncate(datPath, 8); err != nil {
+		t.Fatalf("truncate dat file for corruption: %v", err)
+	}
+}
+
+// CorruptEcxFile appends garbage bytes to a volume's .ecx file on disk so
+// that CheckIndexFile detects a size mismatch during EC index scrub.
+func CorruptEcxFile(t testing.TB, baseDir string, volumeID uint32) {
+	t.Helper()
+	ecxPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.ecx", volumeID))
+	f, err := os.OpenFile(ecxPath, os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		t.Fatalf("open ecx file for corruption: %v", err)
+	}
+	defer f.Close()
+	if _, err := f.Write([]byte{0xDE, 0xAD}); err != nil {
+		t.Fatalf("corrupt ecx file: %v", err)
+	}
+}
+
+// CorruptEcShardFile truncates an EC shard file to 1 byte so that local shard
+// reads fail during an EC scrub.
+func CorruptEcShardFile(t testing.TB, baseDir string, volumeID uint32, shardID int) {
+	t.Helper()
+	shardPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.ec%02d", volumeID, shardID))
+	if err := os.Truncate(shardPath, 1); err != nil {
+		t.Fatalf("truncate EC shard file %s: %v", shardPath, err)
+	}
+}
+
 func ReadBytes(t testing.TB, client *http.Client, volumeURL, fid string) *http.Response {
 	t.Helper()
 

test/volume_server/grpc/erasure_coding_test.go

@@ -752,6 +752,87 @@ func TestEcShardsCopyFromPeerSuccess(t *testing.T) {
 	}
 }
 
+// TestEcIndexConsistencyAfterEncode verifies that every needle indexed in .ecx
+// can be read back correctly from EC shards after VolumeEcShardsGenerate.
+// This catches the race condition fixed in this PR where .ecx could reference
+// data not present in EC shards.
+func TestEcIndexConsistencyAfterEncode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(130)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+
+	// Upload multiple needles of varying sizes
+	type testNeedle struct {
+		fid     string
+		payload []byte
+	}
+	needles := []testNeedle{
+		{framework.NewFileID(volumeID, 1001, 0xAABB0001), []byte("small-needle-1")},
+		{framework.NewFileID(volumeID, 1002, 0xAABB0002), make([]byte, 1024)},     // 1KB
+		{framework.NewFileID(volumeID, 1003, 0xAABB0003), make([]byte, 64*1024)},   // 64KB
+		{framework.NewFileID(volumeID, 1004, 0xAABB0004), make([]byte, 256*1024)},  // 256KB
+		{framework.NewFileID(volumeID, 1005, 0xAABB0005), []byte("small-needle-2")},
+	}
+
+	// Fill larger payloads with recognizable data
+	for i := range needles {
+		for j := range needles[i].payload {
+			needles[i].payload[j] = byte(i*37 + j%251)
+		}
+	}
+
+	for _, n := range needles {
+		resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload)
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s expected 201, got %d", n.fid, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// EC encode
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	// Mount all data shards so reads go through the EC path
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount failed: %v", err)
+	}
+
+	// Read every needle back from EC shards and verify payload
+	for _, n := range needles {
+		readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid)
+		readBody := framework.ReadAllAndClose(t, readResp)
+		if readResp.StatusCode != http.StatusOK {
+			t.Fatalf("EC read %s expected 200, got %d", n.fid, readResp.StatusCode)
+		}
+		if string(readBody) != string(n.payload) {
+			t.Fatalf("EC read %s payload mismatch: got %d bytes, want %d bytes", n.fid, len(readBody), len(n.payload))
+		}
+	}
+}
+
 func TestEcShardsCopyFailsWhenSourceUnavailable(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")

test/volume_server/grpc/scrub_integration_test.go

@@ -0,0 +1,416 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// --- Normal volume scrub tests ---
+
+func TestScrubVolumeFullHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(200)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("data-one")))
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 2, 2), []byte("data-two")))
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 3, 3), []byte("data-three")))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume FULL on healthy volume failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 1 {
+		t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
+	}
+	if resp.GetTotalFiles() != 3 {
+		t.Fatalf("expected total_files=3, got %d", resp.GetTotalFiles())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
+	}
+}
+
+func TestScrubVolumeFullCorruptData(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(201)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("important data")))
+
+	framework.CorruptDatFile(t, clusterHarness.BaseDir(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume FULL on corrupt volume failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after data corruption, got none")
+	}
+	if len(resp.GetDetails()) == 0 {
+		t.Fatalf("expected error details for corrupt volume")
+	}
+}
+
+func TestScrubVolumeMixedHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const healthyVol = uint32(202)
+	const corruptVol = uint32(203)
+	framework.AllocateVolume(t, grpcClient, healthyVol, "")
+	framework.AllocateVolume(t, grpcClient, corruptVol, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(healthyVol, 1, 1), []byte("healthy")))
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(corruptVol, 1, 1), []byte("will corrupt")))
+
+	framework.CorruptIndexFile(t, clusterHarness.BaseDir(), corruptVol)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{healthyVol, corruptVol},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume INDEX on mixed volumes failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 2 {
+		t.Fatalf("expected total_volumes=2, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 1 {
+		t.Fatalf("expected exactly 1 broken volume, got %v", resp.GetBrokenVolumeIds())
+	}
+	if resp.GetBrokenVolumeIds()[0] != corruptVol {
+		t.Fatalf("expected broken volume %d, got %d", corruptVol, resp.GetBrokenVolumeIds()[0])
+	}
+}
+
+func TestScrubVolumeMissingVolumeReturnsError(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{99999},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err == nil {
+		t.Fatalf("ScrubVolume should fail for missing volume")
+	}
+}
+
+// --- EC volume scrub tests ---
+
+// ecSetup creates a volume, uploads data, generates EC shards, and mounts all of them.
+func ecSetup(t *testing.T, grpcClient volume_server_pb.VolumeServerClient, httpClient *http.Client, volumeURL string, volumeID uint32) {
+	t.Helper()
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 1, 0xABCD0001)
+	uploadResp := framework.UploadBytes(t, httpClient, volumeURL, fid, []byte("ec-scrub-test-data-payload"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	allShards := make([]uint32, erasure_coding.TotalShardsCount)
+	for i := range allShards {
+		allShards[i] = uint32(i)
+	}
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   allShards,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount all shards failed: %v", err)
+	}
+}
+
+func TestScrubEcVolumeIndexHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(210)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume INDEX on healthy volume failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 1 {
+		t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
+	}
+}
+
+func TestScrubEcVolumeLocalHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(211)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_LOCAL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume LOCAL on healthy volume failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 1 {
+		t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
+	}
+	if resp.GetTotalFiles() != 1 {
+		t.Fatalf("expected total_files=1, got %d", resp.GetTotalFiles())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
+	}
+	if len(resp.GetBrokenShardInfos()) != 0 {
+		t.Fatalf("expected no broken shards, got %v", resp.GetBrokenShardInfos())
+	}
+}
+
+func TestScrubEcVolumeLocalCorruptShard(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(212)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	// Corrupt shard 0 by truncating it.
+	framework.CorruptEcShardFile(t, clusterHarness.BaseDir(), volumeID, 0)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_LOCAL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume LOCAL on corrupt shard failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after shard corruption")
+	}
+	if len(resp.GetBrokenShardInfos()) == 0 {
+		t.Fatalf("expected broken shard info after shard corruption")
+	}
+	// Verify all reported broken shards belong to the corrupted volume.
+	for _, si := range resp.GetBrokenShardInfos() {
+		if si.GetVolumeId() != volumeID {
+			t.Fatalf("broken shard info for unexpected volume %d, want %d", si.GetVolumeId(), volumeID)
+		}
+	}
+}
+
+func TestScrubEcVolumeAutoSelectWithEcPresent(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeIDA = uint32(213)
+	const volumeIDB = uint32(214)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeIDA)
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeIDB)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Auto-select (empty VolumeIds) should find both EC volumes.
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		Mode: volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume auto-select failed: %v", err)
+	}
+	if resp.GetTotalVolumes() < 2 {
+		t.Fatalf("expected at least 2 EC volumes via auto-select, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v", resp.GetBrokenVolumeIds())
+	}
+}
+
+func TestScrubEcVolumeUnsupportedMode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(215)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode(99),
+	})
+	if err == nil {
+		t.Fatalf("ScrubEcVolume should fail for unsupported mode")
+	}
+}
+
+func TestScrubEcVolumeIndexCorruptEcx(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(216)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 1, 0xABCD0001)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-ecx-corrupt-test"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Generate EC shards (this creates the .ecx file on disk).
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	// Corrupt the .ecx file BEFORE mounting, so the corrupted size is loaded.
+	framework.CorruptEcxFile(t, clusterHarness.BaseDir(), volumeID)
+
+	// Now mount shards - the ecx file size will reflect the corruption.
+	allShards := make([]uint32, erasure_coding.TotalShardsCount)
+	for i := range allShards {
+		allShards[i] = uint32(i)
+	}
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   allShards,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount failed: %v", err)
+	}
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume INDEX on corrupt ecx failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after ECX corruption")
+	}
+}

weed/admin/dash/config_persistence.go

@@ -1091,7 +1091,7 @@ func (cp *ConfigPersistence) loadTaskStateLocked(taskID string) (*maintenance.Ma
 	// Convert protobuf to maintenance task
 	task := cp.protobufToMaintenanceTask(taskStateFile.Task)
 
-	glog.V(2).Infof("Loaded task state for task %s from %s", taskID, taskFilePath)
+	glog.V(3).Infof("Loaded task state for task %s from %s", taskID, taskFilePath)
 	return task, nil
 }
 
@@ -1135,6 +1135,43 @@ func (cp *ConfigPersistence) loadAllTaskStatesLocked() ([]*maintenance.Maintenan
 	return tasks, nil
 }
 
+// DeleteAllTaskStates removes all task state .pb files from disk without reading them.
+// Used at startup to clean up stale files from previous runs — the scanner will
+// re-detect any tasks that are still needed from live cluster state.
+func (cp *ConfigPersistence) DeleteAllTaskStates() error {
+	cp.tasksMu.Lock()
+	defer cp.tasksMu.Unlock()
+
+	if cp.dataDir == "" {
+		return nil
+	}
+
+	tasksDir := filepath.Join(cp.dataDir, TasksSubdir)
+	entries, err := os.ReadDir(tasksDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("failed to read tasks directory: %w", err)
+	}
+
+	var removed int
+	for _, entry := range entries {
+		if !entry.IsDir() && filepath.Ext(entry.Name()) == ".pb" {
+			if err := os.Remove(filepath.Join(tasksDir, entry.Name())); err != nil && !os.IsNotExist(err) {
+				glog.Warningf("Failed to delete task file %s: %v", entry.Name(), err)
+			} else {
+				removed++
+			}
+		}
+	}
+
+	if removed > 0 {
+		glog.Infof("Cleaned up %d stale task files from disk", removed)
+	}
+	return nil
+}
+
 // DeleteTaskState removes a task state file from disk
 func (cp *ConfigPersistence) DeleteTaskState(taskID string) error {
 	cp.tasksMu.Lock()

weed/admin/maintenance/maintenance_queue.go

@@ -33,80 +33,16 @@ func (mq *MaintenanceQueue) SetPersistence(persistence TaskPersistence) {
 	glog.V(1).Infof("Maintenance queue configured with task persistence")
 }
 
-// LoadTasksFromPersistence loads tasks from persistent storage on startup
+// LoadTasksFromPersistence is called on startup. Previous task states are NOT loaded
+// into memory — the maintenance scanner will re-detect current needs from the live
+// cluster state. Stale task files from previous runs are deleted from disk.
 func (mq *MaintenanceQueue) LoadTasksFromPersistence() error {
-	if mq.persistence == nil {
-		glog.V(1).Infof("No task persistence configured, skipping task loading")
-		return nil
-	}
-
-	mq.mutex.Lock()
-	defer mq.mutex.Unlock()
-
-	glog.Infof("Loading tasks from persistence...")
-
-	tasks, err := mq.persistence.LoadAllTaskStates()
-	if err != nil {
-		return fmt.Errorf("failed to load task states: %w", err)
-	}
-
-	glog.Infof("DEBUG LoadTasksFromPersistence: Found %d tasks in persistence", len(tasks))
-
-	// Reset task maps
-	mq.tasks = make(map[string]*MaintenanceTask)
-	mq.pendingTasks = make([]*MaintenanceTask, 0)
-
-	// Load tasks by status
-	for _, task := range tasks {
-		glog.Infof("DEBUG LoadTasksFromPersistence: Loading task %s (type: %s, status: %s, scheduled: %v)", task.ID, task.Type, task.Status, task.ScheduledAt)
-		mq.tasks[task.ID] = task
-
-		switch task.Status {
-		case TaskStatusPending:
-			glog.Infof("DEBUG LoadTasksFromPersistence: Adding task %s to pending queue", task.ID)
-			mq.pendingTasks = append(mq.pendingTasks, task)
-		case TaskStatusAssigned, TaskStatusInProgress:
-			// For assigned/in-progress tasks, we need to check if the worker is still available
-			// If not, we should fail them and make them eligible for retry
-			if task.WorkerID != "" {
-				if _, exists := mq.workers[task.WorkerID]; !exists {
-					glog.Warningf("Task %s was assigned to unavailable worker %s, marking as failed", task.ID, task.WorkerID)
-					task.Status = TaskStatusFailed
-					task.Error = "Worker unavailable after restart"
-					completedTime := time.Now()
-					task.CompletedAt = &completedTime
-
-					// Check if it should be retried
-					if task.RetryCount < task.MaxRetries {
-						task.RetryCount++
-						task.Status = TaskStatusPending
-						task.WorkerID = ""
-						task.StartedAt = nil
-						task.CompletedAt = nil
-						task.Error = ""
-						task.ScheduledAt = time.Now().Add(1 * time.Minute) // Retry after restart delay
-						glog.Infof("DEBUG LoadTasksFromPersistence: Retrying task %s, adding to pending queue", task.ID)
-						mq.pendingTasks = append(mq.pendingTasks, task)
-					}
-				}
-			}
-		}
-
-		// Sync task with ActiveTopology for capacity tracking
-		if mq.integration != nil {
-			mq.integration.SyncTask(task)
+	if mq.persistence != nil {
+		if err := mq.persistence.DeleteAllTaskStates(); err != nil {
+			glog.Warningf("Failed to clean up old task files: %v", err)
 		}
 	}
-
-	// Sort pending tasks by priority and schedule time
-	sort.Slice(mq.pendingTasks, func(i, j int) bool {
-		if mq.pendingTasks[i].Priority != mq.pendingTasks[j].Priority {
-			return mq.pendingTasks[i].Priority > mq.pendingTasks[j].Priority
-		}
-		return mq.pendingTasks[i].ScheduledAt.Before(mq.pendingTasks[j].ScheduledAt)
-	})
-
-	glog.Infof("Loaded %d tasks from persistence (%d pending)", len(tasks), len(mq.pendingTasks))
+	glog.Infof("Task queue initialized (previous tasks will be re-detected by scanner)")
 	return nil
 }
 
@@ -119,6 +55,14 @@ func (mq *MaintenanceQueue) saveTaskState(task *MaintenanceTask) {
 	}
 }
 
+func (mq *MaintenanceQueue) deleteTaskState(taskID string) {
+	if mq.persistence != nil {
+		if err := mq.persistence.DeleteTaskState(taskID); err != nil {
+			glog.V(2).Infof("Failed to delete task state for %s: %v", taskID, err)
+		}
+	}
+}
+
 // cleanupCompletedTasks removes old completed tasks beyond the retention limit
 func (mq *MaintenanceQueue) cleanupCompletedTasks() {
 	if mq.persistence != nil {
@@ -128,10 +72,24 @@ func (mq *MaintenanceQueue) cleanupCompletedTasks() {
 	}
 }
 
+const MaxTasksPerType = 100
+
 // AddTask adds a new maintenance task to the queue with deduplication
 func (mq *MaintenanceQueue) AddTask(task *MaintenanceTask) {
 	mq.mutex.Lock()
 
+	// Enforce per-type capacity limit (only counting active tasks)
+	if mq.countActiveTasksByType(task.Type) >= MaxTasksPerType {
+		// Purge terminal tasks first, then recheck
+		mq.purgeTerminalTasksLocked()
+		if mq.countActiveTasksByType(task.Type) >= MaxTasksPerType {
+			mq.mutex.Unlock()
+			glog.V(1).Infof("Task skipped (type %s at capacity %d): volume %d on %s",
+				task.Type, MaxTasksPerType, task.VolumeID, task.Server)
+			return
+		}
+	}
+
 	// Enforce one queued/active task per volume (across all task types).
 	if mq.hasQueuedOrActiveTaskForVolume(task.VolumeID) {
 		mq.mutex.Unlock()
@@ -197,6 +155,30 @@ func (mq *MaintenanceQueue) AddTask(task *MaintenanceTask) {
 		taskSnapshot.ID, taskSnapshot.Type, taskSnapshot.VolumeID, taskSnapshot.Server, taskSnapshot.Priority, scheduleInfo, taskSnapshot.Reason)
 }
 
+// countActiveTasksByType returns the number of active (non-terminal) tasks of a given type. Caller must hold mq.mutex.
+func (mq *MaintenanceQueue) countActiveTasksByType(taskType MaintenanceTaskType) int {
+	count := 0
+	for _, t := range mq.tasks {
+		if t.Type == taskType {
+			switch t.Status {
+			case TaskStatusPending, TaskStatusAssigned, TaskStatusInProgress:
+				count++
+			}
+		}
+	}
+	return count
+}
+
+// purgeTerminalTasksLocked removes terminal tasks from the in-memory map. Caller must hold mq.mutex.
+func (mq *MaintenanceQueue) purgeTerminalTasksLocked() {
+	for id, task := range mq.tasks {
+		switch task.Status {
+		case TaskStatusCompleted, TaskStatusFailed, TaskStatusCancelled:
+			delete(mq.tasks, id)
+		}
+	}
+}
+
 // hasQueuedOrActiveTaskForVolume checks if any pending/assigned/in-progress task already exists for this volume.
 // Caller must hold mq.mutex.
 func (mq *MaintenanceQueue) hasQueuedOrActiveTaskForVolume(volumeID uint32) bool {
@@ -273,6 +255,9 @@ func (mq *MaintenanceQueue) CancelPendingTasksByType(taskType MaintenanceTaskTyp
 
 // AddTasksFromResults converts detection results to tasks and adds them to the queue
 func (mq *MaintenanceQueue) AddTasksFromResults(results []*TaskDetectionResult) {
+	// Purge terminal tasks from memory before adding new ones
+	mq.purgeTerminalTasks()
+
 	for _, result := range results {
 		// Validate that task has proper typed parameters
 		if result.TypedParams == nil {
@@ -297,6 +282,21 @@ func (mq *MaintenanceQueue) AddTasksFromResults(results []*TaskDetectionResult)
 	}
 }
 
+// purgeTerminalTasks removes completed/failed/cancelled tasks from memory.
+// Terminal tasks are already deleted from disk by CompleteTask, so this
+// only needs to clean up the in-memory map.
+func (mq *MaintenanceQueue) purgeTerminalTasks() {
+	mq.mutex.Lock()
+	before := len(mq.tasks)
+	mq.purgeTerminalTasksLocked()
+	purged := before - len(mq.tasks)
+	mq.mutex.Unlock()
+
+	if purged > 0 {
+		glog.V(1).Infof("Purged %d terminal tasks from memory", purged)
+	}
+}
+
 // GetNextTask returns the next available task for a worker
 func (mq *MaintenanceQueue) GetNextTask(workerID string, capabilities []MaintenanceTaskType) *MaintenanceTask {
 	// Use read lock for initial checks and search
@@ -570,7 +570,6 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 		}
 	}
 	taskStatus := task.Status
-	taskCount := len(mq.tasks)
 	// Snapshot task state while lock is still held to avoid data race
 	var taskToSaveSnapshot *MaintenanceTask
 	if taskToSave != nil {
@@ -578,9 +577,18 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 	}
 	mq.mutex.Unlock()
 
-	// Save task state to persistence outside the lock
+	// Only persist non-terminal tasks (retries). Completed/failed tasks stay
+	// in memory for the UI but are not written to disk — they would just
+	// accumulate and slow down future startups.
 	if taskToSaveSnapshot != nil {
-		mq.saveTaskState(taskToSaveSnapshot)
+		switch taskStatus {
+		case TaskStatusPending:
+			// Retry — save so the task survives a restart
+			mq.saveTaskState(taskToSaveSnapshot)
+		case TaskStatusCompleted, TaskStatusFailed, TaskStatusCancelled:
+			// Terminal — delete the file if one exists from a previous state
+			mq.deleteTaskState(taskToSaveSnapshot.ID)
+		}
 	}
 
 	if logFn != nil {
@@ -591,13 +599,6 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 	if taskStatus != TaskStatusPending {
 		mq.removePendingOperation(taskID)
 	}
-
-	// Periodically cleanup old completed tasks (when total task count is a multiple of 10)
-	if taskStatus == TaskStatusCompleted {
-		if taskCount%10 == 0 {
-			go mq.cleanupCompletedTasks()
-		}
-	}
 }
 
 // isNonRetriableError returns true for errors that will never succeed on retry,

weed/admin/maintenance/maintenance_queue_test.go

@@ -693,10 +693,11 @@ func (m *MockPersistence) SaveTaskState(task *MaintenanceTask) error
 func (m *MockPersistence) LoadTaskState(taskID string) (*MaintenanceTask, error)    { return nil, nil }
 func (m *MockPersistence) LoadAllTaskStates() ([]*MaintenanceTask, error)           { return m.tasks, nil }
 func (m *MockPersistence) DeleteTaskState(taskID string) error                      { return nil }
+func (m *MockPersistence) DeleteAllTaskStates() error                               { return nil }
 func (m *MockPersistence) CleanupCompletedTasks() error                             { return nil }
 func (m *MockPersistence) SaveTaskPolicy(taskType string, policy *TaskPolicy) error { return nil }
 
-func TestMaintenanceQueue_LoadTasksCapacitySync(t *testing.T) {
+func TestMaintenanceQueue_LoadTasksStartsEmpty(t *testing.T) {
 	// Setup
 	policy := &MaintenancePolicy{
 		TaskPolicies: map[string]*worker_pb.TaskPolicy{
@@ -704,56 +705,25 @@ func TestMaintenanceQueue_LoadTasksCapacitySync(t *testing.T) {
 		},
 	}
 	mq := NewMaintenanceQueue(policy)
-	integration := NewMaintenanceIntegration(mq, policy)
-	mq.SetIntegration(integration)
-	at := integration.GetActiveTopology()
 
-	topologyInfo := &master_pb.TopologyInfo{
-		DataCenterInfos: []*master_pb.DataCenterInfo{
-			{
-				Id: "dc1",
-				RackInfos: []*master_pb.RackInfo{
-					{
-						Id: "rack1",
-						DataNodeInfos: []*master_pb.DataNodeInfo{
-							{
-								Id: "server1",
-								DiskInfos: map[string]*master_pb.DiskInfo{
-									"hdd1": {DiskId: 1, VolumeCount: 1, MaxVolumeCount: 10},
-									"hdd2": {DiskId: 2, VolumeCount: 0, MaxVolumeCount: 10},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-	at.UpdateTopology(topologyInfo)
-
-	// Setup mock persistence with a pending task
-	taskID := "load_test_123"
+	// Setup mock persistence with tasks — these should NOT be loaded
 	mockTask := &MaintenanceTask{
-		ID:     taskID,
+		ID:     "old_task_123",
 		Type:   "balance",
 		Status: TaskStatusPending,
-		TypedParams: &worker_pb.TaskParams{
-			TaskId:  taskID,
-			Sources: []*worker_pb.TaskSource{{Node: "server1", DiskId: 1}},
-			Targets: []*worker_pb.TaskTarget{{Node: "server1", DiskId: 2}},
-		},
 	}
 	mq.SetPersistence(&MockPersistence{tasks: []*MaintenanceTask{mockTask}})
 
-	// Load tasks
+	// LoadTasksFromPersistence should be a no-op — scanner will re-detect
 	err := mq.LoadTasksFromPersistence()
 	if err != nil {
-		t.Fatalf("Failed to load tasks: %v", err)
+		t.Fatalf("LoadTasksFromPersistence failed: %v", err)
 	}
 
-	// Verify capacity is reserved in ActiveTopology after loading (9 left)
-	if at.GetEffectiveAvailableCapacity("server1", 2) != 9 {
-		t.Errorf("Expected capacity 9 after loading tasks, got %d", at.GetEffectiveAvailableCapacity("server1", 2))
+	// Queue should be empty — tasks will be re-detected by scanner
+	stats := mq.GetStats()
+	if stats.TotalTasks != 0 {
+		t.Errorf("Expected 0 tasks after startup, got %d", stats.TotalTasks)
 	}
 }
 

weed/admin/maintenance/maintenance_scanner.go

@@ -32,21 +32,18 @@ func NewMaintenanceScanner(adminClient AdminClient, policy *MaintenancePolicy, q
 
 // ScanForMaintenanceTasks analyzes the cluster and generates maintenance tasks
 func (ms *MaintenanceScanner) ScanForMaintenanceTasks() ([]*TaskDetectionResult, error) {
-	// Get volume health metrics
-	volumeMetrics, err := ms.getVolumeHealthMetrics()
+	// Get volume health metrics directly in task-system format, along with topology info
+	taskMetrics, topologyInfo, err := ms.getVolumeHealthMetrics()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get volume health metrics: %w", err)
 	}
 
 	// Use task system for all task types
 	if ms.integration != nil {
-		// Convert metrics to task system format
-		taskMetrics := ms.convertToTaskMetrics(volumeMetrics)
-
 		// Update topology information for complete cluster view (including empty servers)
 		// This must happen before task detection to ensure EC placement can consider all servers
-		if ms.lastTopologyInfo != nil {
-			if err := ms.integration.UpdateTopologyInfo(ms.lastTopologyInfo); err != nil {
+		if topologyInfo != nil {
+			if err := ms.integration.UpdateTopologyInfo(topologyInfo); err != nil {
 				glog.Errorf("Failed to update topology info for empty servers: %v", err)
 				// Don't fail the scan - continue with just volume-bearing servers
 			} else {
@@ -70,9 +67,12 @@ func (ms *MaintenanceScanner) ScanForMaintenanceTasks() ([]*TaskDetectionResult,
 	return []*TaskDetectionResult{}, nil
 }
 
-// getVolumeHealthMetrics collects health information for all volumes
-func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics, error) {
-	var metrics []*VolumeHealthMetrics
+// getVolumeHealthMetrics collects health information for all volumes.
+// Returns metrics in task-system format directly (no intermediate copy) and
+// the topology info for updating the active topology.
+func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*types.VolumeHealthMetrics, *master_pb.TopologyInfo, error) {
+	var metrics []*types.VolumeHealthMetrics
+	var topologyInfo *master_pb.TopologyInfo
 
 	glog.V(1).Infof("Collecting volume health metrics from master")
 	err := ms.adminClient.WithMasterClient(func(client master_pb.SeaweedClient) error {
@@ -89,30 +89,28 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics,
 
 		volumeSizeLimitBytes := uint64(resp.VolumeSizeLimitMb) * 1024 * 1024 // Convert MB to bytes
 
-		// Track all nodes discovered in topology
-		var allNodesInTopology []string
-		var nodesWithVolumes []string
-		var nodesWithoutVolumes []string
+		// Track node counts for summary logging (avoid accumulating full ID slices)
+		var totalNodes, nodesWithVolumes, nodesWithoutVolumes int
 
 		for _, dc := range resp.TopologyInfo.DataCenterInfos {
-			glog.V(2).Infof("Processing datacenter: %s", dc.Id)
+			glog.V(3).Infof("Processing datacenter: %s", dc.Id)
 			for _, rack := range dc.RackInfos {
-				glog.V(2).Infof("Processing rack: %s in datacenter: %s", rack.Id, dc.Id)
+				glog.V(3).Infof("Processing rack: %s in datacenter: %s", rack.Id, dc.Id)
 				for _, node := range rack.DataNodeInfos {
-					allNodesInTopology = append(allNodesInTopology, node.Id)
-					glog.V(2).Infof("Found volume server in topology: %s (disks: %d)", node.Id, len(node.DiskInfos))
+					totalNodes++
+					glog.V(3).Infof("Found volume server in topology: %s (disks: %d)", node.Id, len(node.DiskInfos))
 
 					hasVolumes := false
 					// Process each disk on this node
 					for diskType, diskInfo := range node.DiskInfos {
 						if len(diskInfo.VolumeInfos) > 0 {
 							hasVolumes = true
-							glog.V(2).Infof("Volume server %s disk %s has %d volumes", node.Id, diskType, len(diskInfo.VolumeInfos))
+							glog.V(3).Infof("Volume server %s disk %s has %d volumes", node.Id, diskType, len(diskInfo.VolumeInfos))
 						}
 
 						// Process volumes on this specific disk
 						for _, volInfo := range diskInfo.VolumeInfos {
-							metric := &VolumeHealthMetrics{
+							metric := &types.VolumeHealthMetrics{
 								VolumeID:         volInfo.Id,
 								Server:           node.Id,
 								ServerAddress:    node.Address,
@@ -138,7 +136,7 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics,
 							}
 							metric.Age = time.Since(metric.LastModified)
 
-							glog.V(3).Infof("Volume %d on %s:%s (ID %d): size=%d, limit=%d, fullness=%.2f",
+							glog.V(4).Infof("Volume %d on %s:%s (ID %d): size=%d, limit=%d, fullness=%.2f",
 								metric.VolumeID, metric.Server, metric.DiskType, metric.DiskId, metric.Size, volumeSizeLimitBytes, metric.FullnessRatio)
 
 							metrics = append(metrics, metric)
@@ -146,29 +144,27 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics,
 					}
 
 					if hasVolumes {
-						nodesWithVolumes = append(nodesWithVolumes, node.Id)
+						nodesWithVolumes++
 					} else {
-						nodesWithoutVolumes = append(nodesWithoutVolumes, node.Id)
+						nodesWithoutVolumes++
 						glog.V(1).Infof("Volume server %s found in topology but has no volumes", node.Id)
 					}
 				}
 			}
 		}
 
-		glog.Infof("Topology discovery complete:")
-		glog.Infof("  - Total volume servers in topology: %d (%v)", len(allNodesInTopology), allNodesInTopology)
-		glog.Infof("  - Volume servers with volumes: %d (%v)", len(nodesWithVolumes), nodesWithVolumes)
-		glog.Infof("  - Volume servers without volumes: %d (%v)", len(nodesWithoutVolumes), nodesWithoutVolumes)
+		glog.Infof("Topology discovery: %d volume servers (%d with volumes, %d without)",
+			totalNodes, nodesWithVolumes, nodesWithoutVolumes)
 
-		// Store topology info for volume shard tracker
-		ms.lastTopologyInfo = resp.TopologyInfo
+		// Return topology info as a local value (not retained on the scanner struct)
+		topologyInfo = resp.TopologyInfo
 
 		return nil
 	})
 
 	if err != nil {
 		glog.Errorf("Failed to get volume health metrics: %v", err)
-		return nil, err
+		return nil, nil, err
 	}
 
 	glog.V(1).Infof("Successfully collected metrics for %d actual volumes with disk ID information", len(metrics))
@@ -176,13 +172,13 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics,
 	// Count actual replicas and identify EC volumes
 	ms.enrichVolumeMetrics(metrics)
 
-	return metrics, nil
+	return metrics, topologyInfo, nil
 }
 
 // enrichVolumeMetrics adds additional information like replica counts
-func (ms *MaintenanceScanner) enrichVolumeMetrics(metrics []*VolumeHealthMetrics) {
+func (ms *MaintenanceScanner) enrichVolumeMetrics(metrics []*types.VolumeHealthMetrics) {
 	// Group volumes by ID to count replicas
-	volumeGroups := make(map[uint32][]*VolumeHealthMetrics)
+	volumeGroups := make(map[uint32][]*types.VolumeHealthMetrics)
 	for _, metric := range metrics {
 		volumeGroups[metric.VolumeID] = append(volumeGroups[metric.VolumeID], metric)
 	}
@@ -193,41 +189,9 @@ func (ms *MaintenanceScanner) enrichVolumeMetrics(metrics []*VolumeHealthMetrics
 		for _, replica := range replicas {
 			replica.ReplicaCount = replicaCount
 		}
-		glog.V(3).Infof("Volume %d has %d replicas", volumeID, replicaCount)
+		glog.V(4).Infof("Volume %d has %d replicas", volumeID, replicaCount)
 	}
 
 	// TODO: Identify EC volumes by checking volume structure
 	// This would require querying volume servers for EC shard information
 }
-
-// convertToTaskMetrics converts existing volume metrics to task system format
-func (ms *MaintenanceScanner) convertToTaskMetrics(metrics []*VolumeHealthMetrics) []*types.VolumeHealthMetrics {
-	var simplified []*types.VolumeHealthMetrics
-
-	for _, metric := range metrics {
-		simplified = append(simplified, &types.VolumeHealthMetrics{
-			VolumeID:         metric.VolumeID,
-			Server:           metric.Server,
-			ServerAddress:    metric.ServerAddress,
-			DiskType:         metric.DiskType,
-			DiskId:           metric.DiskId,
-			DataCenter:       metric.DataCenter,
-			Rack:             metric.Rack,
-			Collection:       metric.Collection,
-			Size:             metric.Size,
-			DeletedBytes:     metric.DeletedBytes,
-			GarbageRatio:     metric.GarbageRatio,
-			LastModified:     metric.LastModified,
-			Age:              metric.Age,
-			ReplicaCount:     metric.ReplicaCount,
-			ExpectedReplicas: metric.ExpectedReplicas,
-			IsReadOnly:       metric.IsReadOnly,
-			HasRemoteCopy:    metric.HasRemoteCopy,
-			IsECVolume:       metric.IsECVolume,
-			FullnessRatio:    metric.FullnessRatio,
-		})
-	}
-
-	glog.V(2).Infof("Converted %d volume metrics with disk ID information for task detection", len(simplified))
-	return simplified
-}

weed/admin/maintenance/maintenance_types.go

@@ -130,6 +130,7 @@ type TaskPersistence interface {
 	LoadTaskState(taskID string) (*MaintenanceTask, error)
 	LoadAllTaskStates() ([]*MaintenanceTask, error)
 	DeleteTaskState(taskID string) error
+	DeleteAllTaskStates() error
 	CleanupCompletedTasks() error
 
 	// Policy persistence
@@ -206,12 +207,11 @@ type MaintenanceQueue struct {
 
 // MaintenanceScanner analyzes the cluster and generates maintenance tasks
 type MaintenanceScanner struct {
-	adminClient      AdminClient
-	policy           *MaintenancePolicy
-	queue            *MaintenanceQueue
-	lastScan         map[MaintenanceTaskType]time.Time
-	integration      *MaintenanceIntegration
-	lastTopologyInfo *master_pb.TopologyInfo
+	adminClient AdminClient
+	policy      *MaintenancePolicy
+	queue       *MaintenanceQueue
+	lastScan    map[MaintenanceTaskType]time.Time
+	integration *MaintenanceIntegration
 }
 
 // TaskDetectionResult represents the result of scanning for maintenance needs

weed/admin/static/js/admin.js

@@ -1267,7 +1267,7 @@ async function submitUploadFile() {
         });
 
         // Send request
-        xhr.open('POST', '/api/files/upload');
+        xhr.open('POST', basePath('/api/files/upload'));
         xhr.send(formData);
 
     } catch (error) {
@@ -1320,7 +1320,7 @@ function exportFileList() {
 // Download file
 function downloadFile(filePath) {
     // Create download link using admin API
-    const downloadUrl = `/api/files/download?path=${encodeURIComponent(filePath)}`;
+    const downloadUrl = basePath(`/api/files/download?path=${encodeURIComponent(filePath)}`);
     window.open(downloadUrl, '_blank');
 }
 
@@ -1786,7 +1786,7 @@ function createFileViewerContent(file, content) {
     if (file.mime.startsWith('image/')) {
         return `
             <div class="text-center">
-                <img src="/api/files/download?path=${encodeURIComponent(file.full_path)}" 
+                <img src="${basePath('/api/files/download?path=' + encodeURIComponent(file.full_path))}"
                      class="img-fluid" alt="${file.name}" style="max-height: 500px;">
             </div>
         `;
@@ -1804,7 +1804,7 @@ function createFileViewerContent(file, content) {
     } else if (file.mime === 'application/pdf') {
         return `
             <div class="text-center">
-                <embed src="/api/files/download?path=${encodeURIComponent(file.full_path)}" 
+                <embed src="${basePath('/api/files/download?path=' + encodeURIComponent(file.full_path))}"
                        type="application/pdf" width="100%" height="500px">
             </div>
         `;

weed/admin/topology/topology_management.go

@@ -87,7 +87,7 @@ func (at *ActiveTopology) UpdateTopology(topologyInfo *master_pb.TopologyInfo) e
 					}
 
 					diskKey := fmt.Sprintf("%s:%d", nodeInfo.Id, diskInfo.DiskId)
-					glog.V(2).Infof("UpdateTopology: adding disk key=%q nodeId=%q diskId=%d diskType=%q address=%q grpcPort=%d volumes=%d maxVolumes=%d",
+					glog.V(3).Infof("UpdateTopology: adding disk key=%q nodeId=%q diskId=%d diskType=%q address=%q grpcPort=%d volumes=%d maxVolumes=%d",
 						diskKey, nodeInfo.Id, diskInfo.DiskId, diskType, nodeInfo.Address, nodeInfo.GrpcPort, diskInfo.VolumeCount, diskInfo.MaxVolumeCount)
 					node.disks[diskInfo.DiskId] = disk
 					at.disks[diskKey] = disk

weed/admin/view/app/object_store_users_templ.go

@@ -41,7 +41,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component {
 		var templ_7745c5c3_Var2 string
 		templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalUsers))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 38, Col: 71}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 38, Col: 71}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 		if templ_7745c5c3_Err != nil {
@@ -54,7 +54,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component {
 		var templ_7745c5c3_Var3 string
 		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.Users)))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 58, Col: 71}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 58, Col: 71}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
 		if templ_7745c5c3_Err != nil {
@@ -67,7 +67,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component {
 		var templ_7745c5c3_Var4 string
 		templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("15:04"))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 78, Col: 69}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 78, Col: 69}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
 		if templ_7745c5c3_Err != nil {
@@ -85,7 +85,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component {
 			var templ_7745c5c3_Var5 string
 			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(user.Username)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 127, Col: 74}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 127, Col: 74}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 			if templ_7745c5c3_Err != nil {
@@ -108,7 +108,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component {
 			var templ_7745c5c3_Var6 string
 			templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(user.Email)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 133, Col: 59}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 133, Col: 59}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 			if templ_7745c5c3_Err != nil {
@@ -121,7 +121,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component {
 			var templ_7745c5c3_Var7 string
 			templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(user.AccessKey)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 135, Col: 88}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 135, Col: 88}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
 			if templ_7745c5c3_Err != nil {
@@ -134,7 +134,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component {
 			var templ_7745c5c3_Var8 string
 			templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(user.Username)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 140, Col: 121}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 140, Col: 121}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
 			if templ_7745c5c3_Err != nil {
@@ -152,7 +152,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component {
 				var templ_7745c5c3_Var9 string
 				templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(user.Username)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 145, Col: 117}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 145, Col: 117}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
 				if templ_7745c5c3_Err != nil {
@@ -171,7 +171,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component {
 				var templ_7745c5c3_Var10 string
 				templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(user.Username)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 151, Col: 126}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 151, Col: 126}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
 				if templ_7745c5c3_Err != nil {
@@ -190,7 +190,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component {
 				var templ_7745c5c3_Var11 string
 				templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(user.Username)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 157, Col: 119}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 157, Col: 119}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
 				if templ_7745c5c3_Err != nil {
@@ -219,7 +219,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component {
 		var templ_7745c5c3_Var12 string
 		templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04:05"))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 189, Col: 81}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 189, Col: 81}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
 		if templ_7745c5c3_Err != nil {

weed/command/admin.go

@@ -392,7 +392,19 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
 	addr := fmt.Sprintf(":%d", *options.port)
 	var handler http.Handler = r
 	if urlPrefix != "" {
-		handler = http.StripPrefix(urlPrefix, r)
+		stripped := http.StripPrefix(urlPrefix, r)
+		handler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			// Redirect /prefix (no trailing slash) to /prefix/
+			if req.URL.Path == urlPrefix {
+				target := urlPrefix + "/"
+				if req.URL.RawQuery != "" {
+					target += "?" + req.URL.RawQuery
+				}
+				http.Redirect(w, req, target, http.StatusFound)
+				return
+			}
+			stripped.ServeHTTP(w, req)
+		})
 	}
 	server := &http.Server{
 		Addr:    addr,

weed/command/filer_backup.go

@@ -11,6 +11,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
 	"github.com/seaweedfs/seaweedfs/weed/replication/source"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -143,6 +144,10 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 		sourceFiler.ToGrpcAddress(),
 		sourcePath,
 		*backupOption.proxyByFiler)
+
+	if err := repl_util.InitializeSSEForReplication(filerSource); err != nil {
+		return fmt.Errorf("SSE initialization failed: %v", err)
+	}
 	dataSink.SetSourceFiler(filerSource)
 
 	var processEventFn func(*filer_pb.SubscribeMetadataResponse) error

weed/command/filer_sync.go

@@ -15,12 +15,14 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/replication"
 	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/replication/sink/filersink"
 	"github.com/seaweedfs/seaweedfs/weed/replication/source"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	statsCollect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
+	util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
 	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
 	"google.golang.org/grpc"
 )
@@ -164,6 +166,21 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 		}
 	}
 
+	// per-cluster HTTPS clients for volume server connections
+	var httpClientA, httpClientB *util_http_client.HTTPClient
+	if *syncOptions.aSecurity != "" {
+		var err error
+		if httpClientA, err = security.LoadHTTPClientFromFile(*syncOptions.aSecurity); err != nil {
+			glog.Fatalf("load HTTPS client config for filer A: %v", err)
+		}
+	}
+	if *syncOptions.bSecurity != "" {
+		var err error
+		if httpClientB, err = security.LoadHTTPClientFromFile(*syncOptions.bSecurity); err != nil {
+			glog.Fatalf("load HTTPS client config for filer B: %v", err)
+		}
+	}
+
 	grace.SetupProfiling(*syncCpuProfile, *syncMemProfile)
 
 	filerA := pb.ServerAddress(*syncOptions.filerA)
@@ -238,7 +255,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 				*syncOptions.bDoDeleteFiles,
 				aFilerSignature,
 				bFilerSignature,
-				&syncStateA2B)
+				&syncStateA2B,
+				httpClientA,
+				httpClientB)
 			if err != nil {
 				glog.Errorf("sync from %s to %s: %v", *syncOptions.filerA, *syncOptions.filerB, err)
 				time.Sleep(1747 * time.Millisecond)
@@ -279,7 +298,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 					*syncOptions.aDoDeleteFiles,
 					bFilerSignature,
 					aFilerSignature,
-					&syncStateB2A)
+					&syncStateB2A,
+					httpClientB,
+					httpClientA)
 				if err != nil {
 					glog.Errorf("sync from %s to %s: %v", *syncOptions.filerB, *syncOptions.filerA, err)
 					time.Sleep(2147 * time.Millisecond)
@@ -308,7 +329,8 @@ func initOffsetFromTsMs(grpcDialOption grpc.DialOption, targetFiler pb.ServerAdd
 }
 
 func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceExcludePaths []string, sourceReadChunkFromFiler bool, targetGrpcDialOption grpc.DialOption, targetFiler pb.ServerAddress, targetPath string,
-	replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, concurrency int, chunkConcurrency int, doDeleteFiles bool, sourceFilerSignature int32, targetFilerSignature int32, statePtr *atomic.Pointer[syncState]) error {
+	replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, concurrency int, chunkConcurrency int, doDeleteFiles bool, sourceFilerSignature int32, targetFilerSignature int32, statePtr *atomic.Pointer[syncState],
+	sourceHttpClient *util_http_client.HTTPClient, sinkHttpClient *util_http_client.HTTPClient) error {
 
 	// if first time, start from now
 	// if has previously synced, resume from that point of time
@@ -323,9 +345,15 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
 	filerSource := &source.FilerSource{}
 	filerSource.DoInitialize(sourceFiler.ToHttpAddress(), sourceFiler.ToGrpcAddress(), sourcePath, sourceReadChunkFromFiler)
 	filerSource.SetGrpcDialOption(sourceGrpcDialOption)
+	if sourceHttpClient != nil {
+		filerSource.SetHttpClient(sourceHttpClient)
+	}
 	filerSink := &filersink.FilerSink{}
 	filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, targetGrpcDialOption, sinkWriteChunkByFiler)
 	filerSink.SetChunkConcurrency(chunkConcurrency)
+	if sinkHttpClient != nil {
+		filerSink.SetUploader(operation.NewUploaderWithHttpClient(sinkHttpClient))
+	}
 	filerSink.SetSourceFiler(filerSource)
 
 	persistEventFn := genProcessFunction(sourcePath, targetPath, sourceExcludePaths, nil, nil, nil, filerSink, doDeleteFiles, debug)

weed/command/master.go

@@ -96,7 +96,7 @@ func init() {
 	m.metricsIntervalSec = cmdMaster.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
 	m.metricsHttpPort = cmdMaster.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	m.metricsHttpIp = cmdMaster.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
-	m.raftResumeState = cmdMaster.Flag.Bool("resumeState", false, "resume previous state on start master server")
+	m.raftResumeState = cmdMaster.Flag.Bool("resumeState", true, "resume previous state on start master server")
 	m.heartbeatInterval = cmdMaster.Flag.Duration("heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
 	m.electionTimeout = cmdMaster.Flag.Duration("electionTimeout", 10*time.Second, "election timeout of master servers")
 	m.raftHashicorp = cmdMaster.Flag.Bool("raftHashicorp", false, "use hashicorp raft")
@@ -208,6 +208,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		DataDir:           util.ResolvePath(metaDir),
 		Topo:              ms.Topo,
 		RaftResumeState:   *masterOption.raftResumeState,
+		SingleMaster:      isSingleMaster,
 		HeartbeatInterval: *masterOption.heartbeatInterval,
 		ElectionTimeout:   *masterOption.electionTimeout,
 		RaftBootstrap:     *masterOption.raftBootstrap,

weed/command/mini.go

@@ -162,7 +162,7 @@ func initMiniMasterFlags() {
 	miniMasterOptions.garbageThreshold = cmdMini.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	miniMasterOptions.metricsAddress = cmdMini.Flag.String("master.metrics.address", "", "Prometheus gateway address")
 	miniMasterOptions.metricsIntervalSec = cmdMini.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
-	miniMasterOptions.raftResumeState = cmdMini.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
+	miniMasterOptions.raftResumeState = cmdMini.Flag.Bool("master.resumeState", true, "resume previous state on start master server")
 	miniMasterOptions.heartbeatInterval = cmdMini.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
 	miniMasterOptions.electionTimeout = cmdMini.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
 	miniMasterOptions.raftHashicorp = cmdMini.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")

weed/command/scaffold/filer.toml

@@ -13,6 +13,9 @@
 # recursive_delete will delete all sub folders and files, similar to "rm -Rf"
 recursive_delete = false
 #max_file_name_length = 255
+# for S3: how long to wait before deleting an empty folder.
+# increase this if using tools like Spark that create temporary directories.
+#s3.empty_folder_cleanup_delay = "2m"
 
 ####################################################
 # The following are filer store options

weed/command/server.go

@@ -102,7 +102,7 @@ func init() {
 	masterOptions.garbageThreshold = cmdServer.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterOptions.metricsAddress = cmdServer.Flag.String("master.metrics.address", "", "Prometheus gateway address")
 	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
-	masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
+	masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", true, "resume previous state on start master server")
 	masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
 	masterOptions.raftBootstrap = cmdServer.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster")
 	masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")

weed/command/shell.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 
@@ -60,7 +61,7 @@ func runShell(command *Command, args []string) bool {
 		filerAddress = viper.GetString("cluster." + cluster + ".filer")
 	}
 	shellOptions.FilerAddress = pb.ServerAddress(filerAddress)
-	fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, shellOptions.FilerAddress)
+	fmt.Fprintf(os.Stderr, "master: %s filer: %s\n", *shellOptions.Masters, shellOptions.FilerAddress)
 
 	shell.RunShell(shellOptions)
 

weed/filer/empty_folder_cleanup/cleanup_queue.go

@@ -153,6 +153,27 @@ func (q *CleanupQueue) Pop() (string, string, bool) {
 	return item.folder, item.triggeredBy, true
 }
 
+// PopOlderThan removes and returns the oldest folder only if it has been in the queue
+// for longer than the specified duration. Returns empty string and false if no item qualifies.
+func (q *CleanupQueue) PopOlderThan(minAge time.Duration) (string, string, bool) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	front := q.items.Front()
+	if front == nil {
+		return "", "", false
+	}
+
+	item := front.Value.(*queueItem)
+	if time.Since(item.queueTime) <= minAge {
+		return "", "", false
+	}
+
+	q.items.Remove(front)
+	delete(q.itemsMap, item.folder)
+	return item.folder, item.triggeredBy, true
+}
+
 // Peek returns the oldest folder without removing it.
 // Returns the folder and queue time if available, or empty values if queue is empty.
 func (q *CleanupQueue) Peek() (folder string, triggeredBy string, queueTime time.Time, ok bool) {

weed/filer/empty_folder_cleanup/empty_folder_cleaner.go

@@ -18,8 +18,8 @@ const (
 	DefaultMaxCountCheck  = 1000
 	DefaultCacheExpiry    = 5 * time.Minute
 	DefaultQueueMaxSize   = 1000
-	DefaultQueueMaxAge    = 5 * time.Second
-	DefaultProcessorSleep = 10 * time.Second // How often to check queue
+	DefaultQueueMaxAge    = 2 * time.Minute
+	DefaultProcessorSleep = 30 * time.Second // How often to check queue
 )
 
 // FilerOperations defines the filer operations needed by EmptyFolderCleaner
@@ -70,15 +70,20 @@ type EmptyFolderCleaner struct {
 	stopCh  chan struct{}
 }
 
-// NewEmptyFolderCleaner creates a new EmptyFolderCleaner
-func NewEmptyFolderCleaner(filer FilerOperations, lockRing *lock_manager.LockRing, host pb.ServerAddress, bucketPath string) *EmptyFolderCleaner {
+// NewEmptyFolderCleaner creates a new EmptyFolderCleaner.
+// cleanupDelay controls how long an empty folder must remain in the queue before deletion.
+// If zero, DefaultQueueMaxAge is used.
+func NewEmptyFolderCleaner(filer FilerOperations, lockRing *lock_manager.LockRing, host pb.ServerAddress, bucketPath string, cleanupDelay time.Duration) *EmptyFolderCleaner {
+	if cleanupDelay <= 0 {
+		cleanupDelay = DefaultQueueMaxAge
+	}
 	efc := &EmptyFolderCleaner{
 		filer:                 filer,
 		lockRing:              lockRing,
 		host:                  host,
 		folderCounts:          make(map[string]*folderState),
 		bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
-		cleanupQueue:          NewCleanupQueue(DefaultQueueMaxSize, DefaultQueueMaxAge),
+		cleanupQueue:          NewCleanupQueue(DefaultQueueMaxSize, cleanupDelay),
 		maxCountCheck:         DefaultMaxCountCheck,
 		cacheExpiry:           DefaultCacheExpiry,
 		processorSleep:        DefaultProcessorSleep,
@@ -207,27 +212,22 @@ func (efc *EmptyFolderCleaner) cleanupProcessor() {
 
 // processCleanupQueue processes items from the cleanup queue
 func (efc *EmptyFolderCleaner) processCleanupQueue() {
-	// Check if we should process
-	if !efc.cleanupQueue.ShouldProcess() {
-		if efc.cleanupQueue.Len() > 0 {
-			glog.Infof("EmptyFolderCleaner: pending queue not processed yet (len=%d, oldest_age=%v, max_size=%d, max_age=%v)",
-				efc.cleanupQueue.Len(), efc.cleanupQueue.OldestAge(), efc.cleanupQueue.maxSize, efc.cleanupQueue.maxAge)
-		}
+	if efc.cleanupQueue.Len() == 0 {
 		return
 	}
 
-	glog.V(3).Infof("EmptyFolderCleaner: processing cleanup queue (len=%d, age=%v)",
+	glog.V(3).Infof("EmptyFolderCleaner: processing cleanup queue (len=%d, oldest_age=%v)",
 		efc.cleanupQueue.Len(), efc.cleanupQueue.OldestAge())
 
-	// Process all items that are ready
-	for efc.cleanupQueue.Len() > 0 {
+	// Only process items that have been queued longer than maxAge
+	for {
 		// Check if still enabled
 		if !efc.IsEnabled() {
 			return
 		}
 
-		// Pop the oldest item
-		folder, triggeredBy, ok := efc.cleanupQueue.Pop()
+		// Only pop items old enough — newer items stay in the queue
+		folder, triggeredBy, ok := efc.cleanupQueue.PopOlderThan(efc.cleanupQueue.maxAge)
 		if !ok {
 			break
 		}
@@ -322,9 +322,18 @@ func (efc *EmptyFolderCleaner) executeCleanup(folder string, triggeredBy string)
 	delete(efc.folderCounts, folder)
 	efc.mu.Unlock()
 
-	// Note: No need to recursively check parent folder here.
-	// The deletion of this folder will generate a metadata event,
-	// which will trigger OnDeleteEvent for the parent folder.
+	// After deleting this folder, immediately try to clean the parent.
+	// Relying solely on cascading metadata events would re-enter the full
+	// delay queue for each ancestor level, causing multi-minute cascading
+	// waits (e.g. 3 levels × 2m = 6m+).  Instead, walk up eagerly.
+	parentDir, _ := util.FullPath(folder).DirAndName()
+	if parentDir != "" && parentDir != folder &&
+		efc.bucketPath != "" && isUnderBucketPath(parentDir, efc.bucketPath) {
+		// Remove any pending queue entry for the parent so we don't
+		// double-process it later from a stale event.
+		efc.cleanupQueue.Remove(parentDir)
+		efc.executeCleanup(parentDir, triggeredBy)
+	}
 }
 
 // countItems counts items in a folder (up to maxCountCheck)

weed/filer/empty_folder_cleanup/empty_folder_cleaner_test.go

@@ -655,7 +655,7 @@ func TestEmptyFolderCleaner_queueFIFOOrder(t *testing.T) {
 	cleaner.Stop()
 }
 
-func TestEmptyFolderCleaner_processCleanupQueue_drainsAllOnceTriggered(t *testing.T) {
+func TestEmptyFolderCleaner_processCleanupQueue_onlyProcessesAgedItems(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
 	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
@@ -670,6 +670,7 @@ func TestEmptyFolderCleaner_processCleanupQueue_drainsAllOnceTriggered(t *testin
 		},
 	}
 
+	maxAge := 100 * time.Millisecond
 	cleaner := &EmptyFolderCleaner{
 		filer:          mock,
 		lockRing:       lockRing,
@@ -677,25 +678,27 @@ func TestEmptyFolderCleaner_processCleanupQueue_drainsAllOnceTriggered(t *testin
 		bucketPath:     "/buckets",
 		enabled:        true,
 		folderCounts:   make(map[string]*folderState),
-		cleanupQueue:   NewCleanupQueue(2, time.Hour),
+		cleanupQueue:   NewCleanupQueue(1000, maxAge),
 		maxCountCheck:  1000,
 		cacheExpiry:    time.Minute,
 		processorSleep: time.Second,
 		stopCh:         make(chan struct{}),
 	}
 
-	now := time.Now()
-	cleaner.cleanupQueue.Add("/buckets/test/folder1", "i1", now)
-	cleaner.cleanupQueue.Add("/buckets/test/folder2", "i2", now.Add(time.Millisecond))
-	cleaner.cleanupQueue.Add("/buckets/test/folder3", "i3", now.Add(2*time.Millisecond))
+	// Add old items (well past maxAge) and a fresh item
+	old := time.Now().Add(-time.Second)
+	cleaner.cleanupQueue.Add("/buckets/test/folder1", "i1", old)
+	cleaner.cleanupQueue.Add("/buckets/test/folder2", "i2", old.Add(time.Millisecond))
+	cleaner.cleanupQueue.Add("/buckets/test/folder3", "i3", time.Now()) // fresh, should NOT be processed
 
 	cleaner.processCleanupQueue()
 
-	if got := cleaner.cleanupQueue.Len(); got != 0 {
-		t.Fatalf("expected queue to be drained, got len=%d", got)
+	// Only the two old items should have been processed
+	if len(deleted) != 2 {
+		t.Fatalf("expected 2 deleted folders (aged items only), got %d: %v", len(deleted), deleted)
 	}
-	if len(deleted) != 3 {
-		t.Fatalf("expected 3 deleted folders, got %d", len(deleted))
+	if got := cleaner.cleanupQueue.Len(); got != 1 {
+		t.Fatalf("expected 1 item remaining in queue, got %d", got)
 	}
 }
 

weed/filer/filer.go

@@ -61,7 +61,8 @@ type Filer struct {
 	MaxFilenameLength   uint32
 	deletionQuit        chan struct{}
 	DeletionRetryQueue  *DeletionRetryQueue
-	EmptyFolderCleaner  *empty_folder_cleanup.EmptyFolderCleaner
+	EmptyFolderCleaner       *empty_folder_cleanup.EmptyFolderCleaner
+	EmptyFolderCleanupDelay time.Duration
 }
 
 func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer {
@@ -123,7 +124,7 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
 	glog.V(0).Infof("%s aggregate from peers %+v", self, snapshot)
 
 	// Initialize the empty folder cleaner using the same LockRing as Dlm for consistent hashing
-	f.EmptyFolderCleaner = empty_folder_cleanup.NewEmptyFolderCleaner(f, f.Dlm.LockRing, self, f.DirBucketsPath)
+	f.EmptyFolderCleaner = empty_folder_cleanup.NewEmptyFolderCleaner(f, f.Dlm.LockRing, self, f.DirBucketsPath, f.EmptyFolderCleanupDelay)
 
 	f.MetaAggregator = NewMetaAggregator(f, self, f.GrpcDialOption)
 	f.MasterClient.SetOnPeerUpdateFn(func(update *master_pb.ClusterNodeUpdate, startFrom time.Time) {

weed/filer/meta_aggregator.go

@@ -2,6 +2,7 @@ package filer
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"strings"
@@ -139,16 +140,37 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
 	if peerSignature != f.Signature {
 		if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil {
 			lastTsNs = prevTsNs
-			defer func(prevTsNs int64) {
-				if lastTsNs != prevTsNs && lastTsNs != lastPersistTime.UnixNano() {
-					if err := ma.updateOffset(f, peer, peerSignature, lastTsNs); err == nil {
-						glog.V(0).Infof("last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
-					} else {
-						glog.Errorf("failed to save last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
-					}
-				}
-			}(prevTsNs)
+		} else if errors.Is(err, ErrKvNotFound) {
+			// No stored offset — this is the first time connecting to this peer.
+			// Traverse the peer's full metadata tree so we get pre-existing data.
+			// Record time before traversal and subtract a safety margin to
+			// account for clock skew between this filer and the peer. Any
+			// duplicate events replayed during the overlap are harmless since
+			// Replay does upserts. We use wall-clock time (same domain as the
+			// metadata stream TsNs) rather than entry Mtime which is a
+			// different concept and can be set to arbitrary values.
+			preTraverseTime := time.Now()
+			glog.V(0).Infof("no previous offset for peer %s, starting full metadata sync", peer)
+			if traverseErr := ma.traversePeerMetadata(f, peer); traverseErr != nil {
+				return lastTsNs, fmt.Errorf("initial metadata sync from %s: %v", peer, traverseErr)
+			}
+			lastTsNs = preTraverseTime.Add(-time.Minute).UnixNano()
+			if err := ma.updateOffset(f, peer, peerSignature, lastTsNs); err != nil {
+				return lastTsNs, fmt.Errorf("save bootstrap offset for peer %s: %w", peer, err)
+			}
+			glog.V(0).Infof("completed full metadata sync from peer %s, will stream changes from %v", peer, time.Unix(0, lastTsNs))
+		} else {
+			return lastTsNs, fmt.Errorf("read offset for peer %s: %w", peer, err)
 		}
+		defer func(prevTsNs int64) {
+			if lastTsNs != prevTsNs && lastTsNs != lastPersistTime.UnixNano() {
+				if err := ma.updateOffset(f, peer, peerSignature, lastTsNs); err == nil {
+					glog.V(0).Infof("last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
+				} else {
+					glog.Errorf("failed to save last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
+				}
+			}
+		}(lastTsNs)
 
 		glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
 		var counter int64
@@ -279,6 +301,59 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
 	return lastTsNs, err
 }
 
+// traversePeerMetadata does a full BFS traversal of a peer filer's metadata
+// and inserts all entries into the local store. This is used when a filer
+// connects to a peer for the first time and needs to bootstrap pre-existing data.
+func (ma *MetaAggregator) traversePeerMetadata(f *Filer, peer pb.ServerAddress) error {
+	return pb.WithFilerClient(true, 0, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		stream, err := client.TraverseBfsMetadata(ctx, &filer_pb.TraverseBfsMetadataRequest{
+			Directory:        "/",
+			ExcludedPrefixes: []string{SystemLogDir},
+		})
+		if err != nil {
+			return fmt.Errorf("traverse bfs metadata: %w", err)
+		}
+		var count int64
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr == io.EOF {
+				break
+			}
+			if recvErr != nil {
+				return fmt.Errorf("traverse bfs metadata recv: %w", recvErr)
+			}
+			if resp.Entry == nil {
+				continue
+			}
+			fullpath := util.Join(resp.Directory, resp.Entry.Name)
+			entry := FromPbEntry(resp.Directory, resp.Entry)
+			if insertErr := f.Store.InsertEntry(context.Background(), entry); insertErr != nil {
+				// Entry may already exist (root dir, or partial previous bootstrap).
+				existing, findErr := f.Store.FindEntry(context.Background(), entry.FullPath)
+				if findErr != nil {
+					return fmt.Errorf("insert entry %s: %w", fullpath, insertErr)
+				}
+				// Only overwrite if the peer's entry is newer.
+				if entry.Attr.Mtime.After(existing.Attr.Mtime) {
+					if updateErr := f.Store.UpdateEntry(context.Background(), entry); updateErr != nil {
+						return fmt.Errorf("update entry %s: %w", fullpath, updateErr)
+					}
+				} else {
+					glog.V(1).Infof("skip older peer entry %s (peer mtime %v <= local mtime %v)", fullpath, entry.Attr.Mtime, existing.Attr.Mtime)
+				}
+			}
+			count++
+			if count%10000 == 0 {
+				glog.V(0).Infof("synced %d entries from peer %s", count, peer)
+			}
+		}
+		glog.V(0).Infof("synced %d entries total from peer %s", count, peer)
+		return nil
+	})
+}
+
 func (ma *MetaAggregator) readFilerStoreSignature(peer pb.ServerAddress) (sig int32, err error) {
 	err = pb.WithFilerClient(false, 0, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
@@ -308,7 +383,7 @@ func (ma *MetaAggregator) readOffset(f *Filer, peer pb.ServerAddress, peerSignat
 	value, err := f.Store.KvGet(context.Background(), key)
 
 	if err != nil {
-		return 0, fmt.Errorf("readOffset %s : %v", peer, err)
+		return 0, fmt.Errorf("readOffset %s : %w", peer, err)
 	}
 
 	lastTsNs = int64(util.BytesToUint64(value))

weed/filer/redis2/universal_redis_store.go

@@ -197,15 +197,18 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dir
 	}
 
 	// fetch entry meta
+	var entry *filer.Entry
 	for _, fileName := range members {
 		path := util.NewFullPath(string(dirPath), fileName)
-		entry, err := store.FindEntry(ctx, path)
+		entry, err = store.FindEntry(ctx, path)
 		lastFileName = fileName
 		if err != nil {
 			glog.V(0).InfofCtx(ctx, "list %s : %v", path, err)
 			if err == filer_pb.ErrNotFound {
+				err = nil
 				continue
 			}
+			break
 		} else {
 			if entry.TtlSec > 0 {
 				if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {

weed/iam/policy/aws_iam_compliance_test.go

@@ -1,8 +1,10 @@
 package policy
 
 import (
+	"strings"
 	"testing"
 
+	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -110,6 +112,92 @@ func TestAWSIAMMatch(t *testing.T) {
 	}
 }
 
+func TestMatchesActionsMultipartExpansion(t *testing.T) {
+	engine := &PolicyEngine{initialized: true}
+	evalCtx := &EvaluationContext{}
+
+	tests := []struct {
+		name            string
+		actions         []string
+		requestedAction string
+		expected        bool
+	}{
+		{
+			name:            "PutObject directly matches PutObject",
+			actions:         []string{"s3:PutObject"},
+			requestedAction: "s3:PutObject",
+			expected:        true,
+		},
+		{
+			name:            "PutObject implicitly allows CreateMultipartUpload",
+			actions:         []string{"s3:PutObject"},
+			requestedAction: "s3:CreateMultipartUpload",
+			expected:        true,
+		},
+		{
+			name:            "PutObject implicitly allows UploadPart",
+			actions:         []string{"s3:PutObject"},
+			requestedAction: "s3:UploadPart",
+			expected:        true,
+		},
+		{
+			name:            "PutObject implicitly allows CompleteMultipartUpload",
+			actions:         []string{"s3:PutObject"},
+			requestedAction: "s3:CompleteMultipartUpload",
+			expected:        true,
+		},
+		{
+			name:            "PutObject implicitly allows AbortMultipartUpload",
+			actions:         []string{"s3:PutObject"},
+			requestedAction: "s3:AbortMultipartUpload",
+			expected:        true,
+		},
+		{
+			name:            "PutObject implicitly allows ListMultipartUploadParts",
+			actions:         []string{"s3:PutObject"},
+			requestedAction: "s3:ListMultipartUploadParts",
+			expected:        true,
+		},
+		{
+			name:            "PutObject implicitly allows ListBucketMultipartUploads",
+			actions:         []string{"s3:PutObject"},
+			requestedAction: "s3:ListBucketMultipartUploads",
+			expected:        true,
+		},
+		{
+			name:            "PutObject does not allow GetObject",
+			actions:         []string{"s3:PutObject"},
+			requestedAction: "s3:GetObject",
+			expected:        false,
+		},
+		{
+			name:            "GetObject does not allow CreateMultipartUpload",
+			actions:         []string{"s3:GetObject"},
+			requestedAction: "s3:CreateMultipartUpload",
+			expected:        false,
+		},
+		{
+			name:            "wildcard s3:Put* implicitly allows multipart via PutObject match",
+			actions:         []string{"s3:Put*"},
+			requestedAction: "s3:CreateMultipartUpload",
+			expected:        true,
+		},
+		{
+			name:            "case-insensitive multipart action lookup",
+			actions:         []string{"s3:PutObject"},
+			requestedAction: "S3:CREATEMULTIPARTUPLOAD",
+			expected:        true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := engine.matchesActions(tt.actions, tt.requestedAction, evalCtx)
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}
+
 func TestExpandPolicyVariables(t *testing.T) {
 	evalCtx := &EvaluationContext{
 		RequestContext: map[string]interface{}{
@@ -200,7 +288,7 @@ func TestAWSWildcardMatch(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			result := AwsWildcardMatch(tt.pattern, tt.value)
+			result := wildcard.MatchesWildcard(strings.ToLower(tt.pattern), strings.ToLower(tt.value))
 			assert.Equal(t, tt.expected, result, "AWS wildcard match should match expected")
 		})
 	}

weed/iam/policy/policy_engine.go

@@ -5,12 +5,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
-	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
 )
 
 // Effect represents the policy evaluation result
@@ -21,10 +21,7 @@ const (
 	EffectDeny  Effect = "Deny"
 )
 
-// Package-level regex cache for performance optimization
 var (
-	regexCache            = make(map[string]*regexp.Regexp)
-	regexCacheMu          sync.RWMutex
 	policyVariablePattern = regexp.MustCompile(`\$\{([^}]+)\}`)
 	safePolicyVariables   = map[string]bool{
 		// AWS standard identity variables
@@ -600,12 +597,31 @@ func (e *PolicyEngine) statementMatches(statement *Statement, evalCtx *Evaluatio
 	return true
 }
 
-// matchesActions checks if any action in the list matches the requested action
+// multipartActionSet contains lowercased S3 multipart upload actions that are
+// implicitly granted when s3:PutObject is allowed, since multipart upload is an
+// implementation detail of putting objects. Keys are lowercased for
+// case-insensitive lookup (AWS IAM actions are case-insensitive).
+var multipartActionSet = map[string]bool{
+	"s3:createmultipartupload":      true,
+	"s3:uploadpart":                 true,
+	"s3:completemultipartupload":    true,
+	"s3:abortmultipartupload":       true,
+	"s3:listmultipartuploadparts":   true,
+	"s3:listbucketmultipartuploads": true,
+}
+
+// matchesActions checks if any action in the list matches the requested action.
+// It also implicitly grants multipart upload actions when s3:PutObject is allowed,
+// mirroring the behavior in the S3 API policy engine (see PR #8445).
 func (e *PolicyEngine) matchesActions(actions []string, requestedAction string, evalCtx *EvaluationContext) bool {
+	isMultipart := multipartActionSet[strings.ToLower(requestedAction)]
 	for _, action := range actions {
 		if awsIAMMatch(action, requestedAction, evalCtx) {
 			return true
 		}
+		if isMultipart && awsIAMMatch(action, "s3:PutObject", evalCtx) {
+			return true
+		}
 	}
 	return false
 }
@@ -1045,8 +1061,7 @@ func (e *PolicyEngine) EvaluateStringCondition(block map[string]interface{}, eva
 				for _, expected := range expectedStrings {
 					expandedExpected := expandPolicyVariables(expected, evalCtx)
 					if useWildcard {
-						// Use filepath.Match for case-sensitive wildcard matching, as required by StringLike
-						if matched, _ := filepath.Match(expandedExpected, contextValue); matched {
+						if wildcard.MatchesWildcard(expandedExpected, contextValue) {
 							contextValueMatchedSet = true
 							break
 						}
@@ -1087,13 +1102,11 @@ func (e *PolicyEngine) EvaluateStringCondition(block map[string]interface{}, eva
 				for _, expected := range expectedStrings {
 					expandedExpected := expandPolicyVariables(expected, evalCtx)
 					if useWildcard {
-						// Use filepath.Match for case-sensitive wildcard matching, as required by StringLike
-						if matched, _ := filepath.Match(expandedExpected, contextValue); matched {
+						if wildcard.MatchesWildcard(expandedExpected, contextValue) {
 							contextValueMatchedSet = true
 							break
 						}
 					} else {
-						// For StringEquals/StringNotEquals, also support policy variables but be case-sensitive
 						if expandedExpected == contextValue {
 							contextValueMatchedSet = true
 							break
@@ -1210,7 +1223,7 @@ func awsIAMMatch(pattern, value string, evalCtx *EvaluationContext) bool {
 
 	// Step 4: Handle AWS-style wildcards (case-insensitive)
 	if strings.Contains(expandedPattern, "*") || strings.Contains(expandedPattern, "?") {
-		return AwsWildcardMatch(expandedPattern, value)
+		return wildcard.MatchesWildcard(strings.ToLower(expandedPattern), strings.ToLower(value))
 	}
 
 	return false
@@ -1246,44 +1259,6 @@ func expandPolicyVariables(pattern string, evalCtx *EvaluationContext) string {
 	return result
 }
 
-// AwsWildcardMatch performs case-insensitive wildcard matching like AWS IAM
-func AwsWildcardMatch(pattern, value string) bool {
-	// Create regex pattern key for caching
-	// First escape all regex metacharacters, then replace wildcards
-	regexPattern := regexp.QuoteMeta(pattern)
-	regexPattern = strings.ReplaceAll(regexPattern, "\\*", ".*")
-	regexPattern = strings.ReplaceAll(regexPattern, "\\?", ".")
-	regexPattern = "^" + regexPattern + "$"
-	regexKey := "(?i)" + regexPattern
-
-	// Try to get compiled regex from cache
-	regexCacheMu.RLock()
-	regex, found := regexCache[regexKey]
-	regexCacheMu.RUnlock()
-
-	if !found {
-		// Compile and cache the regex
-		compiledRegex, err := regexp.Compile(regexKey)
-		if err != nil {
-			// Fallback to simple case-insensitive comparison if regex fails
-			return strings.EqualFold(pattern, value)
-		}
-
-		// Store in cache with write lock
-		regexCacheMu.Lock()
-		// Double-check in case another goroutine added it
-		if existingRegex, exists := regexCache[regexKey]; exists {
-			regex = existingRegex
-		} else {
-			regexCache[regexKey] = compiledRegex
-			regex = compiledRegex
-		}
-		regexCacheMu.Unlock()
-	}
-
-	return regex.MatchString(value)
-}
-
 // evaluateStringConditionIgnoreCase evaluates string conditions with case insensitivity
 func (e *PolicyEngine) evaluateStringConditionIgnoreCase(block map[string]interface{}, evalCtx *EvaluationContext, shouldMatch bool, useWildcard bool, forAllValues bool) bool {
 	for key, expectedValues := range block {
@@ -1328,7 +1303,7 @@ func (e *PolicyEngine) evaluateStringConditionIgnoreCase(block map[string]interf
 				case string:
 					expandedPattern := expandPolicyVariables(v, evalCtx)
 					if useWildcard {
-						if AwsWildcardMatch(expandedPattern, ctxStr) {
+						if wildcard.MatchesWildcard(strings.ToLower(expandedPattern), strings.ToLower(ctxStr)) {
 							itemMatchedSet = true
 						}
 					} else {
@@ -1350,7 +1325,7 @@ func (e *PolicyEngine) evaluateStringConditionIgnoreCase(block map[string]interf
 					for _, valStr := range slice {
 						expandedPattern := expandPolicyVariables(valStr, evalCtx)
 						if useWildcard {
-							if AwsWildcardMatch(expandedPattern, ctxStr) {
+							if wildcard.MatchesWildcard(strings.ToLower(expandedPattern), strings.ToLower(ctxStr)) {
 								itemMatchedSet = true
 								break
 							}
@@ -1390,7 +1365,7 @@ func (e *PolicyEngine) evaluateStringConditionIgnoreCase(block map[string]interf
 				case string:
 					expandedPattern := expandPolicyVariables(v, evalCtx)
 					if useWildcard {
-						if AwsWildcardMatch(expandedPattern, ctxStr) {
+						if wildcard.MatchesWildcard(strings.ToLower(expandedPattern), strings.ToLower(ctxStr)) {
 							itemMatchedSet = true
 						}
 					} else {
@@ -1412,7 +1387,7 @@ func (e *PolicyEngine) evaluateStringConditionIgnoreCase(block map[string]interf
 					for _, valStr := range slice {
 						expandedPattern := expandPolicyVariables(valStr, evalCtx)
 						if useWildcard {
-							if AwsWildcardMatch(expandedPattern, ctxStr) {
+							if wildcard.MatchesWildcard(strings.ToLower(expandedPattern), strings.ToLower(ctxStr)) {
 								itemMatchedSet = true
 								break
 							}

weed/iam/providers/provider.go

@@ -4,10 +4,11 @@ import (
 	"context"
 	"fmt"
 	"net/mail"
+	"strings"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/iam/policy"
+	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
 )
 
 // IdentityProvider defines the interface for external identity providers
@@ -225,7 +226,7 @@ func (r *MappingRule) Matches(claims *TokenClaims) bool {
 // matchValue checks if a value matches the rule value (with wildcard support)
 // Uses AWS IAM-compliant case-insensitive wildcard matching for consistency with policy engine
 func (r *MappingRule) matchValue(value string) bool {
-	matched := policy.AwsWildcardMatch(r.Value, value)
+	matched := wildcard.MatchesWildcard(strings.ToLower(r.Value), strings.ToLower(value))
 	glog.V(3).Infof("AWS IAM pattern match result: '%s' matches '%s' = %t", value, r.Value, matched)
 	return matched
 }

weed/mq/kafka/consumer/rebalance_timeout.go

@@ -116,11 +116,9 @@ func (rtm *RebalanceTimeoutManager) IsRebalanceStuck(group *ConsumerGroup, maxRe
 	return time.Since(group.LastActivity) > maxRebalanceDuration
 }
 
-// ForceCompleteRebalance forces completion of a stuck rebalance
+// ForceCompleteRebalance forces completion of a stuck rebalance.
+// IMPORTANT: The caller must already hold group.Mu.Lock().
 func (rtm *RebalanceTimeoutManager) ForceCompleteRebalance(group *ConsumerGroup) {
-	group.Mu.Lock()
-	defer group.Mu.Unlock()
-
 	// If stuck in preparing rebalance, move to completing
 	if group.State == GroupStatePreparingRebalance {
 		group.State = GroupStateCompletingRebalance

weed/mq/kafka/consumer/rebalance_timeout_test.go

@@ -185,20 +185,19 @@ func TestRebalanceTimeoutManager_ForceCompleteRebalance(t *testing.T) {
 		State: MemberStatePending,
 	}
 	group.Members["member1"] = member
-	group.Mu.Unlock()
 
+	// ForceCompleteRebalance expects the caller to hold group.Mu.Lock()
 	rtm.ForceCompleteRebalance(group)
 
-	group.Mu.RLock()
 	if group.State != GroupStateCompletingRebalance {
 		t.Errorf("Expected group state to be CompletingRebalance, got %s", group.State.String())
 	}
-	group.Mu.RUnlock()
+	group.Mu.Unlock()
 
 	// Test forcing completion from CompletingRebalance
+	group.Mu.Lock()
 	rtm.ForceCompleteRebalance(group)
 
-	group.Mu.RLock()
 	if group.State != GroupStateStable {
 		t.Errorf("Expected group state to be Stable, got %s", group.State.String())
 	}
@@ -206,7 +205,7 @@ func TestRebalanceTimeoutManager_ForceCompleteRebalance(t *testing.T) {
 	if member.State != MemberStateStable {
 		t.Errorf("Expected member state to be Stable, got %s", member.State.String())
 	}
-	group.Mu.RUnlock()
+	group.Mu.Unlock()
 }
 
 func TestRebalanceTimeoutManager_GetRebalanceStatus(t *testing.T) {

weed/mq/kafka/protocol/fetch.go

@@ -55,7 +55,10 @@ func (h *Handler) handleFetch(ctx context.Context, correlationID uint32, apiVers
 			for _, partition := range topic.Partitions {
 				hwm, err := h.seaweedMQHandler.GetLatestOffset(topic.Name, partition.PartitionID)
 				if err != nil {
-					continue
+					// HWM lookup failed (e.g. partition deactivated between consumer
+					// sessions). Assume data may be available rather than blocking in
+					// the long-poll loop — the actual fetch will determine the truth.
+					return true
 				}
 				// Normalize fetch offset
 				effectiveOffset := partition.FetchOffset

weed/mq/kafka/protocol/fetch_partition_reader.go

@@ -2,6 +2,7 @@ package protocol
 
 import (
 	"context"
+	"math"
 	"sync"
 	"time"
 
@@ -136,13 +137,19 @@ func (pr *partitionReader) serveFetchRequest(ctx context.Context, req *partition
 	}()
 
 	// Get high water mark
+	hwmUnknown := false
 	hwm, hwmErr := pr.handler.seaweedMQHandler.GetLatestOffset(pr.topicName, pr.partitionID)
 	if hwmErr != nil {
-		glog.Errorf("[%s] CRITICAL: Failed to get HWM for %s[%d]: %v",
+		// HWM lookup can fail when the partition has been deactivated between consumer
+		// sessions. Proceed with the fetch anyway — the broker will return the correct
+		// data (or empty) based on its own state. Use math.MaxInt64 as sentinel so
+		// FetchMultipleBatches doesn't artificially cap recordsAvailable, and we
+		// don't hit the early-return below. The actual HWM will be derived from
+		// the fetch result (newOffset) after the read.
+		glog.Warningf("[%s] HWM lookup failed for %s[%d]: %v — will attempt fetch anyway",
 			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, hwmErr)
-		result.recordBatch = []byte{}
-		result.highWaterMark = 0
-		return
+		hwm = math.MaxInt64
+		hwmUnknown = true
 	}
 	result.highWaterMark = hwm
 
@@ -170,10 +177,20 @@ func (pr *partitionReader) serveFetchRequest(ctx context.Context, req *partition
 	// Fetch on-demand - no pre-fetching to avoid overwhelming the broker
 	recordBatch, newOffset := pr.readRecords(ctx, req.requestedOffset, req.maxBytes, req.maxWaitMs, hwm)
 
+	// When HWM was unknown, derive a reasonable value from the fetch result
+	// so the client sees a meaningful high water mark instead of MaxInt64.
+	if hwmUnknown {
+		if newOffset > req.requestedOffset {
+			result.highWaterMark = newOffset // best estimate: end of what we read
+		} else {
+			result.highWaterMark = req.requestedOffset // no data found
+		}
+	}
+
 	// Log what we got back - DETAILED for diagnostics
 	if len(recordBatch) == 0 {
 		glog.V(2).Infof("[%s] FETCH %s[%d]: readRecords returned EMPTY (offset=%d, hwm=%d)",
-			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, req.requestedOffset, hwm)
+			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, req.requestedOffset, result.highWaterMark)
 		result.recordBatch = []byte{}
 	} else {
 		result.recordBatch = recordBatch
@@ -228,15 +245,40 @@ func (pr *partitionReader) readRecords(ctx context.Context, fromOffset int64, ma
 		return fetchResult.RecordBatches, fetchResult.NextOffset
 	}
 
-	// Multi-batch failed - try single batch WITHOUT the timeout constraint
-	// to ensure we get at least some data even if multi-batch timed out
+	// Multi-batch failed - try single batch with a fresh timeout
 	glog.Warningf("[%s] Multi-batch fetch failed for %s[%d] offset=%d after %v, falling back to single-batch (err: %v)",
 		pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, fromOffset, fetchDuration, err)
 
-	// Use original context for fallback, NOT the timed-out fetchCtx
-	// This ensures the fallback has a fresh chance to fetch data
+	// Compute the remaining time budget for the fallback. If the parent
+	// context carries a deadline, honour it; otherwise derive remaining
+	// from maxWaitMs minus elapsed time. This prevents the fallback from
+	// restarting the full budget after the multi-batch fetch already
+	// consumed part of it.
+	var remaining time.Duration
+	if deadline, ok := ctx.Deadline(); ok {
+		remaining = time.Until(deadline)
+	} else {
+		remaining = time.Duration(maxWaitMs)*time.Millisecond - time.Since(fetchStartTime)
+	}
+	if remaining <= 0 {
+		// Budget exhausted — skip the fallback entirely.
+		glog.V(2).Infof("[%s] No remaining budget for fallback on %s[%d] (maxWait=%dms, elapsed=%v)",
+			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, maxWaitMs, time.Since(fetchStartTime))
+		return []byte{}, fromOffset
+	}
+	// Clamp: floor of 2s so disk reads via gRPC have a realistic chance,
+	// but never exceed 10s to bound data-plane blocking.
+	fallbackTimeout := remaining
+	if fallbackTimeout < 2*time.Second {
+		fallbackTimeout = 2 * time.Second
+	}
+	if fallbackTimeout > 10*time.Second {
+		fallbackTimeout = 10 * time.Second
+	}
+	fallbackCtx, fallbackCancel := context.WithTimeout(ctx, fallbackTimeout)
+	defer fallbackCancel()
 	fallbackStartTime := time.Now()
-	smqRecords, err := pr.handler.seaweedMQHandler.GetStoredRecords(ctx, pr.topicName, pr.partitionID, fromOffset, 10)
+	smqRecords, err := pr.handler.seaweedMQHandler.GetStoredRecords(fallbackCtx, pr.topicName, pr.partitionID, fromOffset, 10)
 	fallbackDuration := time.Since(fallbackStartTime)
 
 	if fallbackDuration > 2*time.Second {

weed/operation/upload_content.go

@@ -135,6 +135,15 @@ func newUploader(httpClient HTTPClient) *Uploader {
 	}
 }
 
+// NewUploaderWithHttpClient creates an Uploader that uses the provided HTTP
+// client instead of the global one. This is used by filer.sync to upload to
+// remote clusters that use different TLS certificates.
+func NewUploaderWithHttpClient(httpClient HTTPClient) *Uploader {
+	return &Uploader{
+		httpClient: httpClient,
+	}
+}
+
 func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, host string, auth security.EncodedJwt, err error), uploadOption *UploadOption, genFileUrlFn func(host, fileId string) string, data []byte) (fileId string, uploadResult *UploadResult, err error) {
 	doUploadFunc := func() error {
 		var host string

weed/replication/repl_util/replication_util.go

@@ -2,14 +2,73 @@ package repl_util
 
 import (
 	"context"
+	"io"
 
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/replication/source"
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
-func CopyFromChunkViews(chunkViews *filer.IntervalList[*filer.ChunkView], filerSource *source.FilerSource, writeFunc func(data []byte) error) error {
+// CopyFromChunkViews copies chunk data with optional SSE decryption.
+// If entry has SSE-encrypted chunks, data is decrypted before writing.
+func CopyFromChunkViews(chunkViews *filer.IntervalList[*filer.ChunkView], filerSource *source.FilerSource, writeFunc func(data []byte) error, entry *filer_pb.Entry) error {
+	if entry != nil {
+		sseType, err := detectSSEType(entry)
+		if err != nil {
+			return err
+		}
+		if sseType != filer_pb.SSEType_NONE {
+			return copyWithDecryption(filerSource, entry, writeFunc)
+		}
+	}
+	return copyChunkViews(chunkViews, filerSource, writeFunc)
+}
+
+func copyWithDecryption(filerSource *source.FilerSource, entry *filer_pb.Entry, writeFunc func(data []byte) error) error {
+	reader := filer.NewFileReader(filerSource, entry)
+	decrypted, err := MaybeDecryptReader(reader, entry)
+	if err != nil {
+		CloseReader(reader)
+		return err
+	}
+	defer CloseMaybeDecryptedReader(reader, decrypted)
+	buf := make([]byte, 128*1024)
+	for {
+		n, readErr := decrypted.Read(buf)
+		if n > 0 {
+			if writeErr := writeFunc(buf[:n]); writeErr != nil {
+				return writeErr
+			}
+		}
+		if readErr == io.EOF {
+			return nil
+		}
+		if readErr != nil {
+			return readErr
+		}
+	}
+}
+
+// CloseReader closes r if it implements io.Closer.
+func CloseReader(r io.Reader) {
+	if closer, ok := r.(io.Closer); ok {
+		closer.Close()
+	}
+}
+
+// CloseMaybeDecryptedReader closes the decrypted reader if it implements io.Closer,
+// otherwise falls back to closing the original reader.
+func CloseMaybeDecryptedReader(original, decrypted io.Reader) {
+	if closer, ok := decrypted.(io.Closer); ok {
+		closer.Close()
+	} else {
+		CloseReader(original)
+	}
+}
+
+func copyChunkViews(chunkViews *filer.IntervalList[*filer.ChunkView], filerSource *source.FilerSource, writeFunc func(data []byte) error) error {
 
 	for x := chunkViews.Front(); x != nil; x = x.Next {
 		chunk := x.Value

weed/replication/repl_util/sse_init.go

@@ -0,0 +1,55 @@
+package repl_util
+
+import (
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/kms"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var (
+	sseInitMu          sync.Mutex
+	sseInitialized     bool
+)
+
+// InitializeSSEForReplication sets up SSE-S3 and SSE-KMS decryption so that
+// replication sinks can transparently decrypt encrypted objects.
+// SSE-S3 is initialized from the filer (KEK stored on filer).
+// SSE-KMS is initialized from Viper config (security.toml [kms] section or
+// WEED_KMS_* environment variables).
+// SSE-C cannot be decrypted (customer key not available) and will error at
+// decryption time.
+//
+// Safe to call multiple times; only the first successful initialization takes
+// effect. Failed attempts do not prevent future retries.
+func InitializeSSEForReplication(filerSource filer_pb.FilerClient) error {
+	sseInitMu.Lock()
+	defer sseInitMu.Unlock()
+	if sseInitialized {
+		return nil
+	}
+
+	// Initialize SSE-S3 key manager from filer
+	if err := s3api.GetSSES3KeyManager().InitializeWithFiler(filerSource); err != nil {
+		return err
+	}
+
+	// Attempt KMS initialization from Viper config.
+	// KMS configuration is typically in the S3 config file which the
+	// replication commands don't load directly. Support loading from
+	// security.toml [kms] section or WEED_KMS_* environment variables.
+	loader := kms.NewConfigLoader(util.GetViper())
+	if err := loader.LoadConfigurations(); err != nil {
+		glog.Warningf("KMS initialization from config failed: %v (SSE-KMS decryption will not be available)", err)
+	} else if err := loader.ValidateConfiguration(); err != nil {
+		glog.Warningf("KMS configuration validation failed: %v (SSE-KMS decryption will not be available)", err)
+	} else {
+		glog.V(0).Infof("KMS initialized for replication")
+	}
+
+	sseInitialized = true
+	return nil
+}

weed/replication/repl_util/sse_reader.go

@@ -0,0 +1,150 @@
+package repl_util
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+)
+
+// MaybeDecryptReader wraps reader with SSE decryption if the entry has encrypted chunks.
+// Returns the original reader unchanged if no SSE encryption is detected.
+func MaybeDecryptReader(reader io.Reader, entry *filer_pb.Entry) (io.Reader, error) {
+	if entry == nil {
+		return reader, nil
+	}
+
+	sseType, err := detectSSEType(entry)
+	if err != nil {
+		return nil, err
+	}
+	if sseType == filer_pb.SSEType_NONE {
+		return reader, nil
+	}
+
+	switch sseType {
+	case filer_pb.SSEType_SSE_S3:
+		return decryptSSES3(reader, entry)
+	case filer_pb.SSEType_SSE_KMS:
+		return decryptSSEKMS(reader, entry)
+	case filer_pb.SSEType_SSE_C:
+		return nil, fmt.Errorf("SSE-C encrypted object cannot be decrypted during replication (customer key not available)")
+	}
+
+	return nil, fmt.Errorf("unsupported SSE type: %v", sseType)
+}
+
+// MaybeDecryptContent decrypts inline entry content if SSE-encrypted.
+// Returns the original content unchanged if no SSE encryption is detected.
+func MaybeDecryptContent(content []byte, entry *filer_pb.Entry) ([]byte, error) {
+	if entry == nil || len(content) == 0 {
+		return content, nil
+	}
+
+	sseType, err := detectSSEType(entry)
+	if err != nil {
+		return nil, err
+	}
+	if sseType == filer_pb.SSEType_NONE {
+		return content, nil
+	}
+
+	reader := bytes.NewReader(content)
+	decrypted, err := MaybeDecryptReader(reader, entry)
+	if err != nil {
+		return nil, err
+	}
+	return io.ReadAll(decrypted)
+}
+
+func detectSSEType(entry *filer_pb.Entry) (filer_pb.SSEType, error) {
+	// Check chunk metadata first
+	var detected filer_pb.SSEType
+	for _, chunk := range entry.GetChunks() {
+		if chunk.SseType != filer_pb.SSEType_NONE {
+			if detected == filer_pb.SSEType_NONE {
+				detected = chunk.SseType
+			} else if chunk.SseType != detected {
+				return filer_pb.SSEType_NONE, fmt.Errorf("mixed SSE types in chunks: %v and %v", detected, chunk.SseType)
+			}
+		}
+	}
+	if detected != filer_pb.SSEType_NONE {
+		return detected, nil
+	}
+
+	// Fall back to extended metadata for inline objects (no chunks)
+	if entry.Extended != nil {
+		hasS3 := len(entry.Extended[s3_constants.SeaweedFSSSES3Key]) > 0
+		hasKMS := len(entry.Extended[s3_constants.SeaweedFSSSEKMSKey]) > 0
+		hasC := len(entry.Extended[s3_constants.SeaweedFSSSEIV]) > 0
+		count := 0
+		if hasS3 {
+			count++
+		}
+		if hasKMS {
+			count++
+		}
+		if hasC {
+			count++
+		}
+		if count > 1 {
+			return filer_pb.SSEType_NONE, fmt.Errorf("conflicting SSE metadata in entry: multiple SSE key types present")
+		}
+		if hasS3 {
+			return filer_pb.SSEType_SSE_S3, nil
+		}
+		if hasKMS {
+			return filer_pb.SSEType_SSE_KMS, nil
+		}
+		if hasC {
+			return filer_pb.SSEType_SSE_C, nil
+		}
+	}
+	return filer_pb.SSEType_NONE, nil
+}
+
+func decryptSSES3(reader io.Reader, entry *filer_pb.Entry) (io.Reader, error) {
+	if entry.Extended == nil {
+		return nil, fmt.Errorf("SSE-S3 encrypted entry has no extended metadata")
+	}
+
+	keyData := entry.Extended[s3_constants.SeaweedFSSSES3Key]
+	if len(keyData) == 0 {
+		return nil, fmt.Errorf("SSE-S3 key metadata not found in entry")
+	}
+
+	keyManager := s3api.GetSSES3KeyManager()
+	sseS3Key, err := s3api.DeserializeSSES3Metadata(keyData, keyManager)
+	if err != nil {
+		return nil, fmt.Errorf("deserialize SSE-S3 metadata: %w", err)
+	}
+
+	iv, err := s3api.GetSSES3IV(entry, sseS3Key, keyManager)
+	if err != nil {
+		return nil, fmt.Errorf("get SSE-S3 IV: %w", err)
+	}
+
+	return s3api.CreateSSES3DecryptedReader(reader, sseS3Key, iv)
+}
+
+func decryptSSEKMS(reader io.Reader, entry *filer_pb.Entry) (io.Reader, error) {
+	if entry.Extended == nil {
+		return nil, fmt.Errorf("SSE-KMS encrypted entry has no extended metadata")
+	}
+
+	kmsMetadata := entry.Extended[s3_constants.SeaweedFSSSEKMSKey]
+	if len(kmsMetadata) == 0 {
+		return nil, fmt.Errorf("SSE-KMS key metadata not found in entry")
+	}
+
+	sseKMSKey, err := s3api.DeserializeSSEKMSMetadata(kmsMetadata)
+	if err != nil {
+		return nil, fmt.Errorf("deserialize SSE-KMS metadata: %w", err)
+	}
+
+	return s3api.CreateSSEKMSDecryptedReader(reader, sseKMSKey)
+}

weed/replication/repl_util/sse_reader_test.go

@@ -0,0 +1,534 @@
+package repl_util
+
+import (
+	"bytes"
+	"context"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/kms"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestDetectSSEType(t *testing.T) {
+	tests := []struct {
+		name      string
+		entry     *filer_pb.Entry
+		wantType  filer_pb.SSEType
+		wantError bool
+	}{
+		{
+			name:     "no chunks no extended",
+			entry:    &filer_pb.Entry{},
+			wantType: filer_pb.SSEType_NONE,
+		},
+		{
+			name: "plaintext chunks",
+			entry: &filer_pb.Entry{
+				Chunks: []*filer_pb.FileChunk{
+					{SseType: filer_pb.SSEType_NONE},
+					{SseType: filer_pb.SSEType_NONE},
+				},
+			},
+			wantType: filer_pb.SSEType_NONE,
+		},
+		{
+			name: "uniform SSE-S3 chunks",
+			entry: &filer_pb.Entry{
+				Chunks: []*filer_pb.FileChunk{
+					{SseType: filer_pb.SSEType_SSE_S3},
+					{SseType: filer_pb.SSEType_SSE_S3},
+				},
+			},
+			wantType: filer_pb.SSEType_SSE_S3,
+		},
+		{
+			name: "uniform SSE-KMS chunks",
+			entry: &filer_pb.Entry{
+				Chunks: []*filer_pb.FileChunk{
+					{SseType: filer_pb.SSEType_SSE_KMS},
+				},
+			},
+			wantType: filer_pb.SSEType_SSE_KMS,
+		},
+		{
+			name: "mixed chunk SSE types",
+			entry: &filer_pb.Entry{
+				Chunks: []*filer_pb.FileChunk{
+					{SseType: filer_pb.SSEType_SSE_S3},
+					{SseType: filer_pb.SSEType_SSE_KMS},
+				},
+			},
+			wantError: true,
+		},
+		{
+			name: "inline SSE-S3 via extended",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.SeaweedFSSSES3Key: {0x01},
+				},
+			},
+			wantType: filer_pb.SSEType_SSE_S3,
+		},
+		{
+			name: "inline SSE-KMS via extended",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.SeaweedFSSSEKMSKey: {0x01},
+				},
+			},
+			wantType: filer_pb.SSEType_SSE_KMS,
+		},
+		{
+			name: "inline SSE-C via extended",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.SeaweedFSSSEIV: {0x01},
+				},
+			},
+			wantType: filer_pb.SSEType_SSE_C,
+		},
+		{
+			name: "conflicting extended metadata",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.SeaweedFSSSES3Key:  {0x01},
+					s3_constants.SeaweedFSSSEKMSKey: {0x02},
+				},
+			},
+			wantError: true,
+		},
+		{
+			name: "chunks take precedence over extended",
+			entry: &filer_pb.Entry{
+				Chunks: []*filer_pb.FileChunk{
+					{SseType: filer_pb.SSEType_SSE_S3},
+				},
+				Extended: map[string][]byte{
+					s3_constants.SeaweedFSSSEKMSKey: {0x01},
+				},
+			},
+			wantType: filer_pb.SSEType_SSE_S3,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := detectSSEType(tt.entry)
+			if tt.wantError {
+				if err == nil {
+					t.Fatalf("expected error, got type %v", got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if got != tt.wantType {
+				t.Errorf("got %v, want %v", got, tt.wantType)
+			}
+		})
+	}
+}
+
+func TestMaybeDecryptReader_Plaintext(t *testing.T) {
+	content := []byte("hello world")
+	entry := &filer_pb.Entry{}
+	reader := bytes.NewReader(content)
+
+	got, err := MaybeDecryptReader(reader, entry)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	result, err := io.ReadAll(got)
+	if err != nil {
+		t.Fatalf("ReadAll error: %v", err)
+	}
+	if !bytes.Equal(result, content) {
+		t.Errorf("got %q, want %q", result, content)
+	}
+}
+
+func TestMaybeDecryptReader_NilEntry(t *testing.T) {
+	content := []byte("hello")
+	reader := bytes.NewReader(content)
+
+	got, err := MaybeDecryptReader(reader, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	result, err := io.ReadAll(got)
+	if err != nil {
+		t.Fatalf("ReadAll error: %v", err)
+	}
+	if !bytes.Equal(result, content) {
+		t.Errorf("got %q, want %q", result, content)
+	}
+}
+
+func TestMaybeDecryptReader_SSEC_Error(t *testing.T) {
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSEIV: {0x01},
+		},
+	}
+	reader := bytes.NewReader([]byte("data"))
+
+	_, err := MaybeDecryptReader(reader, entry)
+	if err == nil {
+		t.Fatal("expected error for SSE-C")
+	}
+}
+
+func TestMaybeDecryptContent_Plaintext(t *testing.T) {
+	content := []byte("hello world")
+	entry := &filer_pb.Entry{}
+
+	got, err := MaybeDecryptContent(content, entry)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !bytes.Equal(got, content) {
+		t.Errorf("got %q, want %q", got, content)
+	}
+}
+
+func TestMaybeDecryptContent_NilEntry(t *testing.T) {
+	content := []byte("data")
+	got, err := MaybeDecryptContent(content, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !bytes.Equal(got, content) {
+		t.Errorf("got %q, want %q", got, content)
+	}
+}
+
+func TestMaybeDecryptContent_Empty(t *testing.T) {
+	got, err := MaybeDecryptContent(nil, &filer_pb.Entry{})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if got != nil {
+		t.Errorf("expected nil, got %v", got)
+	}
+}
+
+func TestMaybeDecryptContent_SSEC_Error(t *testing.T) {
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSEIV: {0x01},
+		},
+	}
+
+	_, err := MaybeDecryptContent([]byte("data"), entry)
+	if err == nil {
+		t.Fatal("expected error for SSE-C")
+	}
+}
+
+func TestMaybeDecryptContent_MixedExtended_Error(t *testing.T) {
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSES3Key:  {0x01},
+			s3_constants.SeaweedFSSSEKMSKey: {0x02},
+		},
+	}
+
+	_, err := MaybeDecryptContent([]byte("data"), entry)
+	if err == nil {
+		t.Fatal("expected error for conflicting SSE metadata")
+	}
+}
+
+// --- SSE-S3 integration tests ---
+// These tests exercise the full MaybeDecryptReader/MaybeDecryptContent path
+// for SSE-S3: detectSSEType → decryptSSES3 → DeserializeSSES3Metadata →
+// GetSSES3IV → CreateSSES3DecryptedReader. A test KEK is injected via
+// WEED_S3_SSE_KEK env var and a mock filer client.
+
+// testFilerClient is a minimal filer_pb.FilerClient mock that returns
+// ErrNotFound for all lookups (no KEK on filer — we use env var instead).
+type testFilerClient struct{}
+
+func (c *testFilerClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return fmt.Errorf("%w", filer_pb.ErrNotFound)
+}
+func (c *testFilerClient) AdjustedUrl(loc *filer_pb.Location) string { return loc.Url }
+func (c *testFilerClient) GetDataCenter() string                     { return "" }
+
+// setupTestSSES3 initializes the global SSE-S3 key manager with a test KEK
+// via the WEED_S3_SSE_KEK env var and returns the KEK bytes + cleanup func.
+func setupTestSSES3(t *testing.T) (kek []byte, cleanup func()) {
+	t.Helper()
+
+	kek = make([]byte, 32)
+	if _, err := io.ReadFull(rand.Reader, kek); err != nil {
+		t.Fatal(err)
+	}
+
+	// Force Viper to pick up the new env var
+	os.Setenv("WEED_S3_SSE_KEK", hex.EncodeToString(kek))
+
+	// Reset Viper cache so it reads the new env var
+	v := util.GetViper()
+	v.AutomaticEnv()
+
+	// Re-initialize the global key manager with the KEK from env
+	km := s3api.GetSSES3KeyManager()
+	if err := km.InitializeWithFiler(&testFilerClient{}); err != nil {
+		os.Unsetenv("WEED_S3_SSE_KEK")
+		t.Fatalf("InitializeWithFiler: %v", err)
+	}
+
+	return kek, func() {
+		os.Unsetenv("WEED_S3_SSE_KEK")
+		// Re-initialize with no KEK to clear the super key
+		km.InitializeWithFiler(&testFilerClient{})
+	}
+}
+
+func TestMaybeDecryptReader_SSES3(t *testing.T) {
+	_, cleanup := setupTestSSES3(t)
+	defer cleanup()
+
+	plaintext := []byte("SSE-S3 encrypted content for testing round-trip decryption")
+
+	// Generate a DEK and encrypt
+	sseKey, err := s3api.GenerateSSES3Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+	encReader, encIV, err := s3api.CreateSSES3EncryptedReader(bytes.NewReader(plaintext), sseKey)
+	if err != nil {
+		t.Fatalf("encrypt: %v", err)
+	}
+	ciphertext, err := io.ReadAll(encReader)
+	if err != nil {
+		t.Fatalf("read ciphertext: %v", err)
+	}
+
+	// Build serialized SSE-S3 metadata (uses the global key manager to
+	// envelope-encrypt the DEK with the test KEK)
+	sseKey.IV = encIV
+	metadataBytes, err := s3api.SerializeSSES3Metadata(sseKey)
+	if err != nil {
+		t.Fatalf("serialize metadata: %v", err)
+	}
+
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSES3Key: metadataBytes,
+		},
+	}
+
+	// Test full path: MaybeDecryptReader → decryptSSES3 → DeserializeSSES3Metadata → CreateSSES3DecryptedReader
+	decrypted, err := MaybeDecryptReader(bytes.NewReader(ciphertext), entry)
+	if err != nil {
+		t.Fatalf("MaybeDecryptReader: %v", err)
+	}
+	result, err := io.ReadAll(decrypted)
+	if err != nil {
+		t.Fatalf("ReadAll: %v", err)
+	}
+	if !bytes.Equal(result, plaintext) {
+		t.Errorf("SSE-S3 round-trip failed: got %q, want %q", result, plaintext)
+	}
+}
+
+func TestMaybeDecryptContent_SSES3(t *testing.T) {
+	_, cleanup := setupTestSSES3(t)
+	defer cleanup()
+
+	plaintext := []byte("inline SSE-S3 content")
+
+	// Generate a DEK and encrypt inline content
+	sseKey, err := s3api.GenerateSSES3Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+	encReader, encIV, err := s3api.CreateSSES3EncryptedReader(bytes.NewReader(plaintext), sseKey)
+	if err != nil {
+		t.Fatalf("encrypt: %v", err)
+	}
+	ciphertext, err := io.ReadAll(encReader)
+	if err != nil {
+		t.Fatalf("read ciphertext: %v", err)
+	}
+
+	sseKey.IV = encIV
+	metadataBytes, err := s3api.SerializeSSES3Metadata(sseKey)
+	if err != nil {
+		t.Fatalf("serialize metadata: %v", err)
+	}
+
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSES3Key: metadataBytes,
+		},
+	}
+
+	// Test full path: MaybeDecryptContent → MaybeDecryptReader → decryptSSES3
+	result, err := MaybeDecryptContent(ciphertext, entry)
+	if err != nil {
+		t.Fatalf("MaybeDecryptContent: %v", err)
+	}
+	if !bytes.Equal(result, plaintext) {
+		t.Errorf("SSE-S3 round-trip failed: got %q, want %q", result, plaintext)
+	}
+}
+
+// --- SSE-KMS integration tests ---
+
+// testKMSProvider is a minimal KMSProvider mock for testing.
+type testKMSProvider struct {
+	keyID     string
+	plaintext []byte // the DEK plaintext returned by Decrypt
+}
+
+func (p *testKMSProvider) GenerateDataKey(_ context.Context, _ *kms.GenerateDataKeyRequest) (*kms.GenerateDataKeyResponse, error) {
+	return nil, nil
+}
+
+func (p *testKMSProvider) Decrypt(_ context.Context, _ *kms.DecryptRequest) (*kms.DecryptResponse, error) {
+	return &kms.DecryptResponse{
+		KeyID:     p.keyID,
+		Plaintext: append([]byte(nil), p.plaintext...), // return a copy
+	}, nil
+}
+
+func (p *testKMSProvider) DescribeKey(_ context.Context, _ *kms.DescribeKeyRequest) (*kms.DescribeKeyResponse, error) {
+	return nil, nil
+}
+
+func (p *testKMSProvider) GetKeyID(_ context.Context, keyIdentifier string) (string, error) {
+	return p.keyID, nil
+}
+
+func (p *testKMSProvider) Close() error { return nil }
+
+func TestMaybeDecryptReader_SSEKMS(t *testing.T) {
+	plaintext := []byte("SSE-KMS encrypted content for testing")
+
+	// Generate a random DEK and IV
+	dek := make([]byte, 32)
+	iv := make([]byte, aes.BlockSize)
+	if _, err := io.ReadFull(rand.Reader, dek); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		t.Fatal(err)
+	}
+
+	// Encrypt with AES-CTR (same cipher mode as SSE-KMS)
+	block, err := aes.NewCipher(dek)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ciphertext := make([]byte, len(plaintext))
+	cipher.NewCTR(block, iv).XORKeyStream(ciphertext, plaintext)
+
+	// Set up a mock KMS provider that returns our DEK
+	keyID := "test-kms-key-1"
+	encryptedDEK := []byte("fake-encrypted-dek") // mock doesn't validate
+	kms.SetGlobalKMSProvider(&testKMSProvider{
+		keyID:     keyID,
+		plaintext: dek,
+	})
+	defer kms.SetGlobalKMSProvider(nil)
+
+	// Build serialized KMS metadata
+	kmsMetadata := s3api.SSEKMSMetadata{
+		Algorithm:        s3_constants.SSEAlgorithmKMS,
+		KeyID:            keyID,
+		EncryptedDataKey: base64.StdEncoding.EncodeToString(encryptedDEK),
+		IV:               base64.StdEncoding.EncodeToString(iv),
+	}
+	metadataBytes, err := json.Marshal(kmsMetadata)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSEKMSKey: metadataBytes,
+		},
+	}
+
+	// Test MaybeDecryptReader
+	reader := bytes.NewReader(ciphertext)
+	decrypted, err := MaybeDecryptReader(reader, entry)
+	if err != nil {
+		t.Fatalf("MaybeDecryptReader: %v", err)
+	}
+	result, err := io.ReadAll(decrypted)
+	if err != nil {
+		t.Fatalf("ReadAll: %v", err)
+	}
+	if !bytes.Equal(result, plaintext) {
+		t.Errorf("SSE-KMS round-trip failed: got %q, want %q", result, plaintext)
+	}
+}
+
+func TestMaybeDecryptContent_SSEKMS(t *testing.T) {
+	plaintext := []byte("inline SSE-KMS content")
+
+	dek := make([]byte, 32)
+	iv := make([]byte, aes.BlockSize)
+	if _, err := io.ReadFull(rand.Reader, dek); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		t.Fatal(err)
+	}
+
+	block, err := aes.NewCipher(dek)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ciphertext := make([]byte, len(plaintext))
+	cipher.NewCTR(block, iv).XORKeyStream(ciphertext, plaintext)
+
+	keyID := "test-kms-key-2"
+	kms.SetGlobalKMSProvider(&testKMSProvider{
+		keyID:     keyID,
+		plaintext: dek,
+	})
+	defer kms.SetGlobalKMSProvider(nil)
+
+	kmsMetadata := s3api.SSEKMSMetadata{
+		Algorithm:        s3_constants.SSEAlgorithmKMS,
+		KeyID:            keyID,
+		EncryptedDataKey: base64.StdEncoding.EncodeToString([]byte("fake-encrypted-dek")),
+		IV:               base64.StdEncoding.EncodeToString(iv),
+	}
+	metadataBytes, err := json.Marshal(kmsMetadata)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSEKMSKey: metadataBytes,
+		},
+	}
+
+	result, err := MaybeDecryptContent(ciphertext, entry)
+	if err != nil {
+		t.Fatalf("MaybeDecryptContent: %v", err)
+	}
+	if !bytes.Equal(result, plaintext) {
+		t.Errorf("SSE-KMS round-trip failed: got %q, want %q", result, plaintext)
+	}
+}

weed/replication/replicator.go

@@ -8,6 +8,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
 	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
 	"github.com/seaweedfs/seaweedfs/weed/replication/source"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -25,6 +26,10 @@ func NewReplicator(sourceConfig util.Configuration, configPrefix string, dataSin
 	source := &source.FilerSource{}
 	source.Initialize(sourceConfig, configPrefix)
 
+	if err := repl_util.InitializeSSEForReplication(source); err != nil {
+		glog.Warningf("SSE initialization failed: %v (encrypted objects may fail to replicate)", err)
+	}
+
 	dataSink.SetSourceFiler(source)
 
 	return &Replicator{

weed/replication/sink/azuresink/azure_sink.go

@@ -138,6 +138,7 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []
 	_, err := appendBlobClient.Create(ctxCreate, nil)
 
 	needsWrite := true
+	freshlyCreated := false
 	if err != nil {
 		if bloberror.HasCode(err, bloberror.BlobAlreadyExists) {
 			// Handle existing blob - check if overwrite is needed and perform it if necessary
@@ -146,9 +147,13 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []
 			if handleErr != nil {
 				return handleErr
 			}
+			// handleExistingBlob recreates the blob when needsWrite is true
+			freshlyCreated = needsWrite
 		} else {
 			return fmt.Errorf("azure create append blob %s/%s: %w", g.container, key, err)
 		}
+	} else {
+		freshlyCreated = true
 	}
 
 	// If we don't need to write (blob is up-to-date), return early
@@ -156,6 +161,23 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []
 		return nil
 	}
 
+	// cleanupOnError deletes a freshly created blob when content write fails,
+	// preventing empty blobs from being left behind.
+	cleanupOnError := func(writeErr error) error {
+		if !freshlyCreated {
+			return writeErr
+		}
+		glog.Warningf("azure sink: cleaning up empty blob %s/%s after write failure: %v", g.container, key, writeErr)
+		ctxCleanup, cancelCleanup := context.WithTimeout(context.Background(), azure.DefaultAzureOpTimeout)
+		defer cancelCleanup()
+		if _, delErr := appendBlobClient.Delete(ctxCleanup, nil); delErr != nil {
+			if !bloberror.HasCode(delErr, bloberror.BlobNotFound) {
+				glog.Warningf("azure sink: failed to clean up blob %s/%s: %v", g.container, key, delErr)
+			}
+		}
+		return writeErr
+	}
+
 	writeFunc := func(data []byte) error {
 		ctxWrite, cancelWrite := context.WithTimeout(context.Background(), azure.DefaultAzureOpTimeout)
 		defer cancelWrite()
@@ -164,11 +186,18 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []
 	}
 
 	if len(entry.Content) > 0 {
-		return writeFunc(entry.Content)
+		content, err := repl_util.MaybeDecryptContent(entry.Content, entry)
+		if err != nil {
+			return cleanupOnError(fmt.Errorf("decrypt inline SSE content: %w", err))
+		}
+		if err := writeFunc(content); err != nil {
+			return cleanupOnError(err)
+		}
+		return nil
 	}
 
-	if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc); err != nil {
-		return err
+	if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc, entry); err != nil {
+		return cleanupOnError(err)
 	}
 
 	return nil

weed/replication/sink/b2sink/b2_sink.go

@@ -2,6 +2,7 @@ package B2Sink
 
 import (
 	"context"
+	"fmt"
 	"strings"
 
 	"github.com/kurin/blazer/b2"
@@ -116,10 +117,14 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int
 	}
 
 	if len(entry.Content) > 0 {
-		return writeFunc(entry.Content)
+		content, err := repl_util.MaybeDecryptContent(entry.Content, entry)
+		if err != nil {
+			return fmt.Errorf("decrypt inline SSE content: %w", err)
+		}
+		return writeFunc(content)
 	}
 
-	if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc); err != nil {
+	if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc, entry); err != nil {
 		return err
 	}
 

weed/replication/sink/filersink/fetch_write.go

@@ -178,7 +178,7 @@ func (fs *FilerSink) replicateOneManifestChunk(ctx context.Context, sourceChunk
 }
 
 func (fs *FilerSink) uploadManifestChunk(path string, sourceMtime int64, sourceFileId string, manifestData []byte) (fileId string, err error) {
-	uploader, err := operation.NewUploader()
+	uploader, err := fs.getUploader()
 	if err != nil {
 		glog.V(0).Infof("upload manifest data %v: %v", sourceFileId, err)
 		return "", fmt.Errorf("upload manifest data: %w", err)
@@ -235,7 +235,7 @@ func (fs *FilerSink) uploadManifestChunk(path string, sourceMtime int64, sourceF
 }
 
 func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string, sourceMtime int64) (fileId string, err error) {
-	uploader, err := operation.NewUploader()
+	uploader, err := fs.getUploader()
 	if err != nil {
 		glog.V(0).Infof("upload source data %v: %v", sourceChunk.GetFileIdString(), err)
 		return "", fmt.Errorf("upload data: %w", err)

weed/replication/sink/filersink/filer_sink.go

@@ -6,6 +6,7 @@ import (
 	"math"
 	"sync"
 
+	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 
@@ -50,6 +51,7 @@ type FilerSink struct {
 	executor          *util.LimitedConcurrentExecutor
 	signature         int32
 	activeTransfers   sync.Map // chunkFileId -> *ChunkTransferStatus
+	uploader          *operation.Uploader
 }
 
 func init() {
@@ -88,6 +90,21 @@ func (fs *FilerSink) SetSourceFiler(s *source.FilerSource) {
 	fs.filerSource = s
 }
 
+// SetUploader sets a custom uploader for this sink, used when the target
+// cluster requires different TLS certificates than the global config.
+// Must be called during initialization, before any replication goroutines
+// start, since it writes fs.uploader without synchronization.
+func (fs *FilerSink) SetUploader(uploader *operation.Uploader) {
+	fs.uploader = uploader
+}
+
+func (fs *FilerSink) getUploader() (*operation.Uploader, error) {
+	if fs.uploader != nil {
+		return fs.uploader, nil
+	}
+	return operation.NewUploader()
+}
+
 func (fs *FilerSink) DoInitialize(address, grpcAddress string, dir string,
 	replication string, collection string, ttlSec int, diskType string, grpcDialOption grpc.DialOption, writeChunkByFiler bool) (err error) {
 	fs.address = address

weed/replication/sink/gcssink/gcs_sink.go

@@ -115,23 +115,37 @@ func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []in
 	totalSize := filer.FileSize(entry)
 	chunkViews := filer.ViewFromChunks(context.Background(), g.filerSource.LookupFileId, entry.GetChunks(), 0, int64(totalSize))
 
-	wc := g.client.Bucket(g.bucket).Object(key).NewWriter(context.Background())
-	defer wc.Close()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	wc := g.client.Bucket(g.bucket).Object(key).NewWriter(ctx)
 
 	writeFunc := func(data []byte) error {
 		_, writeErr := wc.Write(data)
 		return writeErr
 	}
 
+	var writeErr error
 	if len(entry.Content) > 0 {
-		return writeFunc(entry.Content)
+		content, decErr := repl_util.MaybeDecryptContent(entry.Content, entry)
+		if decErr != nil {
+			writeErr = fmt.Errorf("decrypt inline SSE content: %w", decErr)
+		} else {
+			writeErr = writeFunc(content)
+		}
+	} else {
+		writeErr = repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc, entry)
 	}
 
-	if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc); err != nil {
-		return err
+	if writeErr != nil {
+		// Cancel the context to abort the GCS upload without touching
+		// any existing object at this key.
+		cancel()
+		wc.Close()
+		return writeErr
 	}
 
-	return nil
+	return wc.Close()
 
 }
 

weed/replication/sink/localsink/local_sink.go

@@ -2,6 +2,7 @@ package localsink
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -94,36 +95,55 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa
 
 	mode := os.FileMode(entry.Attributes.FileMode)
 	shortFileName := util.ToShortFileName(key)
-	if err := os.Remove(shortFileName); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	dstFile, err := os.OpenFile(shortFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
-	if err != nil {
-		return err
-	}
-	defer dstFile.Close()
 
-	fi, err := dstFile.Stat()
+	// Write to a temp file in the same directory, then atomically rename
+	// on success. This prevents leaving a truncated/empty file if
+	// decryption or chunk copy fails.
+	tmpFile, err := os.CreateTemp(dir, ".seaweedfs-tmp-*")
 	if err != nil {
 		return err
 	}
-	if fi.Mode() != mode {
-		glog.V(4).Infof("Modify file mode: %o -> %o", fi.Mode(), mode)
-		if err := dstFile.Chmod(mode); err != nil {
-			return err
+	tmpName := tmpFile.Name()
+	defer func() {
+		// Clean up temp file on any error (rename removes it on success)
+		if tmpFile != nil {
+			tmpFile.Close()
+			os.Remove(tmpName)
 		}
+	}()
+
+	if err := tmpFile.Chmod(mode); err != nil {
+		return err
 	}
 
 	writeFunc := func(data []byte) error {
-		_, writeErr := dstFile.Write(data)
+		_, writeErr := tmpFile.Write(data)
 		return writeErr
 	}
 
 	if len(entry.Content) > 0 {
-		return writeFunc(entry.Content)
+		content, err := repl_util.MaybeDecryptContent(entry.Content, entry)
+		if err != nil {
+			return fmt.Errorf("decrypt inline SSE content: %w", err)
+		}
+		if err := writeFunc(content); err != nil {
+			return err
+		}
+	} else {
+		if err := repl_util.CopyFromChunkViews(chunkViews, localsink.filerSource, writeFunc, entry); err != nil {
+			return err
+		}
 	}
 
-	if err := repl_util.CopyFromChunkViews(chunkViews, localsink.filerSource, writeFunc); err != nil {
+	// Close before rename so the data is flushed
+	if err := tmpFile.Close(); err != nil {
+		return err
+	}
+	tmpFile = nil // prevent deferred cleanup
+
+	// Atomic rename into final destination
+	if err := os.Rename(tmpName, shortFileName); err != nil {
+		os.Remove(tmpName)
 		return err
 	}
 

weed/replication/sink/s3sink/s3_sink.go

@@ -3,6 +3,7 @@ package S3Sink
 import (
 	"encoding/base64"
 	"fmt"
+	"net/url"
 	"strconv"
 	"strings"
 
@@ -17,6 +18,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
 	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
 	"github.com/seaweedfs/seaweedfs/weed/replication/source"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -160,6 +162,14 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures
 
 	reader := filer.NewFileReader(s3sink.filerSource, entry)
 
+	// Decrypt SSE-encrypted objects so the destination receives plaintext
+	decryptedReader, err := repl_util.MaybeDecryptReader(reader, entry)
+	if err != nil {
+		repl_util.CloseReader(reader)
+		return fmt.Errorf("decrypt SSE object: %w", err)
+	}
+	defer repl_util.CloseMaybeDecryptedReader(reader, decryptedReader)
+
 	// Create an uploader with the session and custom options
 	uploader := s3manager.NewUploaderWithClient(s3sink.conn, func(u *s3manager.Uploader) {
 		u.PartSize = int64(s3sink.uploaderPartSizeMb * 1024 * 1024)
@@ -188,20 +198,16 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures
 		entry.Extended[s3_constants.AmzUserMetaMtime] = []byte(strconv.FormatInt(entry.Attributes.Mtime, 10))
 	}
 	// process tagging
-	tags := ""
-	for k, v := range entry.Extended {
-		if len(tags) > 0 {
-			tags = tags + "&"
-		}
-		tags = tags + k + "=" + string(v)
-	}
+	tags := buildTaggingString(entry.Extended)
 
 	// Upload the file to S3.
 	uploadInput := s3manager.UploadInput{
-		Bucket:  aws.String(s3sink.bucket),
-		Key:     aws.String(key),
-		Body:    reader,
-		Tagging: aws.String(tags),
+		Bucket: aws.String(s3sink.bucket),
+		Key:    aws.String(key),
+		Body:   decryptedReader,
+	}
+	if tags != "" {
+		uploadInput.Tagging = aws.String(tags)
 	}
 	if len(entry.Attributes.Md5) > 0 {
 		uploadInput.ContentMD5 = aws.String(base64.StdEncoding.EncodeToString([]byte(entry.Attributes.Md5)))
@@ -223,3 +229,18 @@ func cleanKey(key string) string {
 	}
 	return key
 }
+
+// buildTaggingString builds the S3 Tagging header value from entry extended metadata.
+// Only keys with the AmzObjectTaggingPrefix ("X-Amz-Tagging-") are included as object
+// tags. The prefix is stripped and values are URL-encoded to produce a valid S3 tagging
+// query string.
+func buildTaggingString(extended map[string][]byte) string {
+	tagValues := url.Values{}
+	for k, v := range extended {
+		if strings.HasPrefix(k, s3_constants.AmzObjectTaggingPrefix) {
+			tagKey := k[len(s3_constants.AmzObjectTaggingPrefix):]
+			tagValues.Set(tagKey, string(v))
+		}
+	}
+	return tagValues.Encode()
+}

weed/replication/sink/s3sink/s3_sink_test.go

@@ -0,0 +1,59 @@
+package S3Sink
+
+import (
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+)
+
+func TestBuildTaggingString_ShouldStripTagPrefix(t *testing.T) {
+	extended := map[string][]byte{
+		s3_constants.AmzObjectTaggingPrefix + "env": []byte("production"),
+	}
+
+	tagging := buildTaggingString(extended)
+
+	if strings.Contains(tagging, s3_constants.AmzObjectTaggingPrefix) {
+		t.Errorf("tagging should not contain storage prefix %q, got %q", s3_constants.AmzObjectTaggingPrefix, tagging)
+	}
+
+	parsed, err := url.ParseQuery(tagging)
+	if err != nil {
+		t.Fatalf("tagging should be valid URL query: %v", err)
+	}
+	if v := parsed.Get("env"); v != "production" {
+		t.Errorf("expected tag env=production, got %q", v)
+	}
+}
+
+func TestBuildTaggingString_ShouldURLEncodeValues(t *testing.T) {
+	extended := map[string][]byte{
+		s3_constants.AmzObjectTaggingPrefix + "path": []byte("/a/b=c&d"),
+	}
+
+	tagging := buildTaggingString(extended)
+
+	parsed, err := url.ParseQuery(tagging)
+	if err != nil {
+		t.Fatalf("tagging should be valid URL query: %v", err)
+	}
+	if v := parsed.Get("path"); v != "/a/b=c&d" {
+		t.Errorf("expected tag value /a/b=c&d after decoding, got %q", v)
+	}
+}
+
+func TestBuildTaggingString_EmptyWhenNoTags(t *testing.T) {
+	extended := map[string][]byte{
+		"Content-Encoding":            []byte("gzip"),
+		s3_constants.AmzUserMetaMtime: []byte("12345"),
+		s3_constants.SeaweedFSSSES3Key: []byte(`{"algorithm":"AES256","encryptedDEK":"abc"}`),
+	}
+
+	tagging := buildTaggingString(extended)
+
+	if tagging != "" {
+		t.Errorf("expected empty tagging when no tag keys present, got %q", tagging)
+	}
+}

weed/replication/source/filer_source.go

@@ -15,6 +15,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
 )
 
 type FilerSource struct {
@@ -25,6 +26,7 @@ type FilerSource struct {
 	proxyByFiler   bool
 	dataCenter     string
 	signature      int32
+	httpClient     *util_http_client.HTTPClient
 }
 
 func (fs *FilerSource) Initialize(configuration util.Configuration, prefix string) error {
@@ -54,6 +56,10 @@ func (fs *FilerSource) SetGrpcDialOption(option grpc.DialOption) {
 	fs.grpcDialOption = option
 }
 
+func (fs *FilerSource) SetHttpClient(client *util_http_client.HTTPClient) {
+	fs.httpClient = client
+}
+
 func (fs *FilerSource) LookupFileId(ctx context.Context, part string) (fileUrls []string, err error) {
 
 	vid2Locations := make(map[string]*filer_pb.Locations)
@@ -104,9 +110,15 @@ func (fs *FilerSource) LookupFileId(ctx context.Context, part string) (fileUrls
 }
 
 func (fs *FilerSource) ReadPart(fileId string, offset int64) (filename string, header http.Header, resp *http.Response, err error) {
+	downloadFn := util_http.DownloadFile
+	if fs.httpClient != nil {
+		downloadFn = func(fileUrl string, jwt string, offset ...int64) (string, http.Header, *http.Response, error) {
+			return util_http.DownloadFileWithClient(fs.httpClient, fileUrl, jwt, offset...)
+		}
+	}
 
 	if fs.proxyByFiler {
-		filename, header, resp, err = util_http.DownloadFile("http://"+fs.address+"/?proxyChunkId="+fileId, "", offset)
+		filename, header, resp, err = downloadFn("http://"+fs.address+"/?proxyChunkId="+fileId, "", offset)
 		if err != nil {
 			glog.V(0).Infof("read part %s via filer proxy %s offset %d: %v", fileId, fs.address, offset, err)
 		} else {
@@ -121,7 +133,7 @@ func (fs *FilerSource) ReadPart(fileId string, offset int64) (filename string, h
 	}
 
 	for _, fileUrl := range fileUrls {
-		filename, header, resp, err = util_http.DownloadFile(fileUrl, "", offset)
+		filename, header, resp, err = downloadFn(fileUrl, "", offset)
 		if err != nil {
 			glog.V(0).Infof("fail to read part %s from %s offset %d: %v", fileId, fileUrl, offset, err)
 		} else {

weed/s3api/auth_signature_v4.go

@@ -194,6 +194,8 @@ func buildPathWithForwardedPrefix(forwardedPrefix, urlPath string) string {
 	var joined string
 	if strings.HasSuffix(forwardedPrefix, "/") && strings.HasPrefix(urlPath, "/") {
 		joined = forwardedPrefix + urlPath[1:]
+	} else if urlPath == "" {
+		joined = forwardedPrefix
 	} else if !strings.HasSuffix(forwardedPrefix, "/") && !strings.HasPrefix(urlPath, "/") {
 		joined = forwardedPrefix + "/" + urlPath
 	} else {

weed/s3api/auth_signature_v4_test.go

@@ -139,6 +139,12 @@ func TestBuildPathWithForwardedPrefix(t *testing.T) {
 			urlPath:         "bucket/obj",
 			expected:        "/storage/bucket/obj",
 		},
+		{
+			name:            "empty urlPath with prefix",
+			forwardedPrefix: "/s3",
+			urlPath:         "",
+			expected:        "/s3",
+		},
 	}
 
 	for _, tt := range tests {

weed/s3api/iceberg/commit_helpers.go

@@ -200,7 +200,7 @@ func (s *Server) finalizeCreateOnCommit(ctx context.Context, input createOnCommi
 		markerBucket = metadataBucket
 	}
 	if markerErr := s.deleteStageCreateMarkers(ctx, markerBucket, input.namespace, input.tableName); markerErr != nil {
-		glog.V(1).Infof("Iceberg: failed to cleanup stage-create markers for %s.%s after finalize: %v", encodeNamespace(input.namespace), input.tableName, markerErr)
+		glog.V(1).Infof("Iceberg: failed to cleanup stage-create markers for %s.%s after finalize: %v", flattenNamespacePath(input.namespace), input.tableName, markerErr)
 	}
 
 	return &CommitTableResponse{

weed/s3api/iceberg/handlers_commit.go

@@ -83,7 +83,7 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
 		})
 		if err != nil {
 			if isS3TablesNotFound(err) {
-				location := fmt.Sprintf("s3://%s/%s/%s", bucketName, encodeNamespace(namespace), tableName)
+				location := fmt.Sprintf("s3://%s/%s", bucketName, path.Join(flattenNamespacePath(namespace), tableName))
 				tableUUID := generatedLegacyUUID
 				baseMetadataVersion := 0
 				baseMetadataLocation := ""
@@ -195,7 +195,7 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
 
 		location := tableLocationFromMetadataLocation(getResp.MetadataLocation)
 		if location == "" {
-			location = fmt.Sprintf("s3://%s/%s/%s", bucketName, encodeNamespace(namespace), tableName)
+			location = fmt.Sprintf("s3://%s/%s", bucketName, path.Join(flattenNamespacePath(namespace), tableName))
 		}
 		tableUUID := uuid.Nil
 		if getResp.Metadata != nil && getResp.Metadata.Iceberg != nil && getResp.Metadata.Iceberg.TableUUID != "" {

weed/s3api/iceberg/handlers_namespace.go

@@ -39,10 +39,21 @@ func (s *Server) handleListNamespaces(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// The Iceberg REST spec allows a "parent" query parameter for hierarchical
+	// namespace listing. Convert it to the dot-separated prefix used by S3 Tables.
+	var prefix string
+	if parent := r.URL.Query().Get("parent"); parent != "" {
+		parentParts := parseNamespace(parent)
+		if len(parentParts) > 0 {
+			prefix = flattenNamespacePath(parentParts) + "."
+		}
+	}
+
 	// Use S3 Tables manager to list namespaces
 	var resp s3tables.ListNamespacesResponse
 	req := &s3tables.ListNamespacesRequest{
 		TableBucketARN:    bucketARN,
+		Prefix:            prefix,
 		ContinuationToken: pageToken,
 		MaxNamespaces:     pageSize,
 	}

weed/s3api/iceberg/handlers_table.go

@@ -120,7 +120,7 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 
 	// Generate UUID for the new table
 	tableUUID := uuid.New()
-	tablePath := path.Join(encodeNamespace(namespace), req.Name)
+	tablePath := path.Join(flattenNamespacePath(namespace), req.Name)
 	location := strings.TrimSuffix(req.Location, "/")
 	if location == "" {
 		if req.Properties != nil {
@@ -179,7 +179,7 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 		}
 		stagedMetadataLocation := fmt.Sprintf("s3://%s/%s/metadata/%s", metadataBucket, stagedTablePath, metadataFileName)
 		if markerErr := s.writeStageCreateMarker(r.Context(), bucketName, namespace, tableName, tableUUID, location, stagedMetadataLocation); markerErr != nil {
-			glog.V(1).Infof("Iceberg: failed to persist stage-create marker for %s.%s: %v", encodeNamespace(namespace), tableName, markerErr)
+			glog.V(1).Infof("Iceberg: failed to persist stage-create marker for %s.%s: %v", flattenNamespacePath(namespace), tableName, markerErr)
 		}
 		result := LoadTableResult{
 			MetadataLocation: metadataLocation,
@@ -266,7 +266,7 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 		finalLocation = metadataLocation
 	}
 	if markerErr := s.deleteStageCreateMarkers(r.Context(), bucketName, namespace, tableName); markerErr != nil {
-		glog.V(1).Infof("Iceberg: failed to cleanup stage-create markers for %s.%s after create: %v", encodeNamespace(namespace), tableName, markerErr)
+		glog.V(1).Infof("Iceberg: failed to cleanup stage-create markers for %s.%s after create: %v", flattenNamespacePath(namespace), tableName, markerErr)
 	}
 
 	result := LoadTableResult{
@@ -324,7 +324,7 @@ func (s *Server) handleLoadTable(w http.ResponseWriter, r *http.Request) {
 func buildLoadTableResult(getResp s3tables.GetTableResponse, bucketName string, namespace []string, tableName string) LoadTableResult {
 	location := tableLocationFromMetadataLocation(getResp.MetadataLocation)
 	if location == "" {
-		location = fmt.Sprintf("s3://%s/%s/%s", bucketName, encodeNamespace(namespace), tableName)
+		location = fmt.Sprintf("s3://%s/%s", bucketName, path.Join(flattenNamespacePath(namespace), tableName))
 	}
 	tableUUID := uuid.Nil
 	if getResp.Metadata != nil && getResp.Metadata.Iceberg != nil && getResp.Metadata.Iceberg.TableUUID != "" {

weed/s3api/iceberg/utils.go

@@ -32,11 +32,19 @@ func parseNamespace(encoded string) []string {
 	return result
 }
 
-// encodeNamespace encodes namespace parts for response.
+// encodeNamespace encodes namespace parts using the Iceberg REST protocol's
+// unit separator (0x1F) convention. This is only appropriate for protocol-level
+// encoding (e.g. URL path parameters), NOT for filesystem/S3 paths.
 func encodeNamespace(parts []string) string {
 	return strings.Join(parts, "\x1F")
 }
 
+// flattenNamespacePath joins namespace parts with "." for use in S3 location
+// and filer paths, matching the S3 Tables storage layer convention.
+func flattenNamespacePath(parts []string) string {
+	return strings.Join(parts, ".")
+}
+
 func parseS3Location(location string) (bucketName, tablePath string, err error) {
 	if !strings.HasPrefix(location, "s3://") {
 		return "", "", fmt.Errorf("unsupported location: %s", location)

weed/s3api/policy_engine/engine.go

@@ -162,6 +162,16 @@ func (engine *PolicyEngine) evaluateStatement(stmt *CompiledStatement, args *Pol
 	if !matchedAction {
 		matchedAction = engine.matchesDynamicPatterns(stmt.DynamicActionPatterns, args.Action, args)
 	}
+	// Multipart upload actions (CreateMultipartUpload, UploadPart, CompleteMultipartUpload, etc.)
+	// are implicitly allowed by s3:PutObject, since multipart upload is an implementation
+	// detail of putting objects. Check if this is a multipart action and the statement
+	// grants s3:PutObject.
+	if !matchedAction && multipartActionSet[args.Action] {
+		matchedAction = engine.matchesPatterns(stmt.ActionPatterns, "s3:PutObject")
+		if !matchedAction {
+			matchedAction = engine.matchesDynamicPatterns(stmt.DynamicActionPatterns, "s3:PutObject", args)
+		}
+	}
 	if !matchedAction {
 		return false
 	}

weed/s3api/policy_engine/engine_test.go

@@ -981,3 +981,80 @@ func TestExistingObjectTagDenyPolicy(t *testing.T) {
 		})
 	}
 }
+
+// TestMultipartUploadInheritsPutObjectPermission verifies that granting s3:PutObject
+// in a bucket policy implicitly allows multipart upload operations.
+// See https://github.com/seaweedfs/seaweedfs/discussions/8751
+func TestMultipartUploadInheritsPutObjectPermission(t *testing.T) {
+	engine := NewPolicyEngine()
+
+	policyJSON := `{
+		"Version": "2012-10-17",
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Principal": "*",
+				"Action": "s3:PutObject",
+				"Resource": "arn:aws:s3:::test-bucket/*"
+			}
+		]
+	}`
+
+	if err := engine.SetBucketPolicy("test-bucket", policyJSON); err != nil {
+		t.Fatalf("Failed to set policy: %v", err)
+	}
+
+	multipartActions := []string{
+		"s3:CreateMultipartUpload",
+		"s3:UploadPart",
+		"s3:UploadPartCopy",
+		"s3:CompleteMultipartUpload",
+		"s3:AbortMultipartUpload",
+		"s3:ListMultipartUploadParts",
+		"s3:ListBucketMultipartUploads",
+	}
+
+	for _, action := range multipartActions {
+		t.Run(action, func(t *testing.T) {
+			args := &PolicyEvaluationArgs{
+				Action:    action,
+				Resource:  "arn:aws:s3:::test-bucket/myfile.dat",
+				Principal: "*",
+				Conditions: map[string][]string{
+					"aws:SourceIp": {"10.0.0.1"},
+				},
+			}
+			result := engine.EvaluatePolicy("test-bucket", args)
+			if result != PolicyResultAllow {
+				t.Errorf("Expected s3:PutObject to implicitly allow %s, got %v", action, result)
+			}
+		})
+	}
+
+	// ListBucketMultipartUploads is a bucket-level action; the object-only
+	// resource "arn:aws:s3:::test-bucket/*" should NOT match the bucket ARN.
+	t.Run("s3:ListBucketMultipartUploads bucket ARN", func(t *testing.T) {
+		args := &PolicyEvaluationArgs{
+			Action:    "s3:ListBucketMultipartUploads",
+			Resource:  "arn:aws:s3:::test-bucket",
+			Principal: "*",
+		}
+		result := engine.EvaluatePolicy("test-bucket", args)
+		if result == PolicyResultAllow {
+			t.Error("Object-only resource should not match bucket ARN for ListBucketMultipartUploads")
+		}
+	})
+
+	// s3:PutObject must NOT implicitly grant unrelated actions
+	t.Run("s3:DeleteObject not inherited", func(t *testing.T) {
+		args := &PolicyEvaluationArgs{
+			Action:    "s3:DeleteObject",
+			Resource:  "arn:aws:s3:::test-bucket/myfile.dat",
+			Principal: "*",
+		}
+		result := engine.EvaluatePolicy("test-bucket", args)
+		if result == PolicyResultAllow {
+			t.Error("s3:PutObject should NOT implicitly allow s3:DeleteObject")
+		}
+	})
+}

weed/s3api/policy_engine/types.go

@@ -43,6 +43,7 @@ var (
 	multipartActionSet = map[string]bool{
 		s3const.S3_ACTION_CREATE_MULTIPART:       true,
 		s3const.S3_ACTION_UPLOAD_PART:            true,
+		s3const.S3_ACTION_UPLOAD_PART_COPY:       true,
 		s3const.S3_ACTION_COMPLETE_MULTIPART:     true,
 		s3const.S3_ACTION_ABORT_MULTIPART:        true,
 		s3const.S3_ACTION_LIST_PARTS:             true,

weed/s3api/s3_constants/s3_action_strings.go

@@ -32,6 +32,7 @@ const (
 	S3_ACTION_UPLOAD_PART            = "s3:UploadPart"
 	S3_ACTION_COMPLETE_MULTIPART     = "s3:CompleteMultipartUpload"
 	S3_ACTION_ABORT_MULTIPART        = "s3:AbortMultipartUpload"
+	S3_ACTION_UPLOAD_PART_COPY       = "s3:UploadPartCopy"
 	S3_ACTION_LIST_PARTS             = "s3:ListMultipartUploadParts"
 	S3_ACTION_LIST_MULTIPART_UPLOADS = "s3:ListBucketMultipartUploads"
 

weed/s3api/s3_end_to_end_test.go

@@ -214,6 +214,153 @@ func TestS3MultipartUploadWithJWT(t *testing.T) {
 	}
 }
 
+// TestS3ListObjectsV2PrefixCondition tests that ListObjectsV2 requests with a prefix
+// query parameter correctly populate s3:prefix in the policy evaluation context and
+// use bucket-level resource ARNs, so that policies with s3:prefix conditions work.
+// This reproduces the bug reported in https://github.com/seaweedfs/seaweedfs/issues/8969
+func TestS3ListObjectsV2PrefixCondition(t *testing.T) {
+	// Set up IAM system
+	iamManager := integration.NewIAMManager()
+	config := &integration.IAMConfig{
+		STS: &sts.STSConfig{
+			TokenDuration:    sts.FlexibleDuration{Duration: time.Hour},
+			MaxSessionLength: sts.FlexibleDuration{Duration: time.Hour * 12},
+			Issuer:           "test-sts",
+			SigningKey:       []byte("test-signing-key-32-characters-long"),
+		},
+		Policy: &policy.PolicyEngineConfig{
+			DefaultEffect: "Deny",
+			StoreType:     "memory",
+		},
+		Roles: &integration.RoleStoreConfig{
+			StoreType: "memory",
+		},
+	}
+
+	err := iamManager.Initialize(config, func() string { return "localhost:8888" })
+	require.NoError(t, err)
+
+	setupTestProviders(t, iamManager)
+
+	s3IAMIntegration := NewS3IAMIntegration(iamManager, "localhost:8888")
+	require.NotNil(t, s3IAMIntegration)
+
+	ctx := context.Background()
+
+	// Create a role with a policy that allows ListBucket only with a specific s3:prefix condition.
+	// This is the pattern used by Lakekeeper-vended STS credentials (issue #8969).
+	prefixPolicy := &policy.PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []policy.Statement{
+			{
+				Sid:    "AllowListUnderWarehouse",
+				Effect: "Allow",
+				Action: []string{"s3:ListBucket"},
+				Resource: []string{
+					"arn:aws:s3:::examples",
+				},
+				Condition: map[string]map[string]interface{}{
+					"StringLike": {
+						"s3:prefix": []string{"warehouse/*"},
+					},
+				},
+			},
+			{
+				Sid:      "AllowSTSSessionValidation",
+				Effect:   "Allow",
+				Action:   []string{"sts:ValidateSession"},
+				Resource: []string{"*"},
+			},
+		},
+	}
+
+	iamManager.CreatePolicy(ctx, "", "PrefixRestrictedPolicy", prefixPolicy)
+	iamManager.CreateRole(ctx, "", "PrefixRestrictedRole", &integration.RoleDefinition{
+		RoleName: "PrefixRestrictedRole",
+		TrustPolicy: &policy.PolicyDocument{
+			Version: "2012-10-17",
+			Statement: []policy.Statement{
+				{
+					Effect:    "Allow",
+					Principal: map[string]interface{}{"Federated": "test-oidc"},
+					Action:    []string{"sts:AssumeRoleWithWebIdentity"},
+				},
+			},
+		},
+		AttachedPolicies: []string{"PrefixRestrictedPolicy"},
+	})
+
+	// Assume role to get a session token
+	validJWTToken := createTestJWTEndToEnd(t, "https://test-issuer.com", "test-user-123", "test-signing-key")
+	response, err := iamManager.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
+		RoleArn:          "arn:aws:iam::role/PrefixRestrictedRole",
+		WebIdentityToken: validJWTToken,
+		RoleSessionName:  "prefix-test-session",
+	})
+	require.NoError(t, err)
+
+	sessionToken := response.Credentials.SessionToken
+	require.NotEmpty(t, sessionToken)
+
+	// Authenticate to get IAM identity
+	authReq := httptest.NewRequest("GET", "/examples", http.NoBody)
+	authReq.Header.Set("Authorization", "Bearer "+sessionToken)
+	identity, errCode := s3IAMIntegration.AuthenticateJWT(ctx, authReq)
+	require.Equal(t, s3err.ErrNone, errCode, "Authentication should succeed")
+
+	tests := []struct {
+		name     string
+		url      string
+		bucket   string
+		objKey   string
+		expected s3err.ErrorCode
+	}{
+		{
+			name:     "ListObjectsV2 with matching prefix query param and empty objectKey",
+			url:      "/examples?list-type=2&prefix=warehouse/data",
+			bucket:   "examples",
+			objKey:   "",
+			expected: s3err.ErrNone,
+		},
+		{
+			name:     "ListObjectsV2 with matching prefix propagated as objectKey",
+			url:      "/examples?list-type=2&prefix=warehouse/data",
+			bucket:   "examples",
+			objKey:   "warehouse/data",
+			expected: s3err.ErrNone,
+		},
+		{
+			name:     "ListObjectsV1 with matching prefix query param",
+			url:      "/examples?prefix=warehouse/files",
+			bucket:   "examples",
+			objKey:   "",
+			expected: s3err.ErrNone,
+		},
+		{
+			name:     "ListObjectsV2 with non-matching prefix should be denied",
+			url:      "/examples?list-type=2&prefix=other/path",
+			bucket:   "examples",
+			objKey:   "",
+			expected: s3err.ErrAccessDenied,
+		},
+		{
+			name:     "ListObjectsV2 with no prefix should be denied",
+			url:      "/examples?list-type=2",
+			bucket:   "examples",
+			objKey:   "",
+			expected: s3err.ErrAccessDenied,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			req := httptest.NewRequest("GET", tt.url, http.NoBody)
+			result := s3IAMIntegration.AuthorizeAction(ctx, identity, Action("List"), tt.bucket, tt.objKey, req)
+			assert.Equal(t, tt.expected, result, "unexpected authorization result for %s", tt.name)
+		})
+	}
+}
+
 // TestS3CORSWithJWT tests CORS preflight requests with IAM
 func TestS3CORSWithJWT(t *testing.T) {
 	s3Server, iamManager := setupCompleteS3IAMSystem(t)

weed/s3api/s3_iam_middleware.go

@@ -250,20 +250,28 @@ func (s3iam *S3IAMIntegration) AuthorizeAction(ctx context.Context, identity *IA
 		return s3err.ErrAccessDenied
 	}
 
-	// Build resource ARN for the S3 operation
-	resourceArn := buildS3ResourceArn(bucket, objectKey)
-
 	// Extract request context for policy conditions
 	requestContext := extractRequestContext(r)
 
-	// Add s3:prefix to request context based on object key
-	// This ensures that policy conditions referencing s3:prefix (like StringLike)
-	// work correctly for both ListObjects (where objectKey is the prefix) and
-	// object operations (where we treat the object key as the prefix for matching)
-	if objectKey != "" && objectKey != "/" {
-		requestContext["s3:prefix"] = objectKey
+	// For list operations, populate the s3:prefix condition key and ensure the
+	// resource ARN stays at bucket level (matching AWS ListBucket semantics).
+	// See https://github.com/seaweedfs/seaweedfs/issues/8969
+	resourceObjectKey := objectKey
+	if action == "List" {
+		listPrefix := r.URL.Query().Get("prefix")
+		if listPrefix != "" {
+			requestContext["s3:prefix"] = listPrefix
+		} else if objectKey != "" && objectKey != "/" {
+			requestContext["s3:prefix"] = objectKey
+		} else {
+			requestContext["s3:prefix"] = ""
+		}
+		resourceObjectKey = ""
 	}
 
+	// Build resource ARN for the S3 operation
+	resourceArn := buildS3ResourceArn(bucket, resourceObjectKey)
+
 	// Add identity claims to request context for policy variables
 	// Only add claim keys if they don't already exist (to avoid overwriting request-derived context)
 	if identity.Claims != nil {

weed/security/tls.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
@@ -209,6 +210,39 @@ func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption {
 	return grpc.WithTransportCredentials(wrapped)
 }
 
+// LoadHTTPClientFromFile creates an HTTP client using the https.client TLS
+// settings from the given security config file. Returns nil if HTTPS is not
+// enabled in the config. This is used by filer.sync to create per-cluster
+// HTTP clients when clusters use different certificates.
+func LoadHTTPClientFromFile(configFile string) (*util_http_client.HTTPClient, error) {
+	v := viper.New()
+	v.SetConfigFile(configFile)
+	if err := v.ReadInConfig(); err != nil {
+		return nil, fmt.Errorf("failed to read security config %s: %v", configFile, err)
+	}
+
+	if !v.GetBool("https.client.enabled") {
+		return nil, nil
+	}
+
+	configDir := filepath.Dir(configFile)
+	resolvePath := func(key string) string {
+		p := v.GetString(key)
+		if p != "" && !filepath.IsAbs(p) {
+			return filepath.Join(configDir, p)
+		}
+		return p
+	}
+
+	return util_http_client.NewHttpClientWithTLS(
+		resolvePath("https.client.cert"),
+		resolvePath("https.client.key"),
+		resolvePath("https.client.ca"),
+		v.GetBool("https.client.insecure_skip_verify"),
+		util_http_client.AddDialContext,
+	)
+}
+
 func LoadClientTLSHTTP(clientCertFile string) *tls.Config {
 	clientCerts, err := os.ReadFile(clientCertFile)
 	if err != nil {

weed/server/filer_server.go

@@ -248,6 +248,10 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 			glog.Fatalf("%s bootstrap from %+v: %v", option.Host, existingNodes, err)
 		}
 	}
+	v.SetDefault("filer.options.s3.empty_folder_cleanup_delay", "2m")
+	if d, err := time.ParseDuration(v.GetString("filer.options.s3.empty_folder_cleanup_delay")); err == nil {
+		fs.filer.EmptyFolderCleanupDelay = d
+	}
 	fs.filer.AggregateFromPeers(option.Host, existingNodes, startFromTime)
 
 	fs.filer.LoadFilerConf()

weed/server/master_grpc_server_raft.go

@@ -41,7 +41,7 @@ func (ms *MasterServer) RaftListClusterServers(ctx context.Context, req *master_
 		// Add the current server itself (Peers() only returns other peers)
 		resp.ClusterServers = append(resp.ClusterServers, &master_pb.RaftListClusterServersResponse_ClusterServers{
 			Id:       currentServerName,
-			Address:  string(ms.option.Master),
+			Address:  ms.option.Master.ToGrpcAddress(),
 			Suffrage: "Voter",
 			IsLeader: currentServerName == leader,
 		})

weed/server/raft_server.go

@@ -32,6 +32,7 @@ type RaftServerOption struct {
 	DataDir           string
 	Topo              *topology.Topology
 	RaftResumeState   bool
+	SingleMaster      bool
 	HeartbeatInterval time.Duration
 	ElectionTimeout   time.Duration
 	RaftBootstrap     bool
@@ -176,10 +177,38 @@ func NewRaftServer(option *RaftServerOption) (*RaftServer, error) {
 	if err := s.raftServer.LoadSnapshot(); err != nil {
 		return nil, err
 	}
+
+	// In single-master mode resuming state, the log is not empty so the
+	// normal self-join path won't promote to leader. The server will
+	// self-elect after the election timeout, so use a tiny timeout to
+	// make this near-instant, then restore the original after election.
+	fastResume := option.SingleMaster && option.RaftResumeState && !s.raftServer.IsLogEmpty()
+	if fastResume {
+		s.raftServer.SetElectionTimeout(time.Millisecond)
+	}
+
 	if err := s.raftServer.Start(); err != nil {
 		return nil, err
 	}
 
+	if fastResume {
+		go func() {
+			defer s.raftServer.SetElectionTimeout(option.ElectionTimeout)
+			ticker := time.NewTicker(100 * time.Millisecond)
+			defer ticker.Stop()
+			timeout := time.After(option.ElectionTimeout)
+			for s.raftServer.Leader() == "" {
+				select {
+				case <-timeout:
+					glog.Warningf("Fast resume timed out waiting for leader election, restoring election timeout to %v", option.ElectionTimeout)
+					return
+				case <-ticker.C:
+				}
+			}
+			glog.V(0).Infof("Resumed as leader with election timeout restored to %v", option.ElectionTimeout)
+		}()
+	}
+
 	for name, peer := range s.peers {
 		if err := s.raftServer.AddPeer(name, peer.ToGrpcAddress()); err != nil {
 			return nil, err
@@ -273,3 +302,4 @@ func (s *RaftServer) DoJoinCommand() {
 	}
 
 }
+

weed/server/volume_grpc_erasure_coding.go

@@ -86,16 +86,30 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_
 		os.Remove(v.IndexFileName() + ".ecx")
 	}()
 
+	// IMPORTANT: Generate .ecx BEFORE EC shards to prevent a race condition.
+	// If .ecx were generated after EC shards, any write (e.g. from WriteNeedleBlob
+	// during replica sync) between the two steps would add entries to .idx that
+	// end up in .ecx but whose data is NOT in the EC shards — causing "shard too
+	// short" and "size mismatch" errors on reads.
+	//
+	// By generating .ecx first, it reflects the .idx state at or before the .dat
+	// is read for EC encoding. If a write sneaks in after .ecx but before/during
+	// EC encoding, the shards contain MORE data than .ecx references, which is
+	// harmless (the extra data is simply not indexed).
+
+	// write .ecx file from the current .idx
+	if err := erasure_coding.WriteSortedFileFromIdx(v.IndexFileName(), ".ecx"); err != nil {
+		return nil, fmt.Errorf("WriteSortedFileFromIdx %s: %v", v.IndexFileName(), err)
+	}
+
+	// snapshot .dat file size before encoding — must match what .ecx references
+	datSize, _, _ := v.FileStat()
+
 	// write .ec00 ~ .ec[TotalShards-1] files using context
 	if err := erasure_coding.WriteEcFilesWithContext(baseFileName, ecCtx); err != nil {
 		return nil, fmt.Errorf("WriteEcFilesWithContext %s: %v", baseFileName, err)
 	}
 
-	// write .ecx file
-	if err := erasure_coding.WriteSortedFileFromIdx(v.IndexFileName(), ".ecx"); err != nil {
-		return nil, fmt.Errorf("WriteSortedFileFromIdx %s: %v", v.IndexFileName(), err)
-	}
-
 	// write .vif files
 	var expireAtSec uint64
 	if v.Ttl != nil {
@@ -106,8 +120,6 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_
 	}
 	volumeInfo := &volume_server_pb.VolumeInfo{Version: uint32(v.Version())}
 	volumeInfo.ExpireAtSec = expireAtSec
-
-	datSize, _, _ := v.FileStat()
 	volumeInfo.DatFileSize = int64(datSize)
 
 	// Validate EC configuration before saving to .vif

weed/shell/command.go

@@ -17,4 +17,5 @@ type CommandTag string
 
 const (
 	ResourceHeavy CommandTag = "resourceHeavy"
+	Hidden        CommandTag = "hidden"
 )

weed/shell/command_s3_accesskey_create.go

@@ -0,0 +1,96 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/iam"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3AccessKeyCreate{})
+}
+
+type commandS3AccessKeyCreate struct {
+}
+
+func (c *commandS3AccessKeyCreate) Name() string {
+	return "s3.accesskey.create"
+}
+
+func (c *commandS3AccessKeyCreate) Help() string {
+	return `create an additional access key for an S3 IAM user
+
+	s3.accesskey.create -user <username>
+	s3.accesskey.create -user <username> -access_key <key> -secret_key <secret>
+
+	Generates a new credential pair for an existing user. If -access_key and
+	-secret_key are omitted, they are generated automatically.
+`
+}
+
+func (c *commandS3AccessKeyCreate) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AccessKeyCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	user := f.String("user", "", "user name")
+	accessKey := f.String("access_key", "", "access key (generated if omitted)")
+	secretKey := f.String("secret_key", "", "secret key (generated if omitted)")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+
+	ak := *accessKey
+	sk := *secretKey
+
+	if ak == "" && sk == "" {
+		var err error
+		ak, err = iam.GenerateRandomString(iam.AccessKeyIdLength, iam.CharsetUpper)
+		if err != nil {
+			return fmt.Errorf("generate access key: %v", err)
+		}
+		sk, err = iam.GenerateSecretAccessKey()
+		if err != nil {
+			return fmt.Errorf("generate secret key: %v", err)
+		}
+	} else if ak == "" || sk == "" {
+		return fmt.Errorf("both -access_key and -secret_key must be provided together, or omit both to auto-generate")
+	}
+
+	err := pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		_, err := client.CreateAccessKey(ctx, &iam_pb.CreateAccessKeyRequest{
+			Username: *user,
+			Credential: &iam_pb.Credential{
+				AccessKey: ak,
+				SecretKey: sk,
+				Status:    iam.AccessKeyStatusActive,
+			},
+		})
+		return err
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(writer, "Created access key for user %q\n", *user)
+	fmt.Fprintf(writer, "Access Key: %s\n", ak)
+	fmt.Fprintf(writer, "Secret Key: %s\n", sk)
+	fmt.Fprintln(writer)
+	fmt.Fprintln(writer, "Save these credentials - the secret key cannot be retrieved later.")
+	return nil
+}

weed/shell/command_s3_accesskey_delete.go

@@ -0,0 +1,68 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3AccessKeyDelete{})
+}
+
+type commandS3AccessKeyDelete struct {
+}
+
+func (c *commandS3AccessKeyDelete) Name() string {
+	return "s3.accesskey.delete"
+}
+
+func (c *commandS3AccessKeyDelete) Help() string {
+	return `delete an access key from an S3 IAM user
+
+	s3.accesskey.delete -user <username> -access_key <key>
+`
+}
+
+func (c *commandS3AccessKeyDelete) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AccessKeyDelete) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	user := f.String("user", "", "user name")
+	accessKey := f.String("access_key", "", "access key to delete")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+	if *accessKey == "" {
+		return fmt.Errorf("-access_key is required")
+	}
+
+	err := pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		_, err := client.DeleteAccessKey(ctx, &iam_pb.DeleteAccessKeyRequest{
+			Username:  *user,
+			AccessKey: *accessKey,
+		})
+		return err
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(writer, "Deleted access key %s from user %q\n", *accessKey, *user)
+	return nil
+}

weed/shell/command_s3_accesskey_list.go

@@ -0,0 +1,75 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"text/tabwriter"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3AccessKeyList{})
+}
+
+type commandS3AccessKeyList struct {
+}
+
+func (c *commandS3AccessKeyList) Name() string {
+	return "s3.accesskey.list"
+}
+
+func (c *commandS3AccessKeyList) Help() string {
+	return `list access keys for an S3 IAM user
+
+	s3.accesskey.list -user <username>
+`
+}
+
+func (c *commandS3AccessKeyList) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AccessKeyList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	user := f.String("user", "", "user name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+
+	return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{Username: *user})
+		if err != nil {
+			return err
+		}
+
+		if len(resp.Identity.Credentials) == 0 {
+			fmt.Fprintf(writer, "No access keys for user %q.\n", *user)
+			return nil
+		}
+
+		tw := tabwriter.NewWriter(writer, 0, 4, 2, ' ', 0)
+		fmt.Fprintln(tw, "ACCESS KEY\tSTATUS")
+		for _, cred := range resp.Identity.Credentials {
+			st := cred.Status
+			if st == "" {
+				st = "Active"
+			}
+			fmt.Fprintf(tw, "%s\t%s\n", cred.AccessKey, st)
+		}
+		return tw.Flush()
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+}

weed/shell/command_s3_accesskey_rotate.go

@@ -0,0 +1,105 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/iam"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3AccessKeyRotate{})
+}
+
+type commandS3AccessKeyRotate struct {
+}
+
+func (c *commandS3AccessKeyRotate) Name() string {
+	return "s3.accesskey.rotate"
+}
+
+func (c *commandS3AccessKeyRotate) Help() string {
+	return `rotate an access key for an S3 IAM user
+
+	s3.accesskey.rotate -user <username> -access_key <old_key>
+
+	Creates a new credential pair and deletes the old one. There is a brief
+	window where both keys are valid.
+`
+}
+
+func (c *commandS3AccessKeyRotate) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AccessKeyRotate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	user := f.String("user", "", "user name")
+	oldKey := f.String("access_key", "", "access key to rotate")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+	if *oldKey == "" {
+		return fmt.Errorf("-access_key is required")
+	}
+
+	newAK, err := iam.GenerateRandomString(iam.AccessKeyIdLength, iam.CharsetUpper)
+	if err != nil {
+		return fmt.Errorf("generate access key: %v", err)
+	}
+	newSK, err := iam.GenerateSecretAccessKey()
+	if err != nil {
+		return fmt.Errorf("generate secret key: %v", err)
+	}
+
+	err = pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		// Create new key first so there's no gap without credentials
+		_, err := client.CreateAccessKey(ctx, &iam_pb.CreateAccessKeyRequest{
+			Username: *user,
+			Credential: &iam_pb.Credential{
+				AccessKey: newAK,
+				SecretKey: newSK,
+				Status:    iam.AccessKeyStatusActive,
+			},
+		})
+		if err != nil {
+			return fmt.Errorf("create new key: %v", err)
+		}
+
+		// Delete old key
+		_, err = client.DeleteAccessKey(ctx, &iam_pb.DeleteAccessKeyRequest{
+			Username:  *user,
+			AccessKey: *oldKey,
+		})
+		if err != nil {
+			return fmt.Errorf("delete old key (new key %s was already created): %v", newAK, err)
+		}
+
+		return nil
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(writer, "Rotated access key for user %q\n", *user)
+	fmt.Fprintf(writer, "Old Key:    %s (deleted)\n", *oldKey)
+	fmt.Fprintf(writer, "Access Key: %s\n", newAK)
+	fmt.Fprintf(writer, "Secret Key: %s\n", newSK)
+	fmt.Fprintln(writer)
+	fmt.Fprintln(writer, "Save these credentials - the secret key cannot be retrieved later.")
+	return nil
+}

weed/shell/command_s3_anonymous_get.go

@@ -0,0 +1,89 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3AnonymousGet{})
+}
+
+type commandS3AnonymousGet struct {
+}
+
+func (c *commandS3AnonymousGet) Name() string {
+	return "s3.anonymous.get"
+}
+
+func (c *commandS3AnonymousGet) Help() string {
+	return `show anonymous access for a bucket
+
+	s3.anonymous.get -bucket <bucket_name>
+`
+}
+
+func (c *commandS3AnonymousGet) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AnonymousGet) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	bucket := f.String("bucket", "", "bucket name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *bucket == "" {
+		return fmt.Errorf("-bucket is required")
+	}
+
+	return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{Username: anonymousUserName})
+		if err != nil {
+			st, ok := status.FromError(err)
+			if ok && st.Code() == codes.NotFound {
+				fmt.Fprintf(writer, "Bucket: %s\nAccess: none\n", *bucket)
+				return nil
+			}
+			return err
+		}
+		if resp.Identity == nil {
+			fmt.Fprintf(writer, "Bucket: %s\nAccess: none\n", *bucket)
+			return nil
+		}
+
+		var actions []string
+		for _, a := range resp.Identity.Actions {
+			parts := strings.SplitN(a, ":", 2)
+			if len(parts) == 2 && parts[1] == *bucket {
+				actions = append(actions, parts[0])
+			}
+		}
+
+		fmt.Fprintf(writer, "Bucket: %s\n", *bucket)
+		if len(actions) == 0 {
+			fmt.Fprintln(writer, "Access: none")
+		} else {
+			sort.Strings(actions)
+			fmt.Fprintf(writer, "Access: %s\n", strings.Join(actions, ", "))
+		}
+
+		return nil
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+}

weed/shell/command_s3_anonymous_list.go

@@ -0,0 +1,91 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3AnonymousList{})
+}
+
+type commandS3AnonymousList struct {
+}
+
+func (c *commandS3AnonymousList) Name() string {
+	return "s3.anonymous.list"
+}
+
+func (c *commandS3AnonymousList) Help() string {
+	return `list all buckets with anonymous access
+
+	s3.anonymous.list
+`
+}
+
+func (c *commandS3AnonymousList) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AnonymousList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{Username: anonymousUserName})
+		if err != nil {
+			st, ok := status.FromError(err)
+			if ok && st.Code() == codes.NotFound {
+				fmt.Fprintln(writer, "No anonymous access configured.")
+				return nil
+			}
+			return err
+		}
+		if resp.Identity == nil {
+			fmt.Fprintln(writer, "No anonymous access configured.")
+			return nil
+		}
+
+		// Group actions by bucket
+		bucketActions := map[string][]string{}
+		for _, a := range resp.Identity.Actions {
+			parts := strings.SplitN(a, ":", 2)
+			if len(parts) == 2 {
+				bucketActions[parts[1]] = append(bucketActions[parts[1]], parts[0])
+			}
+		}
+
+		if len(bucketActions) == 0 {
+			fmt.Fprintln(writer, "No anonymous access configured.")
+			return nil
+		}
+
+		// Sort bucket names
+		buckets := make([]string, 0, len(bucketActions))
+		for b := range bucketActions {
+			buckets = append(buckets, b)
+		}
+		sort.Strings(buckets)
+
+		tw := tabwriter.NewWriter(writer, 0, 4, 2, ' ', 0)
+		fmt.Fprintln(tw, "BUCKET\tACCESS")
+		for _, b := range buckets {
+			actions := bucketActions[b]
+			sort.Strings(actions)
+			fmt.Fprintf(tw, "%s\t%s\n", b, strings.Join(actions, ", "))
+		}
+		return tw.Flush()
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+}

weed/shell/command_s3_anonymous_set.go

@@ -0,0 +1,144 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+const anonymousUserName = "anonymous"
+
+func init() {
+	Commands = append(Commands, &commandS3AnonymousSet{})
+}
+
+type commandS3AnonymousSet struct {
+}
+
+func (c *commandS3AnonymousSet) Name() string {
+	return "s3.anonymous.set"
+}
+
+func (c *commandS3AnonymousSet) Help() string {
+	return `set anonymous (public) access on a bucket
+
+	s3.anonymous.set -bucket <bucket_name> -access Read,List
+	s3.anonymous.set -bucket <bucket_name> -access none
+
+	Supported actions: Read, Write, List, Tagging, Admin
+	Use "none" to remove all anonymous access for the bucket.
+
+	This manages the special "anonymous" user's actions. It does not
+	use IAM policies — it sets legacy per-bucket actions directly.
+`
+}
+
+func (c *commandS3AnonymousSet) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AnonymousSet) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	bucket := f.String("bucket", "", "bucket name")
+	access := f.String("access", "", "comma-separated actions: Read,Write,List,Tagging,Admin or none")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *bucket == "" {
+		return fmt.Errorf("-bucket is required")
+	}
+	if *access == "" {
+		return fmt.Errorf("-access is required")
+	}
+
+	return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		// Get or create anonymous user
+		identity, isNew, err := getOrCreateAnonymousUser(ctx, client)
+		if err != nil {
+			return err
+		}
+
+		// Remove existing actions for this bucket
+		var kept []string
+		for _, a := range identity.Actions {
+			parts := strings.SplitN(a, ":", 2)
+			if len(parts) != 2 || parts[1] != *bucket {
+				kept = append(kept, a)
+			}
+		}
+
+		// Add new actions unless "none"
+		canonicalActions := map[string]string{
+			"read": "Read", "write": "Write", "list": "List",
+			"tagging": "Tagging", "admin": "Admin",
+		}
+		if strings.ToLower(strings.TrimSpace(*access)) != "none" {
+			seen := make(map[string]struct{})
+			for _, action := range strings.Split(*access, ",") {
+				action = strings.TrimSpace(action)
+				if action != "" {
+					canonical, ok := canonicalActions[strings.ToLower(action)]
+					if !ok {
+						return fmt.Errorf("invalid action %q: supported actions are Read, Write, List, Tagging, Admin", action)
+					}
+					if _, dup := seen[canonical]; dup {
+						continue
+					}
+					seen[canonical] = struct{}{}
+					kept = append(kept, canonical+":"+*bucket)
+				}
+			}
+		}
+
+		identity.Actions = kept
+
+		if isNew {
+			_, err = client.CreateUser(ctx, &iam_pb.CreateUserRequest{Identity: identity})
+		} else {
+			_, err = client.UpdateUser(ctx, &iam_pb.UpdateUserRequest{
+				Username: anonymousUserName,
+				Identity: identity,
+			})
+		}
+		if err != nil {
+			return err
+		}
+
+		fmt.Fprintf(writer, "Set anonymous access on bucket %q to: %s\n", *bucket, *access)
+		return nil
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+}
+
+func getOrCreateAnonymousUser(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) (*iam_pb.Identity, bool, error) {
+	resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{Username: anonymousUserName})
+	if err == nil {
+		if resp.Identity == nil {
+			return nil, false, fmt.Errorf("anonymous user returned nil identity")
+		}
+		return resp.Identity, false, nil
+	}
+
+	st, ok := status.FromError(err)
+	if ok && st != nil && st.Code() == codes.NotFound {
+		return &iam_pb.Identity{
+			Name:    anonymousUserName,
+			Actions: []string{},
+		}, true, nil
+	}
+
+	return nil, false, fmt.Errorf("failed to get anonymous user: %w", err)
+}

weed/shell/command_s3_bucket_access.go

@@ -61,8 +61,8 @@ func (c *commandS3BucketAccess) Help() string {
 `
 }
 
-func (c *commandS3BucketAccess) HasTag(CommandTag) bool {
-	return false
+func (c *commandS3BucketAccess) HasTag(tag CommandTag) bool {
+	return tag == Hidden
 }
 
 func (c *commandS3BucketAccess) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

weed/shell/command_s3_config_show.go

@@ -0,0 +1,116 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"text/tabwriter"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3ConfigShow{})
+}
+
+type commandS3ConfigShow struct {
+}
+
+func (c *commandS3ConfigShow) Name() string {
+	return "s3.config.show"
+}
+
+func (c *commandS3ConfigShow) Help() string {
+	return `show a summary of the current S3 IAM configuration
+
+	s3.config.show
+
+	Displays counts and a brief listing of users, policies, service accounts,
+	and groups. Use s3.iam.export for the full JSON dump.
+`
+}
+
+func (c *commandS3ConfigShow) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3ConfigShow) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		resp, err := client.GetConfiguration(ctx, &iam_pb.GetConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		cfg := resp.Configuration
+		if cfg == nil {
+			fmt.Fprintln(writer, "No S3 IAM configuration found.")
+			return nil
+		}
+
+		fmt.Fprintf(writer, "S3 IAM Configuration Summary\n")
+		fmt.Fprintf(writer, "============================\n\n")
+
+		// Users
+		fmt.Fprintf(writer, "Users: %d\n", len(cfg.Identities))
+		if len(cfg.Identities) > 0 {
+			tw := tabwriter.NewWriter(writer, 0, 4, 2, ' ', 0)
+			fmt.Fprintln(tw, "  NAME\tSTATUS\tSOURCE\tKEYS\tPOLICIES")
+			for _, id := range cfg.Identities {
+				status := "enabled"
+				if id.Disabled {
+					status = "disabled"
+				}
+				source := "dynamic"
+				if id.IsStatic {
+					source = "static"
+				}
+				policies := "-"
+				if len(id.PolicyNames) > 0 {
+					policies = joinMax(id.PolicyNames, 3)
+				}
+				fmt.Fprintf(tw, "  %s\t%s\t%s\t%d\t%s\n",
+					id.Name, status, source, len(id.Credentials), policies)
+			}
+			tw.Flush()
+		}
+		fmt.Fprintln(writer)
+
+		// Policies
+		fmt.Fprintf(writer, "Policies: %d\n", len(cfg.Policies))
+		if len(cfg.Policies) > 0 {
+			for _, p := range cfg.Policies {
+				fmt.Fprintf(writer, "  %s\n", p.Name)
+			}
+		}
+		fmt.Fprintln(writer)
+
+		// Service Accounts
+		fmt.Fprintf(writer, "Service Accounts: %d\n", len(cfg.ServiceAccounts))
+		if len(cfg.ServiceAccounts) > 0 {
+			for _, sa := range cfg.ServiceAccounts {
+				status := "enabled"
+				if sa.Disabled {
+					status = "disabled"
+				}
+				fmt.Fprintf(writer, "  %s (parent: %s, %s)\n", sa.Id, sa.ParentUser, status)
+			}
+		}
+		fmt.Fprintln(writer)
+
+		// Groups
+		fmt.Fprintf(writer, "Groups: %d\n", len(cfg.Groups))
+		if len(cfg.Groups) > 0 {
+			for _, g := range cfg.Groups {
+				fmt.Fprintf(writer, "  %s (%d members)\n", g.Name, len(g.Members))
+			}
+		}
+
+		return nil
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+}

weed/shell/command_s3_configure.go

@@ -39,8 +39,8 @@ func (c *commandS3Configure) Help() string {
 	`
 }
 
-func (c *commandS3Configure) HasTag(CommandTag) bool {
-	return false
+func (c *commandS3Configure) HasTag(tag CommandTag) bool {
+	return tag == Hidden
 }
 
 func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

weed/shell/command_s3_iam_export.go

@@ -0,0 +1,80 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3IAMExport{})
+}
+
+type commandS3IAMExport struct {
+}
+
+func (c *commandS3IAMExport) Name() string {
+	return "s3.iam.export"
+}
+
+func (c *commandS3IAMExport) Help() string {
+	return `export the full S3 IAM configuration as JSON
+
+	s3.iam.export
+	s3.iam.export -file backup.json
+
+	Exports all users, credentials, policies, service accounts, and groups.
+	Without -file, prints to stdout.
+`
+}
+
+func (c *commandS3IAMExport) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3IAMExport) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	file := f.String("file", "", "output file path (stdout if omitted)")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		resp, err := client.GetConfiguration(ctx, &iam_pb.GetConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+
+		var out io.Writer = writer
+		if *file != "" {
+			fp, err := os.OpenFile(*file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+			if err != nil {
+				return fmt.Errorf("create file: %v", err)
+			}
+			defer fp.Close()
+			out = fp
+		}
+
+		if err := filer.ProtoToText(out, resp.Configuration); err != nil {
+			return err
+		}
+		fmt.Fprintln(out)
+
+		if *file != "" {
+			fmt.Fprintf(writer, "Exported IAM configuration to %s\n", *file)
+		}
+		return nil
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+}

weed/shell/command_s3_iam_import.go

@@ -0,0 +1,88 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3IAMImport{})
+}
+
+type commandS3IAMImport struct {
+}
+
+func (c *commandS3IAMImport) Name() string {
+	return "s3.iam.import"
+}
+
+func (c *commandS3IAMImport) Help() string {
+	return `import S3 IAM configuration from a JSON file
+
+	s3.iam.import -file backup.json -apply
+
+	Replaces the entire IAM configuration (users, credentials, policies,
+	service accounts, groups) with the contents of the file.
+
+	Requires -apply to confirm, since this overwrites the current configuration.
+`
+}
+
+func (c *commandS3IAMImport) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3IAMImport) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	file := f.String("file", "", "input JSON file")
+	apply := f.Bool("apply", false, "confirm overwrite of the entire IAM configuration")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *file == "" {
+		return fmt.Errorf("-file is required")
+	}
+	if !*apply {
+		return fmt.Errorf("this overwrites the entire IAM configuration; use -apply to confirm")
+	}
+
+	data, err := os.ReadFile(*file)
+	if err != nil {
+		return fmt.Errorf("read file: %w", err)
+	}
+
+	config := &iam_pb.S3ApiConfiguration{}
+	if err := filer.ParseS3ConfigurationFromBytes(data, config); err != nil {
+		return fmt.Errorf("parse configuration: %w", err)
+	}
+
+	err = pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		_, err := client.PutConfiguration(ctx, &iam_pb.PutConfigurationRequest{
+			Configuration: config,
+		})
+		return err
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+	if err != nil {
+		return fmt.Errorf("put IAM configuration: %w", err)
+	}
+
+	fmt.Fprintf(writer, "Imported IAM configuration from %s\n", *file)
+	fmt.Fprintf(writer, "  Users:            %d\n", len(config.Identities))
+	fmt.Fprintf(writer, "  Policies:         %d\n", len(config.Policies))
+	fmt.Fprintf(writer, "  Service Accounts: %d\n", len(config.ServiceAccounts))
+	fmt.Fprintf(writer, "  Groups:           %d\n", len(config.Groups))
+	return nil
+}

weed/shell/command_s3_policy_attach.go

@@ -0,0 +1,93 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3PolicyAttach{})
+}
+
+type commandS3PolicyAttach struct {
+}
+
+func (c *commandS3PolicyAttach) Name() string {
+	return "s3.policy.attach"
+}
+
+func (c *commandS3PolicyAttach) Help() string {
+	return `attach a policy to an S3 IAM user
+
+	s3.policy.attach -policy <policy_name> -user <username>
+
+	The policy must already exist (create it with s3.policy -put).
+`
+}
+
+func (c *commandS3PolicyAttach) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3PolicyAttach) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	policy := f.String("policy", "", "policy name")
+	user := f.String("user", "", "user name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *policy == "" {
+		return fmt.Errorf("-policy is required")
+	}
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+
+	return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		// Verify the policy exists
+		_, err := client.GetPolicy(ctx, &iam_pb.GetPolicyRequest{Name: *policy})
+		if err != nil {
+			return fmt.Errorf("get policy %q: %w", *policy, err)
+		}
+
+		// Get the user
+		resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{Username: *user})
+		if err != nil {
+			return fmt.Errorf("get user %q: %w", *user, err)
+		}
+		if resp.Identity == nil {
+			return fmt.Errorf("user %q returned empty identity", *user)
+		}
+
+		// Check if already attached
+		for _, p := range resp.Identity.PolicyNames {
+			if p == *policy {
+				return json.NewEncoder(writer).Encode(map[string]string{"policy": *policy, "user": *user})
+			}
+		}
+
+		resp.Identity.PolicyNames = append(resp.Identity.PolicyNames, *policy)
+		_, err = client.UpdateUser(ctx, &iam_pb.UpdateUserRequest{
+			Username: *user,
+			Identity: resp.Identity,
+		})
+		if err != nil {
+			return err
+		}
+
+		return json.NewEncoder(writer).Encode(map[string]string{"policy": *policy, "user": *user})
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+}

weed/shell/command_s3_policy_detach.go

@@ -0,0 +1,90 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3PolicyDetach{})
+}
+
+type commandS3PolicyDetach struct {
+}
+
+func (c *commandS3PolicyDetach) Name() string {
+	return "s3.policy.detach"
+}
+
+func (c *commandS3PolicyDetach) Help() string {
+	return `detach a policy from an S3 IAM user
+
+	s3.policy.detach -policy <policy_name> -user <username>
+`
+}
+
+func (c *commandS3PolicyDetach) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3PolicyDetach) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	policy := f.String("policy", "", "policy name")
+	user := f.String("user", "", "user name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *policy == "" {
+		return fmt.Errorf("-policy is required")
+	}
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+
+	return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{Username: *user})
+		if err != nil {
+			return fmt.Errorf("get user %q: %w", *user, err)
+		}
+		if resp.Identity == nil {
+			return fmt.Errorf("user %q returned empty identity", *user)
+		}
+
+		found := false
+		var kept []string
+		for _, p := range resp.Identity.PolicyNames {
+			if p == *policy {
+				found = true
+			} else {
+				kept = append(kept, p)
+			}
+		}
+		if !found {
+			return fmt.Errorf("policy %q is not attached to user %q", *policy, *user)
+		}
+
+		resp.Identity.PolicyNames = kept
+		_, err = client.UpdateUser(ctx, &iam_pb.UpdateUserRequest{
+			Username: *user,
+			Identity: resp.Identity,
+		})
+		if err != nil {
+			return err
+		}
+
+		return json.NewEncoder(writer).Encode(map[string]string{"policy": *policy, "user": *user})
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+}

weed/shell/command_s3_serviceaccount_create.go

@@ -0,0 +1,132 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/iam"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3ServiceAccountCreate{})
+}
+
+type commandS3ServiceAccountCreate struct {
+}
+
+func (c *commandS3ServiceAccountCreate) Name() string {
+	return "s3.serviceaccount.create"
+}
+
+func (c *commandS3ServiceAccountCreate) Help() string {
+	return `create a service account for an S3 IAM user
+
+	s3.serviceaccount.create -user <parent_user> -description "my app"
+	s3.serviceaccount.create -user <parent_user> -actions Read,List -expiry 24h
+
+	Service accounts are linked to a parent user and can have restricted
+	permissions (a subset of the parent's actions).
+`
+}
+
+func (c *commandS3ServiceAccountCreate) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3ServiceAccountCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	user := f.String("user", "", "parent user name")
+	description := f.String("description", "", "optional description")
+	actions := f.String("actions", "", "comma-separated actions (subset of parent)")
+	expiry := f.Duration("expiry", 0, "expiration duration (e.g. 24h, 0 = no expiration)")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+
+	ak, err := iam.GenerateRandomString(iam.AccessKeyIdLength, iam.CharsetUpper)
+	if err != nil {
+		return fmt.Errorf("generate access key: %v", err)
+	}
+	sk, err := iam.GenerateSecretAccessKey()
+	if err != nil {
+		return fmt.Errorf("generate secret key: %v", err)
+	}
+
+	sa := &iam_pb.ServiceAccount{
+		ParentUser:  *user,
+		Description: *description,
+		Credential: &iam_pb.Credential{
+			AccessKey: ak,
+			SecretKey: sk,
+			Status:    iam.AccessKeyStatusActive,
+		},
+		CreatedAt: time.Now().Unix(),
+	}
+
+	validActions := map[string]string{
+		"read": "Read", "write": "Write", "list": "List",
+		"tagging": "Tagging", "admin": "Admin",
+	}
+	if *actions != "" {
+		seen := make(map[string]struct{})
+		for _, a := range strings.Split(*actions, ",") {
+			a = strings.TrimSpace(a)
+			if a != "" {
+				canonical, ok := validActions[strings.ToLower(a)]
+				if !ok {
+					return fmt.Errorf("invalid action %q: supported actions are Read, Write, List, Tagging, Admin", a)
+				}
+				if _, dup := seen[canonical]; dup {
+					continue
+				}
+				seen[canonical] = struct{}{}
+				sa.Actions = append(sa.Actions, canonical)
+			}
+		}
+	}
+
+	if *expiry < 0 {
+		return fmt.Errorf("-expiry must be >= 0")
+	}
+	if *expiry > 0 {
+		sa.Expiration = time.Now().Add(*expiry).Unix()
+	}
+
+	err = pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		_, err := client.CreateServiceAccount(ctx, &iam_pb.CreateServiceAccountRequest{
+			ServiceAccount: sa,
+		})
+		return err
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(writer, "Created service account for user %q\n", *user)
+	fmt.Fprintln(writer, "Note: use s3.serviceaccount.list to find the server-assigned ID.")
+	fmt.Fprintf(writer, "Access Key: %s\n", ak)
+	fmt.Fprintf(writer, "Secret Key: %s\n", sk)
+	if *description != "" {
+		fmt.Fprintf(writer, "Desc:       %s\n", *description)
+	}
+	if *expiry > 0 {
+		fmt.Fprintf(writer, "Expires:    %s\n", time.Unix(sa.Expiration, 0).Format(time.RFC3339))
+	}
+	fmt.Fprintln(writer)
+	fmt.Fprintln(writer, "Save these credentials - the secret key cannot be retrieved later.")
+	return nil
+}

weed/shell/command_s3_serviceaccount_delete.go

@@ -0,0 +1,61 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3ServiceAccountDelete{})
+}
+
+type commandS3ServiceAccountDelete struct {
+}
+
+func (c *commandS3ServiceAccountDelete) Name() string {
+	return "s3.serviceaccount.delete"
+}
+
+func (c *commandS3ServiceAccountDelete) Help() string {
+	return `delete a service account
+
+	s3.serviceaccount.delete -id <service_account_id>
+`
+}
+
+func (c *commandS3ServiceAccountDelete) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3ServiceAccountDelete) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	id := f.String("id", "", "service account ID")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *id == "" {
+		return fmt.Errorf("-id is required")
+	}
+
+	err := pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		_, err := client.DeleteServiceAccount(ctx, &iam_pb.DeleteServiceAccountRequest{Id: *id})
+		return err
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(writer, "Deleted service account %q\n", *id)
+	return nil
+}

weed/shell/command_s3_serviceaccount_list.go

@@ -0,0 +1,85 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"text/tabwriter"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3ServiceAccountList{})
+}
+
+type commandS3ServiceAccountList struct {
+}
+
+func (c *commandS3ServiceAccountList) Name() string {
+	return "s3.serviceaccount.list"
+}
+
+func (c *commandS3ServiceAccountList) Help() string {
+	return `list service accounts
+
+	s3.serviceaccount.list
+	s3.serviceaccount.list -user <parent_user>
+
+	Lists all service accounts, optionally filtered by parent user.
+`
+}
+
+func (c *commandS3ServiceAccountList) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3ServiceAccountList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	user := f.String("user", "", "filter by parent user (optional)")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		resp, err := client.ListServiceAccounts(ctx, &iam_pb.ListServiceAccountsRequest{})
+		if err != nil {
+			return err
+		}
+
+		var filtered []*iam_pb.ServiceAccount
+		for _, sa := range resp.ServiceAccounts {
+			if *user == "" || sa.ParentUser == *user {
+				filtered = append(filtered, sa)
+			}
+		}
+
+		if len(filtered) == 0 {
+			fmt.Fprintln(writer, "No service accounts found.")
+			return nil
+		}
+
+		tw := tabwriter.NewWriter(writer, 0, 4, 2, ' ', 0)
+		fmt.Fprintln(tw, "ID\tPARENT\tSTATUS\tDESCRIPTION")
+		for _, sa := range filtered {
+			st := "enabled"
+			if sa.Disabled {
+				st = "disabled"
+			}
+			desc := sa.Description
+			if len(desc) > 40 {
+				desc = desc[:37] + "..."
+			}
+			fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", sa.Id, sa.ParentUser, st, desc)
+		}
+		return tw.Flush()
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+}

weed/shell/command_s3_serviceaccount_show.go

@@ -0,0 +1,93 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3ServiceAccountShow{})
+}
+
+type commandS3ServiceAccountShow struct {
+}
+
+func (c *commandS3ServiceAccountShow) Name() string {
+	return "s3.serviceaccount.show"
+}
+
+func (c *commandS3ServiceAccountShow) Help() string {
+	return `show details of a service account
+
+	s3.serviceaccount.show -id <service_account_id>
+`
+}
+
+func (c *commandS3ServiceAccountShow) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3ServiceAccountShow) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	id := f.String("id", "", "service account ID")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *id == "" {
+		return fmt.Errorf("-id is required")
+	}
+
+	return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
+		client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		resp, err := client.GetServiceAccount(ctx, &iam_pb.GetServiceAccountRequest{Id: *id})
+		if err != nil {
+			return err
+		}
+		sa := resp.ServiceAccount
+
+		status := "enabled"
+		if sa.Disabled {
+			status = "disabled"
+		}
+
+		fmt.Fprintf(writer, "ID:          %s\n", sa.Id)
+		fmt.Fprintf(writer, "Parent:      %s\n", sa.ParentUser)
+		fmt.Fprintf(writer, "Status:      %s\n", status)
+		if sa.Description != "" {
+			fmt.Fprintf(writer, "Description: %s\n", sa.Description)
+		}
+		if sa.Credential != nil {
+			st := sa.Credential.Status
+			if st == "" {
+				st = "Active"
+			}
+			fmt.Fprintf(writer, "Access Key:  %s (%s)\n", sa.Credential.AccessKey, st)
+		}
+		if len(sa.Actions) > 0 {
+			fmt.Fprintf(writer, "Actions:     %s\n", strings.Join(sa.Actions, ", "))
+		}
+		if sa.Expiration > 0 {
+			fmt.Fprintf(writer, "Expires:     %s\n", time.Unix(sa.Expiration, 0).Format(time.RFC3339))
+		}
+		if sa.CreatedAt > 0 {
+			fmt.Fprintf(writer, "Created:     %s\n", time.Unix(sa.CreatedAt, 0).Format(time.RFC3339))
+		}
+		if sa.CreatedBy != "" {
+			fmt.Fprintf(writer, "Created By:  %s\n", sa.CreatedBy)
+		}
+
+		return nil
+	}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
+}