Fail fs.mergeVolumes on duplicate volume IDs

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

.claude/scheduled_tasks.lock

@@ -1 +0,0 @@
-{"sessionId":"d6574c47-eafc-4a94-9dce-f9ffea22b53c","pid":10111,"acquiredAt":1775248373916}

.github/workflows/container_release_unified.yml

@@ -39,80 +39,7 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-
-  # ── Pre-build Rust volume server binaries natively ──────────────────
-  # Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
-  # emulated cargo build into ~15 minutes of native compilation.
-  build-rust-binaries:
-    runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        include:
-          - target: x86_64-unknown-linux-musl
-            arch: amd64
-          - target: aarch64-unknown-linux-musl
-            arch: arm64
-            cross: true
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-
-      - name: Install protobuf compiler
-        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          targets: ${{ matrix.target }}
-
-      - name: Install musl tools (amd64)
-        if: ${{ !matrix.cross }}
-        run: sudo apt-get install -y musl-tools
-
-      - name: Install cross-compilation tools (arm64)
-        if: matrix.cross
-        run: |
-          sudo apt-get install -y gcc-aarch64-linux-gnu
-          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
-
-      - name: Cache cargo registry and target
-        uses: actions/cache@v5
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            seaweed-volume/target
-          key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
-          restore-keys: |
-            rust-docker-${{ matrix.target }}-
-
-      - name: Build large-disk variant
-        env:
-          SEAWEEDFS_COMMIT: ${{ github.sha }}
-        run: |
-          cd seaweed-volume
-          cargo build --release --target ${{ matrix.target }}
-          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
-
-      - name: Build normal variant
-        env:
-          SEAWEEDFS_COMMIT: ${{ github.sha }}
-        run: |
-          cd seaweed-volume
-          cargo build --release --target ${{ matrix.target }} --no-default-features
-          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: rust-volume-${{ matrix.arch }}
-          path: |
-            weed-volume-large-disk-${{ matrix.arch }}
-            weed-volume-normal-${{ matrix.arch }}
-
-  # ── Build Docker containers ─────────────────────────────────────────
   build:
-    needs: [build-rust-binaries]
     runs-on: ubuntu-latest
     strategy:
       # Build sequentially to avoid rate limits
@@ -125,23 +52,20 @@ jobs:
             dockerfile: ./docker/Dockerfile.go_build
             build_args: ""
             tag_suffix: ""
-            rust_variant: normal
-
-          # Large disk - multi-arch
+            
+          # Large disk - multi-arch  
           - variant: large_disk
             platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
             dockerfile: ./docker/Dockerfile.go_build
             build_args: TAGS=5BytesOffset
             tag_suffix: _large_disk
-            rust_variant: large-disk
-
+            
           # Full tags - multi-arch
           - variant: full
             platforms: linux/amd64,linux/arm64
             dockerfile: ./docker/Dockerfile.go_build
             build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
             tag_suffix: _full
-            rust_variant: normal
 
           # Large disk + full tags - multi-arch
           - variant: large_disk_full
@@ -149,42 +73,19 @@ jobs:
             dockerfile: ./docker/Dockerfile.go_build
             build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
             tag_suffix: _large_disk_full
-            rust_variant: large-disk
-
+            
           # RocksDB large disk - amd64 only
           - variant: rocksdb
             platforms: linux/amd64
             dockerfile: ./docker/Dockerfile.rocksdb_large
             build_args: ""
             tag_suffix: _large_disk_rocksdb
-            rust_variant: large-disk
             
     steps:
       - name: Checkout
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         uses: actions/checkout@v6
-
-      - name: Download pre-built Rust binaries
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-        uses: actions/download-artifact@v4
-        with:
-          pattern: rust-volume-*
-          merge-multiple: true
-          path: ./rust-bins
-
-      - name: Place Rust binaries in Docker context
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-        run: |
-          mkdir -p docker/weed-volume-prebuilt
-          for arch in amd64 arm64; do
-            src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
-            if [ -f "$src" ]; then
-              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
-              echo "Placed pre-built Rust binary for ${arch}"
-            fi
-          done
-          ls -la docker/weed-volume-prebuilt/
-
+        
       - name: Free Disk Space
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         run: |

.github/workflows/rust-volume-server-tests.yml

@@ -93,7 +93,7 @@ jobs:
       - name: Build Go weed binary
         run: |
           cd weed
-          go build -tags 5BytesOffset -o weed .
+          go build -o weed .
           chmod +x weed
           ./weed version
 
@@ -169,7 +169,7 @@ jobs:
       - name: Build Go weed binary
         run: |
           cd weed
-          go build -tags 5BytesOffset -o weed .
+          go build -o weed .
           chmod +x weed
           ./weed version
 

.github/workflows/rust_binaries_release.yml

@@ -39,16 +39,8 @@ jobs:
       - name: Install cross-compilation tools
         if: matrix.cross
         run: |
-          sudo dpkg --add-architecture arm64
-          sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
-          echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
-          echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
-          sudo apt-get update
-          sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
+          sudo apt-get install -y gcc-aarch64-linux-gnu
           echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
-          echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
-          echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
-          echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
 
       - name: Cache cargo registry and target
         uses: actions/cache@v5
@@ -88,7 +80,6 @@ jobs:
           rm weed-volume-normal
 
       - name: Upload release assets
-        if: startsWith(github.ref, 'refs/tags/')
         uses: softprops/action-gh-release@v2
         with:
           files: |
@@ -97,15 +88,6 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Upload artifacts
-        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: rust-volume-${{ matrix.asset_suffix }}
-          path: |
-            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
-            weed-volume_${{ matrix.asset_suffix }}.tar.gz
-
   build-rust-volume-darwin:
     permissions:
       contents: write
@@ -165,7 +147,6 @@ jobs:
           rm weed-volume-normal
 
       - name: Upload release assets
-        if: startsWith(github.ref, 'refs/tags/')
         uses: softprops/action-gh-release@v2
         with:
           files: |
@@ -174,15 +155,6 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Upload artifacts
-        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: rust-volume-${{ matrix.asset_suffix }}
-          path: |
-            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
-            weed-volume_${{ matrix.asset_suffix }}.tar.gz
-
   build-rust-volume-windows:
     permissions:
       contents: write
@@ -234,7 +206,6 @@ jobs:
           rm weed-volume-normal.exe
 
       - name: Upload release assets
-        if: startsWith(github.ref, 'refs/tags/')
         uses: softprops/action-gh-release@v2
         with:
           files: |
@@ -242,12 +213,3 @@ jobs:
             weed-volume_windows_amd64.zip
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Upload artifacts
-        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: rust-volume-windows_amd64
-          path: |
-            weed-volume_large_disk_windows_amd64.zip
-            weed-volume_windows_amd64.zip

.superset/config.json

@@ -1,5 +0,0 @@
-{
-  "setup": [],
-  "teardown": [],
-  "run": []
-}

docker/Dockerfile.go_build

@@ -16,21 +16,15 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
   && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
   && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
 
-# Rust volume server: use pre-built binary from CI when available (placed in
-# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
-# from source.  Pre-building avoids a multi-hour QEMU-emulated cargo build
-# for non-native architectures.
+# Rust volume server builder. Alpine packages avoid depending on the
+# upstream rust:alpine manifest list, which no longer includes linux/386.
 FROM alpine:3.23 as rust_builder
 ARG TARGETARCH
-ARG TAGS
-COPY weed-volume-prebuilt/ /prebuilt/
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
 WORKDIR /build/seaweed-volume
-RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
-      echo "Using pre-built Rust binary for ${TARGETARCH}" && \
-      cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \
-    elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
+ARG TAGS
+RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
       apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
       if [ "$TAGS" = "5BytesOffset" ]; then \
         cargo build --release; \
@@ -57,7 +51,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
 
 # Install dependencies and create non-root user
 RUN apk upgrade --no-cache zlib && \
-    apk add --no-cache fuse curl su-exec libgcc && \
+    apk add --no-cache fuse curl su-exec && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 

docker/compose/s3_static_users_example.json

@@ -1,53 +0,0 @@
-{
-  "identities": [
-    {
-      "name": "admin",
-      "credentials": [
-        {
-          "accessKey": "AKIAIOSFODNN7EXAMPLE",
-          "secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
-        }
-      ],
-      "actions": [
-        "Admin",
-        "Read",
-        "List",
-        "Tagging",
-        "Write"
-      ]
-    },
-    {
-      "name": "steward",
-      "credentials": [
-        {
-          "accessKey": "steward-key",
-          "secretKey": "steward-secret"
-        }
-      ],
-      "actions": [
-        "Read",
-        "List",
-        "Write"
-      ]
-    },
-    {
-      "name": "le001",
-      "credentials": [
-        {
-          "accessKey": "le001-key",
-          "secretKey": "le001-secret"
-        }
-      ],
-      "actions": [
-        "Read",
-        "List"
-      ]
-    },
-    {
-      "name": "anonymous",
-      "actions": [
-        "Read"
-      ]
-    }
-  ]
-}

go.mod

@@ -371,7 +371,7 @@ require (
 	github.com/geoffgarside/ber v1.2.0 // indirect
 	github.com/go-chi/chi/v5 v5.2.5 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect

go.sum

@@ -1076,8 +1076,8 @@ github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
-github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
+github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.18"
+appVersion: "4.17"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.18.0
+version: 4.17.0

seaweed-volume/Cargo.lock

@@ -2561,15 +2561,6 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
 
-[[package]]
-name = "openssl-src"
-version = "300.5.5+3.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f1787d533e03597a7934fd0a765f0d28e94ecc5fb7789f8053b1e699a56f709"
-dependencies = [
- "cc",
-]
-
 [[package]]
 name = "openssl-sys"
 version = "0.9.111"
@@ -2578,7 +2569,6 @@ checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321"
 dependencies = [
  "cc",
  "libc",
- "openssl-src",
  "pkg-config",
  "vcpkg",
 ]
@@ -4664,7 +4654,6 @@ dependencies = [
  "memmap2",
  "mime_guess",
  "multer",
- "openssl",
  "parking_lot 0.12.5",
  "pprof",
  "prometheus",

seaweed-volume/Cargo.toml

@@ -20,7 +20,7 @@ default = ["5bytes"]
 [dependencies]
 # Async runtime
 tokio = { version = "1", features = ["full"] }
-tokio-stream = { version = "0.1", features = ["net"] }
+tokio-stream = "0.1"
 tokio-io-timeout = "1"
 
 # gRPC + protobuf
@@ -65,6 +65,8 @@ reed-solomon-erasure = "6"
 # Logging
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+pprof = { version = "0.15", features = ["prost-codec"] }
+
 # Config
 toml = "0.8"
 serde = { version = "1", features = ["derive"] }
@@ -125,10 +127,6 @@ aws-sdk-s3 = { version = "1.125.0", default-features = false, features = ["sigv4
 aws-credential-types = "1"
 aws-types = "1"
 
-# pprof is Unix-only (requires libc/nix APIs not available on Windows)
-[target.'cfg(unix)'.dependencies]
-pprof = { version = "0.15", features = ["prost-codec"] }
-
 [dev-dependencies]
 tempfile = "3"
 

seaweed-volume/src/main.rs

@@ -10,11 +10,9 @@ use seaweed_volume::security::tls::{
     GrpcClientAuthPolicy, TlsPolicy,
 };
 use seaweed_volume::security::{Guard, SigningKey};
-#[cfg(unix)]
 use seaweed_volume::server::debug::build_debug_router;
 use seaweed_volume::server::grpc_client::load_outgoing_grpc_tls;
 use seaweed_volume::server::grpc_server::VolumeGrpcService;
-#[cfg(unix)]
 use seaweed_volume::server::profiling::CpuProfileSession;
 use seaweed_volume::server::request_id::GrpcRequestIdLayer;
 use seaweed_volume::server::volume_server::{
@@ -26,11 +24,6 @@ use seaweed_volume::storage::types::DiskType;
 
 use tokio_rustls::TlsAcceptor;
 
-#[cfg(unix)]
-type CpuProfileParam = Option<CpuProfileSession>;
-#[cfg(not(unix))]
-type CpuProfileParam = Option<()>;
-
 const GRPC_MAX_MESSAGE_SIZE: usize = 1 << 30;
 const GRPC_KEEPALIVE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
 const GRPC_KEEPALIVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
@@ -49,7 +42,6 @@ fn main() {
 
     let config = config::parse_cli();
     seaweed_volume::server::server_stats::init_process_start();
-    #[cfg(unix)]
     let cpu_profile = match CpuProfileSession::start(&config) {
         Ok(session) => session,
         Err(e) => {
@@ -57,8 +49,6 @@ fn main() {
             std::process::exit(1);
         }
     };
-    #[cfg(not(unix))]
-    let cpu_profile: Option<()> = None;
     info!(
         "SeaweedFS Volume Server (Rust) v{}",
         seaweed_volume::version::full_version()
@@ -267,7 +257,7 @@ where
 
 async fn run(
     config: VolumeServerConfig,
-    #[allow(unused_variables)] cpu_profile: CpuProfileParam,
+    cpu_profile: Option<CpuProfileSession>,
 ) -> Result<(), Box<dyn std::error::Error>> {
     // Initialize the store
     let mut store = Store::new(config.index_type);
@@ -441,12 +431,10 @@ async fn run(
     }
 
     // Build HTTP routers
-    #[allow(unused_mut)]
     let mut admin_router = seaweed_volume::server::volume_server::build_admin_router_with_ui(
         state.clone(),
         config.ui_enabled,
     );
-    #[cfg(unix)]
     if config.pprof {
         admin_router = admin_router.merge(build_debug_router());
     }
@@ -600,35 +588,31 @@ async fn run(
         })
     };
 
-    // Bind the gRPC listener before spawning to propagate bind errors at startup.
-    let grpc_listener = tokio::net::TcpListener::bind(&grpc_addr)
-        .await
-        .unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
-    let grpc_local_addr = grpc_listener
-        .local_addr()
-        .unwrap_or_else(|e| panic!("Failed to get gRPC local addr: {}", e));
-
     let grpc_handle = {
         let grpc_state = state.clone();
+        let grpc_addr = grpc_addr.clone();
         let grpc_tls_acceptor = grpc_tls_acceptor.clone();
         let mut shutdown_rx = shutdown_tx.subscribe();
-        let shutdown_tx_grpc = shutdown_tx.clone();
         tokio::spawn(async move {
+            let addr = grpc_addr.parse().expect("Invalid gRPC address");
             let grpc_service = VolumeGrpcService {
                 state: grpc_state.clone(),
             };
-            let reflection_v1 = tonic_reflection::server::Builder::configure()
-                .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
-                .build_v1()
-                .expect("Failed to build gRPC reflection v1 service");
-            let reflection_v1alpha = tonic_reflection::server::Builder::configure()
-                .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
-                .build_v1alpha()
-                .expect("Failed to build gRPC reflection v1alpha service");
-            let result = if let Some(tls_acceptor) = grpc_tls_acceptor {
-                let incoming = grpc_tls_incoming(grpc_listener, tls_acceptor);
-                info!("gRPC server listening on {} (TLS enabled)", grpc_local_addr);
-                build_grpc_server_builder()
+            if let Some(tls_acceptor) = grpc_tls_acceptor {
+                let listener = tokio::net::TcpListener::bind(&grpc_addr)
+                    .await
+                    .unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
+                let incoming = grpc_tls_incoming(listener, tls_acceptor);
+                let reflection_v1 = tonic_reflection::server::Builder::configure()
+                    .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
+                    .build_v1()
+                    .expect("Failed to build gRPC reflection v1 service");
+                let reflection_v1alpha = tonic_reflection::server::Builder::configure()
+                    .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
+                    .build_v1alpha()
+                    .expect("Failed to build gRPC reflection v1alpha service");
+                info!("gRPC server listening on {} (TLS enabled)", addr);
+                if let Err(e) = build_grpc_server_builder()
                     .layer(GrpcRequestIdLayer)
                     .add_service(reflection_v1)
                     .add_service(reflection_v1alpha)
@@ -637,25 +621,32 @@ async fn run(
                         let _ = shutdown_rx.recv().await;
                     })
                     .await
+                {
+                    error!("gRPC server error: {}", e);
+                }
             } else {
-                let incoming =
-                    tokio_stream::wrappers::TcpListenerStream::new(grpc_listener);
-                info!("gRPC server listening on {}", grpc_local_addr);
-                build_grpc_server_builder()
+                let reflection_v1 = tonic_reflection::server::Builder::configure()
+                    .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
+                    .build_v1()
+                    .expect("Failed to build gRPC reflection v1 service");
+                let reflection_v1alpha = tonic_reflection::server::Builder::configure()
+                    .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
+                    .build_v1alpha()
+                    .expect("Failed to build gRPC reflection v1alpha service");
+                info!("gRPC server listening on {}", addr);
+                if let Err(e) = build_grpc_server_builder()
                     .layer(GrpcRequestIdLayer)
                     .add_service(reflection_v1)
                     .add_service(reflection_v1alpha)
                     .add_service(build_volume_grpc_service(grpc_service))
-                    .serve_with_incoming_shutdown(incoming, async move {
+                    .serve_with_shutdown(addr, async move {
                         let _ = shutdown_rx.recv().await;
                     })
                     .await
-            };
-            if let Err(ref e) = result {
-                error!("gRPC server error: {}", e);
-                let _ = shutdown_tx_grpc.send(());
+                {
+                    error!("gRPC server error: {}", e);
+                }
             }
-            result
         })
     };
 
@@ -730,7 +721,6 @@ async fn run(
         None
     };
 
-    #[cfg(unix)]
     let debug_handle = if config.debug {
         let debug_addr = format!("0.0.0.0:{}", config.debug_port);
         info!("Debug pprof server listening on {}", debug_addr);
@@ -752,8 +742,6 @@ async fn run(
     } else {
         None
     };
-    #[cfg(not(unix))]
-    let debug_handle: Option<tokio::task::JoinHandle<()>> = None;
 
     let metrics_push_handle = {
         let push_state = state.clone();
@@ -764,40 +752,9 @@ async fn run(
         }))
     };
 
-    // Wait for servers. Use select! with &mut so the losing handle is not
-    // dropped, then await it explicitly afterward.
-    let mut server_err: Option<String> = None;
-    let mut http_handle = http_handle;
-    let mut grpc_handle = grpc_handle;
-    let grpc_finished_first = tokio::select! {
-        _ = &mut http_handle => false,
-        _ = &mut grpc_handle => true,
-    };
-    // Inspect the gRPC result (already resolved if it finished first,
-    // otherwise await it now).
-    let grpc_result = if grpc_finished_first {
-        grpc_handle.await
-    } else {
-        // HTTP finished first; gRPC is still running. Await it.
-        grpc_handle.await
-    };
-    match grpc_result {
-        Ok(Ok(())) => {}
-        Ok(Err(e)) => {
-            let msg = format!("gRPC server exited with error: {}", e);
-            error!("{}", msg);
-            server_err = Some(msg);
-            // serve error already sent shutdown inside the task
-        }
-        Err(e) => {
-            let msg = format!("gRPC task panicked: {}", e);
-            error!("{}", msg);
-            server_err = Some(msg);
-            let _ = shutdown_tx.send(());
-        }
-    }
-    // Ensure the HTTP handle completes too.
+    // Wait for all servers
     let _ = http_handle.await;
+    let _ = grpc_handle.await;
     if let Some(h) = public_handle {
         let _ = h.await;
     }
@@ -817,15 +774,10 @@ async fn run(
     // Close all volumes (flush and release file handles) matching Go's Shutdown()
     state.store.write().unwrap().close();
 
-    #[cfg(unix)]
     if let Some(cpu_profile) = cpu_profile {
         cpu_profile.finish().map_err(std::io::Error::other)?;
     }
 
-    if let Some(err_msg) = server_err {
-        return Err(std::io::Error::other(err_msg).into());
-    }
-
     info!("Volume server stopped.");
     Ok(())
 }

seaweed-volume/src/server/grpc_server.rs

@@ -1867,7 +1867,6 @@ impl VolumeServer for VolumeGrpcService {
         {
             let needle_header = resp.needle_header;
             let mut needle_body = resp.needle_body;
-            let resp_version = resp.version;
 
             if needle_header.is_empty() {
                 continue;
@@ -1892,36 +1891,8 @@ impl VolumeServer for VolumeGrpcService {
             // Parse needle from header + body
             let mut n = Needle::default();
             n.read_header(&needle_header);
-
-            if n.size.0 < 0 {
-                return Err(Status::invalid_argument(format!(
-                    "unexpected negative needle size {} for needle {}",
-                    n.size.0, n.id.0
-                )));
-            } else if n.size.0 > 0 {
-                // Normal needle: parse the body fields (DataSize, Data, flags, etc.)
-                n.read_body_v2(&needle_body)
-                    .map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
-            } else {
-                // Delete tombstone (size == 0): body is checksum + timestamp
-                // (V3) or checksum only (V2) + padding. Validate minimum
-                // footer length for the protocol version.
-                use crate::storage::types::{
-                    NEEDLE_CHECKSUM_SIZE, TIMESTAMP_SIZE, VERSION_3, Version,
-                };
-                let version = Version(resp_version as u8);
-                let min_footer = if version >= VERSION_3 {
-                    NEEDLE_CHECKSUM_SIZE + TIMESTAMP_SIZE
-                } else {
-                    NEEDLE_CHECKSUM_SIZE
-                };
-                if needle_body.len() < min_footer {
-                    return Err(Status::invalid_argument(format!(
-                        "tombstone needle {} body too short: got {} bytes, need >= {} for version {}",
-                        n.id.0, needle_body.len(), min_footer, resp_version
-                    )));
-                }
-            }
+            n.read_body_v2(&needle_body)
+                .map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
 
             // Write needle to local volume
             let mut store = state.store.write().unwrap();
@@ -4076,12 +4047,11 @@ fn find_last_append_at_ns(idx_path: &str, dat_path: &str, version: u32) -> Optio
     let mut header = [0u8; 16];
     dat_file.read_exact(&mut header).ok()?;
     let needle_size = i32::from_be_bytes([header[12], header[13], header[14], header[15]]);
-    if needle_size < 0 {
+    if needle_size <= 0 {
         return None;
     }
 
     // Seek to tail: offset + 16 (header) + size -> checksum (4) + timestamp (8)
-    // For delete needles (size == 0), the tail is right after the header.
     let tail_offset = actual_offset as u64 + 16 + needle_size as u64;
     dat_file.seek(SeekFrom::Start(tail_offset)).ok()?;
 

seaweed-volume/src/server/mod.rs

@@ -1,11 +1,9 @@
-#[cfg(unix)]
 pub mod debug;
 pub mod grpc_client;
 pub mod grpc_server;
 pub mod handlers;
 pub mod heartbeat;
 pub mod memory_status;
-#[cfg(unix)]
 pub mod profiling;
 pub mod request_id;
 pub mod server_stats;

seaweed-volume/src/storage/erasure_coding/ec_encoder.rs

@@ -6,7 +6,7 @@
 use std::fs::File;
 use std::io;
 #[cfg(not(unix))]
-use std::io::{Read, Seek, SeekFrom};
+use std::io::{Seek, SeekFrom};
 
 use reed_solomon_erasure::galois_8::ReedSolomon;
 

test/fuse_integration/concurrent_operations_test.go

@@ -249,18 +249,11 @@ func testConcurrentDirectoryOperations(t *testing.T, framework *FuseTestFramewor
 					return
 				}
 
-				// Create file in subdirectory with retry for transient FUSE errors
+				// Create file in subdirectory
 				testFile := filepath.Join(subDir, "test.txt")
 				content := []byte(fmt.Sprintf("Worker %d, Subdir %d", workerID, i))
-				var writeErr error
-				for attempt := 0; attempt < 3; attempt++ {
-					if writeErr = os.WriteFile(testFile, content, 0644); writeErr == nil {
-						break
-					}
-					time.Sleep(100 * time.Millisecond)
-				}
-				if writeErr != nil {
-					addError(fmt.Errorf("worker %d file %d: %v", workerID, i, writeErr))
+				if err := os.WriteFile(testFile, content, 0644); err != nil {
+					addError(fmt.Errorf("worker %d file %d: %v", workerID, i, err))
 					return
 				}
 			}

test/fuse_integration/git_operations_test.go

@@ -85,19 +85,9 @@ func testGitCloneAndPull(t *testing.T, mountPoint, localDir string) {
 	branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
 	gitRun(t, localClone, "push", "origin", branch)
 
-	// The bare repo lives on the FUSE mount and can briefly disappear after
-	// the push completes, and pushed pack objects may not be immediately
-	// consistent on the FUSE layer. Give the mount a chance to settle, then
-	// recover from the local clone if the remote is still missing.
-	if !waitForBareRepoEventually(t, bareRepo, 10*time.Second) {
-		t.Logf("bare repo %s did not stabilise after push; forcing recovery before clone", bareRepo)
-	}
-	refreshDirEntry(t, bareRepo)
-	time.Sleep(1 * time.Second)
-
 	// ---- Phase 3: Clone from mount bare repo into on-mount working dir ----
 	t.Log("Phase 3: clone from mount bare repo to on-mount working dir")
-	ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
+	gitRun(t, "", "clone", bareRepo, mountClone)
 
 	assertFileContains(t, filepath.Join(mountClone, "README.md"), "# Updated")
 	assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v2")
@@ -300,7 +290,7 @@ func waitForBareRepoEventually(t *testing.T, bareRepo string, timeout time.Durat
 	t.Helper()
 	deadline := time.Now().Add(timeout)
 	for time.Now().Before(deadline) {
-		if isBareRepoAccessible(bareRepo) {
+		if isBareRepo(bareRepo) {
 			return true
 		}
 		refreshDirEntry(t, bareRepo)
@@ -322,14 +312,6 @@ func isBareRepo(bareRepo string) bool {
 	return true
 }
 
-func isBareRepoAccessible(bareRepo string) bool {
-	if !isBareRepo(bareRepo) {
-		return false
-	}
-	out, err := tryGitCommand("", "--git-dir="+bareRepo, "rev-parse", "--is-bare-repository")
-	return err == nil && out == "true"
-}
-
 func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
 	t.Helper()
 	require.NoError(t, tryEnsureMountClone(bareRepo, mountClone))
@@ -338,7 +320,7 @@ func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
 // tryEnsureBareRepo verifies the bare repo on the FUSE mount exists.
 // If it has vanished, it re-creates it from the local clone.
 func tryEnsureBareRepo(bareRepo, localClone string) error {
-	if isBareRepoAccessible(bareRepo) {
+	if _, err := os.Stat(filepath.Join(bareRepo, "HEAD")); err == nil {
 		return nil
 	}
 	branch, err := tryGitCommand(localClone, "rev-parse", "--abbrev-ref", "HEAD")
@@ -360,36 +342,6 @@ func tryEnsureBareRepo(bareRepo, localClone string) error {
 	return nil
 }
 
-func ensureMountCloneFromBareWithRecovery(t *testing.T, bareRepo, localClone, mountClone string) {
-	t.Helper()
-	const maxAttempts = 3
-	var lastErr error
-	for attempt := 1; attempt <= maxAttempts; attempt++ {
-		if lastErr = tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone); lastErr == nil {
-			return
-		}
-		if attempt == maxAttempts {
-			require.NoError(t, lastErr, "git clone %s %s failed after %d recovery attempts", bareRepo, mountClone, maxAttempts)
-		}
-		t.Logf("clone recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
-		os.RemoveAll(mountClone)
-		time.Sleep(2 * time.Second)
-	}
-}
-
-func tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone string) error {
-	if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
-		return fmt.Errorf("ensure bare repo: %w", err)
-	}
-	if err := tryEnsureMountClone(bareRepo, mountClone); err != nil {
-		return fmt.Errorf("ensure mount clone: %w", err)
-	}
-	if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
-		return fmt.Errorf("verify mount clone: %w", err)
-	}
-	return nil
-}
-
 // tryEnsureMountClone is like ensureMountClone but returns an error instead
 // of failing the test, for use in recovery loops.
 func tryEnsureMountClone(bareRepo, mountClone string) error {
@@ -598,33 +550,3 @@ func TestTryEnsureBareRepoPreservesCurrentBranch(t *testing.T) {
 	assert.Equal(t, branch, restoredHead, "clone from recovered bare repo should check out the current branch")
 	assertFileContains(t, filepath.Join(restoredClone, "README.md"), "hello recovery")
 }
-
-func TestEnsureMountCloneFromBareWithRecoveryRecreatesMissingBareRepo(t *testing.T) {
-	tempDir, err := os.MkdirTemp("", "git_mount_clone_recovery_")
-	require.NoError(t, err)
-	defer os.RemoveAll(tempDir)
-
-	bareRepo := filepath.Join(tempDir, "repo.git")
-	localClone := filepath.Join(tempDir, "clone")
-	mountClone := filepath.Join(tempDir, "mount-clone")
-
-	gitRun(t, "", "init", "--bare", bareRepo)
-	gitRun(t, "", "clone", bareRepo, localClone)
-	gitRun(t, localClone, "config", "user.email", "test@seaweedfs.test")
-	gitRun(t, localClone, "config", "user.name", "Test")
-
-	writeFile(t, localClone, "README.md", "hello clone recovery\n")
-	gitRun(t, localClone, "add", "README.md")
-	gitRun(t, localClone, "commit", "-m", "initial commit")
-
-	branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
-	gitRun(t, localClone, "push", "origin", branch)
-
-	require.NoError(t, os.RemoveAll(bareRepo))
-
-	ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
-
-	head := gitOutput(t, mountClone, "rev-parse", "--abbrev-ref", "HEAD")
-	assert.Equal(t, branch, head, "recovered clone should stay on the pushed branch")
-	assertFileContains(t, filepath.Join(mountClone, "README.md"), "hello clone recovery")
-}

test/kafka/go.mod

@@ -43,25 +43,25 @@ require (
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/aws/aws-sdk-go v1.55.8 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.32.13 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.13 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.32.9 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect
 	github.com/aws/smithy-go v1.24.2 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -104,7 +104,7 @@ require (
 	github.com/go-chi/chi/v5 v5.2.5 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
 	github.com/go-git/go-billy/v5 v5.6.2 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect

test/kafka/go.sum

@@ -112,44 +112,44 @@ github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc h1:LoL75er
 github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc/go.mod h1:w648aMHEgFYS6xb0KVMMtZ2uMeemhiKCuD2vj6gY52A=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
-github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
+github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
-github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
+github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A=
+github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
 github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
 github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
@@ -284,8 +284,8 @@ github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
-github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
+github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -707,8 +707,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
-github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=

test/s3/distributed_lock/distributed_lock_cluster_test.go

@@ -21,9 +21,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc"
@@ -141,7 +139,6 @@ func startDistributedLockCluster(t *testing.T) *distributedLockCluster {
 		require.NoError(t, cluster.waitForTCP(cluster.filerGRPCAddress(i), 30*time.Second), "wait for filer %d grpc\n%s", i, cluster.tailLog(fmt.Sprintf("filer%d.log", i)))
 	}
 	require.NoError(t, cluster.waitForFilerCount(2, 30*time.Second), "wait for filer group registration")
-	require.NoError(t, cluster.waitForLockRingConverged(30*time.Second), "wait for lock ring convergence")
 
 	for i := 0; i < 2; i++ {
 		require.NoError(t, cluster.startS3(i), "start s3 %d", i)
@@ -414,130 +411,6 @@ func (c *distributedLockCluster) waitForFilerCount(expected int, timeout time.Du
 	return fmt.Errorf("timed out waiting for %d filers in group %q", expected, c.filerGroup)
 }
 
-// waitForLockRingConverged verifies that both filers have a consistent view of the
-// lock ring by acquiring the same lock through each filer and checking mutual exclusion.
-// This guards against the window between master seeing both filers and the filers
-// actually receiving the LockRingUpdate broadcast (delayed by the stabilization timer).
-//
-// A single arbitrary key could false-pass: if the key's real primary is filer0 and
-// filer0 has a stale ring (only sees itself), it still grants correctly because it IS
-// the primary. To catch the stale ring we must also test a key whose real primary is
-// filer1. So we generate one test key per primary filer using the same consistent-hash
-// ring as production, and require mutual exclusion for all of them.
-func (c *distributedLockCluster) waitForLockRingConverged(timeout time.Duration) error {
-	deadline := time.Now().Add(timeout)
-
-	owners := make([]pb.ServerAddress, 0, len(c.filerPorts))
-	for i := range c.filerPorts {
-		owners = append(owners, c.filerServerAddress(i))
-	}
-	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
-	ring.SetServers(owners)
-
-	attempt := 0
-	for time.Now().Before(deadline) {
-		// Generate one unique test key per primary filer
-		testKeys := c.convergenceKeysPerPrimary(ring, owners, attempt)
-		attempt++
-
-		allConverged := true
-		for _, key := range testKeys {
-			converged, _ := c.checkLockMutualExclusion(key)
-			if !converged {
-				allConverged = false
-				break
-			}
-		}
-		if allConverged {
-			return nil
-		}
-		time.Sleep(500 * time.Millisecond)
-	}
-	return fmt.Errorf("lock ring did not converge: both filers independently grant the same lock")
-}
-
-// convergenceKeysPerPrimary returns one lock key per distinct primary filer,
-// using the same consistent-hash ring as production routing.
-func (c *distributedLockCluster) convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
-	keysByOwner := make(map[pb.ServerAddress]string, len(owners))
-	for i := 0; len(keysByOwner) < len(owners) && i < 1024; i++ {
-		candidate := fmt.Sprintf("convergence-%d-%d", attempt, i)
-		primary := ring.GetPrimary(candidate)
-		if _, exists := keysByOwner[primary]; !exists {
-			keysByOwner[primary] = candidate
-		}
-	}
-	keys := make([]string, 0, len(keysByOwner))
-	for _, k := range keysByOwner {
-		keys = append(keys, k)
-	}
-	return keys
-}
-
-// checkLockMutualExclusion acquires a lock via filer0, then tries the same lock via filer1.
-// Returns true if mutual exclusion holds (second attempt is denied).
-func (c *distributedLockCluster) checkLockMutualExclusion(testKey string) (bool, error) {
-	conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
-	if err != nil {
-		return false, err
-	}
-	defer conn0.Close()
-
-	conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
-	if err != nil {
-		return false, err
-	}
-	defer conn1.Close()
-
-	client0 := filer_pb.NewSeaweedFilerClient(conn0)
-	client1 := filer_pb.NewSeaweedFilerClient(conn1)
-
-	// Acquire lock via filer0
-	ctx0, cancel0 := context.WithTimeout(context.Background(), 2*time.Second)
-	defer cancel0()
-	resp0, err := client0.DistributedLock(ctx0, &filer_pb.LockRequest{
-		Name:          testKey,
-		SecondsToLock: 5,
-		Owner:         "convergence-filer0",
-	})
-	if err != nil || resp0.RenewToken == "" {
-		return false, fmt.Errorf("filer0 lock failed: err=%v resp=%v", err, resp0)
-	}
-	defer func() {
-		// Always release the lock we acquired
-		client0.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
-			Name:       testKey,
-			RenewToken: resp0.RenewToken,
-		})
-	}()
-
-	// Try the same lock via filer1 - should be denied
-	ctx1, cancel1 := context.WithTimeout(context.Background(), 2*time.Second)
-	defer cancel1()
-	resp1, err := client1.DistributedLock(ctx1, &filer_pb.LockRequest{
-		Name:          testKey,
-		SecondsToLock: 5,
-		Owner:         "convergence-filer1",
-	})
-	if err != nil {
-		return false, err
-	}
-	if resp1.RenewToken != "" {
-		// Both filers granted the lock - ring not converged. Release the second lock.
-		client1.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
-			Name:       testKey,
-			RenewToken: resp1.RenewToken,
-		})
-		return false, nil
-	}
-	// Verify the denial is specifically because the lock is already held,
-	// not due to a transient error that might give a false positive.
-	if !strings.Contains(resp1.Error, "lock already owned") {
-		return false, nil
-	}
-	return true, nil
-}
-
 func (c *distributedLockCluster) waitForHTTP(url string, timeout time.Duration) error {
 	client := &net.Dialer{Timeout: time.Second}
 	httpClient := &httpClientWithDialer{dialer: client}

test/s3/distributed_lock/distributed_lock_test.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"sort"
 	"strings"
 	"sync"
 	"testing"
@@ -14,9 +15,9 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/smithy-go"
-	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -148,15 +149,14 @@ func (c *distributedLockCluster) findLockOwnerKeys(bucket, prefix string) map[pb
 	for i := range c.filerPorts {
 		owners = append(owners, c.filerServerAddress(i))
 	}
-
-	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
-	ring.SetServers(owners)
+	sort.Slice(owners, func(i, j int) bool {
+		return owners[i] < owners[j]
+	})
 
 	keysByOwner := make(map[pb.ServerAddress]string, len(owners))
 	for i := 0; i < 1024 && len(keysByOwner) < len(owners); i++ {
 		key := fmt.Sprintf("%s-%03d.txt", prefix, i)
-		lockKey := fmt.Sprintf("s3.object.write:/buckets/%s/%s", bucket, s3_constants.NormalizeObjectKey(key))
-		lockOwner := ring.GetPrimary(lockKey)
+		lockOwner := ownerForObjectLock(bucket, key, owners)
 		if _, exists := keysByOwner[lockOwner]; !exists {
 			keysByOwner[lockOwner] = key
 		}
@@ -164,6 +164,15 @@ func (c *distributedLockCluster) findLockOwnerKeys(bucket, prefix string) map[pb
 	return keysByOwner
 }
 
+func ownerForObjectLock(bucket, object string, owners []pb.ServerAddress) pb.ServerAddress {
+	lockKey := fmt.Sprintf("s3.object.write:/buckets/%s/%s", bucket, s3_constants.NormalizeObjectKey(object))
+	hash := util.HashStringToLong(lockKey)
+	if hash < 0 {
+		hash = -hash
+	}
+	return owners[hash%int64(len(owners))]
+}
+
 func lockOwnerLabel(owner pb.ServerAddress) string {
 	replacer := strings.NewReplacer(":", "_", ".", "_")
 	return "owner_" + replacer.Replace(string(owner))

test/s3/iam/s3_sts_get_federation_token_test.go

@@ -1,511 +0,0 @@
-package iam
-
-import (
-	"encoding/xml"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// GetFederationTokenTestResponse represents the STS GetFederationToken response
-type GetFederationTokenTestResponse struct {
-	XMLName xml.Name `xml:"GetFederationTokenResponse"`
-	Result  struct {
-		Credentials struct {
-			AccessKeyId     string `xml:"AccessKeyId"`
-			SecretAccessKey string `xml:"SecretAccessKey"`
-			SessionToken    string `xml:"SessionToken"`
-			Expiration      string `xml:"Expiration"`
-		} `xml:"Credentials"`
-		FederatedUser struct {
-			FederatedUserId string `xml:"FederatedUserId"`
-			Arn             string `xml:"Arn"`
-		} `xml:"FederatedUser"`
-	} `xml:"GetFederationTokenResult"`
-}
-
-func getTestCredentials() (string, string) {
-	accessKey := os.Getenv("STS_TEST_ACCESS_KEY")
-	if accessKey == "" {
-		accessKey = "admin"
-	}
-	secretKey := os.Getenv("STS_TEST_SECRET_KEY")
-	if secretKey == "" {
-		secretKey = "admin"
-	}
-	return accessKey, secretKey
-}
-
-// isGetFederationTokenImplemented checks if the running server supports GetFederationToken
-func isGetFederationTokenImplemented(t *testing.T) bool {
-	accessKey, secretKey := getTestCredentials()
-	resp, err := callSTSAPIWithSigV4(t, url.Values{
-		"Action":  {"GetFederationToken"},
-		"Version": {"2011-06-15"},
-		"Name":    {"probe"},
-	}, accessKey, secretKey)
-	if err != nil {
-		return false
-	}
-	defer resp.Body.Close()
-
-	body, _ := io.ReadAll(resp.Body)
-	var errResp STSErrorTestResponse
-	if xml.Unmarshal(body, &errResp) == nil {
-		if errResp.Error.Code == "InvalidAction" || errResp.Error.Code == "NotImplemented" {
-			return false
-		}
-	}
-	return true
-}
-
-// TestSTSGetFederationTokenValidation tests input validation for the GetFederationToken endpoint
-func TestSTSGetFederationTokenValidation(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	if !isSTSEndpointRunning(t) {
-		t.Fatal("SeaweedFS STS endpoint is not running at", TestSTSEndpoint, "- please run 'make setup-all-tests' first")
-	}
-
-	if !isGetFederationTokenImplemented(t) {
-		t.Fatal("GetFederationToken action is not implemented in the running server")
-	}
-
-	accessKey, secretKey := getTestCredentials()
-
-	t.Run("missing_name", func(t *testing.T) {
-		resp, err := callSTSAPIWithSigV4(t, url.Values{
-			"Action":  {"GetFederationToken"},
-			"Version": {"2011-06-15"},
-			// Name is missing
-		}, accessKey, secretKey)
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		body, _ := io.ReadAll(resp.Body)
-		var errResp STSErrorTestResponse
-		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
-		assert.Equal(t, "MissingParameter", errResp.Error.Code)
-	})
-
-	t.Run("name_too_short", func(t *testing.T) {
-		resp, err := callSTSAPIWithSigV4(t, url.Values{
-			"Action":  {"GetFederationToken"},
-			"Version": {"2011-06-15"},
-			"Name":    {"A"},
-		}, accessKey, secretKey)
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		body, _ := io.ReadAll(resp.Body)
-		var errResp STSErrorTestResponse
-		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
-		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
-	})
-
-	t.Run("name_too_long", func(t *testing.T) {
-		resp, err := callSTSAPIWithSigV4(t, url.Values{
-			"Action":  {"GetFederationToken"},
-			"Version": {"2011-06-15"},
-			"Name":    {strings.Repeat("A", 33)},
-		}, accessKey, secretKey)
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		body, _ := io.ReadAll(resp.Body)
-		var errResp STSErrorTestResponse
-		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
-		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
-	})
-
-	t.Run("name_invalid_characters", func(t *testing.T) {
-		resp, err := callSTSAPIWithSigV4(t, url.Values{
-			"Action":  {"GetFederationToken"},
-			"Version": {"2011-06-15"},
-			"Name":    {"bad name"},
-		}, accessKey, secretKey)
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		body, _ := io.ReadAll(resp.Body)
-		var errResp STSErrorTestResponse
-		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
-		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
-	})
-
-	t.Run("duration_too_short", func(t *testing.T) {
-		resp, err := callSTSAPIWithSigV4(t, url.Values{
-			"Action":          {"GetFederationToken"},
-			"Version":         {"2011-06-15"},
-			"Name":            {"TestApp"},
-			"DurationSeconds": {"100"},
-		}, accessKey, secretKey)
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		body, _ := io.ReadAll(resp.Body)
-		var errResp STSErrorTestResponse
-		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
-		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
-	})
-
-	t.Run("duration_too_long", func(t *testing.T) {
-		resp, err := callSTSAPIWithSigV4(t, url.Values{
-			"Action":          {"GetFederationToken"},
-			"Version":         {"2011-06-15"},
-			"Name":            {"TestApp"},
-			"DurationSeconds": {"200000"},
-		}, accessKey, secretKey)
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		body, _ := io.ReadAll(resp.Body)
-		var errResp STSErrorTestResponse
-		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
-		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
-	})
-
-	t.Run("malformed_policy", func(t *testing.T) {
-		resp, err := callSTSAPIWithSigV4(t, url.Values{
-			"Action":  {"GetFederationToken"},
-			"Version": {"2011-06-15"},
-			"Name":    {"TestApp"},
-			"Policy":  {"not-valid-json"},
-		}, accessKey, secretKey)
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		body, _ := io.ReadAll(resp.Body)
-		var errResp STSErrorTestResponse
-		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
-		assert.Equal(t, "MalformedPolicyDocument", errResp.Error.Code)
-	})
-
-	t.Run("anonymous_rejected", func(t *testing.T) {
-		// GetFederationToken requires SigV4, anonymous should fail
-		resp, err := callSTSAPI(t, url.Values{
-			"Action":  {"GetFederationToken"},
-			"Version": {"2011-06-15"},
-			"Name":    {"TestApp"},
-		})
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		assert.NotEqual(t, http.StatusOK, resp.StatusCode)
-	})
-}
-
-// TestSTSGetFederationTokenRejectTemporaryCredentials tests that temporary
-// credentials (session tokens) are rejected by GetFederationToken
-func TestSTSGetFederationTokenRejectTemporaryCredentials(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	if !isSTSEndpointRunning(t) {
-		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
-	}
-
-	if !isGetFederationTokenImplemented(t) {
-		t.Skip("GetFederationToken not implemented")
-	}
-
-	accessKey, secretKey := getTestCredentials()
-
-	// First, obtain temporary credentials via AssumeRole
-	resp, err := callSTSAPIWithSigV4(t, url.Values{
-		"Action":          {"AssumeRole"},
-		"Version":         {"2011-06-15"},
-		"RoleArn":         {"arn:aws:iam::role/admin"},
-		"RoleSessionName": {"temp-session"},
-	}, accessKey, secretKey)
-	require.NoError(t, err)
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	require.NoError(t, err)
-
-	if resp.StatusCode != http.StatusOK {
-		t.Skipf("AssumeRole failed (may not be configured): status=%d body=%s", resp.StatusCode, string(body))
-	}
-
-	var assumeResp AssumeRoleTestResponse
-	require.NoError(t, xml.Unmarshal(body, &assumeResp), "Parse AssumeRole response: %s", string(body))
-
-	tempAccessKey := assumeResp.Result.Credentials.AccessKeyId
-	tempSecretKey := assumeResp.Result.Credentials.SecretAccessKey
-	tempSessionToken := assumeResp.Result.Credentials.SessionToken
-	require.NotEmpty(t, tempAccessKey)
-	require.NotEmpty(t, tempSessionToken)
-
-	// Now try GetFederationToken with the temporary credentials
-	// Include X-Amz-Security-Token header which marks this as a temp credential call
-	params := url.Values{
-		"Action":  {"GetFederationToken"},
-		"Version": {"2011-06-15"},
-		"Name":    {"ShouldFail"},
-	}
-
-	reqBody := params.Encode()
-	req, err := http.NewRequest(http.MethodPost, TestSTSEndpoint+"/", strings.NewReader(reqBody))
-	require.NoError(t, err)
-
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	req.Header.Set("X-Amz-Security-Token", tempSessionToken)
-
-	creds := credentials.NewStaticCredentials(tempAccessKey, tempSecretKey, tempSessionToken)
-	signer := v4.NewSigner(creds)
-	_, err = signer.Sign(req, strings.NewReader(reqBody), "sts", "us-east-1", time.Now())
-	require.NoError(t, err)
-
-	client := &http.Client{Timeout: 30 * time.Second}
-	resp2, err := client.Do(req)
-	require.NoError(t, err)
-	defer resp2.Body.Close()
-
-	body2, _ := io.ReadAll(resp2.Body)
-	assert.Equal(t, http.StatusForbidden, resp2.StatusCode,
-		"GetFederationToken should reject temporary credentials: %s", string(body2))
-	assert.Contains(t, string(body2), "temporary credentials",
-		"Error should mention temporary credentials")
-}
-
-// TestSTSGetFederationTokenSuccess tests a successful GetFederationToken call
-// and verifies the returned credentials can be used to access S3
-func TestSTSGetFederationTokenSuccess(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	if !isSTSEndpointRunning(t) {
-		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
-	}
-
-	if !isGetFederationTokenImplemented(t) {
-		t.Skip("GetFederationToken not implemented")
-	}
-
-	accessKey, secretKey := getTestCredentials()
-
-	t.Run("basic_success", func(t *testing.T) {
-		resp, err := callSTSAPIWithSigV4(t, url.Values{
-			"Action":  {"GetFederationToken"},
-			"Version": {"2011-06-15"},
-			"Name":    {"AppClient"},
-		}, accessKey, secretKey)
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		body, err := io.ReadAll(resp.Body)
-		require.NoError(t, err)
-		t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
-
-		if resp.StatusCode != http.StatusOK {
-			var errResp STSErrorTestResponse
-			_ = xml.Unmarshal(body, &errResp)
-			t.Fatalf("GetFederationToken failed: code=%s message=%s", errResp.Error.Code, errResp.Error.Message)
-		}
-
-		var stsResp GetFederationTokenTestResponse
-		require.NoError(t, xml.Unmarshal(body, &stsResp), "Parse response: %s", string(body))
-
-		creds := stsResp.Result.Credentials
-		assert.NotEmpty(t, creds.AccessKeyId)
-		assert.NotEmpty(t, creds.SecretAccessKey)
-		assert.NotEmpty(t, creds.SessionToken)
-		assert.NotEmpty(t, creds.Expiration)
-
-		fedUser := stsResp.Result.FederatedUser
-		assert.Contains(t, fedUser.Arn, "federated-user/AppClient")
-		assert.Contains(t, fedUser.FederatedUserId, "AppClient")
-	})
-
-	t.Run("with_custom_duration", func(t *testing.T) {
-		resp, err := callSTSAPIWithSigV4(t, url.Values{
-			"Action":          {"GetFederationToken"},
-			"Version":         {"2011-06-15"},
-			"Name":            {"DurationTest"},
-			"DurationSeconds": {"3600"},
-		}, accessKey, secretKey)
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		body, _ := io.ReadAll(resp.Body)
-		t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
-
-		if resp.StatusCode == http.StatusOK {
-			var stsResp GetFederationTokenTestResponse
-			require.NoError(t, xml.Unmarshal(body, &stsResp))
-			assert.NotEmpty(t, stsResp.Result.Credentials.AccessKeyId)
-
-			// Verify expiration is roughly 1 hour from now
-			expTime, err := time.Parse(time.RFC3339, stsResp.Result.Credentials.Expiration)
-			require.NoError(t, err)
-			diff := time.Until(expTime)
-			assert.InDelta(t, 3600, diff.Seconds(), 60,
-				"Expiration should be ~1 hour from now")
-		}
-	})
-
-	t.Run("with_36_hour_duration", func(t *testing.T) {
-		// GetFederationToken allows up to 36 hours (unlike AssumeRole's 12h max)
-		resp, err := callSTSAPIWithSigV4(t, url.Values{
-			"Action":          {"GetFederationToken"},
-			"Version":         {"2011-06-15"},
-			"Name":            {"LongDuration"},
-			"DurationSeconds": {"129600"}, // 36 hours
-		}, accessKey, secretKey)
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		body, _ := io.ReadAll(resp.Body)
-		if resp.StatusCode == http.StatusOK {
-			var stsResp GetFederationTokenTestResponse
-			require.NoError(t, xml.Unmarshal(body, &stsResp))
-
-			expTime, err := time.Parse(time.RFC3339, stsResp.Result.Credentials.Expiration)
-			require.NoError(t, err)
-			diff := time.Until(expTime)
-			assert.InDelta(t, 129600, diff.Seconds(), 60,
-				"Expiration should be ~36 hours from now")
-		} else {
-			// Duration should not cause a rejection
-			var errResp STSErrorTestResponse
-			_ = xml.Unmarshal(body, &errResp)
-			assert.NotContains(t, errResp.Error.Message, "DurationSeconds",
-				"36-hour duration should be accepted by GetFederationToken")
-		}
-	})
-}
-
-// TestSTSGetFederationTokenWithSessionPolicy tests that vended credentials
-// are scoped down by an inline session policy
-func TestSTSGetFederationTokenWithSessionPolicy(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	if !isSTSEndpointRunning(t) {
-		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
-	}
-
-	if !isGetFederationTokenImplemented(t) {
-		t.Skip("GetFederationToken not implemented")
-	}
-
-	accessKey, secretKey := getTestCredentials()
-
-	// Create a test bucket using admin credentials
-	adminSess, err := session.NewSession(&aws.Config{
-		Region:           aws.String("us-east-1"),
-		Endpoint:         aws.String(TestSTSEndpoint),
-		DisableSSL:       aws.Bool(true),
-		S3ForcePathStyle: aws.Bool(true),
-		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
-	})
-	require.NoError(t, err)
-
-	adminS3 := s3.New(adminSess)
-	bucket := fmt.Sprintf("fed-token-test-%d", time.Now().UnixNano())
-
-	_, err = adminS3.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
-	require.NoError(t, err)
-	defer adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)})
-
-	_, err = adminS3.PutObject(&s3.PutObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String("test.txt"),
-		Body:   strings.NewReader("hello"),
-	})
-	require.NoError(t, err)
-	defer adminS3.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String("test.txt")})
-
-	// Get federated credentials with a session policy that only allows GetObject
-	sessionPolicy := fmt.Sprintf(`{
-		"Version": "2012-10-17",
-		"Statement": [{
-			"Effect": "Allow",
-			"Action": ["s3:GetObject"],
-			"Resource": ["arn:aws:s3:::%s/*"]
-		}]
-	}`, bucket)
-
-	resp, err := callSTSAPIWithSigV4(t, url.Values{
-		"Action":  {"GetFederationToken"},
-		"Version": {"2011-06-15"},
-		"Name":    {"ScopedClient"},
-		"Policy":  {sessionPolicy},
-	}, accessKey, secretKey)
-	require.NoError(t, err)
-	defer resp.Body.Close()
-
-	body, _ := io.ReadAll(resp.Body)
-	t.Logf("GetFederationToken response: status=%d body=%s", resp.StatusCode, string(body))
-
-	if resp.StatusCode != http.StatusOK {
-		t.Skipf("GetFederationToken failed (may need IAM policy config): %s", string(body))
-	}
-
-	var stsResp GetFederationTokenTestResponse
-	require.NoError(t, xml.Unmarshal(body, &stsResp))
-
-	fedCreds := stsResp.Result.Credentials
-	require.NotEmpty(t, fedCreds.AccessKeyId)
-	require.NotEmpty(t, fedCreds.SessionToken)
-
-	// Create S3 client with the federated credentials
-	fedSess, err := session.NewSession(&aws.Config{
-		Region:           aws.String("us-east-1"),
-		Endpoint:         aws.String(TestSTSEndpoint),
-		DisableSSL:       aws.Bool(true),
-		S3ForcePathStyle: aws.Bool(true),
-		Credentials: credentials.NewStaticCredentials(
-			fedCreds.AccessKeyId, fedCreds.SecretAccessKey, fedCreds.SessionToken),
-	})
-	require.NoError(t, err)
-
-	fedS3 := s3.New(fedSess)
-
-	// GetObject should succeed (allowed by session policy)
-	getResp, err := fedS3.GetObject(&s3.GetObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String("test.txt"),
-	})
-	if err == nil {
-		defer getResp.Body.Close()
-		t.Log("GetObject with federated credentials succeeded (as expected)")
-	} else {
-		t.Logf("GetObject with federated credentials: %v (session policy enforcement may vary)", err)
-	}
-
-	// PutObject should be denied (not allowed by session policy)
-	_, err = fedS3.PutObject(&s3.PutObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String("denied.txt"),
-		Body:   strings.NewReader("should fail"),
-	})
-	if err != nil {
-		t.Log("PutObject correctly denied with federated credentials")
-		assert.Contains(t, err.Error(), "AccessDenied",
-			"PutObject should be denied by session policy")
-	} else {
-		// Clean up if unexpectedly succeeded
-		adminS3.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String("denied.txt")})
-		t.Log("PutObject unexpectedly succeeded — session policy enforcement may not be active")
-	}
-}

test/s3/sse/Makefile

@@ -98,8 +98,8 @@ start-seaweedfs: check-binary
 	# Create S3 configuration with SSE-KMS support
 	@printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}],"kms":{"type":"%s","configs":{"keyId":"%s","encryptionContext":{},"bucketKey":false}}}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" "$(KMS_TYPE)" "$(KMS_KEY_ID)" > /tmp/seaweedfs-sse-s3.json
 	
-	# Start weed mini (WEED_S3_SSE_KEY enables SSE-S3 encryption)
-	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key $(SEAWEEDFS_BINARY) mini \
+	# Start weed mini
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(SEAWEEDFS_BINARY) mini \
 		-dir=/tmp/seaweedfs-test-sse \
 		-s3.port=$(S3_PORT) \
 		-s3.config=/tmp/seaweedfs-sse-s3.json \
@@ -337,9 +337,8 @@ start-seaweedfs-ci: check-binary
 	     s3-config-template.json > /tmp/seaweedfs-s3.json
 	
 	# Start weed mini with embedded S3 using the JSON config (with verbose logging)
-	# WEED_S3_SSE_KEY enables SSE-S3 encryption for testing (KEK derived via HKDF)
 	@echo "Starting weed mini with embedded S3..."
-	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
 		-dir=/tmp/seaweedfs-test-sse \
 		-s3.port=$(S3_PORT) \
 		-s3.config=/tmp/seaweedfs-s3.json \
@@ -483,7 +482,7 @@ test-volume-encryption: build-weed
 	     -e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \
 	     s3-config-template.json > /tmp/seaweedfs-s3.json
 	@echo "Starting weed mini with S3 volume encryption..."
-	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
 		-dir=/tmp/seaweedfs-test-sse \
 		-s3.port=$(S3_PORT) \
 		-s3.config=/tmp/seaweedfs-s3.json \

test/s3/versioning/s3_versioning_pagination_stress_test.go

@@ -289,157 +289,6 @@ func TestVersioningPaginationMultipleObjectsManyVersions(t *testing.T) {
 	})
 }
 
-// TestVersioningPaginationDeepDirectoryHierarchy tests that paginated ListObjectVersions
-// correctly skips directory subtrees before the key-marker. This reproduces the
-// real-world scenario where Veeam backup objects are spread across many subdirectories
-// (e.g., Mailboxes/<uuid>/ItemsData/<file>) and pagination becomes exponentially
-// slower as the marker advances through the tree.
-//
-// Run with: ENABLE_STRESS_TESTS=true go test -v -run TestVersioningPaginationDeepDirectoryHierarchy -timeout 10m
-func TestVersioningPaginationDeepDirectoryHierarchy(t *testing.T) {
-	if os.Getenv("ENABLE_STRESS_TESTS") != "true" {
-		t.Skip("Skipping stress test. Set ENABLE_STRESS_TESTS=true to run.")
-	}
-
-	client := getS3Client(t)
-	bucketName := getNewBucketName()
-
-	createBucket(t, client, bucketName)
-	defer deleteBucket(t, client, bucketName)
-
-	enableVersioning(t, client, bucketName)
-	checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
-
-	// Create a deep directory structure mimicking Veeam 365 backup layout:
-	// Backup/Organizations/<org>/Mailboxes/<mailbox>/ItemsData/<file>
-	numMailboxes := 20
-	filesPerMailbox := 5
-	totalObjects := numMailboxes * filesPerMailbox
-	orgPrefix := "Backup/Organizations/org-001/Mailboxes"
-
-	t.Logf("Creating %d objects across %d subdirectories (depth=6)...", totalObjects, numMailboxes)
-	startTime := time.Now()
-
-	allKeys := make([]string, 0, totalObjects)
-	for i := 0; i < numMailboxes; i++ {
-		mailboxId := fmt.Sprintf("mbx-%03d", i)
-		for j := 0; j < filesPerMailbox; j++ {
-			key := fmt.Sprintf("%s/%s/ItemsData/file-%03d.dat", orgPrefix, mailboxId, j)
-			_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
-				Bucket: aws.String(bucketName),
-				Key:    aws.String(key),
-				Body:   strings.NewReader(fmt.Sprintf("content-%d-%d", i, j)),
-			})
-			require.NoError(t, err)
-			allKeys = append(allKeys, key)
-		}
-	}
-	t.Logf("Created %d objects in %v", totalObjects, time.Since(startTime))
-
-	// Test 1: Paginate through all versions with a broad prefix and small maxKeys.
-	// This forces multiple pages that must skip earlier subdirectory trees.
-	t.Run("PaginateAcrossSubdirectories", func(t *testing.T) {
-		maxKeys := int32(10) // Force many pages to exercise marker skipping
-		var allVersions []types.ObjectVersion
-		var keyMarker, versionIdMarker *string
-		pageCount := 0
-		pageStartTimes := make([]time.Duration, 0)
-
-		for {
-			pageStart := time.Now()
-			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
-				Bucket:          aws.String(bucketName),
-				Prefix:          aws.String(orgPrefix + "/"),
-				MaxKeys:         aws.Int32(maxKeys),
-				KeyMarker:       keyMarker,
-				VersionIdMarker: versionIdMarker,
-			})
-			pageDuration := time.Since(pageStart)
-			require.NoError(t, err)
-			pageCount++
-			pageStartTimes = append(pageStartTimes, pageDuration)
-
-			allVersions = append(allVersions, resp.Versions...)
-			t.Logf("Page %d: %d versions in %v (marker: %v)",
-				pageCount, len(resp.Versions), pageDuration,
-				keyMarker)
-
-			if resp.IsTruncated == nil || !*resp.IsTruncated {
-				break
-			}
-			keyMarker = resp.NextKeyMarker
-			versionIdMarker = resp.NextVersionIdMarker
-		}
-
-		assert.Greater(t, pageCount, 1, "Should require multiple pages")
-
-		// Verify listed keys exactly match the keys we created (same elements, same order)
-		listedKeys := make([]string, 0, len(allVersions))
-		for _, v := range allVersions {
-			listedKeys = append(listedKeys, *v.Key)
-		}
-		assert.Equal(t, allKeys, listedKeys,
-			"Listed version keys should exactly match created keys")
-
-		// Check that later pages don't take dramatically longer than earlier ones.
-		// Before the fix, later pages were exponentially slower because they
-		// re-traversed the entire tree. With the fix, all pages should be similar.
-		if len(pageStartTimes) >= 4 {
-			firstQuarter := pageStartTimes[0]
-			lastQuarter := pageStartTimes[len(pageStartTimes)-1]
-			t.Logf("First page: %v, Last page: %v, Ratio: %.1fx",
-				firstQuarter, lastQuarter,
-				float64(lastQuarter)/float64(firstQuarter))
-			// Allow generous 10x ratio to avoid flakiness; before the fix
-			// the ratio was 100x+ on large datasets
-			if lastQuarter > firstQuarter*10 && lastQuarter > 500*time.Millisecond {
-				t.Errorf("Last page took %.1fx longer than first page (%v vs %v) — possible pagination regression",
-					float64(lastQuarter)/float64(firstQuarter), lastQuarter, firstQuarter)
-			}
-		}
-	})
-
-	// Test 2: Paginate with delimiter to verify CommonPrefixes interaction
-	t.Run("PaginateWithDelimiterAcrossSubdirs", func(t *testing.T) {
-		maxKeys := int32(5)
-		var allPrefixes []string
-		var keyMarker *string
-		pageCount := 0
-
-		for {
-			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
-				Bucket:    aws.String(bucketName),
-				Prefix:    aws.String(orgPrefix + "/"),
-				Delimiter: aws.String("/"),
-				MaxKeys:   aws.Int32(maxKeys),
-				KeyMarker: keyMarker,
-			})
-			require.NoError(t, err)
-			pageCount++
-
-			for _, cp := range resp.CommonPrefixes {
-				allPrefixes = append(allPrefixes, *cp.Prefix)
-			}
-
-			if resp.IsTruncated == nil || !*resp.IsTruncated {
-				break
-			}
-			keyMarker = resp.NextKeyMarker
-		}
-
-		assert.Greater(t, pageCount, 1, "Should require multiple pages with maxKeys=%d", maxKeys)
-
-		// Build the exact expected prefixes
-		expectedPrefixes := make([]string, 0, numMailboxes)
-		for i := 0; i < numMailboxes; i++ {
-			expectedPrefixes = append(expectedPrefixes,
-				fmt.Sprintf("%s/mbx-%03d/", orgPrefix, i))
-		}
-		assert.Equal(t, expectedPrefixes, allPrefixes,
-			"CommonPrefixes should exactly match expected mailbox prefixes")
-	})
-}
-
 // listAllVersions is a helper to list all versions of a specific object using pagination
 func listAllVersions(t *testing.T, client *s3.Client, bucketName, objectKey string) []types.ObjectVersion {
 	var allVersions []types.ObjectVersion

test/s3tables/lakekeeper/lakekeeper_test.go

@@ -115,13 +115,10 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 
 	bindIP := testutil.FindBindIP()
 
-	// Allocate all ports in a single batch to prevent the OS from recycling
-	// a released port, which can cause two services to get the same port.
-	ports := testutil.MustAllocatePorts(t, 8)
-	masterPort, masterGrpcPort := ports[0], ports[1]
-	volumePort, volumeGrpcPort := ports[2], ports[3]
-	filerPort, filerGrpcPort := ports[4], ports[5]
-	s3Port, s3GrpcPort := ports[6], ports[7]
+	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
+	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
+	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
+	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
 
 	return &TestEnvironment{
 		seaweedDir:     seaweedDir,

test/s3tables/polaris/polaris_env_test.go

@@ -89,14 +89,11 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 
 	bindIP := testutil.FindBindIP()
 
-	// Allocate all ports in a single batch to prevent the OS from recycling
-	// a released port, which can cause two services to get the same port.
-	ports := testutil.MustAllocatePorts(t, 10)
-	masterPort, masterGrpcPort := ports[0], ports[1]
-	volumePort, volumeGrpcPort := ports[2], ports[3]
-	filerPort, filerGrpcPort := ports[4], ports[5]
-	s3Port, s3GrpcPort := ports[6], ports[7]
-	polarisPort, polarisAdminPort := ports[8], ports[9]
+	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
+	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
+	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
+	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
+	polarisPort, polarisAdminPort := testutil.MustFreePortPair(t, "Polaris")
 
 	return &TestEnvironment{
 		seaweedDir:       seaweedDir,

test/s3tables/sts_integration/sts_integration_test.go

@@ -93,13 +93,10 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 
 	bindIP := testutil.FindBindIP()
 
-	// Allocate all ports in a single batch to prevent the OS from recycling
-	// a released port, which can cause two services to get the same port.
-	ports := testutil.MustAllocatePorts(t, 8)
-	masterPort, masterGrpcPort := ports[0], ports[1]
-	volumePort, volumeGrpcPort := ports[2], ports[3]
-	filerPort, filerGrpcPort := ports[4], ports[5]
-	s3Port, s3GrpcPort := ports[6], ports[7]
+	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
+	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
+	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
+	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3") // Changed to use testutil.MustFreePortPair
 
 	return &TestEnvironment{
 		seaweedDir:      seaweedDir,

test/s3tables/testutil/docker.go

@@ -15,44 +15,33 @@ func HasDocker() bool {
 	return cmd.Run() == nil
 }
 
-// MustFreePortPair is a convenience wrapper for tests that only need a single pair.
-// Prefer MustAllocatePorts when allocating multiple pairs to guarantee uniqueness.
 func MustFreePortPair(t *testing.T, name string) (int, int) {
-	ports := MustAllocatePorts(t, 2)
-	return ports[0], ports[1]
-}
-
-// MustAllocatePorts allocates count unique free ports atomically.
-// All listeners are held open until every port is obtained, preventing
-// the OS from recycling a port between successive allocations.
-func MustAllocatePorts(t *testing.T, count int) []int {
-	t.Helper()
-	ports, err := AllocatePorts(count)
+	httpPort, grpcPort, err := findAvailablePortPair()
 	if err != nil {
-		t.Fatalf("Failed to allocate %d free ports: %v", count, err)
+		t.Fatalf("Failed to get free port pair for %s: %v", name, err)
 	}
-	return ports
+	return httpPort, grpcPort
 }
 
-// AllocatePorts allocates count unique free ports atomically.
-func AllocatePorts(count int) ([]int, error) {
-	listeners := make([]net.Listener, 0, count)
-	ports := make([]int, 0, count)
-	for i := 0; i < count; i++ {
-		l, err := net.Listen("tcp", "0.0.0.0:0")
-		if err != nil {
-			for _, ll := range listeners {
-				_ = ll.Close()
-			}
-			return nil, err
-		}
-		listeners = append(listeners, l)
-		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
+func findAvailablePortPair() (int, int, error) {
+	httpPort, err := GetFreePort()
+	if err != nil {
+		return 0, 0, err
 	}
-	for _, l := range listeners {
-		_ = l.Close()
+	grpcPort, err := GetFreePort()
+	if err != nil {
+		return 0, 0, err
 	}
-	return ports, nil
+	return httpPort, grpcPort, nil
+}
+
+func GetFreePort() (int, error) {
+	listener, err := net.Listen("tcp", "0.0.0.0:0")
+	if err != nil {
+		return 0, err
+	}
+	defer listener.Close()
+	return listener.Addr().(*net.TCPAddr).Port, nil
 }
 
 func WaitForService(url string, timeout time.Duration) bool {

test/volume_server/framework/cluster_mixed.go

@@ -1,368 +0,0 @@
-package framework
-
-import (
-	"fmt"
-	"net"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strconv"
-	"sync"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-// MixedVolumeCluster wraps a Go master + a mix of Go and Rust volume servers
-// for integration testing. Go servers occupy indices [0, goCount) and Rust
-// servers occupy indices [goCount, goCount+rustCount).
-type MixedVolumeCluster struct {
-	testingTB testing.TB
-	profile   matrix.Profile
-
-	weedBinary       string // Go weed binary (master + Go volume servers)
-	rustVolumeBinary string // Rust volume binary
-
-	baseDir   string
-	configDir string
-	logsDir   string
-	keepLogs  bool
-
-	masterPort     int
-	masterGrpcPort int
-
-	volumePorts     []int
-	volumeGrpcPorts []int
-	volumePubPorts  []int
-	isRust          []bool // which servers are Rust
-
-	masterCmd  *exec.Cmd
-	volumeCmds []*exec.Cmd
-
-	cleanupOnce sync.Once
-}
-
-// StartMixedVolumeCluster starts a cluster with 1 Go master, goCount Go volume
-// servers, and rustCount Rust volume servers. Go servers come first in the index.
-func StartMixedVolumeCluster(t testing.TB, profile matrix.Profile, goCount, rustCount int) *MixedVolumeCluster {
-	t.Helper()
-
-	if goCount < 0 || rustCount < 0 {
-		t.Fatalf("goCount and rustCount must be non-negative, got go=%d rust=%d", goCount, rustCount)
-	}
-	total := goCount + rustCount
-	if total < 2 {
-		t.Fatalf("need at least 2 volume servers, got %d", total)
-	}
-
-	weedBinary, err := FindOrBuildWeedBinary()
-	if err != nil {
-		t.Fatalf("resolve weed binary: %v", err)
-	}
-
-	// Only build the Rust binary when Rust servers are requested.
-	var rustBinary string
-	if rustCount > 0 {
-		rustBinary, err = FindOrBuildRustBinary()
-		if err != nil {
-			t.Skipf("skipping mixed cluster test: rust binary unavailable: %v", err)
-		}
-	}
-
-	baseDir, keepLogs, err := newWorkDir()
-	if err != nil {
-		t.Fatalf("create temp test directory: %v", err)
-	}
-
-	configDir := filepath.Join(baseDir, "config")
-	logsDir := filepath.Join(baseDir, "logs")
-	masterDataDir := filepath.Join(baseDir, "master")
-
-	dirs := []string{configDir, logsDir, masterDataDir}
-	for i := 0; i < total; i++ {
-		dirs = append(dirs, filepath.Join(baseDir, fmt.Sprintf("volume%d", i)))
-	}
-	for _, dir := range dirs {
-		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
-			t.Fatalf("create %s: %v", dir, mkErr)
-		}
-	}
-
-	if err = writeSecurityConfig(configDir, profile); err != nil {
-		t.Fatalf("write security config: %v", err)
-	}
-
-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
-	if err != nil {
-		t.Fatalf("allocate master port pair: %v", err)
-	}
-
-	// 2 ports per server (admin, grpc); add 1 more when public port is split out.
-	portsPerServer := 2
-	if profile.SplitPublicPort {
-		portsPerServer = 3
-	}
-	ports, err := allocatePorts(total * portsPerServer)
-	if err != nil {
-		t.Fatalf("allocate volume ports: %v", err)
-	}
-
-	isRust := make([]bool, total)
-	for i := goCount; i < total; i++ {
-		isRust[i] = true
-	}
-
-	c := &MixedVolumeCluster{
-		testingTB:        t,
-		profile:          profile,
-		weedBinary:       weedBinary,
-		rustVolumeBinary: rustBinary,
-		baseDir:          baseDir,
-		configDir:        configDir,
-		logsDir:          logsDir,
-		keepLogs:         keepLogs,
-		masterPort:       masterPort,
-		masterGrpcPort:   masterGrpcPort,
-		volumePorts:      make([]int, total),
-		volumeGrpcPorts:  make([]int, total),
-		volumePubPorts:   make([]int, total),
-		isRust:           isRust,
-		volumeCmds:       make([]*exec.Cmd, total),
-	}
-
-	for i := 0; i < total; i++ {
-		baseIdx := i * portsPerServer
-		c.volumePorts[i] = ports[baseIdx]
-		c.volumeGrpcPorts[i] = ports[baseIdx+1]
-		if profile.SplitPublicPort {
-			c.volumePubPorts[i] = ports[baseIdx+2]
-		} else {
-			c.volumePubPorts[i] = c.volumePorts[i] // reuse admin port
-		}
-	}
-
-	// Start master
-	if err = c.startMaster(masterDataDir); err != nil {
-		c.Stop()
-		t.Fatalf("start master: %v", err)
-	}
-	helper := &Cluster{logsDir: logsDir}
-	if err = helper.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
-		masterLog := helper.tailLog("master.log")
-		c.Stop()
-		t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
-	}
-
-	// Start volume servers
-	for i := 0; i < total; i++ {
-		volumeDataDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
-		if isRust[i] {
-			err = c.startRustVolume(i, volumeDataDir)
-		} else {
-			err = c.startGoVolume(i, volumeDataDir)
-		}
-		if err != nil {
-			logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
-			c.Stop()
-			t.Fatalf("start volume server %d (rust=%v): %v\nlog tail:\n%s", i, isRust[i], err, logTail)
-		}
-
-		// Rust uses /healthz, Go uses /status
-		healthURL := c.VolumeAdminURL(i) + "/status"
-		if isRust[i] {
-			healthURL = c.VolumeAdminURL(i) + "/healthz"
-		}
-		if err = helper.waitForHTTP(healthURL); err != nil {
-			logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
-			c.Stop()
-			t.Fatalf("wait for volume server %d readiness: %v\nlog tail:\n%s", i, err, logTail)
-		}
-		if err = helper.waitForTCP(c.VolumeGRPCAddress(i)); err != nil {
-			logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
-			c.Stop()
-			t.Fatalf("wait for volume server %d grpc readiness: %v\nlog tail:\n%s", i, err, logTail)
-		}
-	}
-
-	t.Cleanup(func() {
-		c.Stop()
-	})
-
-	return c
-}
-
-func (c *MixedVolumeCluster) Stop() {
-	if c == nil {
-		return
-	}
-	c.cleanupOnce.Do(func() {
-		for i := len(c.volumeCmds) - 1; i >= 0; i-- {
-			stopProcess(c.volumeCmds[i])
-		}
-		stopProcess(c.masterCmd)
-		if !c.keepLogs && !c.testingTB.Failed() {
-			_ = os.RemoveAll(c.baseDir)
-		} else if c.baseDir != "" {
-			c.testingTB.Logf("mixed volume server integration logs kept at %s", c.baseDir)
-		}
-	})
-}
-
-func (c *MixedVolumeCluster) startMaster(dataDir string) error {
-	logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
-	if err != nil {
-		return err
-	}
-
-	args := []string{
-		"-config_dir=" + c.configDir,
-		"master",
-		"-ip=127.0.0.1",
-		"-port=" + strconv.Itoa(c.masterPort),
-		"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
-		"-mdir=" + dataDir,
-		"-peers=none",
-		"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
-		"-defaultReplication=000",
-	}
-
-	c.masterCmd = exec.Command(c.weedBinary, args...)
-	c.masterCmd.Dir = c.baseDir
-	c.masterCmd.Stdout = logFile
-	c.masterCmd.Stderr = logFile
-	if err = c.masterCmd.Start(); err != nil {
-		logFile.Close()
-		return err
-	}
-	logFile.Close() // child inherited the fd
-	return nil
-}
-
-func (c *MixedVolumeCluster) startGoVolume(index int, dataDir string) error {
-	logName := fmt.Sprintf("volume%d.log", index)
-	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
-	if err != nil {
-		return err
-	}
-
-	args := []string{
-		"-config_dir=" + c.configDir,
-		"volume",
-		"-ip=127.0.0.1",
-		"-port=" + strconv.Itoa(c.volumePorts[index]),
-		"-port.grpc=" + strconv.Itoa(c.volumeGrpcPorts[index]),
-		"-port.public=" + strconv.Itoa(c.volumePubPorts[index]),
-		"-dir=" + dataDir,
-		"-max=16",
-		"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
-		"-readMode=" + c.profile.ReadMode,
-		"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
-		"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
-	}
-	if c.profile.InflightUploadTimeout > 0 {
-		args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
-	}
-	if c.profile.InflightDownloadTimeout > 0 {
-		args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
-	}
-
-	cmd := exec.Command(c.weedBinary, args...)
-	cmd.Dir = c.baseDir
-	cmd.Stdout = logFile
-	cmd.Stderr = logFile
-	if err = cmd.Start(); err != nil {
-		logFile.Close()
-		return err
-	}
-	logFile.Close() // child inherited the fd
-	c.volumeCmds[index] = cmd
-	return nil
-}
-
-func (c *MixedVolumeCluster) startRustVolume(index int, dataDir string) error {
-	logName := fmt.Sprintf("volume%d.log", index)
-	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
-	if err != nil {
-		return err
-	}
-
-	args := rustVolumeArgs(
-		c.profile,
-		c.configDir,
-		c.masterPort,
-		c.volumePorts[index],
-		c.volumeGrpcPorts[index],
-		c.volumePubPorts[index],
-		dataDir,
-	)
-
-	cmd := exec.Command(c.rustVolumeBinary, args...)
-	cmd.Dir = c.baseDir
-	cmd.Stdout = logFile
-	cmd.Stderr = logFile
-	if err = cmd.Start(); err != nil {
-		logFile.Close()
-		return err
-	}
-	logFile.Close() // child inherited the fd
-	c.volumeCmds[index] = cmd
-	return nil
-}
-
-// --- accessor methods (mirror MultiVolumeCluster) ---
-
-func (c *MixedVolumeCluster) MasterAddress() string {
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
-}
-
-func (c *MixedVolumeCluster) MasterURL() string {
-	return "http://" + c.MasterAddress()
-}
-
-func (c *MixedVolumeCluster) VolumeAdminAddress(index int) string {
-	if index < 0 || index >= len(c.volumePorts) {
-		return ""
-	}
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePorts[index]))
-}
-
-func (c *MixedVolumeCluster) VolumePublicAddress(index int) string {
-	if index < 0 || index >= len(c.volumePubPorts) {
-		return ""
-	}
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPorts[index]))
-}
-
-func (c *MixedVolumeCluster) VolumeGRPCAddress(index int) string {
-	if index < 0 || index >= len(c.volumeGrpcPorts) {
-		return ""
-	}
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPorts[index]))
-}
-
-func (c *MixedVolumeCluster) VolumeAdminURL(index int) string {
-	return "http://" + c.VolumeAdminAddress(index)
-}
-
-func (c *MixedVolumeCluster) VolumePublicURL(index int) string {
-	return "http://" + c.VolumePublicAddress(index)
-}
-
-func (c *MixedVolumeCluster) BaseDir() string {
-	return c.baseDir
-}
-
-// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
-func (c *MixedVolumeCluster) VolumeServerAddress(index int) string {
-	if index < 0 || index >= len(c.volumePorts) {
-		return ""
-	}
-	return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(index), c.volumeGrpcPorts[index])
-}
-
-func (c *MixedVolumeCluster) IsRust(index int) bool {
-	if index < 0 || index >= len(c.isRust) {
-		return false
-	}
-	return c.isRust[index]
-}

test/volume_server/grpc/ec_decode_ecj_test.go

@@ -1,294 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
-)
-
-// TestEcDecodePreservesDeletedNeedles verifies that needles deleted via
-// VolumeEcBlobDelete (recorded in .ecj) are correctly excluded from the
-// decoded volume produced by VolumeEcShardsToVolume.
-func TestEcDecodePreservesDeletedNeedles(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartVolumeCluster(t, matrix.P1())
-	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const (
-		volumeID = uint32(140)
-		keyA     = uint64(990020)
-		cookieA  = uint32(0xDA001122)
-		keyB     = uint64(990021)
-		cookieB  = uint32(0xDA003344)
-	)
-
-	framework.AllocateVolume(t, client, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fidA := framework.NewFileID(volumeID, keyA, cookieA)
-	fidB := framework.NewFileID(volumeID, keyB, cookieB)
-	payloadA := []byte("needle-A-should-be-deleted-after-decode")
-	payloadB := []byte("needle-B-should-survive-decode")
-
-	// Upload two needles.
-	for _, tc := range []struct {
-		fid     string
-		payload []byte
-	}{
-		{fidA, payloadA},
-		{fidB, payloadB},
-	} {
-		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, tc.payload)
-		_ = framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode != http.StatusCreated {
-			t.Fatalf("upload %s: expected 201, got %d", tc.fid, resp.StatusCode)
-		}
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
-	defer cancel()
-
-	// EC encode.
-	_, err := client.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
-		VolumeId: volumeID, Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsGenerate: %v", err)
-	}
-
-	// Mount all data shards so the EC volume is usable.
-	_, err = client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
-		VolumeId: volumeID, Collection: "",
-		ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsMount: %v", err)
-	}
-
-	// Delete needle A via EC path (writes to .ecj).
-	_, err = client.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
-		VolumeId: volumeID, Collection: "",
-		FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcBlobDelete needle A: %v", err)
-	}
-
-	// Unmount the normal volume so decode writes fresh files.
-	_, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VolumeUnmount: %v", err)
-	}
-
-	// Decode EC shards back to a normal volume.
-	_, err = client.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
-		VolumeId: volumeID, Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsToVolume: %v", err)
-	}
-
-	// Re-mount the decoded volume.
-	_, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VolumeMount: %v", err)
-	}
-
-	// Needle A should be gone (deleted via .ecj before decode).
-	respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA)
-	bodyA := framework.ReadAllAndClose(t, respA)
-	if respA.StatusCode >= 500 {
-		t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
-	}
-	if respA.StatusCode != http.StatusNotFound {
-		t.Fatalf("needle A should be 404 after decode, got %d", respA.StatusCode)
-	}
-
-	// Needle B should still be readable.
-	respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB)
-	bodyB := framework.ReadAllAndClose(t, respB)
-	if respB.StatusCode != http.StatusOK {
-		t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
-	}
-	if string(bodyB) != string(payloadB) {
-		t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
-	}
-}
-
-// TestEcDecodeCollectsEcjFromPeer verifies that .ecj deletion entries from a
-// peer server that contributes no new data shards are still collected during
-// decode. This is the regression test for the fix in collectEcShards that
-// always copies .ecj from every shard location.
-//
-// Scenario:
-//   - Server 0 holds all 10 data shards (decode target).
-//   - Server 1 holds a copy of shard 0 (no new shards for server 0).
-//   - A needle is deleted ONLY on server 1 (server 0's .ecj is empty).
-//   - During decode on server 0, server 1's .ecj must be collected and applied.
-func TestEcDecodeCollectsEcjFromPeer(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartMultiVolumeClusterAuto(t, matrix.P1(), 2)
-	conn0, client0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
-	defer conn0.Close()
-	conn1, client1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
-	defer conn1.Close()
-
-	const (
-		volumeID = uint32(141)
-		keyA     = uint64(990030)
-		cookieA  = uint32(0xDB001122)
-		keyB     = uint64(990031)
-		cookieB  = uint32(0xDB003344)
-	)
-
-	// Allocate and upload on server 0.
-	framework.AllocateVolume(t, client0, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fidA := framework.NewFileID(volumeID, keyA, cookieA)
-	fidB := framework.NewFileID(volumeID, keyB, cookieB)
-	payloadB := []byte("needle-B-should-survive-peer-ecj-decode")
-
-	resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA, []byte("needle-A-deleted-on-peer"))
-	_ = framework.ReadAllAndClose(t, resp)
-	if resp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload A: expected 201, got %d", resp.StatusCode)
-	}
-	resp = framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB, payloadB)
-	_ = framework.ReadAllAndClose(t, resp)
-	if resp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload B: expected 201, got %d", resp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
-	defer cancel()
-
-	// EC encode on server 0.
-	_, err := client0.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
-		VolumeId: volumeID, Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsGenerate on server 0: %v", err)
-	}
-
-	// Build the SourceDataNode address for server 0 (format: host:adminPort.grpcPort).
-	sourceDataNode := cluster.VolumeAdminAddress(0) + "." +
-		strings.Split(cluster.VolumeGRPCAddress(0), ":")[1]
-
-	// Copy shard 0 + ecx + ecj from server 0 → server 1.
-	_, err = client1.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
-		VolumeId:       volumeID,
-		Collection:     "",
-		SourceDataNode: sourceDataNode,
-		ShardIds:       []uint32{0},
-		CopyEcxFile:    true,
-		CopyEcjFile:    true,
-		CopyVifFile:    true,
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsCopy 0→1: %v", err)
-	}
-
-	// Mount shard 0 on server 1 so the EC volume can accept deletions.
-	_, err = client1.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
-		VolumeId: volumeID, Collection: "",
-		ShardIds: []uint32{0},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsMount on server 1: %v", err)
-	}
-
-	// Delete needle A on server 1 only (creates .ecj entry on server 1).
-	_, err = client1.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
-		VolumeId: volumeID, Collection: "",
-		FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcBlobDelete needle A on server 1: %v", err)
-	}
-
-	// Mount all data shards on server 0 (the decode target).
-	_, err = client0.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
-		VolumeId: volumeID, Collection: "",
-		ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsMount on server 0: %v", err)
-	}
-
-	// Collect .ecj from server 1 → server 0 with NO new shard IDs.
-	// This is the critical path: server 1 has shard 0 which server 0 already
-	// has, so needToCopyShardsInfo would be empty. Before the fix in
-	// collectEcShards, this copy would be skipped entirely, losing server 1's
-	// deletion entries.
-	server1DataNode := cluster.VolumeAdminAddress(1) + "." +
-		strings.Split(cluster.VolumeGRPCAddress(1), ":")[1]
-
-	_, err = client0.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
-		VolumeId:       volumeID,
-		Collection:     "",
-		SourceDataNode: server1DataNode,
-		ShardIds:       []uint32{}, // No new shards — just .ecj.
-		CopyEcxFile:    false,
-		CopyEcjFile:    true,
-		CopyVifFile:    false,
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsCopy .ecj from server 1→0: %v", err)
-	}
-
-	// Unmount the normal volume before decode.
-	_, err = client0.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VolumeUnmount on server 0: %v", err)
-	}
-
-	// Decode on server 0. RebuildEcxFile should see needle A's deletion from
-	// the .ecj that was collected from server 1.
-	_, err = client0.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
-		VolumeId: volumeID, Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsToVolume on server 0: %v", err)
-	}
-
-	// Re-mount the decoded normal volume.
-	_, err = client0.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VolumeMount on server 0: %v", err)
-	}
-
-	// Needle A should be gone — its deletion was in server 1's .ecj.
-	respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA)
-	bodyA := framework.ReadAllAndClose(t, respA)
-	if respA.StatusCode >= 500 {
-		t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
-	}
-	if respA.StatusCode != http.StatusNotFound {
-		t.Fatalf("needle A should be 404 (ecj from peer), got %d", respA.StatusCode)
-	}
-
-	// Needle B should still be readable.
-	respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB)
-	bodyB := framework.ReadAllAndClose(t, respB)
-	if respB.StatusCode != http.StatusOK {
-		t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
-	}
-	if string(bodyB) != string(payloadB) {
-		t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
-	}
-}

test/volume_server/grpc/mixed_balance_test.go

@@ -1,567 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-)
-
-// deleteAndWaitForTombstone issues an HTTP DELETE for the given fid on
-// volumeURL, asserts a successful response, then polls with GET until
-// the file returns 404 (tombstone visible) or the timeout elapses.
-func deleteAndWaitForTombstone(t *testing.T, httpClient *http.Client, volumeURL, fid string) {
-	t.Helper()
-	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/%s", volumeURL, fid), nil)
-	if err != nil {
-		t.Fatalf("build delete request for %s: %v", fid, err)
-	}
-	resp := framework.DoRequest(t, httpClient, req)
-	framework.ReadAllAndClose(t, resp)
-	if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK {
-		t.Fatalf("delete %s: expected 200 or 202, got %d", fid, resp.StatusCode)
-	}
-	// Poll until GET returns 404 (tombstone flushed to disk).
-	deadline := time.Now().Add(5 * time.Second)
-	for time.Now().Before(deadline) {
-		getResp := framework.ReadBytes(t, httpClient, volumeURL, fid)
-		status := getResp.StatusCode
-		framework.ReadAllAndClose(t, getResp)
-		if status == http.StatusNotFound {
-			return
-		}
-		time.Sleep(50 * time.Millisecond)
-	}
-	t.Fatalf("delete %s: tombstone not visible after 5s", fid)
-}
-
-// TestMixedBalanceCopyGoToRust verifies that VolumeCopy works from a Go volume
-// server to a Rust volume server. This is the core operation behind volume
-// balancing in a mixed Go+Rust cluster.
-func TestMixedBalanceCopyGoToRust(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
-
-	// server 0 = Go, server 1 = Rust
-	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
-	defer conn0.Close()
-	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
-	defer conn1.Close()
-
-	httpClient := framework.NewHTTPClient()
-
-	const volumeID = uint32(50)
-
-	// Allocate volume on Go server and upload test data
-	framework.AllocateVolume(t, goClient, volumeID, "")
-
-	testFiles := []struct {
-		key    uint64
-		cookie uint32
-		data   []byte
-	}{
-		{1, 0xAABBCCDD, []byte("hello from Go server")},
-		{2, 0x11223344, []byte("second file for balance test")},
-		{3, 0xDEADBEEF, make([]byte, 4096)}, // larger file
-	}
-
-	for _, f := range testFiles {
-		fid := framework.NewFileID(volumeID, f.key, f.cookie)
-		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, f.data)
-		body := framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode != http.StatusCreated {
-			t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
-		}
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	// Read source volume status before copy
-	sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
-	}
-	t.Logf("Source: dat=%d idx=%d files=%d version=%d",
-		sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
-		sourceStatus.GetFileCount(), sourceStatus.GetVersion())
-
-	// Copy volume from Go (server 0) to Rust (server 1)
-	copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
-		VolumeId:       volumeID,
-		SourceDataNode: cluster.VolumeServerAddress(0),
-	})
-	if err != nil {
-		t.Fatalf("VolumeCopy call failed: %v", err)
-	}
-
-	var lastAppendAtNs uint64
-	for {
-		resp, recvErr := copyStream.Recv()
-		if recvErr != nil {
-			if recvErr != io.EOF {
-				t.Fatalf("VolumeCopy recv error: %v", recvErr)
-			}
-			break
-		}
-		if resp.GetLastAppendAtNs() != 0 {
-			lastAppendAtNs = resp.GetLastAppendAtNs()
-		}
-	}
-	t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
-
-	// Verify: read volume status from Rust server
-	targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
-	}
-	t.Logf("Target: dat=%d idx=%d files=%d version=%d",
-		targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
-		targetStatus.GetFileCount(), targetStatus.GetVersion())
-
-	if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
-		t.Fatalf("dat file size mismatch: source=%d target=%d",
-			sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
-	}
-	if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
-		t.Fatalf("idx file size mismatch: source=%d target=%d",
-			sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
-	}
-	if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
-		t.Fatalf("file count mismatch: source=%d target=%d",
-			sourceStatus.GetFileCount(), targetStatus.GetFileCount())
-	}
-
-	// Verify data can be read from Rust server
-	for _, f := range testFiles {
-		fid := framework.NewFileID(volumeID, f.key, f.cookie)
-		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
-		body := framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode != http.StatusOK {
-			t.Fatalf("read %s from Rust server: expected 200, got %d", fid, resp.StatusCode)
-		}
-		if !bytes.Equal(body, f.data) {
-			t.Fatalf("read %s from Rust server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
-		}
-	}
-}
-
-// TestMixedBalanceCopyRustToGo verifies that VolumeCopy works from a Rust
-// volume server to a Go volume server.
-func TestMixedBalanceCopyRustToGo(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
-
-	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
-	defer conn0.Close()
-	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
-	defer conn1.Close()
-
-	httpClient := framework.NewHTTPClient()
-
-	const volumeID = uint32(51)
-
-	// Allocate volume on Rust server and upload test data
-	framework.AllocateVolume(t, rustClient, volumeID, "")
-
-	testFiles := []struct {
-		key    uint64
-		cookie uint32
-		data   []byte
-	}{
-		{1, 0xAABBCCDD, []byte("hello from Rust server")},
-		{2, 0x11223344, []byte("second file for reverse balance")},
-	}
-
-	for _, f := range testFiles {
-		fid := framework.NewFileID(volumeID, f.key, f.cookie)
-		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid, f.data)
-		body := framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode != http.StatusCreated {
-			t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
-		}
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	sourceStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
-	}
-
-	// Copy volume from Rust (server 1) to Go (server 0)
-	copyStream, err := goClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
-		VolumeId:       volumeID,
-		SourceDataNode: cluster.VolumeServerAddress(1),
-	})
-	if err != nil {
-		t.Fatalf("VolumeCopy call failed: %v", err)
-	}
-	for {
-		_, recvErr := copyStream.Recv()
-		if recvErr != nil {
-			if recvErr != io.EOF {
-				t.Fatalf("VolumeCopy recv error: %v", recvErr)
-			}
-			break
-		}
-	}
-
-	targetStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
-	}
-
-	if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
-		t.Fatalf("dat file size mismatch: source=%d target=%d",
-			sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
-	}
-	if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
-		t.Fatalf("idx file size mismatch: source=%d target=%d",
-			sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
-	}
-	if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
-		t.Fatalf("file count mismatch: source=%d target=%d",
-			sourceStatus.GetFileCount(), targetStatus.GetFileCount())
-	}
-
-	// Verify data can be read from Go server
-	for _, f := range testFiles {
-		fid := framework.NewFileID(volumeID, f.key, f.cookie)
-		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid)
-		body := framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode != http.StatusOK {
-			t.Fatalf("read %s from Go server: expected 200, got %d", fid, resp.StatusCode)
-		}
-		if !bytes.Equal(body, f.data) {
-			t.Fatalf("read %s from Go server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
-		}
-	}
-}
-
-// TestMixedBalanceCopyWithDeletes verifies that VolumeCopy correctly handles
-// volumes that have both active and deleted needles.
-func TestMixedBalanceCopyWithDeletes(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
-
-	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
-	defer conn0.Close()
-	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
-	defer conn1.Close()
-
-	httpClient := framework.NewHTTPClient()
-
-	const volumeID = uint32(52)
-	framework.AllocateVolume(t, goClient, volumeID, "")
-
-	// Upload files
-	for i := uint64(1); i <= 5; i++ {
-		fid := framework.NewFileID(volumeID, i, 0x12340000+uint32(i))
-		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("file-%d", i)))
-		framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode != http.StatusCreated {
-			t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
-		}
-	}
-
-	// Delete some files and wait for tombstones to be visible
-	for _, key := range []uint64{2, 4} {
-		fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
-		deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0), fid)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("ReadVolumeFileStatus: %v", err)
-	}
-	t.Logf("Source after deletes: dat=%d idx=%d files=%d",
-		sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
-		sourceStatus.GetFileCount())
-
-	// Copy volume from Go to Rust
-	copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
-		VolumeId:       volumeID,
-		SourceDataNode: cluster.VolumeServerAddress(0),
-	})
-	if err != nil {
-		t.Fatalf("VolumeCopy: %v", err)
-	}
-	var lastAppendAtNs uint64
-	for {
-		resp, recvErr := copyStream.Recv()
-		if recvErr != nil {
-			if recvErr != io.EOF {
-				t.Fatalf("VolumeCopy recv error: %v", recvErr)
-			}
-			break
-		}
-		if resp.GetLastAppendAtNs() != 0 {
-			lastAppendAtNs = resp.GetLastAppendAtNs()
-		}
-	}
-	if lastAppendAtNs == 0 {
-		t.Fatalf("VolumeCopy did not return a lastAppendAtNs timestamp")
-	}
-	t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
-
-	targetStatusAfterCopy, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("ReadVolumeFileStatus on Rust: %v", err)
-	}
-	t.Logf("Target after copy: dat=%d idx=%d files=%d",
-		targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterCopy.GetIdxFileSize(),
-		targetStatusAfterCopy.GetFileCount())
-
-	if sourceStatus.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
-		t.Fatalf("dat file size mismatch: source=%d target=%d",
-			sourceStatus.GetDatFileSize(), targetStatusAfterCopy.GetDatFileSize())
-	}
-	if sourceStatus.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
-		t.Fatalf("idx file size mismatch: source=%d target=%d",
-			sourceStatus.GetIdxFileSize(), targetStatusAfterCopy.GetIdxFileSize())
-	}
-
-	// Tail from the copy checkpoint — source is unchanged (deletes happened
-	// before copy), so tailing should not append any data.
-	_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
-		VolumeId:           volumeID,
-		SinceNs:            lastAppendAtNs,
-		IdleTimeoutSeconds: 3,
-		SourceVolumeServer: cluster.VolumeServerAddress(0),
-	})
-	if err != nil {
-		t.Fatalf("VolumeTailReceiver: %v", err)
-	}
-
-	targetStatusAfterTail, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("ReadVolumeFileStatus after tail: %v", err)
-	}
-	if targetStatusAfterTail.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
-		t.Fatalf("dat grew after tail: before=%d after=%d",
-			targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterTail.GetDatFileSize())
-	}
-	if targetStatusAfterTail.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
-		t.Fatalf("idx grew after tail: before=%d after=%d",
-			targetStatusAfterCopy.GetIdxFileSize(), targetStatusAfterTail.GetIdxFileSize())
-	}
-
-	// Verify surviving files are readable from Rust
-	for _, key := range []uint64{1, 3, 5} {
-		fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
-		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
-		body := framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode != http.StatusOK {
-			t.Fatalf("read surviving file %d from Rust: expected 200, got %d", key, resp.StatusCode)
-		}
-		expected := fmt.Sprintf("file-%d", key)
-		if string(body) != expected {
-			t.Fatalf("data mismatch for file %d: got %q want %q", key, body, expected)
-		}
-	}
-
-	// Verify deleted files return 404
-	for _, key := range []uint64{2, 4} {
-		fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
-		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
-		framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode != http.StatusNotFound {
-			t.Fatalf("read deleted file %d from Rust: expected 404, got %d", key, resp.StatusCode)
-		}
-	}
-}
-
-// TestMixedBalanceFullMoveGoToRust exercises the complete volume balance move
-// flow: mark readonly → copy → tail → verify sizes → delete source.
-// This mirrors the steps in balance_task.go Execute().
-func TestMixedBalanceFullMoveGoToRust(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
-
-	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
-	defer conn0.Close()
-	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
-	defer conn1.Close()
-
-	httpClient := framework.NewHTTPClient()
-
-	const volumeID = uint32(53)
-	framework.AllocateVolume(t, goClient, volumeID, "")
-
-	// Upload test data
-	for i := uint64(1); i <= 5; i++ {
-		fid := framework.NewFileID(volumeID, i, 0xABCD0000+uint32(i))
-		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("balance-move-file-%d", i)))
-		framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode != http.StatusCreated {
-			t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
-		}
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
-	defer cancel()
-
-	// Step 1: Copy volume to Rust server (source stays writable so we can
-	// delete after copy to exercise the tail tombstone path)
-	copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
-		VolumeId:       volumeID,
-		SourceDataNode: cluster.VolumeServerAddress(0),
-	})
-	if err != nil {
-		t.Fatalf("VolumeCopy: %v", err)
-	}
-	var lastAppendAtNs uint64
-	for {
-		resp, recvErr := copyStream.Recv()
-		if recvErr != nil {
-			if recvErr != io.EOF {
-				t.Fatalf("VolumeCopy recv: %v", recvErr)
-			}
-			break
-		}
-		if resp.GetLastAppendAtNs() != 0 {
-			lastAppendAtNs = resp.GetLastAppendAtNs()
-		}
-	}
-	t.Logf("Copy done, lastAppendAtNs=%d", lastAppendAtNs)
-
-	// Step 2: Delete file 3 on the source AFTER copy. This creates a
-	// tombstone needle that the tail step must propagate to the Rust server.
-	deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0),
-		framework.NewFileID(volumeID, 3, 0xABCD0003))
-
-	// Step 3: Mark source readonly so no further writes arrive during tail
-	_, err = goClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("mark readonly: %v", err)
-	}
-
-	// Read source status (the reference for post-tail verification)
-	sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("read source status: %v", err)
-	}
-	t.Logf("Source: dat=%d idx=%d files=%d",
-		sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
-		sourceStatus.GetFileCount())
-
-	// Step 4: Tail for updates — this must pick up the delete tombstone
-	_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
-		VolumeId:           volumeID,
-		SinceNs:            lastAppendAtNs,
-		IdleTimeoutSeconds: 5,
-		SourceVolumeServer: cluster.VolumeServerAddress(0),
-	})
-	if err != nil {
-		t.Fatalf("VolumeTailReceiver: %v", err)
-	}
-
-	// Step 5: Verify dat/idx sizes match after tail.
-	// We compare file sizes (byte-level correctness) rather than file_count
-	// because the tail writes the tombstone via the write path (which
-	// increments file_count) while the source used the delete path (which
-	// only increments deletion_count). This is consistent with Go behavior.
-	targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("read target status: %v", err)
-	}
-	t.Logf("Target: dat=%d idx=%d files=%d",
-		targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
-		targetStatus.GetFileCount())
-
-	if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
-		t.Fatalf("dat size mismatch after tail: source=%d target=%d",
-			sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
-	}
-	if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
-		t.Fatalf("idx size mismatch after tail: source=%d target=%d",
-			sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
-	}
-
-	// Step 6: Delete volume from source
-	_, err = goClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("delete source volume: %v", err)
-	}
-
-	// Verify source volume is gone
-	_, err = goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
-		VolumeId: volumeID,
-	})
-	if err == nil {
-		t.Fatalf("expected error reading deleted source volume, got nil")
-	}
-	t.Logf("Source volume deleted successfully (error as expected: %v)", err)
-
-	// Verify all surviving data is readable from Rust (the new home)
-	for _, key := range []uint64{1, 2, 4, 5} {
-		fid := framework.NewFileID(volumeID, key, 0xABCD0000+uint32(key))
-		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
-		body := framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode != http.StatusOK {
-			t.Fatalf("read file %d from Rust after move: expected 200, got %d", key, resp.StatusCode)
-		}
-		expected := fmt.Sprintf("balance-move-file-%d", key)
-		if string(body) != expected {
-			t.Fatalf("data mismatch for file %d after move: got %q want %q", key, body, expected)
-		}
-	}
-
-	// Verify deleted file is 404 on Rust (tombstone propagated via tail)
-	{
-		fid := framework.NewFileID(volumeID, 3, 0xABCD0003)
-		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
-		framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode != http.StatusNotFound {
-			t.Fatalf("deleted file 3 on Rust: expected 404, got %d", resp.StatusCode)
-		}
-	}
-
-	t.Logf("Full balance move completed: volume %d moved from Go to Rust, source purged", volumeID)
-}

weed/admin/dash/admin_data.go

@@ -45,7 +45,6 @@ type ObjectStoreUser struct {
 	SecretKey   string   `json:"secret_key"`
 	Permissions []string `json:"permissions"`
 	PolicyNames []string `json:"policy_names"`
-	IsStatic    bool     `json:"is_static"` // loaded from static config file, not editable
 }
 
 type ObjectStoreUsersData struct {

weed/admin/dash/admin_server.go

@@ -867,24 +867,6 @@ func (s *AdminServer) DeleteS3Bucket(bucketName string) error {
 	})
 }
 
-// IsStaticUser checks if a user is a static identity by loading the
-// configuration from the credential manager and checking the IsStatic flag.
-func (s *AdminServer) IsStaticUser(username string) bool {
-	if s.credentialManager == nil {
-		return false
-	}
-	s3cfg, err := s.credentialManager.LoadConfiguration(context.Background())
-	if err != nil {
-		return false
-	}
-	for _, ident := range s3cfg.Identities {
-		if ident.Name == username {
-			return ident.IsStatic
-		}
-	}
-	return false
-}
-
 // GetObjectStoreUsers retrieves object store users from identity.json
 func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUser, error) {
 	if s.credentialManager == nil {
@@ -908,7 +890,6 @@ func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUse
 		user := ObjectStoreUser{
 			Username:    identity.Name,
 			Permissions: identity.Actions,
-			IsStatic:    identity.IsStatic,
 		}
 
 		// Set email from account if available
@@ -962,7 +943,7 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 			leaderCount++
 		}
 
-		masterMap[masterInfo.Address] = masterInfo
+		masterMap[master.Address] = masterInfo
 	}
 
 	// Then, get additional master information from Raft cluster
@@ -974,11 +955,11 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 
 		// Process each raft server
 		for _, server := range resp.ClusterServers {
-			// Raft stores gRPC addresses, convert to HTTP address
-			httpAddress := pb.GrpcAddressToServerAddress(server.Address)
+			address := server.Address
+			httpAddress := pb.ServerAddress(address).ToHttpAddress()
 
 			// Update existing master info or create new one
-			if masterInfo, exists := masterMap[httpAddress]; exists {
+			if masterInfo, exists := masterMap[address]; exists {
 				// Update existing master with raft data
 				masterInfo.IsLeader = server.IsLeader
 				masterInfo.Suffrage = server.Suffrage
@@ -989,7 +970,7 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 					IsLeader: server.IsLeader,
 					Suffrage: server.Suffrage,
 				}
-				masterMap[httpAddress] = masterInfo
+				masterMap[address] = masterInfo
 			}
 
 			if server.IsLeader {
@@ -1665,6 +1646,159 @@ func (as *AdminServer) GetConfigPersistence() *ConfigPersistence {
 	return as.configPersistence
 }
 
+// convertJSONToMaintenanceConfig converts JSON map to protobuf MaintenanceConfig
+func convertJSONToMaintenanceConfig(jsonConfig map[string]interface{}) (*maintenance.MaintenanceConfig, error) {
+	config := &maintenance.MaintenanceConfig{}
+
+	// Helper function to get int32 from interface{}
+	getInt32 := func(key string) (int32, error) {
+		if val, ok := jsonConfig[key]; ok {
+			switch v := val.(type) {
+			case int:
+				return int32(v), nil
+			case int32:
+				return v, nil
+			case int64:
+				return int32(v), nil
+			case float64:
+				return int32(v), nil
+			default:
+				return 0, fmt.Errorf("invalid type for %s: expected number, got %T", key, v)
+			}
+		}
+		return 0, nil
+	}
+
+	// Helper function to get bool from interface{}
+	getBool := func(key string) bool {
+		if val, ok := jsonConfig[key]; ok {
+			if b, ok := val.(bool); ok {
+				return b
+			}
+		}
+		return false
+	}
+
+	var err error
+
+	// Convert basic fields
+	config.Enabled = getBool("enabled")
+
+	if config.ScanIntervalSeconds, err = getInt32("scan_interval_seconds"); err != nil {
+		return nil, err
+	}
+	if config.WorkerTimeoutSeconds, err = getInt32("worker_timeout_seconds"); err != nil {
+		return nil, err
+	}
+	if config.TaskTimeoutSeconds, err = getInt32("task_timeout_seconds"); err != nil {
+		return nil, err
+	}
+	if config.RetryDelaySeconds, err = getInt32("retry_delay_seconds"); err != nil {
+		return nil, err
+	}
+	if config.MaxRetries, err = getInt32("max_retries"); err != nil {
+		return nil, err
+	}
+	if config.CleanupIntervalSeconds, err = getInt32("cleanup_interval_seconds"); err != nil {
+		return nil, err
+	}
+	if config.TaskRetentionSeconds, err = getInt32("task_retention_seconds"); err != nil {
+		return nil, err
+	}
+
+	// Convert policy if present
+	if policyData, ok := jsonConfig["policy"]; ok {
+		if policyMap, ok := policyData.(map[string]interface{}); ok {
+			policy := &maintenance.MaintenancePolicy{}
+
+			if globalMaxConcurrent, err := getInt32FromMap(policyMap, "global_max_concurrent"); err != nil {
+				return nil, err
+			} else {
+				policy.GlobalMaxConcurrent = globalMaxConcurrent
+			}
+
+			if defaultRepeatIntervalSeconds, err := getInt32FromMap(policyMap, "default_repeat_interval_seconds"); err != nil {
+				return nil, err
+			} else {
+				policy.DefaultRepeatIntervalSeconds = defaultRepeatIntervalSeconds
+			}
+
+			if defaultCheckIntervalSeconds, err := getInt32FromMap(policyMap, "default_check_interval_seconds"); err != nil {
+				return nil, err
+			} else {
+				policy.DefaultCheckIntervalSeconds = defaultCheckIntervalSeconds
+			}
+
+			// Convert task policies if present
+			if taskPoliciesData, ok := policyMap["task_policies"]; ok {
+				if taskPoliciesMap, ok := taskPoliciesData.(map[string]interface{}); ok {
+					policy.TaskPolicies = make(map[string]*maintenance.TaskPolicy)
+
+					for taskType, taskPolicyData := range taskPoliciesMap {
+						if taskPolicyMap, ok := taskPolicyData.(map[string]interface{}); ok {
+							taskPolicy := &maintenance.TaskPolicy{}
+
+							taskPolicy.Enabled = getBoolFromMap(taskPolicyMap, "enabled")
+
+							if maxConcurrent, err := getInt32FromMap(taskPolicyMap, "max_concurrent"); err != nil {
+								return nil, err
+							} else {
+								taskPolicy.MaxConcurrent = maxConcurrent
+							}
+
+							if repeatIntervalSeconds, err := getInt32FromMap(taskPolicyMap, "repeat_interval_seconds"); err != nil {
+								return nil, err
+							} else {
+								taskPolicy.RepeatIntervalSeconds = repeatIntervalSeconds
+							}
+
+							if checkIntervalSeconds, err := getInt32FromMap(taskPolicyMap, "check_interval_seconds"); err != nil {
+								return nil, err
+							} else {
+								taskPolicy.CheckIntervalSeconds = checkIntervalSeconds
+							}
+
+							policy.TaskPolicies[taskType] = taskPolicy
+						}
+					}
+				}
+			}
+
+			config.Policy = policy
+		}
+	}
+
+	return config, nil
+}
+
+// Helper functions for map conversion
+func getInt32FromMap(m map[string]interface{}, key string) (int32, error) {
+	if val, ok := m[key]; ok {
+		switch v := val.(type) {
+		case int:
+			return int32(v), nil
+		case int32:
+			return v, nil
+		case int64:
+			return int32(v), nil
+		case float64:
+			return int32(v), nil
+		default:
+			return 0, fmt.Errorf("invalid type for %s: expected number, got %T", key, v)
+		}
+	}
+	return 0, nil
+}
+
+func getBoolFromMap(m map[string]interface{}, key string) bool {
+	if val, ok := m[key]; ok {
+		if b, ok := val.(bool); ok {
+			return b
+		}
+	}
+	return false
+}
+
 type collectionStats struct {
 	PhysicalSize int64
 	LogicalSize  int64

weed/admin/dash/bucket_management.go

@@ -361,6 +361,26 @@ func normalizeQuotaUnit(unit string) (string, error) {
 	}
 }
 
+// Helper function to convert bytes to appropriate unit and size
+func convertBytesToQuota(bytes int64) (int64, string) {
+	if bytes == 0 {
+		return 0, "MB"
+	}
+
+	// Convert to TB if >= 1TB
+	if bytes >= 1024*1024*1024*1024 && bytes%(1024*1024*1024*1024) == 0 {
+		return bytes / (1024 * 1024 * 1024 * 1024), "TB"
+	}
+
+	// Convert to GB if >= 1GB
+	if bytes >= 1024*1024*1024 && bytes%(1024*1024*1024) == 0 {
+		return bytes / (1024 * 1024 * 1024), "GB"
+	}
+
+	// Convert to MB (default)
+	return bytes / (1024 * 1024), "MB"
+}
+
 // SetBucketQuota sets the quota for a bucket
 func (s *AdminServer) SetBucketQuota(bucketName string, quotaBytes int64, quotaEnabled bool) error {
 	return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

weed/admin/dash/ec_shard_management.go

@@ -506,6 +506,18 @@ func getShardCount(ecIndexBits uint32) int {
 	return count
 }
 
+// getMissingShards returns a slice of missing shard IDs for a volume
+// Assumes default 10+4 EC configuration (14 total shards)
+func getMissingShards(ecIndexBits uint32) []int {
+	var missing []int
+	for i := 0; i < erasure_coding.TotalShardsCount; i++ {
+		if (ecIndexBits & (1 << uint(i))) == 0 {
+			missing = append(missing, i)
+		}
+	}
+	return missing
+}
+
 // sortEcShards sorts EC shards based on the specified field and order
 func sortEcShards(shards []EcShardWithInfo, sortBy string, sortOrder string) {
 	sort.Slice(shards, func(i, j int) bool {

weed/admin/dash/mq_management.go

@@ -430,6 +430,67 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
 	return offsets, nil
 }
 
+// convertRecordTypeToSchemaFields converts a protobuf RecordType to SchemaFieldInfo slice
+func convertRecordTypeToSchemaFields(recordType *schema_pb.RecordType) []SchemaFieldInfo {
+	var schemaFields []SchemaFieldInfo
+
+	if recordType == nil || recordType.Fields == nil {
+		return schemaFields
+	}
+
+	for _, field := range recordType.Fields {
+		schemaField := SchemaFieldInfo{
+			Name:     field.Name,
+			Type:     getFieldTypeString(field.Type),
+			Required: field.IsRequired,
+		}
+		schemaFields = append(schemaFields, schemaField)
+	}
+
+	return schemaFields
+}
+
+// getFieldTypeString converts a protobuf Type to a human-readable string
+func getFieldTypeString(fieldType *schema_pb.Type) string {
+	if fieldType == nil {
+		return "unknown"
+	}
+
+	switch kind := fieldType.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		return getScalarTypeString(kind.ScalarType)
+	case *schema_pb.Type_RecordType:
+		return "record"
+	case *schema_pb.Type_ListType:
+		elementType := getFieldTypeString(kind.ListType.ElementType)
+		return fmt.Sprintf("list<%s>", elementType)
+	default:
+		return "unknown"
+	}
+}
+
+// getScalarTypeString converts a protobuf ScalarType to a string
+func getScalarTypeString(scalarType schema_pb.ScalarType) string {
+	switch scalarType {
+	case schema_pb.ScalarType_BOOL:
+		return "bool"
+	case schema_pb.ScalarType_INT32:
+		return "int32"
+	case schema_pb.ScalarType_INT64:
+		return "int64"
+	case schema_pb.ScalarType_FLOAT:
+		return "float"
+	case schema_pb.ScalarType_DOUBLE:
+		return "double"
+	case schema_pb.ScalarType_BYTES:
+		return "bytes"
+	case schema_pb.ScalarType_STRING:
+		return "string"
+	default:
+		return "unknown"
+	}
+}
+
 // convertTopicPublishers converts protobuf TopicPublisher slice to PublisherInfo slice
 func convertTopicPublishers(publishers []*mq_pb.TopicPublisher) []PublisherInfo {
 	publisherInfos := make([]PublisherInfo, 0, len(publishers))

weed/admin/dash/plugin_api.go

@@ -2,6 +2,8 @@ package dash
 
 import (
 	"context"
+	"crypto/rand"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -849,6 +851,43 @@ func normalizeTimeout(timeoutSeconds int, defaultTimeout, maxTimeout time.Durati
 	return timeout
 }
 
+func buildJobSpecFromProposal(jobType string, proposal *plugin_pb.JobProposal, index int) *plugin_pb.JobSpec {
+	now := timestamppb.Now()
+	suffix := make([]byte, 4)
+	if _, err := rand.Read(suffix); err != nil {
+		// Fallback to simpler ID if rand fails
+		suffix = []byte(fmt.Sprintf("%d", index))
+	}
+	jobID := fmt.Sprintf("%s-%d-%s", jobType, now.AsTime().UnixNano(), hex.EncodeToString(suffix))
+
+	jobSpec := &plugin_pb.JobSpec{
+		JobId:      jobID,
+		JobType:    jobType,
+		Priority:   plugin_pb.JobPriority_JOB_PRIORITY_NORMAL,
+		CreatedAt:  now,
+		Labels:     make(map[string]string),
+		Parameters: make(map[string]*plugin_pb.ConfigValue),
+		DedupeKey:  "",
+	}
+
+	if proposal != nil {
+		jobSpec.Summary = proposal.Summary
+		jobSpec.Detail = proposal.Detail
+		if proposal.Priority != plugin_pb.JobPriority_JOB_PRIORITY_UNSPECIFIED {
+			jobSpec.Priority = proposal.Priority
+		}
+		jobSpec.DedupeKey = proposal.DedupeKey
+		jobSpec.Parameters = plugin.CloneConfigValueMap(proposal.Parameters)
+		if proposal.Labels != nil {
+			for k, v := range proposal.Labels {
+				jobSpec.Labels[k] = v
+			}
+		}
+	}
+
+	return jobSpec
+}
+
 func applyDescriptorDefaultsToPersistedConfig(
 	config *plugin_pb.PersistedJobTypeConfig,
 	descriptor *plugin_pb.JobTypeDescriptor,

weed/admin/dash/plugin_api_test.go

@@ -115,6 +115,32 @@ func TestExpirePluginJobAPI(t *testing.T) {
 	})
 }
 
+func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
+	t.Parallel()
+
+	proposal := &plugin_pb.JobProposal{
+		ProposalId: "vacuum-2",
+		DedupeKey:  "vacuum:2",
+		JobType:    "vacuum",
+	}
+
+	jobA := buildJobSpecFromProposal("vacuum", proposal, 0)
+	jobB := buildJobSpecFromProposal("vacuum", proposal, 1)
+
+	if jobA.JobId == proposal.ProposalId {
+		t.Fatalf("job id must not reuse proposal id: %s", jobA.JobId)
+	}
+	if jobB.JobId == proposal.ProposalId {
+		t.Fatalf("job id must not reuse proposal id: %s", jobB.JobId)
+	}
+	if jobA.JobId == jobB.JobId {
+		t.Fatalf("job ids must be unique across jobs: %s", jobA.JobId)
+	}
+	if jobA.DedupeKey != proposal.DedupeKey {
+		t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey)
+	}
+}
+
 func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testing.T) {
 	t.Parallel()
 

weed/admin/dash/types.go

@@ -447,6 +447,11 @@ type QueueStats = maintenance.QueueStats
 type WorkerDetailsData = maintenance.WorkerDetailsData
 type WorkerPerformance = maintenance.WorkerPerformance
 
+// GetTaskIcon returns the icon CSS class for a task type from its UI provider
+func GetTaskIcon(taskType MaintenanceTaskType) string {
+	return maintenance.GetTaskIcon(taskType)
+}
+
 // Status constants (these are still static)
 const (
 	TaskStatusPending    = maintenance.TaskStatusPending

weed/admin/dash/user_management.go

@@ -3,13 +3,13 @@ package dash
 import (
 	"context"
 	"crypto/rand"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"strings"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/credential"
-	"github.com/seaweedfs/seaweedfs/weed/iam"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 )
 
@@ -175,7 +175,7 @@ func (s *AdminServer) GetObjectStoreUserDetails(username string) (*UserDetails,
 
 	ctx := context.Background()
 
-	// Get user using credential manager (resolves static users via filer gRPC)
+	// Get user using credential manager
 	identity, err := s.credentialManager.GetUser(ctx, username)
 	if err != nil {
 		if err == credential.ErrUserNotFound {
@@ -435,13 +435,10 @@ func generateAccessKey() string {
 }
 
 func generateSecretKey() string {
-	// Use the IAM helper to generate URL-safe secret keys (no +, / characters)
-	// that won't break S3 signature authentication
-	key, err := iam.GenerateSecretAccessKey()
-	if err != nil {
-		panic(fmt.Sprintf("failed to generate secret key: %v", err))
-	}
-	return key
+	// Generate 40-character secret key (AWS standard)
+	b := make([]byte, 30) // 30 bytes = 40 characters in base64
+	rand.Read(b)
+	return base64.StdEncoding.EncodeToString(b)
 }
 
 func generateAccountId() string {

weed/admin/dash/user_management_test.go

@@ -74,16 +74,9 @@ func TestGenerateSecretKey(t *testing.T) {
 	key1 := generateSecretKey()
 	key2 := generateSecretKey()
 
-	// Check length (IAM standard secret key length)
-	if len(key1) != 42 {
-		t.Errorf("Expected secret key length 42, got %d", len(key1))
-	}
-
-	// Check that key contains only URL-safe characters (no +, /)
-	for _, c := range key1 {
-		if c == '+' || c == '/' || c == '=' {
-			t.Errorf("Secret key contains non-URL-safe character: %c", c)
-		}
+	// Check length (base64 encoding of 30 bytes = 40 characters)
+	if len(key1) != 40 {
+		t.Errorf("Expected secret key length 40, got %d", len(key1))
 	}
 
 	// Check uniqueness

weed/admin/dash/worker_grpc_server.go

@@ -787,6 +787,15 @@ func (s *WorkerGrpcServer) RequestTaskLogsFromAllWorkers(taskID string, maxEntri
 	return results, nil
 }
 
+// convertTaskParameters converts task parameters to protobuf format
+func convertTaskParameters(params map[string]interface{}) map[string]string {
+	result := make(map[string]string)
+	for key, value := range params {
+		result[key] = fmt.Sprintf("%v", value)
+	}
+	return result
+}
+
 func findClientAddress(ctx context.Context) string {
 	// fmt.Printf("FromContext %+v\n", ctx)
 	pr, ok := peer.FromContext(ctx)

weed/admin/handlers/cluster_handlers.go

@@ -312,6 +312,29 @@ func (h *ClusterHandlers) ShowClusterFilers(w http.ResponseWriter, r *http.Reque
 	}
 }
 
+// ShowClusterBrokers renders the cluster message brokers page
+func (h *ClusterHandlers) ShowClusterBrokers(w http.ResponseWriter, r *http.Request) {
+	// Get cluster brokers data
+	brokersData, err := h.adminServer.GetClusterBrokers()
+	if err != nil {
+		writeJSONError(w, http.StatusInternalServerError, "Failed to get cluster brokers: "+err.Error())
+		return
+	}
+
+	username := usernameOrDefault(r)
+	brokersData.Username = username
+
+	// Render HTML template
+	w.Header().Set("Content-Type", "text/html")
+	brokersComponent := app.ClusterBrokers(*brokersData)
+	viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
+	layoutComponent := layout.Layout(viewCtx, brokersComponent)
+	if err := layoutComponent.Render(r.Context(), w); err != nil {
+		writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
+		return
+	}
+}
+
 // GetClusterTopology returns the cluster topology as JSON
 func (h *ClusterHandlers) GetClusterTopology(w http.ResponseWriter, r *http.Request) {
 	topology, err := h.adminServer.GetClusterTopology()

weed/admin/handlers/mq_handlers.go

@@ -78,6 +78,34 @@ func (h *MessageQueueHandlers) ShowTopics(w http.ResponseWriter, r *http.Request
 	}
 }
 
+// ShowSubscribers renders the message queue subscribers page
+func (h *MessageQueueHandlers) ShowSubscribers(w http.ResponseWriter, r *http.Request) {
+	// Get subscribers data
+	subscribersData, err := h.adminServer.GetSubscribers()
+	if err != nil {
+		writeJSONError(w, http.StatusInternalServerError, "Failed to get subscribers: "+err.Error())
+		return
+	}
+
+	// Set username
+	username := dash.UsernameFromContext(r.Context())
+	if username == "" {
+		username = "admin"
+	}
+	subscribersData.Username = username
+
+	// Render HTML template
+	w.Header().Set("Content-Type", "text/html")
+	subscribersComponent := app.Subscribers(*subscribersData)
+	viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
+	layoutComponent := layout.Layout(viewCtx, subscribersComponent)
+	err = layoutComponent.Render(r.Context(), w)
+	if err != nil {
+		writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
+		return
+	}
+}
+
 // ShowTopicDetails renders the topic details page
 func (h *MessageQueueHandlers) ShowTopicDetails(w http.ResponseWriter, r *http.Request) {
 	// Get topic parameters from URL

weed/admin/handlers/user_handlers.go

@@ -93,11 +93,6 @@ func (h *UserHandlers) UpdateUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if h.adminServer.IsStaticUser(username) {
-		writeJSONError(w, http.StatusForbidden, "Cannot modify static user "+username+" (loaded from config file)")
-		return
-	}
-
 	var req dash.UpdateUserRequest
 	if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
 		writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
@@ -125,11 +120,6 @@ func (h *UserHandlers) DeleteUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if h.adminServer.IsStaticUser(username) {
-		writeJSONError(w, http.StatusForbidden, "Cannot delete static user "+username+" (loaded from config file)")
-		return
-	}
-
 	err := h.adminServer.DeleteObjectStoreUser(username)
 	if err != nil {
 		glog.Errorf("Failed to delete user %s: %v", username, err)

weed/admin/maintenance/config_verification.go

@@ -0,0 +1,124 @@
+package maintenance
+
+import (
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+)
+
+// VerifyProtobufConfig demonstrates that the protobuf configuration system is working
+func VerifyProtobufConfig() error {
+	// Create configuration manager
+	configManager := NewMaintenanceConfigManager()
+	config := configManager.GetConfig()
+
+	// Verify basic configuration
+	if !config.Enabled {
+		return fmt.Errorf("expected config to be enabled by default")
+	}
+
+	if config.ScanIntervalSeconds != 30*60 {
+		return fmt.Errorf("expected scan interval to be 1800 seconds, got %d", config.ScanIntervalSeconds)
+	}
+
+	// Verify policy configuration
+	if config.Policy == nil {
+		return fmt.Errorf("expected policy to be configured")
+	}
+
+	if config.Policy.GlobalMaxConcurrent != 4 {
+		return fmt.Errorf("expected global max concurrent to be 4, got %d", config.Policy.GlobalMaxConcurrent)
+	}
+
+	// Verify task policies
+	vacuumPolicy := config.Policy.TaskPolicies["vacuum"]
+	if vacuumPolicy == nil {
+		return fmt.Errorf("expected vacuum policy to be configured")
+	}
+
+	if !vacuumPolicy.Enabled {
+		return fmt.Errorf("expected vacuum policy to be enabled")
+	}
+
+	// Verify typed configuration access
+	vacuumConfig := vacuumPolicy.GetVacuumConfig()
+	if vacuumConfig == nil {
+		return fmt.Errorf("expected vacuum config to be accessible")
+	}
+
+	if vacuumConfig.GarbageThreshold != 0.3 {
+		return fmt.Errorf("expected garbage threshold to be 0.3, got %f", vacuumConfig.GarbageThreshold)
+	}
+
+	// Verify helper functions work
+	if !IsTaskEnabled(config.Policy, "vacuum") {
+		return fmt.Errorf("expected vacuum task to be enabled via helper function")
+	}
+
+	maxConcurrent := GetMaxConcurrent(config.Policy, "vacuum")
+	if maxConcurrent != 2 {
+		return fmt.Errorf("expected vacuum max concurrent to be 2, got %d", maxConcurrent)
+	}
+
+	// Verify erasure coding configuration
+	ecPolicy := config.Policy.TaskPolicies["erasure_coding"]
+	if ecPolicy == nil {
+		return fmt.Errorf("expected EC policy to be configured")
+	}
+
+	ecConfig := ecPolicy.GetErasureCodingConfig()
+	if ecConfig == nil {
+		return fmt.Errorf("expected EC config to be accessible")
+	}
+
+	// Verify configurable EC fields only
+	if ecConfig.FullnessRatio <= 0 || ecConfig.FullnessRatio > 1 {
+		return fmt.Errorf("expected EC config to have valid fullness ratio (0-1), got %f", ecConfig.FullnessRatio)
+	}
+
+	return nil
+}
+
+// GetProtobufConfigSummary returns a summary of the current protobuf configuration
+func GetProtobufConfigSummary() string {
+	configManager := NewMaintenanceConfigManager()
+	config := configManager.GetConfig()
+
+	summary := fmt.Sprintf("SeaweedFS Protobuf Maintenance Configuration:\n")
+	summary += fmt.Sprintf("  Enabled: %v\n", config.Enabled)
+	summary += fmt.Sprintf("  Scan Interval: %d seconds\n", config.ScanIntervalSeconds)
+	summary += fmt.Sprintf("  Max Retries: %d\n", config.MaxRetries)
+	summary += fmt.Sprintf("  Global Max Concurrent: %d\n", config.Policy.GlobalMaxConcurrent)
+	summary += fmt.Sprintf("  Task Policies: %d configured\n", len(config.Policy.TaskPolicies))
+
+	for taskType, policy := range config.Policy.TaskPolicies {
+		summary += fmt.Sprintf("    %s: enabled=%v, max_concurrent=%d\n",
+			taskType, policy.Enabled, policy.MaxConcurrent)
+	}
+
+	return summary
+}
+
+// CreateCustomConfig demonstrates creating a custom protobuf configuration
+func CreateCustomConfig() *worker_pb.MaintenanceConfig {
+	return &worker_pb.MaintenanceConfig{
+		Enabled:             true,
+		ScanIntervalSeconds: 60 * 60, // 1 hour
+		MaxRetries:          5,
+		Policy: &worker_pb.MaintenancePolicy{
+			GlobalMaxConcurrent: 8,
+			TaskPolicies: map[string]*worker_pb.TaskPolicy{
+				"custom_vacuum": {
+					Enabled:       true,
+					MaxConcurrent: 4,
+					TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
+						VacuumConfig: &worker_pb.VacuumTaskConfig{
+							GarbageThreshold:  0.5,
+							MinVolumeAgeHours: 48,
+						},
+					},
+				},
+			},
+		},
+	}
+}

weed/admin/maintenance/maintenance_config_proto.go

@@ -1,9 +1,24 @@
 package maintenance
 
 import (
+	"fmt"
+	"time"
+
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
 )
 
+// MaintenanceConfigManager handles protobuf-based configuration
+type MaintenanceConfigManager struct {
+	config *worker_pb.MaintenanceConfig
+}
+
+// NewMaintenanceConfigManager creates a new config manager with defaults
+func NewMaintenanceConfigManager() *MaintenanceConfigManager {
+	return &MaintenanceConfigManager{
+		config: DefaultMaintenanceConfigProto(),
+	}
+}
+
 // DefaultMaintenanceConfigProto returns default configuration as protobuf
 func DefaultMaintenanceConfigProto() *worker_pb.MaintenanceConfig {
 	return &worker_pb.MaintenanceConfig{
@@ -19,3 +34,253 @@ func DefaultMaintenanceConfigProto() *worker_pb.MaintenanceConfig {
 		Policy: nil,
 	}
 }
+
+// GetConfig returns the current configuration
+func (mcm *MaintenanceConfigManager) GetConfig() *worker_pb.MaintenanceConfig {
+	return mcm.config
+}
+
+// Type-safe configuration accessors
+
+// GetVacuumConfig returns vacuum-specific configuration for a task type
+func (mcm *MaintenanceConfigManager) GetVacuumConfig(taskType string) *worker_pb.VacuumTaskConfig {
+	if policy := mcm.getTaskPolicy(taskType); policy != nil {
+		if vacuumConfig := policy.GetVacuumConfig(); vacuumConfig != nil {
+			return vacuumConfig
+		}
+	}
+	// Return defaults if not configured
+	return &worker_pb.VacuumTaskConfig{
+		GarbageThreshold:  0.3,
+		MinVolumeAgeHours: 24,
+	}
+}
+
+// GetErasureCodingConfig returns EC-specific configuration for a task type
+func (mcm *MaintenanceConfigManager) GetErasureCodingConfig(taskType string) *worker_pb.ErasureCodingTaskConfig {
+	if policy := mcm.getTaskPolicy(taskType); policy != nil {
+		if ecConfig := policy.GetErasureCodingConfig(); ecConfig != nil {
+			return ecConfig
+		}
+	}
+	// Return defaults if not configured
+	return &worker_pb.ErasureCodingTaskConfig{
+		FullnessRatio:    0.95,
+		QuietForSeconds:  3600,
+		MinVolumeSizeMb:  100,
+		CollectionFilter: "",
+	}
+}
+
+// GetBalanceConfig returns balance-specific configuration for a task type
+func (mcm *MaintenanceConfigManager) GetBalanceConfig(taskType string) *worker_pb.BalanceTaskConfig {
+	if policy := mcm.getTaskPolicy(taskType); policy != nil {
+		if balanceConfig := policy.GetBalanceConfig(); balanceConfig != nil {
+			return balanceConfig
+		}
+	}
+	// Return defaults if not configured
+	return &worker_pb.BalanceTaskConfig{
+		ImbalanceThreshold: 0.2,
+		MinServerCount:     2,
+	}
+}
+
+// GetReplicationConfig returns replication-specific configuration for a task type
+func (mcm *MaintenanceConfigManager) GetReplicationConfig(taskType string) *worker_pb.ReplicationTaskConfig {
+	if policy := mcm.getTaskPolicy(taskType); policy != nil {
+		if replicationConfig := policy.GetReplicationConfig(); replicationConfig != nil {
+			return replicationConfig
+		}
+	}
+	// Return defaults if not configured
+	return &worker_pb.ReplicationTaskConfig{
+		TargetReplicaCount: 2,
+	}
+}
+
+// Typed convenience methods for getting task configurations
+
+// GetVacuumTaskConfigForType returns vacuum configuration for a specific task type
+func (mcm *MaintenanceConfigManager) GetVacuumTaskConfigForType(taskType string) *worker_pb.VacuumTaskConfig {
+	return GetVacuumTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
+}
+
+// GetErasureCodingTaskConfigForType returns erasure coding configuration for a specific task type
+func (mcm *MaintenanceConfigManager) GetErasureCodingTaskConfigForType(taskType string) *worker_pb.ErasureCodingTaskConfig {
+	return GetErasureCodingTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
+}
+
+// GetBalanceTaskConfigForType returns balance configuration for a specific task type
+func (mcm *MaintenanceConfigManager) GetBalanceTaskConfigForType(taskType string) *worker_pb.BalanceTaskConfig {
+	return GetBalanceTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
+}
+
+// GetReplicationTaskConfigForType returns replication configuration for a specific task type
+func (mcm *MaintenanceConfigManager) GetReplicationTaskConfigForType(taskType string) *worker_pb.ReplicationTaskConfig {
+	return GetReplicationTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
+}
+
+// Helper methods
+
+func (mcm *MaintenanceConfigManager) getTaskPolicy(taskType string) *worker_pb.TaskPolicy {
+	if mcm.config.Policy != nil && mcm.config.Policy.TaskPolicies != nil {
+		return mcm.config.Policy.TaskPolicies[taskType]
+	}
+	return nil
+}
+
+// IsTaskEnabled returns whether a task type is enabled
+func (mcm *MaintenanceConfigManager) IsTaskEnabled(taskType string) bool {
+	if policy := mcm.getTaskPolicy(taskType); policy != nil {
+		return policy.Enabled
+	}
+	return false
+}
+
+// GetMaxConcurrent returns the max concurrent limit for a task type
+func (mcm *MaintenanceConfigManager) GetMaxConcurrent(taskType string) int32 {
+	if policy := mcm.getTaskPolicy(taskType); policy != nil {
+		return policy.MaxConcurrent
+	}
+	return 1 // Default
+}
+
+// GetRepeatInterval returns the repeat interval for a task type in seconds
+func (mcm *MaintenanceConfigManager) GetRepeatInterval(taskType string) int32 {
+	if policy := mcm.getTaskPolicy(taskType); policy != nil {
+		return policy.RepeatIntervalSeconds
+	}
+	return mcm.config.Policy.DefaultRepeatIntervalSeconds
+}
+
+// GetCheckInterval returns the check interval for a task type in seconds
+func (mcm *MaintenanceConfigManager) GetCheckInterval(taskType string) int32 {
+	if policy := mcm.getTaskPolicy(taskType); policy != nil {
+		return policy.CheckIntervalSeconds
+	}
+	return mcm.config.Policy.DefaultCheckIntervalSeconds
+}
+
+// Duration accessor methods
+
+// GetScanInterval returns the scan interval as a time.Duration
+func (mcm *MaintenanceConfigManager) GetScanInterval() time.Duration {
+	return time.Duration(mcm.config.ScanIntervalSeconds) * time.Second
+}
+
+// GetWorkerTimeout returns the worker timeout as a time.Duration
+func (mcm *MaintenanceConfigManager) GetWorkerTimeout() time.Duration {
+	return time.Duration(mcm.config.WorkerTimeoutSeconds) * time.Second
+}
+
+// GetTaskTimeout returns the task timeout as a time.Duration
+func (mcm *MaintenanceConfigManager) GetTaskTimeout() time.Duration {
+	return time.Duration(mcm.config.TaskTimeoutSeconds) * time.Second
+}
+
+// GetRetryDelay returns the retry delay as a time.Duration
+func (mcm *MaintenanceConfigManager) GetRetryDelay() time.Duration {
+	return time.Duration(mcm.config.RetryDelaySeconds) * time.Second
+}
+
+// GetCleanupInterval returns the cleanup interval as a time.Duration
+func (mcm *MaintenanceConfigManager) GetCleanupInterval() time.Duration {
+	return time.Duration(mcm.config.CleanupIntervalSeconds) * time.Second
+}
+
+// GetTaskRetention returns the task retention period as a time.Duration
+func (mcm *MaintenanceConfigManager) GetTaskRetention() time.Duration {
+	return time.Duration(mcm.config.TaskRetentionSeconds) * time.Second
+}
+
+// ValidateMaintenanceConfigWithSchema validates protobuf maintenance configuration using ConfigField rules
+func ValidateMaintenanceConfigWithSchema(config *worker_pb.MaintenanceConfig) error {
+	if config == nil {
+		return fmt.Errorf("configuration cannot be nil")
+	}
+
+	// Get the schema to access field validation rules
+	schema := GetMaintenanceConfigSchema()
+
+	// Validate each field individually using the ConfigField rules
+	if err := validateFieldWithSchema(schema, "enabled", config.Enabled); err != nil {
+		return err
+	}
+
+	if err := validateFieldWithSchema(schema, "scan_interval_seconds", int(config.ScanIntervalSeconds)); err != nil {
+		return err
+	}
+
+	if err := validateFieldWithSchema(schema, "worker_timeout_seconds", int(config.WorkerTimeoutSeconds)); err != nil {
+		return err
+	}
+
+	if err := validateFieldWithSchema(schema, "task_timeout_seconds", int(config.TaskTimeoutSeconds)); err != nil {
+		return err
+	}
+
+	if err := validateFieldWithSchema(schema, "retry_delay_seconds", int(config.RetryDelaySeconds)); err != nil {
+		return err
+	}
+
+	if err := validateFieldWithSchema(schema, "max_retries", int(config.MaxRetries)); err != nil {
+		return err
+	}
+
+	if err := validateFieldWithSchema(schema, "cleanup_interval_seconds", int(config.CleanupIntervalSeconds)); err != nil {
+		return err
+	}
+
+	if err := validateFieldWithSchema(schema, "task_retention_seconds", int(config.TaskRetentionSeconds)); err != nil {
+		return err
+	}
+
+	// Validate policy fields if present
+	if config.Policy != nil {
+		// Note: These field names might need to be adjusted based on the actual schema
+		if err := validatePolicyField("global_max_concurrent", int(config.Policy.GlobalMaxConcurrent)); err != nil {
+			return err
+		}
+
+		if err := validatePolicyField("default_repeat_interval_seconds", int(config.Policy.DefaultRepeatIntervalSeconds)); err != nil {
+			return err
+		}
+
+		if err := validatePolicyField("default_check_interval_seconds", int(config.Policy.DefaultCheckIntervalSeconds)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// validateFieldWithSchema validates a single field using its ConfigField definition
+func validateFieldWithSchema(schema *MaintenanceConfigSchema, fieldName string, value interface{}) error {
+	field := schema.GetFieldByName(fieldName)
+	if field == nil {
+		// Field not in schema, skip validation
+		return nil
+	}
+
+	return field.ValidateValue(value)
+}
+
+// validatePolicyField validates policy fields (simplified validation for now)
+func validatePolicyField(fieldName string, value int) error {
+	switch fieldName {
+	case "global_max_concurrent":
+		if value < 1 || value > 20 {
+			return fmt.Errorf("Global Max Concurrent must be between 1 and 20, got %d", value)
+		}
+	case "default_repeat_interval":
+		if value < 1 || value > 168 {
+			return fmt.Errorf("Default Repeat Interval must be between 1 and 168 hours, got %d", value)
+		}
+	case "default_check_interval":
+		if value < 1 || value > 168 {
+			return fmt.Errorf("Default Check Interval must be between 1 and 168 hours, got %d", value)
+		}
+	}
+	return nil
+}

weed/admin/maintenance/maintenance_queue.go

@@ -1055,6 +1055,28 @@ func (mq *MaintenanceQueue) getMaxConcurrentForTaskType(taskType MaintenanceTask
 	return 1
 }
 
+// getRunningTasks returns all currently running tasks
+func (mq *MaintenanceQueue) getRunningTasks() []*MaintenanceTask {
+	var runningTasks []*MaintenanceTask
+	for _, task := range mq.tasks {
+		if task.Status == TaskStatusAssigned || task.Status == TaskStatusInProgress {
+			runningTasks = append(runningTasks, task)
+		}
+	}
+	return runningTasks
+}
+
+// getAvailableWorkers returns all workers that can take more work
+func (mq *MaintenanceQueue) getAvailableWorkers() []*MaintenanceWorker {
+	var availableWorkers []*MaintenanceWorker
+	for _, worker := range mq.workers {
+		if worker.Status == "active" && worker.CurrentLoad < worker.MaxConcurrent {
+			availableWorkers = append(availableWorkers, worker)
+		}
+	}
+	return availableWorkers
+}
+
 // trackPendingOperation adds a task to the pending operations tracker
 func (mq *MaintenanceQueue) trackPendingOperation(task *MaintenanceTask) {
 	if mq.integration == nil {

weed/admin/maintenance/maintenance_types.go

@@ -2,11 +2,15 @@ package maintenance
 
 import (
 	"html/template"
+	"sort"
 	"sync"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
+	"github.com/seaweedfs/seaweedfs/weed/worker/types"
 )
 
 // AdminClient interface defines what the maintenance system needs from the admin server
@@ -17,6 +21,51 @@ type AdminClient interface {
 // MaintenanceTaskType represents different types of maintenance operations
 type MaintenanceTaskType string
 
+// GetRegisteredMaintenanceTaskTypes returns all registered task types as MaintenanceTaskType values
+// sorted alphabetically for consistent menu ordering
+func GetRegisteredMaintenanceTaskTypes() []MaintenanceTaskType {
+	typesRegistry := tasks.GetGlobalTypesRegistry()
+	var taskTypes []MaintenanceTaskType
+
+	for workerTaskType := range typesRegistry.GetAllDetectors() {
+		maintenanceTaskType := MaintenanceTaskType(string(workerTaskType))
+		taskTypes = append(taskTypes, maintenanceTaskType)
+	}
+
+	// Sort task types alphabetically to ensure consistent menu ordering
+	sort.Slice(taskTypes, func(i, j int) bool {
+		return string(taskTypes[i]) < string(taskTypes[j])
+	})
+
+	return taskTypes
+}
+
+// GetMaintenanceTaskType returns a specific task type if it's registered, or empty string if not found
+func GetMaintenanceTaskType(taskTypeName string) MaintenanceTaskType {
+	typesRegistry := tasks.GetGlobalTypesRegistry()
+
+	for workerTaskType := range typesRegistry.GetAllDetectors() {
+		if string(workerTaskType) == taskTypeName {
+			return MaintenanceTaskType(taskTypeName)
+		}
+	}
+
+	return MaintenanceTaskType("")
+}
+
+// IsMaintenanceTaskTypeRegistered checks if a task type is registered
+func IsMaintenanceTaskTypeRegistered(taskType MaintenanceTaskType) bool {
+	typesRegistry := tasks.GetGlobalTypesRegistry()
+
+	for workerTaskType := range typesRegistry.GetAllDetectors() {
+		if string(workerTaskType) == string(taskType) {
+			return true
+		}
+	}
+
+	return false
+}
+
 // MaintenanceTaskPriority represents task execution priority
 type MaintenanceTaskPriority int
 
@@ -151,6 +200,14 @@ func GetTaskPolicy(mp *MaintenancePolicy, taskType MaintenanceTaskType) *TaskPol
 	return mp.TaskPolicies[string(taskType)]
 }
 
+// SetTaskPolicy sets the policy for a specific task type
+func SetTaskPolicy(mp *MaintenancePolicy, taskType MaintenanceTaskType, policy *TaskPolicy) {
+	if mp.TaskPolicies == nil {
+		mp.TaskPolicies = make(map[string]*TaskPolicy)
+	}
+	mp.TaskPolicies[string(taskType)] = policy
+}
+
 // IsTaskEnabled returns whether a task type is enabled
 func IsTaskEnabled(mp *MaintenancePolicy, taskType MaintenanceTaskType) bool {
 	policy := GetTaskPolicy(mp, taskType)
@@ -178,6 +235,84 @@ func GetRepeatInterval(mp *MaintenancePolicy, taskType MaintenanceTaskType) int
 	return int(policy.RepeatIntervalSeconds)
 }
 
+// GetVacuumTaskConfig returns the vacuum task configuration
+func GetVacuumTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.VacuumTaskConfig {
+	policy := GetTaskPolicy(mp, taskType)
+	if policy == nil {
+		return nil
+	}
+	return policy.GetVacuumConfig()
+}
+
+// GetErasureCodingTaskConfig returns the erasure coding task configuration
+func GetErasureCodingTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.ErasureCodingTaskConfig {
+	policy := GetTaskPolicy(mp, taskType)
+	if policy == nil {
+		return nil
+	}
+	return policy.GetErasureCodingConfig()
+}
+
+// GetBalanceTaskConfig returns the balance task configuration
+func GetBalanceTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.BalanceTaskConfig {
+	policy := GetTaskPolicy(mp, taskType)
+	if policy == nil {
+		return nil
+	}
+	return policy.GetBalanceConfig()
+}
+
+// GetReplicationTaskConfig returns the replication task configuration
+func GetReplicationTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.ReplicationTaskConfig {
+	policy := GetTaskPolicy(mp, taskType)
+	if policy == nil {
+		return nil
+	}
+	return policy.GetReplicationConfig()
+}
+
+// Note: GetTaskConfig was removed - use typed getters: GetVacuumTaskConfig, GetErasureCodingTaskConfig, GetBalanceTaskConfig, or GetReplicationTaskConfig
+
+// SetVacuumTaskConfig sets the vacuum task configuration
+func SetVacuumTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.VacuumTaskConfig) {
+	policy := GetTaskPolicy(mp, taskType)
+	if policy != nil {
+		policy.TaskConfig = &worker_pb.TaskPolicy_VacuumConfig{
+			VacuumConfig: config,
+		}
+	}
+}
+
+// SetErasureCodingTaskConfig sets the erasure coding task configuration
+func SetErasureCodingTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.ErasureCodingTaskConfig) {
+	policy := GetTaskPolicy(mp, taskType)
+	if policy != nil {
+		policy.TaskConfig = &worker_pb.TaskPolicy_ErasureCodingConfig{
+			ErasureCodingConfig: config,
+		}
+	}
+}
+
+// SetBalanceTaskConfig sets the balance task configuration
+func SetBalanceTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.BalanceTaskConfig) {
+	policy := GetTaskPolicy(mp, taskType)
+	if policy != nil {
+		policy.TaskConfig = &worker_pb.TaskPolicy_BalanceConfig{
+			BalanceConfig: config,
+		}
+	}
+}
+
+// SetReplicationTaskConfig sets the replication task configuration
+func SetReplicationTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.ReplicationTaskConfig) {
+	policy := GetTaskPolicy(mp, taskType)
+	if policy != nil {
+		policy.TaskConfig = &worker_pb.TaskPolicy_ReplicationConfig{
+			ReplicationConfig: config,
+		}
+	}
+}
+
 // SetTaskConfig sets a configuration value for a task type (legacy method - use typed setters above)
 // Note: SetTaskConfig was removed - use typed setters: SetVacuumTaskConfig, SetErasureCodingTaskConfig, SetBalanceTaskConfig, or SetReplicationTaskConfig
 
@@ -340,6 +475,180 @@ type ClusterReplicationTask struct {
 	Metadata        map[string]string `json:"metadata,omitempty"`
 }
 
+// BuildMaintenancePolicyFromTasks creates a maintenance policy with configurations
+// from all registered tasks using their UI providers
+func BuildMaintenancePolicyFromTasks() *MaintenancePolicy {
+	policy := &MaintenancePolicy{
+		TaskPolicies:                 make(map[string]*TaskPolicy),
+		GlobalMaxConcurrent:          4,
+		DefaultRepeatIntervalSeconds: 6 * 3600,  // 6 hours in seconds
+		DefaultCheckIntervalSeconds:  12 * 3600, // 12 hours in seconds
+	}
+
+	// Get all registered task types from the UI registry
+	uiRegistry := tasks.GetGlobalUIRegistry()
+	typesRegistry := tasks.GetGlobalTypesRegistry()
+
+	for taskType, provider := range uiRegistry.GetAllProviders() {
+		// Convert task type to maintenance task type
+		maintenanceTaskType := MaintenanceTaskType(string(taskType))
+
+		// Get the default configuration from the UI provider
+		defaultConfig := provider.GetCurrentConfig()
+
+		// Create task policy from UI configuration
+		taskPolicy := &TaskPolicy{
+			Enabled:               true, // Default enabled
+			MaxConcurrent:         2,    // Default concurrency
+			RepeatIntervalSeconds: policy.DefaultRepeatIntervalSeconds,
+			CheckIntervalSeconds:  policy.DefaultCheckIntervalSeconds,
+		}
+
+		// Extract configuration using TaskConfig interface - no more map conversions!
+		if taskConfig, ok := defaultConfig.(interface{ ToTaskPolicy() *worker_pb.TaskPolicy }); ok {
+			// Use protobuf directly for clean, type-safe config extraction
+			pbTaskPolicy := taskConfig.ToTaskPolicy()
+			taskPolicy.Enabled = pbTaskPolicy.Enabled
+			taskPolicy.MaxConcurrent = pbTaskPolicy.MaxConcurrent
+			if pbTaskPolicy.RepeatIntervalSeconds > 0 {
+				taskPolicy.RepeatIntervalSeconds = pbTaskPolicy.RepeatIntervalSeconds
+			}
+			if pbTaskPolicy.CheckIntervalSeconds > 0 {
+				taskPolicy.CheckIntervalSeconds = pbTaskPolicy.CheckIntervalSeconds
+			}
+		}
+
+		// Also get defaults from scheduler if available (using types.TaskScheduler explicitly)
+		var scheduler types.TaskScheduler = typesRegistry.GetScheduler(taskType)
+		if scheduler != nil {
+			if taskPolicy.MaxConcurrent <= 0 {
+				taskPolicy.MaxConcurrent = int32(scheduler.GetMaxConcurrent())
+			}
+			// Convert default repeat interval to seconds
+			if repeatInterval := scheduler.GetDefaultRepeatInterval(); repeatInterval > 0 {
+				taskPolicy.RepeatIntervalSeconds = int32(repeatInterval.Seconds())
+			}
+		}
+
+		// Also get defaults from detector if available (using types.TaskDetector explicitly)
+		var detector types.TaskDetector = typesRegistry.GetDetector(taskType)
+		if detector != nil {
+			// Convert scan interval to check interval (seconds)
+			if scanInterval := detector.ScanInterval(); scanInterval > 0 {
+				taskPolicy.CheckIntervalSeconds = int32(scanInterval.Seconds())
+			}
+		}
+
+		policy.TaskPolicies[string(maintenanceTaskType)] = taskPolicy
+		glog.V(3).Infof("Built policy for task type %s: enabled=%v, max_concurrent=%d",
+			maintenanceTaskType, taskPolicy.Enabled, taskPolicy.MaxConcurrent)
+	}
+
+	glog.V(2).Infof("Built maintenance policy with %d task configurations", len(policy.TaskPolicies))
+	return policy
+}
+
+// SetPolicyFromTasks sets the maintenance policy from registered tasks
+func SetPolicyFromTasks(policy *MaintenancePolicy) {
+	if policy == nil {
+		return
+	}
+
+	// Build new policy from tasks
+	newPolicy := BuildMaintenancePolicyFromTasks()
+
+	// Copy task policies
+	policy.TaskPolicies = newPolicy.TaskPolicies
+
+	glog.V(1).Infof("Updated maintenance policy with %d task configurations from registered tasks", len(policy.TaskPolicies))
+}
+
+// GetTaskIcon returns the icon CSS class for a task type from its UI provider
+func GetTaskIcon(taskType MaintenanceTaskType) string {
+	typesRegistry := tasks.GetGlobalTypesRegistry()
+	uiRegistry := tasks.GetGlobalUIRegistry()
+
+	// Convert MaintenanceTaskType to TaskType
+	for workerTaskType := range typesRegistry.GetAllDetectors() {
+		if string(workerTaskType) == string(taskType) {
+			// Get the UI provider for this task type
+			provider := uiRegistry.GetProvider(workerTaskType)
+			if provider != nil {
+				return provider.GetIcon()
+			}
+			break
+		}
+	}
+
+	// Default icon if no UI provider found
+	return "fas fa-cog text-muted"
+}
+
+// GetTaskDisplayName returns the display name for a task type from its UI provider
+func GetTaskDisplayName(taskType MaintenanceTaskType) string {
+	typesRegistry := tasks.GetGlobalTypesRegistry()
+	uiRegistry := tasks.GetGlobalUIRegistry()
+
+	// Convert MaintenanceTaskType to TaskType
+	for workerTaskType := range typesRegistry.GetAllDetectors() {
+		if string(workerTaskType) == string(taskType) {
+			// Get the UI provider for this task type
+			provider := uiRegistry.GetProvider(workerTaskType)
+			if provider != nil {
+				return provider.GetDisplayName()
+			}
+			break
+		}
+	}
+
+	// Fallback to the task type string
+	return string(taskType)
+}
+
+// GetTaskDescription returns the description for a task type from its UI provider
+func GetTaskDescription(taskType MaintenanceTaskType) string {
+	typesRegistry := tasks.GetGlobalTypesRegistry()
+	uiRegistry := tasks.GetGlobalUIRegistry()
+
+	// Convert MaintenanceTaskType to TaskType
+	for workerTaskType := range typesRegistry.GetAllDetectors() {
+		if string(workerTaskType) == string(taskType) {
+			// Get the UI provider for this task type
+			provider := uiRegistry.GetProvider(workerTaskType)
+			if provider != nil {
+				return provider.GetDescription()
+			}
+			break
+		}
+	}
+
+	// Fallback to a generic description
+	return "Configure detailed settings for " + string(taskType) + " tasks."
+}
+
+// BuildMaintenanceMenuItems creates menu items for all registered task types
+func BuildMaintenanceMenuItems() []*MaintenanceMenuItem {
+	var menuItems []*MaintenanceMenuItem
+
+	// Get all registered task types
+	registeredTypes := GetRegisteredMaintenanceTaskTypes()
+
+	for _, taskType := range registeredTypes {
+		menuItem := &MaintenanceMenuItem{
+			TaskType:    taskType,
+			DisplayName: GetTaskDisplayName(taskType),
+			Description: GetTaskDescription(taskType),
+			Icon:        GetTaskIcon(taskType),
+			IsEnabled:   IsMaintenanceTaskTypeRegistered(taskType),
+			Path:        "/maintenance/config/" + string(taskType),
+		}
+
+		menuItems = append(menuItems, menuItem)
+	}
+
+	return menuItems
+}
+
 // Helper functions to extract configuration fields
 
 // Note: Removed getVacuumConfigField, getErasureCodingConfigField, getBalanceConfigField, getReplicationConfigField

weed/admin/maintenance/maintenance_worker.go

@@ -0,0 +1,421 @@
+package maintenance
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/worker"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
+	"github.com/seaweedfs/seaweedfs/weed/worker/types"
+
+	// Import task packages to trigger their auto-registration
+	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
+	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
+)
+
+// MaintenanceWorkerService manages maintenance task execution
+// TaskExecutor defines the function signature for task execution
+type TaskExecutor func(*MaintenanceWorkerService, *MaintenanceTask) error
+
+// TaskExecutorFactory creates a task executor for a given worker service
+type TaskExecutorFactory func() TaskExecutor
+
+// Global registry for task executor factories
+var taskExecutorFactories = make(map[MaintenanceTaskType]TaskExecutorFactory)
+var executorRegistryMutex sync.RWMutex
+var executorRegistryInitOnce sync.Once
+
+// initializeExecutorFactories dynamically registers executor factories for all auto-registered task types
+func initializeExecutorFactories() {
+	executorRegistryInitOnce.Do(func() {
+		// Get all registered task types from the global registry
+		typesRegistry := tasks.GetGlobalTypesRegistry()
+
+		var taskTypes []MaintenanceTaskType
+		for workerTaskType := range typesRegistry.GetAllDetectors() {
+			// Convert types.TaskType to MaintenanceTaskType by string conversion
+			maintenanceTaskType := MaintenanceTaskType(string(workerTaskType))
+			taskTypes = append(taskTypes, maintenanceTaskType)
+		}
+
+		// Register generic executor for all task types
+		for _, taskType := range taskTypes {
+			RegisterTaskExecutorFactory(taskType, createGenericTaskExecutor)
+		}
+
+		glog.V(1).Infof("Dynamically registered generic task executor for %d task types: %v", len(taskTypes), taskTypes)
+	})
+}
+
+// RegisterTaskExecutorFactory registers a factory function for creating task executors
+func RegisterTaskExecutorFactory(taskType MaintenanceTaskType, factory TaskExecutorFactory) {
+	executorRegistryMutex.Lock()
+	defer executorRegistryMutex.Unlock()
+	taskExecutorFactories[taskType] = factory
+	glog.V(2).Infof("Registered executor factory for task type: %s", taskType)
+}
+
+// GetTaskExecutorFactory returns the factory for a task type
+func GetTaskExecutorFactory(taskType MaintenanceTaskType) (TaskExecutorFactory, bool) {
+	// Ensure executor factories are initialized
+	initializeExecutorFactories()
+
+	executorRegistryMutex.RLock()
+	defer executorRegistryMutex.RUnlock()
+	factory, exists := taskExecutorFactories[taskType]
+	return factory, exists
+}
+
+// GetSupportedExecutorTaskTypes returns all task types with registered executor factories
+func GetSupportedExecutorTaskTypes() []MaintenanceTaskType {
+	// Ensure executor factories are initialized
+	initializeExecutorFactories()
+
+	executorRegistryMutex.RLock()
+	defer executorRegistryMutex.RUnlock()
+
+	taskTypes := make([]MaintenanceTaskType, 0, len(taskExecutorFactories))
+	for taskType := range taskExecutorFactories {
+		taskTypes = append(taskTypes, taskType)
+	}
+	return taskTypes
+}
+
+// createGenericTaskExecutor creates a generic task executor that uses the task registry
+func createGenericTaskExecutor() TaskExecutor {
+	return func(mws *MaintenanceWorkerService, task *MaintenanceTask) error {
+		return mws.executeGenericTask(task)
+	}
+}
+
+// init does minimal initialization - actual registration happens lazily
+func init() {
+	// Executor factory registration will happen lazily when first accessed
+	glog.V(1).Infof("Maintenance worker initialized - executor factories will be registered on first access")
+}
+
+type MaintenanceWorkerService struct {
+	workerID      string
+	address       string
+	adminServer   string
+	capabilities  []MaintenanceTaskType
+	maxConcurrent int
+	currentTasks  map[string]*MaintenanceTask
+	queue         *MaintenanceQueue
+	adminClient   AdminClient
+	running       bool
+	stopChan      chan struct{}
+
+	// Task execution registry
+	taskExecutors map[MaintenanceTaskType]TaskExecutor
+
+	// Task registry for creating task instances
+	taskRegistry *tasks.TaskRegistry
+}
+
+// NewMaintenanceWorkerService creates a new maintenance worker service
+func NewMaintenanceWorkerService(workerID, address, adminServer string) *MaintenanceWorkerService {
+	// Get all registered maintenance task types dynamically
+	capabilities := GetRegisteredMaintenanceTaskTypes()
+
+	worker := &MaintenanceWorkerService{
+		workerID:      workerID,
+		address:       address,
+		adminServer:   adminServer,
+		capabilities:  capabilities,
+		maxConcurrent: 2, // Default concurrent task limit
+		currentTasks:  make(map[string]*MaintenanceTask),
+		stopChan:      make(chan struct{}),
+		taskExecutors: make(map[MaintenanceTaskType]TaskExecutor),
+		taskRegistry:  tasks.GetGlobalTaskRegistry(), // Use global registry with auto-registered tasks
+	}
+
+	// Initialize task executor registry
+	worker.initializeTaskExecutors()
+
+	glog.V(1).Infof("Created maintenance worker with %d registered task types", len(worker.taskRegistry.GetAll()))
+
+	return worker
+}
+
+// executeGenericTask executes a task using the task registry instead of hardcoded methods
+func (mws *MaintenanceWorkerService) executeGenericTask(task *MaintenanceTask) error {
+	glog.V(2).Infof("Executing generic task %s: %s for volume %d", task.ID, task.Type, task.VolumeID)
+
+	// Validate that task has proper typed parameters
+	if task.TypedParams == nil {
+		return fmt.Errorf("task %s has no typed parameters - task was not properly planned (insufficient destinations)", task.ID)
+	}
+
+	// Convert MaintenanceTask to types.TaskType
+	taskType := types.TaskType(string(task.Type))
+
+	// Create task instance using the registry
+	taskInstance, err := mws.taskRegistry.Get(taskType).Create(task.TypedParams)
+	if err != nil {
+		return fmt.Errorf("failed to create task instance: %w", err)
+	}
+
+	// Update progress to show task has started
+	mws.updateTaskProgress(task.ID, 5)
+
+	// Execute the task
+	err = taskInstance.Execute(context.Background(), task.TypedParams)
+	if err != nil {
+		return fmt.Errorf("task execution failed: %w", err)
+	}
+
+	// Update progress to show completion
+	mws.updateTaskProgress(task.ID, 100)
+
+	glog.V(2).Infof("Generic task %s completed successfully", task.ID)
+	return nil
+}
+
+// initializeTaskExecutors sets up the task execution registry dynamically
+func (mws *MaintenanceWorkerService) initializeTaskExecutors() {
+	mws.taskExecutors = make(map[MaintenanceTaskType]TaskExecutor)
+
+	// Get all registered executor factories and create executors
+	executorRegistryMutex.RLock()
+	defer executorRegistryMutex.RUnlock()
+
+	for taskType, factory := range taskExecutorFactories {
+		executor := factory()
+		mws.taskExecutors[taskType] = executor
+		glog.V(3).Infof("Initialized executor for task type: %s", taskType)
+	}
+
+	glog.V(2).Infof("Initialized %d task executors", len(mws.taskExecutors))
+}
+
+// RegisterTaskExecutor allows dynamic registration of new task executors
+func (mws *MaintenanceWorkerService) RegisterTaskExecutor(taskType MaintenanceTaskType, executor TaskExecutor) {
+	if mws.taskExecutors == nil {
+		mws.taskExecutors = make(map[MaintenanceTaskType]TaskExecutor)
+	}
+	mws.taskExecutors[taskType] = executor
+	glog.V(1).Infof("Registered executor for task type: %s", taskType)
+}
+
+// GetSupportedTaskTypes returns all task types that this worker can execute
+func (mws *MaintenanceWorkerService) GetSupportedTaskTypes() []MaintenanceTaskType {
+	return GetSupportedExecutorTaskTypes()
+}
+
+// Start begins the worker service
+func (mws *MaintenanceWorkerService) Start() error {
+	mws.running = true
+
+	// Register with admin server
+	worker := &MaintenanceWorker{
+		ID:            mws.workerID,
+		Address:       mws.address,
+		Capabilities:  mws.capabilities,
+		MaxConcurrent: mws.maxConcurrent,
+	}
+
+	if mws.queue != nil {
+		mws.queue.RegisterWorker(worker)
+	}
+
+	// Start worker loop
+	go mws.workerLoop()
+
+	glog.Infof("Maintenance worker %s started at %s", mws.workerID, mws.address)
+	return nil
+}
+
+// Stop terminates the worker service
+func (mws *MaintenanceWorkerService) Stop() {
+	mws.running = false
+	close(mws.stopChan)
+
+	// Wait for current tasks to complete or timeout
+	timeout := time.NewTimer(30 * time.Second)
+	defer timeout.Stop()
+
+	for len(mws.currentTasks) > 0 {
+		select {
+		case <-timeout.C:
+			glog.Warningf("Worker %s stopping with %d tasks still running", mws.workerID, len(mws.currentTasks))
+			return
+		case <-time.After(time.Second):
+			// Check again
+		}
+	}
+
+	glog.Infof("Maintenance worker %s stopped", mws.workerID)
+}
+
+// workerLoop is the main worker event loop
+func (mws *MaintenanceWorkerService) workerLoop() {
+	heartbeatTicker := time.NewTicker(30 * time.Second)
+	defer heartbeatTicker.Stop()
+
+	taskRequestTicker := time.NewTicker(5 * time.Second)
+	defer taskRequestTicker.Stop()
+
+	for mws.running {
+		select {
+		case <-mws.stopChan:
+			return
+		case <-heartbeatTicker.C:
+			mws.sendHeartbeat()
+		case <-taskRequestTicker.C:
+			mws.requestTasks()
+		}
+	}
+}
+
+// sendHeartbeat sends heartbeat to admin server
+func (mws *MaintenanceWorkerService) sendHeartbeat() {
+	if mws.queue != nil {
+		mws.queue.UpdateWorkerHeartbeat(mws.workerID)
+	}
+}
+
+// requestTasks requests new tasks from the admin server
+func (mws *MaintenanceWorkerService) requestTasks() {
+	if len(mws.currentTasks) >= mws.maxConcurrent {
+		return // Already at capacity
+	}
+
+	if mws.queue != nil {
+		task := mws.queue.GetNextTask(mws.workerID, mws.capabilities)
+		if task != nil {
+			mws.executeTask(task)
+		}
+	}
+}
+
+// executeTask executes a maintenance task
+func (mws *MaintenanceWorkerService) executeTask(task *MaintenanceTask) {
+	mws.currentTasks[task.ID] = task
+
+	go func() {
+		defer func() {
+			delete(mws.currentTasks, task.ID)
+		}()
+
+		glog.Infof("Worker %s executing task %s: %s", mws.workerID, task.ID, task.Type)
+
+		// Execute task using dynamic executor registry
+		var err error
+		if executor, exists := mws.taskExecutors[task.Type]; exists {
+			err = executor(mws, task)
+		} else {
+			err = fmt.Errorf("unsupported task type: %s", task.Type)
+			glog.Errorf("No executor registered for task type: %s", task.Type)
+		}
+
+		// Report task completion
+		if mws.queue != nil {
+			errorMsg := ""
+			if err != nil {
+				errorMsg = err.Error()
+			}
+			mws.queue.CompleteTask(task.ID, errorMsg)
+		}
+
+		if err != nil {
+			glog.Errorf("Worker %s failed to execute task %s: %v", mws.workerID, task.ID, err)
+		} else {
+			glog.Infof("Worker %s completed task %s successfully", mws.workerID, task.ID)
+		}
+	}()
+}
+
+// updateTaskProgress updates the progress of a task
+func (mws *MaintenanceWorkerService) updateTaskProgress(taskID string, progress float64) {
+	if mws.queue != nil {
+		mws.queue.UpdateTaskProgress(taskID, progress)
+	}
+}
+
+// GetStatus returns the current status of the worker
+func (mws *MaintenanceWorkerService) GetStatus() map[string]interface{} {
+	return map[string]interface{}{
+		"worker_id":      mws.workerID,
+		"address":        mws.address,
+		"running":        mws.running,
+		"capabilities":   mws.capabilities,
+		"max_concurrent": mws.maxConcurrent,
+		"current_tasks":  len(mws.currentTasks),
+		"task_details":   mws.currentTasks,
+	}
+}
+
+// SetQueue sets the maintenance queue for the worker
+func (mws *MaintenanceWorkerService) SetQueue(queue *MaintenanceQueue) {
+	mws.queue = queue
+}
+
+// SetAdminClient sets the admin client for the worker
+func (mws *MaintenanceWorkerService) SetAdminClient(client AdminClient) {
+	mws.adminClient = client
+}
+
+// SetCapabilities sets the worker capabilities
+func (mws *MaintenanceWorkerService) SetCapabilities(capabilities []MaintenanceTaskType) {
+	mws.capabilities = capabilities
+}
+
+// SetMaxConcurrent sets the maximum concurrent tasks
+func (mws *MaintenanceWorkerService) SetMaxConcurrent(max int) {
+	mws.maxConcurrent = max
+}
+
+// SetHeartbeatInterval sets the heartbeat interval (placeholder for future use)
+func (mws *MaintenanceWorkerService) SetHeartbeatInterval(interval time.Duration) {
+	// Future implementation for configurable heartbeat
+}
+
+// SetTaskRequestInterval sets the task request interval (placeholder for future use)
+func (mws *MaintenanceWorkerService) SetTaskRequestInterval(interval time.Duration) {
+	// Future implementation for configurable task requests
+}
+
+// MaintenanceWorkerCommand represents a standalone maintenance worker command
+type MaintenanceWorkerCommand struct {
+	workerService *MaintenanceWorkerService
+}
+
+// NewMaintenanceWorkerCommand creates a new worker command
+func NewMaintenanceWorkerCommand(workerID, address, adminServer string) *MaintenanceWorkerCommand {
+	return &MaintenanceWorkerCommand{
+		workerService: NewMaintenanceWorkerService(workerID, address, adminServer),
+	}
+}
+
+// Run starts the maintenance worker as a standalone service
+func (mwc *MaintenanceWorkerCommand) Run() error {
+	// Generate or load persistent worker ID if not provided
+	if mwc.workerService.workerID == "" {
+		// Get current working directory for worker ID persistence
+		wd, err := os.Getwd()
+		if err != nil {
+			return fmt.Errorf("failed to get working directory: %w", err)
+		}
+
+		workerID, err := worker.GenerateOrLoadWorkerID(wd)
+		if err != nil {
+			return fmt.Errorf("failed to generate or load worker ID: %w", err)
+		}
+		mwc.workerService.workerID = workerID
+	}
+
+	// Start the worker service
+	err := mwc.workerService.Start()
+	if err != nil {
+		return fmt.Errorf("failed to start maintenance worker: %w", err)
+	}
+
+	// Wait for interrupt signal
+	select {}
+}

weed/admin/plugin/plugin.go

@@ -122,7 +122,6 @@ type Plugin struct {
 type streamSession struct {
 	workerID  string
 	outgoing  chan *plugin_pb.AdminToWorkerMessage
-	done      chan struct{}
 	closeOnce sync.Once
 }
 
@@ -275,7 +274,6 @@ func (r *Plugin) WorkerStream(stream plugin_pb.PluginControlService_WorkerStream
 	session := &streamSession{
 		workerID: workerID,
 		outgoing: make(chan *plugin_pb.AdminToWorkerMessage, r.outgoingBuffer),
-		done:     make(chan struct{}),
 	}
 	r.putSession(session)
 	defer r.cleanupSession(workerID)
@@ -910,10 +908,8 @@ func (r *Plugin) sendLoop(
 			return nil
 		case <-r.shutdownCh:
 			return nil
-		case <-session.done:
-			return nil
-		case msg := <-session.outgoing:
-			if msg == nil {
+		case msg, ok := <-session.outgoing:
+			if !ok {
 				return nil
 			}
 			if err := stream.Send(msg); err != nil {
@@ -934,8 +930,6 @@ func (r *Plugin) sendToWorker(workerID string, message *plugin_pb.AdminToWorkerM
 	select {
 	case <-r.shutdownCh:
 		return fmt.Errorf("plugin is shutting down")
-	case <-session.done:
-		return fmt.Errorf("worker %s session is closed", workerID)
 	case session.outgoing <- message:
 		return nil
 	case <-time.After(r.sendTimeout):
@@ -1431,7 +1425,7 @@ func CloneConfigValueMap(in map[string]*plugin_pb.ConfigValue) map[string]*plugi
 
 func (s *streamSession) close() {
 	s.closeOnce.Do(func() {
-		close(s.done)
+		close(s.outgoing)
 	})
 }
 

weed/admin/plugin/plugin_cancel_test.go

@@ -26,7 +26,7 @@ func TestRunDetectionSendsCancelOnContextDone(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4), done: make(chan struct{})}
+	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4)}
 	pluginSvc.putSession(session)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -77,7 +77,7 @@ func TestExecuteJobSendsCancelOnContextDone(t *testing.T) {
 			{JobType: jobType, CanExecute: true, MaxExecutionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4), done: make(chan struct{})}
+	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4)}
 	pluginSvc.putSession(session)
 
 	job := &plugin_pb.JobSpec{JobId: "job-1", JobType: jobType}
@@ -135,8 +135,8 @@ func TestAdminScriptExecutionBlocksOtherDetection(t *testing.T) {
 			{JobType: "vacuum", CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
-	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
+	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
+	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
 	pluginSvc.putSession(adminSession)
 	pluginSvc.putSession(otherSession)
 
@@ -214,8 +214,8 @@ func TestAdminScriptExecutionBlocksOtherExecution(t *testing.T) {
 			{JobType: "vacuum", CanExecute: true, MaxExecutionConcurrency: 1},
 		},
 	})
-	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
-	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
+	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
+	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
 	pluginSvc.putSession(adminSession)
 	pluginSvc.putSession(otherSession)
 

weed/admin/plugin/plugin_detection_test.go

@@ -22,7 +22,7 @@ func TestRunDetectionIncludesLatestSuccessfulRun(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
+	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
 	pluginSvc.putSession(session)
 
 	oldSuccess := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
@@ -80,7 +80,7 @@ func TestRunDetectionOmitsLastSuccessfulRunWhenNoSuccessHistory(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
+	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
 	pluginSvc.putSession(session)
 
 	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
@@ -130,7 +130,7 @@ func TestRunDetectionWithReportCapturesDetectionActivities(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
+	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
 	pluginSvc.putSession(session)
 
 	reportCh := make(chan *DetectionReport, 1)
@@ -210,7 +210,7 @@ func TestRunDetectionAdminScriptUsesLastCompletedRun(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: "worker-admin-script", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
+	session := &streamSession{workerID: "worker-admin-script", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
 	pluginSvc.putSession(session)
 
 	successCompleted := time.Date(2026, 2, 1, 10, 0, 0, 0, time.UTC)

weed/admin/plugin/plugin_scheduler.go

@@ -95,6 +95,16 @@ func (r *Plugin) laneSchedulerLoop(ls *schedulerLaneState) {
 	}
 }
 
+// schedulerLoop is kept for backward compatibility; it delegates to
+// laneSchedulerLoop with the default lane. New code should not call this.
+func (r *Plugin) schedulerLoop() {
+	ls := r.lanes[LaneDefault]
+	if ls == nil {
+		ls = newLaneState(LaneDefault)
+	}
+	r.laneSchedulerLoop(ls)
+}
+
 // runLaneSchedulerIteration runs one scheduling pass for a single lane,
 // processing only the job types assigned to that lane.
 //
@@ -219,6 +229,82 @@ func (r *Plugin) runLaneSchedulerIterationConcurrent(ls *schedulerLaneState, job
 	return hadJobs.Load()
 }
 
+// runSchedulerIteration is kept for backward compatibility. It runs a
+// single iteration across ALL job types (equivalent to the old single-loop
+// behavior). It is only used by the legacy schedulerLoop() fallback.
+func (r *Plugin) runSchedulerIteration() bool {
+	ls := r.lanes[LaneDefault]
+	if ls == nil {
+		ls = newLaneState(LaneDefault)
+	}
+	// For backward compat, the old function processes all job types.
+	r.expireStaleJobs(time.Now().UTC())
+
+	jobTypes := r.registry.DetectableJobTypes()
+	if len(jobTypes) == 0 {
+		r.setSchedulerLoopState("", "idle")
+		return false
+	}
+
+	r.setSchedulerLoopState("", "waiting_for_lock")
+	releaseLock, err := r.acquireAdminLock("plugin scheduler iteration")
+	if err != nil {
+		glog.Warningf("Plugin scheduler failed to acquire lock: %v", err)
+		r.setSchedulerLoopState("", "idle")
+		return false
+	}
+	if releaseLock != nil {
+		defer releaseLock()
+	}
+
+	active := make(map[string]struct{}, len(jobTypes))
+	hadJobs := false
+
+	for _, jobType := range jobTypes {
+		active[jobType] = struct{}{}
+
+		policy, enabled, err := r.loadSchedulerPolicy(jobType)
+		if err != nil {
+			glog.Warningf("Plugin scheduler failed to load policy for %s: %v", jobType, err)
+			continue
+		}
+		if !enabled {
+			r.clearSchedulerJobType(jobType)
+			continue
+		}
+		initialDelay := time.Duration(0)
+		if runInfo := r.snapshotSchedulerRun(jobType); runInfo.lastRunStartedAt.IsZero() {
+			initialDelay = 5 * time.Second
+		}
+		if !r.markDetectionDue(jobType, policy.DetectionInterval, initialDelay) {
+			continue
+		}
+
+		detected := r.runJobTypeIteration(jobType, policy)
+		if detected {
+			hadJobs = true
+		}
+	}
+
+	r.pruneSchedulerState(active)
+	r.pruneDetectorLeases(active)
+	r.setSchedulerLoopState("", "idle")
+	return hadJobs
+}
+
+// wakeLane wakes the scheduler goroutine for a specific lane.
+func (r *Plugin) wakeLane(lane SchedulerLane) {
+	if r == nil {
+		return
+	}
+	if ls, ok := r.lanes[lane]; ok {
+		select {
+		case ls.wakeCh <- struct{}{}:
+		default:
+		}
+	}
+}
+
 // wakeAllLanes wakes all lane scheduler goroutines.
 func (r *Plugin) wakeAllLanes() {
 	if r == nil {

weed/admin/plugin/scheduler_status.go

@@ -210,6 +210,16 @@ func (r *Plugin) setSchedulerLoopStateForJobType(jobType, phase string) {
 	}
 }
 
+func (r *Plugin) recordSchedulerIterationComplete(hadJobs bool) {
+	if r == nil {
+		return
+	}
+	r.schedulerLoopMu.Lock()
+	r.schedulerLoopState.lastIterationHadJobs = hadJobs
+	r.schedulerLoopState.lastIterationCompleted = time.Now().UTC()
+	r.schedulerLoopMu.Unlock()
+}
+
 func (r *Plugin) snapshotSchedulerLoopState() schedulerLoopState {
 	if r == nil {
 		return schedulerLoopState{}

weed/admin/view/app/object_store_users.templ

@@ -125,9 +125,6 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
                                                 <div class="d-flex align-items-center">
                                                     <i class="fas fa-user me-2 text-muted"></i>
                                                     <strong>{user.Username}</strong>
-                                                    if user.IsStatic {
-                                                        <span class="badge bg-secondary ms-2" title="Loaded from config file (read-only)">static</span>
-                                                    }
                                                 </div>
                                             </td>
                                             <td>{user.Email}</td>
@@ -136,28 +133,24 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
                                             </td>
                                             <td>
                                                 <div class="btn-group btn-group-sm" role="group">
-                                                    <button type="button" class="btn btn-outline-info"
+                                                    <button type="button" class="btn btn-outline-info" 
                                                             data-action="show-user-details" data-username={ user.Username }>
                                                         <i class="fas fa-info-circle"></i>
                                                     </button>
-                                                    if !user.IsStatic {
-                                                        <button type="button" class="btn btn-outline-primary"
-                                                                data-action="edit-user" data-username={ user.Username }>
-                                                            <i class="fas fa-edit"></i>
-                                                        </button>
-                                                    }
-                                                    if user.Username != "anonymous" && !user.IsStatic {
+                                                    <button type="button" class="btn btn-outline-primary" 
+                                                            data-action="edit-user" data-username={ user.Username }>
+                                                        <i class="fas fa-edit"></i>
+                                                    </button>
+                                                    if user.Username != "anonymous" {
                                                         <button type="button" class="btn btn-outline-secondary"
                                                                 data-action="manage-access-keys" data-username={ user.Username }>
                                                             <i class="fas fa-key"></i>
                                                         </button>
                                                     }
-                                                    if !user.IsStatic {
-                                                        <button type="button" class="btn btn-outline-danger"
-                                                                data-action="delete-user" data-username={ user.Username }>
-                                                            <i class="fas fa-trash"></i>
-                                                        </button>
-                                                    }
+                                                    <button type="button" class="btn btn-outline-danger" 
+                                                            data-action="delete-user" data-username={ user.Username }>
+                                                        <i class="fas fa-trash"></i>
+                                                    </button>
                                                 </div>
                                             </td>
                                         </tr>

weed/admin/view/app/s3_buckets.templ

@@ -163,7 +163,7 @@ templ S3Buckets(data dash.S3BucketsData) {
                                     for _, bucket := range data.Buckets {
                                         <tr>
                                             <td>
-                                                <a href={dash.PUrl(ctx, fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
+                                                <a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
                                                    class="text-decoration-none">
                                                     <i class="fas fa-cube me-2"></i>
                                                     {bucket.Name}
@@ -236,7 +236,7 @@ templ S3Buckets(data dash.S3BucketsData) {
                                             </td>
                                             <td>
                                                 <div class="btn-group btn-group-sm" role="group">
-                                                    <a href={dash.PUrl(ctx, fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
+                                                    <a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
                                                        class="btn btn-outline-success btn-sm"
                                                        title="Browse Files">
                                                         <i class="fas fa-folder-open"></i>

weed/admin/view/app/s3_buckets_templ.go

@@ -171,9 +171,9 @@ func S3Buckets(data dash.S3BucketsData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var5 templ.SafeURL
-			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
+			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 166, Col: 124}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 166, Col: 123}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 			if templ_7745c5c3_Err != nil {
@@ -439,9 +439,9 @@ func S3Buckets(data dash.S3BucketsData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var19 templ.SafeURL
-			templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
+			templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 239, Col: 128}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 239, Col: 127}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 			if templ_7745c5c3_Err != nil {

weed/admin/view/app/s3tables_buckets.templ

@@ -2,7 +2,6 @@ package app
 
 import (
 	"fmt"
-	"net/url"
 
 	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
@@ -153,7 +152,7 @@ templ S3TablesBuckets(data dash.S3TablesBucketsData) {
 												<div class="btn-group btn-group-sm" role="group">
 													{{ bucketName, parseErr := s3tables.ParseBucketNameFromARN(bucket.ARN) }}
 													if parseErr == nil {
-														<a class="btn btn-outline-primary btn-sm" href={ dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName))) }>
+														<a class="btn btn-outline-primary btn-sm" href={ templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)) }>
 															<i class="fas fa-folder-open"></i>
 														</a>
 													} else {

weed/admin/view/app/s3tables_buckets_templ.go

@@ -10,7 +10,6 @@ import templruntime "github.com/a-h/templ/runtime"
 
 import (
 	"fmt"
-	"net/url"
 
 	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
@@ -49,7 +48,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var2 templ.SafeURL
 			templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("http://localhost:%d/v1/config", data.IcebergPort)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 24, Col: 124}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 23, Col: 124}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 			if templ_7745c5c3_Err != nil {
@@ -67,7 +66,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 		var templ_7745c5c3_Var3 string
 		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 31, Col: 91}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 30, Col: 91}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
 		if templ_7745c5c3_Err != nil {
@@ -85,7 +84,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var4 string
 			templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 38, Col: 109}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 37, Col: 109}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
 			if templ_7745c5c3_Err != nil {
@@ -98,7 +97,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var5 string
 			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 40, Col: 107}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 39, Col: 107}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 			if templ_7745c5c3_Err != nil {
@@ -121,7 +120,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 		var templ_7745c5c3_Var6 string
 		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalBuckets))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 70, Col: 47}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 69, Col: 47}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {
@@ -134,7 +133,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 		var templ_7745c5c3_Var7 string
 		templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04"))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 89, Col: 54}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 88, Col: 54}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
 		if templ_7745c5c3_Err != nil {
@@ -148,7 +147,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var8 string
 			templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 109, Col: 47}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 108, Col: 47}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
 			if templ_7745c5c3_Err != nil {
@@ -172,7 +171,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var9 string
 			templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 147, Col: 28}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 146, Col: 28}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
 			if templ_7745c5c3_Err != nil {
@@ -185,7 +184,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var10 string
 			templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.OwnerAccountID)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 148, Col: 38}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 147, Col: 38}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
 			if templ_7745c5c3_Err != nil {
@@ -198,7 +197,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var11 string
 			templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 149, Col: 52}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 148, Col: 52}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
 			if templ_7745c5c3_Err != nil {
@@ -211,7 +210,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var12 string
 			templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 150, Col: 52}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 149, Col: 52}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
 			if templ_7745c5c3_Err != nil {
@@ -224,7 +223,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var13 string
 			templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04"))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 151, Col: 60}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 150, Col: 60}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
 			if templ_7745c5c3_Err != nil {
@@ -241,9 +240,9 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 					return templ_7745c5c3_Err
 				}
 				var templ_7745c5c3_Var14 templ.SafeURL
-				templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName))))
+				templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 156, Col: 166}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 155, Col: 149}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
 				if templ_7745c5c3_Err != nil {
@@ -266,7 +265,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var15 string
 			templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 164, Col: 122}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 163, Col: 122}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
 			if templ_7745c5c3_Err != nil {
@@ -279,7 +278,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var16 string
 			templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 167, Col: 126}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 166, Col: 126}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
 			if templ_7745c5c3_Err != nil {
@@ -292,7 +291,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var17 string
 			templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 170, Col: 128}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 169, Col: 128}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 			if templ_7745c5c3_Err != nil {
@@ -305,7 +304,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var18 string
 			templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 170, Col: 161}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 169, Col: 161}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 			if templ_7745c5c3_Err != nil {
@@ -343,7 +342,7 @@ CREATE SECRET (
 
 SELECT * FROM iceberg_scan('s3://my-table-bucket/my-namespace/my-table');`)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 220, Col: 74}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 219, Col: 74}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 			if templ_7745c5c3_Err != nil {
@@ -366,7 +365,7 @@ catalog = load_catalog(
 
 namespaces = catalog.list_namespaces()`)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 236, Col: 39}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 235, Col: 39}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
 			if templ_7745c5c3_Err != nil {

weed/admin/view/app/s3tables_tables.templ

@@ -24,7 +24,7 @@ templ S3TablesTables(data dash.S3TablesTablesData) {
 	<div class="mb-3">
 		{{ bucketName, parseErr := s3tables.ParseBucketNameFromARN(data.BucketARN) }}
 		if parseErr == nil {
-			<a href={ dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName))) } class="btn btn-sm btn-outline-secondary">
+			<a href={ templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)) } class="btn btn-sm btn-outline-secondary">
 				<i class="fas fa-arrow-left me-1"></i>Back to Namespaces
 			</a>
 		} else {
@@ -126,7 +126,7 @@ templ S3TablesTables(data dash.S3TablesTablesData) {
 											<td>
 												<div class="btn-group btn-group-sm" role="group">
 													if parseErr == nil {
-														<a class="btn btn-outline-primary btn-sm" href={ dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables/%s", url.PathEscape(bucketName), url.PathEscape(data.Namespace), url.PathEscape(tableName))) } title="View Iceberg Details">
+														<a class="btn btn-outline-primary btn-sm" href={ templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables/%s", url.PathEscape(bucketName), url.PathEscape(data.Namespace), url.PathEscape(tableName))) } title="View Iceberg Details">
 															<i class="fas fa-eye"></i>
 														</a>
 													} else {

weed/admin/view/app/s3tables_tables_templ.go

@@ -48,9 +48,9 @@ func S3TablesTables(data dash.S3TablesTablesData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var2 templ.SafeURL
-			templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName))))
+			templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_tables.templ`, Line: 27, Col: 116}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_tables.templ`, Line: 27, Col: 99}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 			if templ_7745c5c3_Err != nil {
@@ -308,9 +308,9 @@ func S3TablesTables(data dash.S3TablesTablesData) templ.Component {
 					return templ_7745c5c3_Err
 				}
 				var templ_7745c5c3_Var17 templ.SafeURL
-				templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables/%s", url.PathEscape(bucketName), url.PathEscape(data.Namespace), url.PathEscape(tableName))))
+				templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables/%s", url.PathEscape(bucketName), url.PathEscape(data.Namespace), url.PathEscape(tableName))))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_tables.templ`, Line: 129, Col: 238}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_tables.templ`, Line: 129, Col: 237}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 				if templ_7745c5c3_Err != nil {

weed/admin/view/app/template_helpers.go

@@ -6,6 +6,20 @@ import (
 	"strings"
 )
 
+// getStatusColor returns Bootstrap color class for status
+func getStatusColor(status string) string {
+	switch status {
+	case "active", "healthy":
+		return "success"
+	case "warning":
+		return "warning"
+	case "critical", "unreachable":
+		return "danger"
+	default:
+		return "secondary"
+	}
+}
+
 // formatBytes converts bytes to human readable format
 func formatBytes(bytes int64) string {
 	if bytes == 0 {

weed/cluster/cluster.go

@@ -95,6 +95,18 @@ func NewCluster() *Cluster {
 	}
 }
 
+func (cluster *Cluster) getGroupMembers(filerGroup FilerGroupName, nodeType string, createIfNotFound bool) *GroupMembers {
+	switch nodeType {
+	case FilerType:
+		return cluster.filerGroups.getGroupMembers(filerGroup, createIfNotFound)
+	case BrokerType:
+		return cluster.brokerGroups.getGroupMembers(filerGroup, createIfNotFound)
+	case S3Type:
+		return cluster.s3Groups.getGroupMembers(filerGroup, createIfNotFound)
+	}
+	return nil
+}
+
 func (cluster *Cluster) AddClusterNode(ns, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
 	filerGroup := FilerGroupName(ns)
 	switch nodeType {

weed/command/admin.go

@@ -30,7 +30,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-	"github.com/seaweedfs/seaweedfs/weed/util/grace"
 )
 
 var (
@@ -49,10 +48,6 @@ type AdminOptions struct {
 	dataDir          *string
 	icebergPort      *int
 	urlPrefix        *string
-	debug            *bool
-	debugPort        *int
-	cpuProfile       *string
-	memProfile       *string
 }
 
 func init() {
@@ -69,10 +64,6 @@ func init() {
 	a.readOnlyPassword = cmdAdmin.Flag.String("readOnlyPassword", "", "read-only user password (optional, for view-only access; requires adminPassword to be set)")
 	a.icebergPort = cmdAdmin.Flag.Int("iceberg.port", 8181, "Iceberg REST Catalog port (0 to hide in UI)")
 	a.urlPrefix = cmdAdmin.Flag.String("urlPrefix", "", "URL path prefix when running behind a reverse proxy under a subdirectory (e.g. /seaweedfs)")
-	a.debug = cmdAdmin.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
-	a.debugPort = cmdAdmin.Flag.Int("debug.port", 6060, "http port for debugging")
-	a.cpuProfile = cmdAdmin.Flag.String("cpuprofile", "", "cpu profile output file")
-	a.memProfile = cmdAdmin.Flag.String("memprofile", "", "memory profile output file")
 }
 
 var cmdAdmin = &Command{
@@ -149,16 +140,6 @@ var cmdAdmin = &Command{
     - All static assets, API endpoints, and navigation links will use the prefix
     - Session cookies are scoped to the prefix path
 
-  Debugging and Profiling:
-    - Use -debug to start a pprof HTTP server for live profiling (localhost only)
-    - Set -debug.port to choose the pprof port (default 6060)
-    - Profiles are accessible at http://127.0.0.1:<debug.port>/debug/pprof/
-    - Use -cpuprofile and -memprofile to write profiles to files on shutdown
-    - WARNING: -debug exposes runtime internals; use only in trusted environments
-    - Examples:
-      weed admin -debug -debug.port=6060 -master="localhost:9333"
-      weed admin -cpuprofile=cpu.prof -memprofile=mem.prof -master="localhost:9333"
-
   Configuration File:
     - The security.toml file is read from ".", "$HOME/.seaweedfs/",
       "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order
@@ -168,12 +149,6 @@ var cmdAdmin = &Command{
 }
 
 func runAdmin(cmd *Command, args []string) bool {
-	if *a.debug {
-		grace.StartDebugServer(*a.debugPort)
-	}
-
-	grace.SetupProfiling(*a.cpuProfile, *a.memProfile)
-
 	// Load security configuration
 	util.LoadSecurityConfiguration()
 
@@ -536,6 +511,11 @@ func recoveryMiddleware(next http.Handler) http.Handler {
 	})
 }
 
+// GetAdminOptions returns the admin command options for testing
+func GetAdminOptions() *AdminOptions {
+	return &AdminOptions{}
+}
+
 // loadOrGenerateSessionKeys loads or creates authentication/encryption keys for session cookies.
 func loadOrGenerateSessionKeys(dataDir string) ([]byte, []byte, error) {
 	const keyLen = 32

weed/command/download.go

@@ -132,3 +132,16 @@ func fetchContent(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption
 	content, e = io.ReadAll(rc.Body)
 	return
 }
+
+func WriteFile(filename string, data []byte, perm os.FileMode) error {
+	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
+	if err != nil {
+		return err
+	}
+	n, err := f.Write(data)
+	f.Close()
+	if err == nil && n < len(data) {
+		err = io.ErrShortWrite
+	}
+	return err
+}

weed/command/filer.go

@@ -81,7 +81,6 @@ type FilerOptions struct {
 	exposeDirectoryData       *bool
 	tusBasePath               *string
 	certProvider              certprovider.Provider
-	s3ConfigFile              *string // optional path to static S3 identity config
 }
 
 func init() {
@@ -147,7 +146,6 @@ func init() {
 	filerS3Options.iamReadOnly = cmdFiler.Flag.Bool("s3.iam.readOnly", true, "disable IAM write operations on this server")
 	filerS3Options.portIceberg = cmdFiler.Flag.Int("s3.port.iceberg", 8181, "Iceberg REST Catalog server listen port (0 to disable)")
 	filerS3Options.externalUrl = cmdFiler.Flag.String("s3.externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Used for S3 signature verification behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.")
-	filerS3Options.defaultFileMode = cmdFiler.Flag.String("s3.defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666")
 
 	// start webdav on filer
 	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
@@ -344,15 +342,6 @@ func (fo *FilerOptions) startFiler() {
 		glog.V(0).Infof("Initialized credential manager: %s", credentialManager.GetStoreName())
 	}
 
-	// Load static S3 identities from config file if specified
-	if fo.s3ConfigFile != nil && *fo.s3ConfigFile != "" {
-		if credentialManager != nil {
-			if err := credentialManager.LoadS3ConfigFile(*fo.s3ConfigFile); err != nil {
-				glog.Warningf("Failed to load S3 config file for static identities: %v", err)
-			}
-		}
-	}
-
 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
 		Masters:                   fo.masters,
 		FilerGroup:                *fo.filerGroup,

weed/command/filer_sync.go

@@ -53,8 +53,6 @@ type SyncOptions struct {
 	chunkConcurrency *int
 	aDoDeleteFiles   *bool
 	bDoDeleteFiles   *bool
-	aSecurity        *string
-	bSecurity        *string
 	clientId         int32
 	clientEpoch      atomic.Int32
 	debug            *bool
@@ -115,8 +113,6 @@ func init() {
 	syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
 	syncOptions.aDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("a.doDeleteFiles", true, "delete and update files when synchronizing on filer A")
 	syncOptions.bDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("b.doDeleteFiles", true, "delete and update files when synchronizing on filer B")
-	syncOptions.aSecurity = cmdFilerSynchronize.Flag.String("a.security", "", "security.toml file for filer A when clusters use different certificates")
-	syncOptions.bSecurity = cmdFilerSynchronize.Flag.String("b.security", "", "security.toml file for filer B when clusters use different certificates")
 	syncOptions.debug = cmdFilerSynchronize.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
 	syncOptions.debugPort = cmdFilerSynchronize.Flag.Int("debug.port", 6060, "http port for debugging")
 	syncOptions.clientId = util.RandomInt32()
@@ -148,22 +144,6 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	util.LoadSecurityConfiguration()
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
-	// per-filer TLS when clusters use different certificates
-	grpcDialOptionA := grpcDialOption
-	grpcDialOptionB := grpcDialOption
-	if *syncOptions.aSecurity != "" {
-		var err error
-		if grpcDialOptionA, err = security.LoadClientTLSFromFile(*syncOptions.aSecurity, "grpc.client"); err != nil {
-			glog.Fatalf("load security config for filer A: %v", err)
-		}
-	}
-	if *syncOptions.bSecurity != "" {
-		var err error
-		if grpcDialOptionB, err = security.LoadClientTLSFromFile(*syncOptions.bSecurity, "grpc.client"); err != nil {
-			glog.Fatalf("load security config for filer B: %v", err)
-		}
-	}
-
 	grace.SetupProfiling(*syncCpuProfile, *syncMemProfile)
 
 	filerA := pb.ServerAddress(*syncOptions.filerA)
@@ -173,13 +153,13 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	go statsCollect.StartMetricsServer(*syncOptions.metricsHttpIp, *syncOptions.metricsHttpPort)
 
 	// read a filer signature
-	aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOptionA, filerA)
+	aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOption, filerA)
 	if aFilerErr != nil {
 		glog.Errorf("get filer 'a' signature %d error from %s to %s: %v", aFilerSignature, *syncOptions.filerA, *syncOptions.filerB, aFilerErr)
 		return true
 	}
 	// read b filer signature
-	bFilerSignature, bFilerErr := replication.ReadFilerSignature(grpcDialOptionB, filerB)
+	bFilerSignature, bFilerErr := replication.ReadFilerSignature(grpcDialOption, filerB)
 	if bFilerErr != nil {
 		glog.Errorf("get filer 'b' signature %d error from %s to %s: %v", bFilerSignature, *syncOptions.filerA, *syncOptions.filerB, bFilerErr)
 		return true
@@ -209,9 +189,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	go func() {
 		// a->b
 		// set synchronization start timestamp to offset
-		initOffsetError := initOffsetFromTsMs(grpcDialOptionB, filerB, aFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath))
+		initOffsetError := initOffsetFromTsMs(grpcDialOption, filerB, aFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath))
 		if initOffsetError != nil {
-			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError)
+			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError)
 			os.Exit(2)
 		}
 		for {
@@ -219,12 +199,11 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 			err := doSubscribeFilerMetaChanges(
 				syncOptions.clientId,
 				syncOptions.clientEpoch.Load(),
-				grpcDialOptionA,
+				grpcDialOption,
 				filerA,
 				*syncOptions.aPath,
 				util.StringSplit(*syncOptions.aExcludePaths, ","),
 				*syncOptions.aProxyByFiler,
-				grpcDialOptionB,
 				filerB,
 				*syncOptions.bPath,
 				*syncOptions.bReplication,
@@ -249,9 +228,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	if !*syncOptions.isActivePassive {
 		// b->a
 		// set synchronization start timestamp to offset
-		initOffsetError := initOffsetFromTsMs(grpcDialOptionA, filerA, bFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath))
+		initOffsetError := initOffsetFromTsMs(grpcDialOption, filerA, bFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath))
 		if initOffsetError != nil {
-			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError)
+			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError)
 			os.Exit(2)
 		}
 		go func() {
@@ -260,12 +239,11 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 				err := doSubscribeFilerMetaChanges(
 					syncOptions.clientId,
 					syncOptions.clientEpoch.Load(),
-					grpcDialOptionB,
+					grpcDialOption,
 					filerB,
 					*syncOptions.bPath,
 					util.StringSplit(*syncOptions.bExcludePaths, ","),
 					*syncOptions.bProxyByFiler,
-					grpcDialOptionA,
 					filerA,
 					*syncOptions.aPath,
 					*syncOptions.aReplication,
@@ -307,12 +285,12 @@ func initOffsetFromTsMs(grpcDialOption grpc.DialOption, targetFiler pb.ServerAdd
 	return nil
 }
 
-func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceExcludePaths []string, sourceReadChunkFromFiler bool, targetGrpcDialOption grpc.DialOption, targetFiler pb.ServerAddress, targetPath string,
+func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceExcludePaths []string, sourceReadChunkFromFiler bool, targetFiler pb.ServerAddress, targetPath string,
 	replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, concurrency int, chunkConcurrency int, doDeleteFiles bool, sourceFilerSignature int32, targetFilerSignature int32, statePtr *atomic.Pointer[syncState]) error {
 
 	// if first time, start from now
 	// if has previously synced, resume from that point of time
-	sourceFilerOffsetTsNs, err := getOffset(targetGrpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature)
+	sourceFilerOffsetTsNs, err := getOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature)
 	if err != nil {
 		return err
 	}
@@ -322,9 +300,8 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
 	// create filer sink
 	filerSource := &source.FilerSource{}
 	filerSource.DoInitialize(sourceFiler.ToHttpAddress(), sourceFiler.ToGrpcAddress(), sourcePath, sourceReadChunkFromFiler)
-	filerSource.SetGrpcDialOption(sourceGrpcDialOption)
 	filerSink := &filersink.FilerSink{}
-	filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, targetGrpcDialOption, sinkWriteChunkByFiler)
+	filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, grpcDialOption, sinkWriteChunkByFiler)
 	filerSink.SetChunkConcurrency(chunkConcurrency)
 	filerSink.SetSourceFiler(filerSource)
 
@@ -351,7 +328,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
 	if statePtr != nil {
 		statePtr.Store(&syncState{
 			processor:            processor,
-			grpcDialOption:       targetGrpcDialOption,
+			grpcDialOption:       grpcDialOption,
 			targetFiler:          targetFiler,
 			sourcePath:           sourcePath,
 			sourceFilerSignature: sourceFilerSignature,
@@ -359,7 +336,6 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
 	}
 
 	var lastLogTsNs = time.Now().UnixNano()
-	var lastProgressedTsNs int64
 	var clientName = fmt.Sprintf("syncFrom_%s_To_%s", string(sourceFiler), string(targetFiler))
 	processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
 		processor.AddSyncJob(resp)
@@ -373,21 +349,9 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
 		now := time.Now().UnixNano()
 		glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
 		lastLogTsNs = now
-		if offsetTsNs == lastProgressedTsNs {
-			for _, t := range filerSink.ActiveTransfers() {
-				if t.LastErr != "" {
-					glog.V(0).Infof("  %s %s: %d bytes received, %s, last error: %s",
-						t.ChunkFileId, t.Path, t.BytesReceived, t.Status, t.LastErr)
-				} else {
-					glog.V(0).Infof("  %s %s: %d bytes received, %s",
-						t.ChunkFileId, t.Path, t.BytesReceived, t.Status)
-				}
-			}
-		}
-		lastProgressedTsNs = offsetTsNs
 		// collect synchronous offset
 		statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(offsetTsNs))
-		return setOffset(targetGrpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
+		return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
 	})
 
 	prefix := sourcePath
@@ -408,7 +372,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
 		EventErrorType:         pb.RetryForeverOnError,
 	}
 
-	return pb.FollowMetadata(sourceFiler, sourceGrpcDialOption, metadataFollowOption, processEventFnWithOffset)
+	return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
 
 }
 

weed/command/mini.go

@@ -250,7 +250,6 @@ func initMiniS3Flags() {
 	miniS3Options.auditLogConfig = cmdMini.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
 	miniS3Options.allowDeleteBucketNotEmpty = miniS3AllowDeleteBucketNotEmpty
 	miniS3Options.externalUrl = cmdMini.Flag.String("s3.externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Used for S3 signature verification behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.")
-	miniS3Options.defaultFileMode = cmdMini.Flag.String("s3.defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666")
 	// In mini mode, S3 uses the shared debug server started at line 681, not its own separate debug server
 	miniS3Options.debug = new(bool) // explicitly false
 	miniS3Options.debugPort = cmdMini.Flag.Int("s3.debug.port", 6060, "http port for debugging (unused in mini mode)")
@@ -820,10 +819,6 @@ func runMini(cmd *Command, args []string) bool {
 	miniFilerOptions.disableHttp = miniDisableHttp
 	miniMasterOptions.disableHttp = miniDisableHttp
 
-	// Share the S3 static identity config file with the filer so its
-	// credential manager can also serve static users.
-	miniFilerOptions.s3ConfigFile = miniS3Config
-
 	filerAddress := string(pb.NewServerAddress(*miniIp, *miniFilerOptions.port, *miniFilerOptions.portGrpc))
 	miniS3Options.filer = &filerAddress
 	miniWebDavOptions.filer = &filerAddress

weed/command/s3.go

@@ -10,7 +10,6 @@ import (
 	"net/http"
 	"os"
 	"runtime"
-	"strconv"
 	"strings"
 	"time"
 
@@ -37,9 +36,6 @@ var (
 	s3StandaloneOptions S3Options
 )
 
-// S3Options holds CLI flags for the S3 gateway.
-// Flags are registered in multiple commands: s3.go (standalone), server.go, filer.go, and mini.go.
-// When adding a new field, update all four flag registration sites.
 type S3Options struct {
 	filer                     *string
 	bindIp                    *string
@@ -72,7 +68,6 @@ type S3Options struct {
 	debugPort                 *int
 	cipher                    *bool
 	externalUrl               *string
-	defaultFileMode           *string
 }
 
 func init() {
@@ -108,7 +103,6 @@ func init() {
 	s3StandaloneOptions.debugPort = cmdS3.Flag.Int("debug.port", 6060, "http port for debugging")
 	s3StandaloneOptions.cipher = cmdS3.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
 	s3StandaloneOptions.externalUrl = cmdS3.Flag.String("externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Used for S3 signature verification behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.")
-	s3StandaloneOptions.defaultFileMode = cmdS3.Flag.String("defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666")
 }
 
 var cmdS3 = &Command{
@@ -238,17 +232,6 @@ func (s3opt *S3Options) resolveExternalUrl() string {
 	return os.Getenv("S3_EXTERNAL_URL")
 }
 
-func (s3opt *S3Options) parseDefaultFileMode() (uint32, error) {
-	if s3opt.defaultFileMode == nil || *s3opt.defaultFileMode == "" {
-		return 0, nil
-	}
-	mode, err := strconv.ParseUint(*s3opt.defaultFileMode, 8, 32)
-	if err != nil {
-		return 0, fmt.Errorf("invalid defaultFileMode %q: %v", *s3opt.defaultFileMode, err)
-	}
-	return uint32(mode), nil
-}
-
 func (s3opt *S3Options) startS3Server() bool {
 
 	filerAddresses := pb.ServerAddresses(*s3opt.filer).ToAddresses()
@@ -315,11 +298,6 @@ func (s3opt *S3Options) startS3Server() bool {
 		*s3opt.bindIp = "0.0.0.0"
 	}
 
-	defaultFileMode, fileModeErr := s3opt.parseDefaultFileMode()
-	if fileModeErr != nil {
-		glog.Fatalf("S3 API Server startup error: %v", fileModeErr)
-	}
-
 	s3ApiServer, s3ApiServer_err = s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
 		Filers:                    filerAddresses,
 		Masters:                   masterAddresses,
@@ -342,7 +320,6 @@ func (s3opt *S3Options) startS3Server() bool {
 		BindIp:                    *s3opt.bindIp,
 		GrpcPort:                  *s3opt.portGrpc,
 		ExternalUrl:               s3opt.resolveExternalUrl(),
-		DefaultFileMode:           defaultFileMode,
 	})
 	if s3ApiServer_err != nil {
 		glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)

weed/command/scaffold/security.toml

@@ -179,19 +179,6 @@ password = ""
 user = ""
 password = ""
 
-# SSE-S3 server-side encryption key management
-# These settings configure the Key Encryption Key (KEK) for S3 SSE-S3 encryption.
-# Set exactly one of kek or key. If neither is set, SSE-S3 is disabled.
-# Can also be set via env vars: WEED_S3_SSE_KEK, WEED_S3_SSE_KEY
-[s3.sse]
-# hex-encoded 256-bit key, same format as the legacy /etc/s3/sse_kek filer file.
-# Use this to migrate from a filer-stored KEK: copy the value from /etc/s3/sse_kek.
-# Generate a new one with: openssl rand -hex 32
-kek = ""
-# any secret string; a 256-bit key is derived automatically via HKDF-SHA256.
-# Cannot be used while /etc/s3/sse_kek exists on the filer — delete it first.
-key = ""
-
 # white list. It's checking request ip address.
 [guard]
 white_list = ""

weed/command/server.go

@@ -180,7 +180,6 @@ func init() {
 	s3Options.iamReadOnly = cmdServer.Flag.Bool("s3.iam.readOnly", true, "disable IAM write operations on this server")
 	s3Options.cipher = cmdServer.Flag.Bool("s3.encryptVolumeData", false, "encrypt data on volume servers for S3 uploads")
 	s3Options.externalUrl = cmdServer.Flag.String("s3.externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Used for S3 signature verification behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.")
-	s3Options.defaultFileMode = cmdServer.Flag.String("s3.defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666")
 
 	sftpOptions.port = cmdServer.Flag.Int("sftp.port", 2022, "SFTP server listen port")
 	sftpOptions.sshPrivateKey = cmdServer.Flag.String("sftp.sshPrivateKey", "", "path to the SSH private key file for host authentication")
@@ -372,8 +371,6 @@ func runServer(cmd *Command, args []string) bool {
 		} else if *serverIamConfig != "" && *s3Options.iamConfig != *serverIamConfig {
 			glog.V(0).Infof("both -s3.iam.config(%s) and -iam.config(%s) provided; using -s3.iam.config", *s3Options.iamConfig, *serverIamConfig)
 		}
-		// Share the S3 static identity config file with the filer
-		filerOptions.s3ConfigFile = s3Options.config
 		go func() {
 			time.Sleep(2 * time.Second)
 			s3Options.localFilerSocket = filerOptions.localSocket

weed/credential/config_loader.go

@@ -57,6 +57,42 @@ func LoadCredentialConfiguration() (*CredentialConfig, error) {
 	}, nil
 }
 
+// GetCredentialStoreConfig extracts credential store configuration from command line flags
+// This is used when credential store is configured via command line instead of credential.toml
+func GetCredentialStoreConfig(store string, config util.Configuration, prefix string) *CredentialConfig {
+	if store == "" {
+		return nil
+	}
+
+	return &CredentialConfig{
+		Store:  store,
+		Config: config,
+		Prefix: prefix,
+	}
+}
+
+// MergeCredentialConfig merges command line credential config with credential.toml config
+// Command line flags take priority over credential.toml
+func MergeCredentialConfig(cmdLineStore string, cmdLineConfig util.Configuration, cmdLinePrefix string) (*CredentialConfig, error) {
+	// If command line credential store is specified, use it
+	if cmdLineStore != "" {
+		glog.V(0).Infof("Using command line credential configuration: store=%s", cmdLineStore)
+		return GetCredentialStoreConfig(cmdLineStore, cmdLineConfig, cmdLinePrefix), nil
+	}
+
+	// Otherwise, try to load from credential.toml
+	config, err := LoadCredentialConfiguration()
+	if err != nil {
+		return nil, err
+	}
+
+	if config == nil {
+		glog.V(1).Info("No credential store configured")
+	}
+
+	return config, nil
+}
+
 // NewCredentialManagerWithDefaults creates a credential manager with fallback to defaults
 // If explicitStore is provided, it will be used regardless of credential.toml
 // If explicitStore is empty, it tries credential.toml first, then defaults to "filer_etc"

weed/credential/credential_manager.go

@@ -3,18 +3,14 @@ package credential
 import (
 	"context"
 	"fmt"
-	"os"
 	"strings"
-	"sync"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 	"google.golang.org/grpc"
-	"google.golang.org/protobuf/encoding/protojson"
 )
 
 // FilerAddressSetter is an interface for credential stores that need a dynamic filer address
@@ -25,15 +21,6 @@ type FilerAddressSetter interface {
 // CredentialManager manages user credentials using a configurable store
 type CredentialManager struct {
 	Store CredentialStore
-	// staticMu protects staticIdentities and staticNames, which are written
-	// by SetStaticIdentities (startup + config reload) and read concurrently
-	// by LoadConfiguration, SaveConfiguration, GetStaticUsernames, and IsStaticIdentity.
-	staticMu sync.RWMutex
-	// staticIdentities holds identities loaded from a static config file (-s3.config).
-	// These are included in LoadConfiguration so that listing operations
-	// return all configured identities, not just dynamic ones from the store.
-	staticIdentities []*iam_pb.Identity
-	staticNames      map[string]bool
 }
 
 // NewCredentialManager creates a new credential manager with the specified store
@@ -87,97 +74,13 @@ func (cm *CredentialManager) GetStoreName() string {
 	return ""
 }
 
-// SetStaticIdentities registers identities loaded from a static config file.
-// These identities are included in LoadConfiguration and ListUsers results
-// but are never persisted to the dynamic store.
-func (cm *CredentialManager) SetStaticIdentities(identities []*iam_pb.Identity) {
-	filtered := make([]*iam_pb.Identity, 0, len(identities))
-	names := make(map[string]bool, len(identities))
-	for _, ident := range identities {
-		if ident != nil {
-			filtered = append(filtered, ident)
-			names[ident.Name] = true
-		}
-	}
-	cm.staticMu.Lock()
-	cm.staticIdentities = filtered
-	cm.staticNames = names
-	cm.staticMu.Unlock()
-}
-
-// IsStaticIdentity returns true if the named identity was loaded from static config.
-func (cm *CredentialManager) IsStaticIdentity(name string) bool {
-	cm.staticMu.RLock()
-	defer cm.staticMu.RUnlock()
-	return cm.staticNames[name]
-}
-
-// GetStaticIdentity returns the protobuf identity for a static user, or nil.
-func (cm *CredentialManager) GetStaticIdentity(name string) *iam_pb.Identity {
-	cm.staticMu.RLock()
-	defer cm.staticMu.RUnlock()
-	for _, ident := range cm.staticIdentities {
-		if ident.Name == name {
-			return ident
-		}
-	}
-	return nil
-}
-
-// GetStaticUsernames returns the names of all static identities.
-func (cm *CredentialManager) GetStaticUsernames() []string {
-	cm.staticMu.RLock()
-	defer cm.staticMu.RUnlock()
-	names := make([]string, 0, len(cm.staticIdentities))
-	for _, ident := range cm.staticIdentities {
-		names = append(names, ident.Name)
-	}
-	return names
-}
-
-// LoadConfiguration loads the S3 API configuration from the store and merges
-// in any static identities so that listing operations show all users.
+// LoadConfiguration loads the S3 API configuration
 func (cm *CredentialManager) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) {
-	config, err := cm.Store.LoadConfiguration(ctx)
-	if err != nil {
-		return config, err
-	}
-	// Merge static identities that are not already in the dynamic config
-	cm.staticMu.RLock()
-	staticIdents := cm.staticIdentities
-	cm.staticMu.RUnlock()
-	if len(staticIdents) > 0 {
-		dynamicNames := make(map[string]bool, len(config.Identities))
-		for _, ident := range config.Identities {
-			dynamicNames[ident.Name] = true
-		}
-		for _, si := range staticIdents {
-			if !dynamicNames[si.Name] {
-				config.Identities = append(config.Identities, si)
-			}
-		}
-	}
-	return config, nil
+	return cm.Store.LoadConfiguration(ctx)
 }
 
-// SaveConfiguration saves the S3 API configuration.
-// Static identities are filtered out before saving to the store.
-// The caller's config is not mutated.
+// SaveConfiguration saves the S3 API configuration
 func (cm *CredentialManager) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error {
-	cm.staticMu.RLock()
-	staticNames := cm.staticNames
-	cm.staticMu.RUnlock()
-	if len(staticNames) > 0 {
-		var dynamicOnly []*iam_pb.Identity
-		for _, ident := range config.Identities {
-			if !staticNames[ident.Name] {
-				dynamicOnly = append(dynamicOnly, ident)
-			}
-		}
-		configCopy := *config
-		configCopy.Identities = dynamicOnly
-		return cm.Store.SaveConfiguration(ctx, &configCopy)
-	}
 	return cm.Store.SaveConfiguration(ctx, config)
 }
 
@@ -201,12 +104,7 @@ func (cm *CredentialManager) DeleteUser(ctx context.Context, username string) er
 	return cm.Store.DeleteUser(ctx, username)
 }
 
-// ListUsers returns usernames from the dynamic store via cm.Store.ListUsers.
-// On store error the error is returned directly without merging static entries.
-// Static identities (cm.staticIdentities) are NOT included here because
-// internal callers (e.g. DeletePolicy) look up each user in the store and
-// would fail on non-existent static entries. External callers that need the
-// full list should merge GetStaticUsernames separately.
+// ListUsers returns all usernames
 func (cm *CredentialManager) ListUsers(ctx context.Context) ([]string, error) {
 	return cm.Store.ListUsers(ctx)
 }
@@ -271,26 +169,6 @@ func (cm *CredentialManager) UpdatePolicy(ctx context.Context, name string, docu
 	return cm.Store.PutPolicy(ctx, name, document)
 }
 
-// LoadS3ConfigFile reads a static S3 identity config file and registers
-// the identities so they appear in LoadConfiguration and listing results.
-func (cm *CredentialManager) LoadS3ConfigFile(path string) error {
-	content, err := os.ReadFile(path)
-	if err != nil {
-		return fmt.Errorf("read %s: %w", path, err)
-	}
-	config := &iam_pb.S3ApiConfiguration{}
-	opts := protojson.UnmarshalOptions{DiscardUnknown: true, AllowPartial: true}
-	if err := opts.Unmarshal(content, config); err != nil {
-		return fmt.Errorf("parse %s: %w", path, err)
-	}
-	for _, ident := range config.Identities {
-		ident.IsStatic = true
-	}
-	cm.SetStaticIdentities(config.Identities)
-	glog.V(1).Infof("Loaded %d static identities from %s", len(config.Identities), path)
-	return nil
-}
-
 // Shutdown performs cleanup
 func (cm *CredentialManager) Shutdown() {
 	if cm.Store != nil {

weed/credential/filer_etc/filer_etc_policy.go

@@ -207,6 +207,32 @@ func (store *FilerEtcStore) loadPoliciesFromMultiFile(ctx context.Context, polic
 	})
 }
 
+func (store *FilerEtcStore) migratePoliciesToMultiFile(ctx context.Context, policies map[string]policy_engine.PolicyDocument) error {
+	glog.Infof("Migrating IAM policies to multi-file layout...")
+
+	// 1. Save all policies to individual files
+	for name, policy := range policies {
+		if err := store.savePolicy(ctx, name, policy); err != nil {
+			return err
+		}
+	}
+
+	// 2. Rename legacy file
+	return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		_, err := client.AtomicRenameEntry(ctx, &filer_pb.AtomicRenameEntryRequest{
+			OldDirectory: filer.IamConfigDirectory,
+			OldName:      filer.IamPoliciesFile,
+			NewDirectory: filer.IamConfigDirectory,
+			NewName:      IamLegacyPoliciesOldFile,
+		})
+		if err != nil {
+			glog.Errorf("Failed to rename legacy IAM policies file %s/%s to %s: %v",
+				filer.IamConfigDirectory, filer.IamPoliciesFile, IamLegacyPoliciesOldFile, err)
+		}
+		return err
+	})
+}
+
 func (store *FilerEtcStore) savePolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error {
 	if err := validatePolicyName(name); err != nil {
 		return err

weed/credential/migration.go

@@ -0,0 +1,221 @@
+package credential
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// MigrateCredentials migrates credentials from one store to another
+func MigrateCredentials(fromStoreName, toStoreName CredentialStoreTypeName, configuration util.Configuration, fromPrefix, toPrefix string) error {
+	ctx := context.Background()
+
+	// Create source credential manager
+	fromCM, err := NewCredentialManager(fromStoreName, configuration, fromPrefix)
+	if err != nil {
+		return fmt.Errorf("failed to create source credential manager (%s): %v", fromStoreName, err)
+	}
+	defer fromCM.Shutdown()
+
+	// Create destination credential manager
+	toCM, err := NewCredentialManager(toStoreName, configuration, toPrefix)
+	if err != nil {
+		return fmt.Errorf("failed to create destination credential manager (%s): %v", toStoreName, err)
+	}
+	defer toCM.Shutdown()
+
+	// Load configuration from source
+	glog.Infof("Loading configuration from %s store...", fromStoreName)
+	config, err := fromCM.LoadConfiguration(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to load configuration from source store: %w", err)
+	}
+
+	if config == nil || len(config.Identities) == 0 {
+		glog.Info("No identities found in source store")
+		return nil
+	}
+
+	glog.Infof("Found %d identities in source store", len(config.Identities))
+
+	// Migrate each identity
+	var migrated, failed int
+	for _, identity := range config.Identities {
+		glog.V(1).Infof("Migrating user: %s", identity.Name)
+
+		// Check if user already exists in destination
+		existingUser, err := toCM.GetUser(ctx, identity.Name)
+		if err != nil && err != ErrUserNotFound {
+			glog.Errorf("Failed to check if user %s exists in destination: %v", identity.Name, err)
+			failed++
+			continue
+		}
+
+		if existingUser != nil {
+			glog.Warningf("User %s already exists in destination store, skipping", identity.Name)
+			continue
+		}
+
+		// Create user in destination
+		err = toCM.CreateUser(ctx, identity)
+		if err != nil {
+			glog.Errorf("Failed to create user %s in destination store: %v", identity.Name, err)
+			failed++
+			continue
+		}
+
+		migrated++
+		glog.V(1).Infof("Successfully migrated user: %s", identity.Name)
+	}
+
+	glog.Infof("Migration completed: %d migrated, %d failed", migrated, failed)
+
+	if failed > 0 {
+		return fmt.Errorf("migration completed with %d failures", failed)
+	}
+
+	return nil
+}
+
+// ExportCredentials exports credentials from a store to a configuration
+func ExportCredentials(storeName CredentialStoreTypeName, configuration util.Configuration, prefix string) (*iam_pb.S3ApiConfiguration, error) {
+	ctx := context.Background()
+
+	// Create credential manager
+	cm, err := NewCredentialManager(storeName, configuration, prefix)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create credential manager (%s): %v", storeName, err)
+	}
+	defer cm.Shutdown()
+
+	// Load configuration
+	config, err := cm.LoadConfiguration(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load configuration: %w", err)
+	}
+
+	return config, nil
+}
+
+// ImportCredentials imports credentials from a configuration to a store
+func ImportCredentials(storeName CredentialStoreTypeName, configuration util.Configuration, prefix string, config *iam_pb.S3ApiConfiguration) error {
+	ctx := context.Background()
+
+	// Create credential manager
+	cm, err := NewCredentialManager(storeName, configuration, prefix)
+	if err != nil {
+		return fmt.Errorf("failed to create credential manager (%s): %v", storeName, err)
+	}
+	defer cm.Shutdown()
+
+	// Import each identity
+	var imported, failed int
+	for _, identity := range config.Identities {
+		glog.V(1).Infof("Importing user: %s", identity.Name)
+
+		// Check if user already exists
+		existingUser, err := cm.GetUser(ctx, identity.Name)
+		if err != nil && err != ErrUserNotFound {
+			glog.Errorf("Failed to check if user %s exists: %v", identity.Name, err)
+			failed++
+			continue
+		}
+
+		if existingUser != nil {
+			glog.Warningf("User %s already exists, skipping", identity.Name)
+			continue
+		}
+
+		// Create user
+		err = cm.CreateUser(ctx, identity)
+		if err != nil {
+			glog.Errorf("Failed to create user %s: %v", identity.Name, err)
+			failed++
+			continue
+		}
+
+		imported++
+		glog.V(1).Infof("Successfully imported user: %s", identity.Name)
+	}
+
+	glog.Infof("Import completed: %d imported, %d failed", imported, failed)
+
+	if failed > 0 {
+		return fmt.Errorf("import completed with %d failures", failed)
+	}
+
+	return nil
+}
+
+// ValidateCredentials validates that all credentials in a store are accessible
+func ValidateCredentials(storeName CredentialStoreTypeName, configuration util.Configuration, prefix string) error {
+	ctx := context.Background()
+
+	// Create credential manager
+	cm, err := NewCredentialManager(storeName, configuration, prefix)
+	if err != nil {
+		return fmt.Errorf("failed to create credential manager (%s): %v", storeName, err)
+	}
+	defer cm.Shutdown()
+
+	// Load configuration
+	config, err := cm.LoadConfiguration(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to load configuration: %w", err)
+	}
+
+	if config == nil || len(config.Identities) == 0 {
+		glog.Info("No identities found in store")
+		return nil
+	}
+
+	glog.Infof("Validating %d identities...", len(config.Identities))
+
+	// Validate each identity
+	var validated, failed int
+	for _, identity := range config.Identities {
+		// Check if user can be retrieved
+		user, err := cm.GetUser(ctx, identity.Name)
+		if err != nil {
+			glog.Errorf("Failed to retrieve user %s: %v", identity.Name, err)
+			failed++
+			continue
+		}
+
+		if user == nil {
+			glog.Errorf("User %s not found", identity.Name)
+			failed++
+			continue
+		}
+
+		// Validate access keys
+		for _, credential := range identity.Credentials {
+			accessKeyUser, err := cm.GetUserByAccessKey(ctx, credential.AccessKey)
+			if err != nil {
+				glog.Errorf("Failed to retrieve user by access key %s: %v", credential.AccessKey, err)
+				failed++
+				continue
+			}
+
+			if accessKeyUser == nil || accessKeyUser.Name != identity.Name {
+				glog.Errorf("Access key %s does not map to correct user %s", credential.AccessKey, identity.Name)
+				failed++
+				continue
+			}
+		}
+
+		validated++
+		glog.V(1).Infof("Successfully validated user: %s", identity.Name)
+	}
+
+	glog.Infof("Validation completed: %d validated, %d failed", validated, failed)
+
+	if failed > 0 {
+		return fmt.Errorf("validation completed with %d failures", failed)
+	}
+
+	return nil
+}

weed/filer/filer_notify_read.go

@@ -246,6 +246,10 @@ func NewLogFileEntryCollector(f *Filer, startPosition log_buffer.MessagePosition
 	}
 }
 
+func (c *LogFileEntryCollector) hasMore() bool {
+	return c.dayEntryQueue.Len() > 0
+}
+
 func (c *LogFileEntryCollector) collectMore(v *OrderedLogVisitor) (err error) {
 	dayEntry := c.dayEntryQueue.Dequeue()
 	if dayEntry == nil {

weed/filer/filerstore_wrapper.go

@@ -128,9 +128,6 @@ func (fsw *FilerStoreWrapper) Initialize(configuration util.Configuration, prefi
 }
 
 func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) error {
-	if err := ctx.Err(); err != nil {
-		return err
-	}
 	ctx = context.WithoutCancel(ctx)
 	actualStore := fsw.getActualStore(entry.FullPath)
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "insert").Inc()
@@ -158,9 +155,6 @@ func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) err
 // InsertEntryKnownAbsent skips the pre-insert FindEntry path when the caller has
 // already established that the target path does not exist.
 func (fsw *FilerStoreWrapper) InsertEntryKnownAbsent(ctx context.Context, entry *Entry) error {
-	if err := ctx.Err(); err != nil {
-		return err
-	}
 	ctx = context.WithoutCancel(ctx)
 	actualStore := fsw.getActualStore(entry.FullPath)
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "insert").Inc()
@@ -184,9 +178,6 @@ func (fsw *FilerStoreWrapper) InsertEntryKnownAbsent(ctx context.Context, entry
 }
 
 func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) error {
-	if err := ctx.Err(); err != nil {
-		return err
-	}
 	ctx = context.WithoutCancel(ctx)
 	actualStore := fsw.getActualStore(entry.FullPath)
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "update").Inc()
@@ -245,9 +236,6 @@ func (fsw *FilerStoreWrapper) FindEntry(ctx context.Context, fp util.FullPath) (
 }
 
 func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) {
-	if err := ctx.Err(); err != nil {
-		return err
-	}
 	ctx = context.WithoutCancel(ctx)
 	actualStore := fsw.getActualStore(fp)
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "delete").Inc()
@@ -276,9 +264,6 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath)
 }
 
 func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry *Entry) (err error) {
-	if err := ctx.Err(); err != nil {
-		return err
-	}
 	ctx = context.WithoutCancel(ctx)
 	actualStore := fsw.getActualStore(existingEntry.FullPath)
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "delete").Inc()
@@ -303,9 +288,6 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
 }
 
 func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
-	if err := ctx.Err(); err != nil {
-		return err
-	}
 	ctx = context.WithoutCancel(ctx)
 	actualStore := fsw.getActualStore(fp + "/")
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Inc()
@@ -412,17 +394,11 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u
 }
 
 func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) {
-	if err := ctx.Err(); err != nil {
-		return nil, err
-	}
 	ctx = context.WithoutCancel(ctx)
 	return fsw.getDefaultStore().BeginTransaction(ctx)
 }
 
 func (fsw *FilerStoreWrapper) CommitTransaction(ctx context.Context) error {
-	if err := ctx.Err(); err != nil {
-		return err
-	}
 	ctx = context.WithoutCancel(ctx)
 	return fsw.getDefaultStore().CommitTransaction(ctx)
 }
@@ -437,9 +413,6 @@ func (fsw *FilerStoreWrapper) Shutdown() {
 }
 
 func (fsw *FilerStoreWrapper) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
-	if err := ctx.Err(); err != nil {
-		return err
-	}
 	ctx = context.WithoutCancel(ctx)
 	return fsw.getDefaultStore().KvPut(ctx, key, value)
 }
@@ -448,9 +421,6 @@ func (fsw *FilerStoreWrapper) KvGet(ctx context.Context, key []byte) (value []by
 	return fsw.getDefaultStore().KvGet(ctx, key)
 }
 func (fsw *FilerStoreWrapper) KvDelete(ctx context.Context, key []byte) (err error) {
-	if err := ctx.Err(); err != nil {
-		return err
-	}
 	ctx = context.WithoutCancel(ctx)
 	return fsw.getDefaultStore().KvDelete(ctx, key)
 }

weed/filer/filerstore_wrapper_test.go

@@ -2,10 +2,8 @@ package filer
 
 import (
 	"context"
-	"errors"
 	"os"
 	"testing"
-	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/stretchr/testify/assert"
@@ -71,137 +69,3 @@ func TestFilerStoreWrapperMimeNormalization(t *testing.T) {
 		}
 	}
 }
-
-// cancelledCtx returns a context that is already cancelled.
-func cancelledCtx() context.Context {
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	return ctx
-}
-
-// expiredCtx returns a context whose deadline has already passed.
-func expiredCtx() context.Context {
-	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
-	cancel() // release resources immediately as it's already expired
-	return ctx
-}
-
-func TestFilerStoreWrapperWriteOpsRejectCancelledContext(t *testing.T) {
-	newEntry := func(path string) *Entry {
-		return &Entry{
-			FullPath: util.FullPath(path),
-			Attr:     Attr{Mode: 0o660, Mime: "application/octet-stream"},
-		}
-	}
-
-	// Each write operation that should be guarded.
-	writeOps := []struct {
-		name string
-		run  func(*FilerStoreWrapper, context.Context) error
-	}{
-		{"InsertEntry", func(fsw *FilerStoreWrapper, ctx context.Context) error {
-			return fsw.InsertEntry(ctx, newEntry("/test/a"))
-		}},
-		{"InsertEntryKnownAbsent", func(fsw *FilerStoreWrapper, ctx context.Context) error {
-			return fsw.InsertEntryKnownAbsent(ctx, newEntry("/test/b"))
-		}},
-		{"UpdateEntry", func(fsw *FilerStoreWrapper, ctx context.Context) error {
-			_ = fsw.InsertEntry(context.Background(), newEntry("/test/c"))
-			return fsw.UpdateEntry(ctx, newEntry("/test/c"))
-		}},
-		{"DeleteEntry", func(fsw *FilerStoreWrapper, ctx context.Context) error {
-			_ = fsw.InsertEntry(context.Background(), newEntry("/test/d"))
-			return fsw.DeleteEntry(ctx, "/test/d")
-		}},
-		{"DeleteOneEntry", func(fsw *FilerStoreWrapper, ctx context.Context) error {
-			e := newEntry("/test/e")
-			_ = fsw.InsertEntry(context.Background(), e)
-			return fsw.DeleteOneEntry(ctx, e)
-		}},
-		{"DeleteFolderChildren", func(fsw *FilerStoreWrapper, ctx context.Context) error {
-			_ = fsw.InsertEntry(context.Background(), newEntry("/test/folder/child"))
-			return fsw.DeleteFolderChildren(ctx, "/test/folder")
-		}},
-		{"BeginTransaction", func(fsw *FilerStoreWrapper, ctx context.Context) error {
-			_, err := fsw.BeginTransaction(ctx)
-			return err
-		}},
-		{"CommitTransaction", func(fsw *FilerStoreWrapper, ctx context.Context) error {
-			return fsw.CommitTransaction(ctx)
-		}},
-		{"KvPut", func(fsw *FilerStoreWrapper, ctx context.Context) error {
-			return fsw.KvPut(ctx, []byte("k"), []byte("v"))
-		}},
-		{"KvDelete", func(fsw *FilerStoreWrapper, ctx context.Context) error {
-			_ = fsw.KvPut(context.Background(), []byte("k"), []byte("v"))
-			return fsw.KvDelete(ctx, []byte("k"))
-		}},
-	}
-
-	badContexts := []struct {
-		name      string
-		ctx       context.Context
-		wantError error
-	}{
-		{"cancelled", cancelledCtx(), context.Canceled},
-		{"deadline exceeded", expiredCtx(), context.DeadlineExceeded},
-	}
-
-	for _, op := range writeOps {
-		for _, bc := range badContexts {
-			t.Run(op.name+"/"+bc.name, func(t *testing.T) {
-				wrapper := NewFilerStoreWrapper(newStubFilerStore())
-				err := op.run(wrapper, bc.ctx)
-				require.Error(t, err)
-				assert.True(t, errors.Is(err, bc.wantError), "got %v, want %v", err, bc.wantError)
-			})
-		}
-	}
-}
-
-func TestFilerStoreWrapperWriteOpsSucceedWithActiveContext(t *testing.T) {
-	wrapper := NewFilerStoreWrapper(newStubFilerStore())
-	ctx := context.Background()
-	entry := &Entry{
-		FullPath: util.FullPath("/test/obj"),
-		Attr:     Attr{Mode: 0o660},
-	}
-
-	require.NoError(t, wrapper.InsertEntry(ctx, entry))
-	require.NoError(t, wrapper.UpdateEntry(ctx, entry))
-	require.NoError(t, wrapper.DeleteOneEntry(ctx, entry))
-	require.NoError(t, wrapper.InsertEntryKnownAbsent(ctx, entry))
-	require.NoError(t, wrapper.DeleteEntry(ctx, entry.FullPath))
-	require.NoError(t, wrapper.KvPut(ctx, []byte("k"), []byte("v")))
-	require.NoError(t, wrapper.KvDelete(ctx, []byte("k")))
-
-	txCtx, err := wrapper.BeginTransaction(ctx)
-	require.NoError(t, err)
-	require.NoError(t, wrapper.CommitTransaction(txCtx))
-}
-
-func TestFilerStoreWrapperReadOpsSucceedWithCancelledContext(t *testing.T) {
-	wrapper := NewFilerStoreWrapper(newStubFilerStore())
-	entry := &Entry{
-		FullPath: util.FullPath("/test/readable"),
-		Attr:     Attr{Mode: 0o660},
-	}
-	require.NoError(t, wrapper.InsertEntry(context.Background(), entry))
-	require.NoError(t, wrapper.KvPut(context.Background(), []byte("rk"), []byte("rv")))
-
-	ctx := cancelledCtx()
-
-	_, err := wrapper.FindEntry(ctx, entry.FullPath)
-	assert.NoError(t, err)
-
-	_, err = wrapper.KvGet(ctx, []byte("rk"))
-	assert.NoError(t, err)
-}
-
-// RollbackTransaction must succeed even when the context is cancelled or
-// expired, because it is a cleanup operation called after failures.
-func TestFilerStoreWrapperRollbackSucceedsWithCancelledContext(t *testing.T) {
-	wrapper := NewFilerStoreWrapper(newStubFilerStore())
-	assert.NoError(t, wrapper.RollbackTransaction(cancelledCtx()))
-	assert.NoError(t, wrapper.RollbackTransaction(expiredCtx()))
-}

weed/filer/meta_replay.go

@@ -2,6 +2,7 @@ package filer
 
 import (
 	"context"
+	"sync"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -35,3 +36,39 @@ func Replay(filerStore FilerStore, resp *filer_pb.SubscribeMetadataResponse) err
 
 	return nil
 }
+
+// ParallelProcessDirectoryStructure processes each entry in parallel, and also ensure parent directories are processed first.
+// This also assumes the parent directories are in the entryChan already.
+func ParallelProcessDirectoryStructure(entryChan chan *Entry, concurrency int, eachEntryFn func(entry *Entry) error) (firstErr error) {
+
+	executors := util.NewLimitedConcurrentExecutor(concurrency)
+
+	var wg sync.WaitGroup
+	for entry := range entryChan {
+		wg.Add(1)
+		if entry.IsDirectory() {
+			func() {
+				defer wg.Done()
+				if err := eachEntryFn(entry); err != nil {
+					if firstErr == nil {
+						firstErr = err
+					}
+				}
+			}()
+		} else {
+			executors.Execute(func() {
+				defer wg.Done()
+				if err := eachEntryFn(entry); err != nil {
+					if firstErr == nil {
+						firstErr = err
+					}
+				}
+			})
+		}
+		if firstErr != nil {
+			break
+		}
+	}
+	wg.Wait()
+	return
+}

weed/filer/redis/universal_redis_store.go

@@ -173,16 +173,14 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirP
 		members = members[:limit]
 	}
 
-	var entry *filer.Entry
 	// fetch entry meta
 	for _, fileName := range members {
 		path := util.NewFullPath(string(dirPath), fileName)
-		entry, err = store.FindEntry(ctx, path)
+		entry, err := store.FindEntry(ctx, path)
 		lastFileName = fileName
 		if err != nil {
 			glog.V(0).InfofCtx(ctx, "list %s : %v", path, err)
 			if err == filer_pb.ErrNotFound {
-				err = nil
 				continue
 			}
 		} else {

weed/filer/redis3/ItemList.go

@@ -16,6 +16,15 @@ type ItemList struct {
 	prefix    string
 }
 
+func newItemList(client redis.UniversalClient, prefix string, store skiplist.ListStore, batchSize int) *ItemList {
+	return &ItemList{
+		skipList:  skiplist.New(store),
+		batchSize: batchSize,
+		client:    client,
+		prefix:    prefix,
+	}
+}
+
 /*
 Be reluctant to create new nodes. Try to fit into either previous node or next node.
 Prefer to add to previous node.

weed/filer/redis_lua/redis_cluster_store.go

@@ -0,0 +1,48 @@
+package redis_lua
+
+import (
+	"github.com/redis/go-redis/v9"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &RedisLuaClusterStore{})
+}
+
+type RedisLuaClusterStore struct {
+	UniversalRedisLuaStore
+}
+
+func (store *RedisLuaClusterStore) GetName() string {
+	return "redis_lua_cluster"
+}
+
+func (store *RedisLuaClusterStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+
+	configuration.SetDefault(prefix+"useReadOnly", false)
+	configuration.SetDefault(prefix+"routeByLatency", false)
+
+	return store.initialize(
+		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetString(prefix+"keyPrefix"),
+		configuration.GetBool(prefix+"useReadOnly"),
+		configuration.GetBool(prefix+"routeByLatency"),
+		configuration.GetStringSlice(prefix+"superLargeDirectories"),
+	)
+}
+
+func (store *RedisLuaClusterStore) initialize(addresses []string, username string, password string, keyPrefix string, readOnly, routeByLatency bool, superLargeDirectories []string) (err error) {
+	store.Client = redis.NewClusterClient(&redis.ClusterOptions{
+		Addrs:          addresses,
+		Username:       username,
+		Password:       password,
+		ReadOnly:       readOnly,
+		RouteByLatency: routeByLatency,
+	})
+	store.keyPrefix = keyPrefix
+	store.loadSuperLargeDirectories(superLargeDirectories)
+	return
+}

weed/filer/redis_lua/redis_sentinel_store.go

@@ -0,0 +1,48 @@
+package redis_lua
+
+import (
+	"time"
+
+	"github.com/redis/go-redis/v9"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &RedisLuaSentinelStore{})
+}
+
+type RedisLuaSentinelStore struct {
+	UniversalRedisLuaStore
+}
+
+func (store *RedisLuaSentinelStore) GetName() string {
+	return "redis_lua_sentinel"
+}
+
+func (store *RedisLuaSentinelStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"masterName"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetInt(prefix+"database"),
+		configuration.GetString(prefix+"keyPrefix"),
+	)
+}
+
+func (store *RedisLuaSentinelStore) initialize(addresses []string, masterName string, username string, password string, database int, keyPrefix string) (err error) {
+	store.Client = redis.NewFailoverClient(&redis.FailoverOptions{
+		MasterName:      masterName,
+		SentinelAddrs:   addresses,
+		Username:        username,
+		Password:        password,
+		DB:              database,
+		MinRetryBackoff: time.Millisecond * 100,
+		MaxRetryBackoff: time.Minute * 1,
+		ReadTimeout:     time.Second * 30,
+		WriteTimeout:    time.Second * 5,
+	})
+	store.keyPrefix = keyPrefix
+	return
+}

weed/filer/redis_lua/redis_store.go

@@ -0,0 +1,42 @@
+package redis_lua
+
+import (
+	"github.com/redis/go-redis/v9"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &RedisLuaStore{})
+}
+
+type RedisLuaStore struct {
+	UniversalRedisLuaStore
+}
+
+func (store *RedisLuaStore) GetName() string {
+	return "redis_lua"
+}
+
+func (store *RedisLuaStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetString(prefix+"address"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetInt(prefix+"database"),
+		configuration.GetString(prefix+"keyPrefix"),
+		configuration.GetStringSlice(prefix+"superLargeDirectories"),
+	)
+}
+
+func (store *RedisLuaStore) initialize(hostPort string, username string, password string, database int, keyPrefix string, superLargeDirectories []string) (err error) {
+	store.Client = redis.NewClient(&redis.Options{
+		Addr:     hostPort,
+		Username: username,
+		Password: password,
+		DB:       database,
+	})
+	store.keyPrefix = keyPrefix
+	store.loadSuperLargeDirectories(superLargeDirectories)
+	return
+}

weed/filer/redis_lua/stored_procedure/delete_entry.lua

@@ -0,0 +1,19 @@
+-- KEYS[1]: full path of entry
+local fullpath = KEYS[1]
+-- KEYS[2]: full path of entry
+local fullpath_list_key = KEYS[2]
+-- KEYS[3]: dir of the entry
+local dir_list_key = KEYS[3]
+
+-- ARGV[1]: isSuperLargeDirectory
+local isSuperLargeDirectory = ARGV[1] == "1"
+-- ARGV[2]: name of the entry
+local name = ARGV[2]
+
+redis.call("DEL", fullpath, fullpath_list_key)
+
+if not isSuperLargeDirectory and name ~= "" then
+    redis.call("ZREM", dir_list_key, name)
+end
+
+return 0

weed/filer/redis_lua/stored_procedure/delete_folder_children.lua

@@ -0,0 +1,15 @@
+-- KEYS[1]: full path of entry
+local fullpath = KEYS[1]
+
+if fullpath ~= "" and string.sub(fullpath, -1) == "/" then
+    fullpath = string.sub(fullpath, 0, -2)
+end
+
+local files = redis.call("ZRANGE", fullpath .. "\0", "0", "-1")
+
+for _, name in ipairs(files) do
+    local file_path = fullpath .. "/" .. name
+    redis.call("DEL", file_path, file_path .. "\0")
+end
+
+return 0

weed/filer/redis_lua/stored_procedure/init.go

@@ -0,0 +1,25 @@
+package stored_procedure
+
+import (
+	_ "embed"
+
+	"github.com/redis/go-redis/v9"
+)
+
+func init() {
+	InsertEntryScript = redis.NewScript(insertEntry)
+	DeleteEntryScript = redis.NewScript(deleteEntry)
+	DeleteFolderChildrenScript = redis.NewScript(deleteFolderChildren)
+}
+
+//go:embed insert_entry.lua
+var insertEntry string
+var InsertEntryScript *redis.Script
+
+//go:embed delete_entry.lua
+var deleteEntry string
+var DeleteEntryScript *redis.Script
+
+//go:embed delete_folder_children.lua
+var deleteFolderChildren string
+var DeleteFolderChildrenScript *redis.Script

weed/filer/redis_lua/stored_procedure/insert_entry.lua

@@ -0,0 +1,27 @@
+-- KEYS[1]: full path of entry
+local full_path = KEYS[1]
+-- KEYS[2]: dir of the entry
+local dir_list_key = KEYS[2]
+
+-- ARGV[1]: content of the entry
+local entry = ARGV[1]
+-- ARGV[2]: TTL of the entry
+local ttlSec = tonumber(ARGV[2])
+-- ARGV[3]: isSuperLargeDirectory
+local isSuperLargeDirectory = ARGV[3] == "1"
+-- ARGV[4]: zscore of the entry in zset
+local zscore = tonumber(ARGV[4])
+-- ARGV[5]: name of the entry
+local name = ARGV[5]
+
+if ttlSec > 0 then
+    redis.call("SET", full_path, entry, "EX", ttlSec)
+else
+    redis.call("SET", full_path, entry)
+end
+
+if not isSuperLargeDirectory and name ~= "" then
+    redis.call("ZADD", dir_list_key, "NX", zscore, name)
+end
+
+return 0

weed/filer/redis_lua/universal_redis_store.go

@@ -0,0 +1,206 @@
+package redis_lua
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/filer/redis_lua/stored_procedure"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	DIR_LIST_MARKER = "\x00"
+)
+
+type UniversalRedisLuaStore struct {
+	Client                  redis.UniversalClient
+	keyPrefix               string
+	superLargeDirectoryHash map[string]bool
+}
+
+func (store *UniversalRedisLuaStore) isSuperLargeDirectory(dir string) (isSuperLargeDirectory bool) {
+	_, isSuperLargeDirectory = store.superLargeDirectoryHash[dir]
+	return
+}
+
+func (store *UniversalRedisLuaStore) loadSuperLargeDirectories(superLargeDirectories []string) {
+	// set directory hash
+	store.superLargeDirectoryHash = make(map[string]bool)
+	for _, dir := range superLargeDirectories {
+		store.superLargeDirectoryHash[dir] = true
+	}
+}
+
+func (store *UniversalRedisLuaStore) getKey(key string) string {
+	if store.keyPrefix == "" {
+		return key
+	}
+	return store.keyPrefix + key
+}
+
+func (store *UniversalRedisLuaStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *UniversalRedisLuaStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *UniversalRedisLuaStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *UniversalRedisLuaStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		value = util.MaybeGzipData(value)
+	}
+
+	dir, name := entry.FullPath.DirAndName()
+
+	err = stored_procedure.InsertEntryScript.Run(ctx, store.Client,
+		[]string{store.getKey(string(entry.FullPath)), store.getKey(genDirectoryListKey(dir))},
+		value, entry.TtlSec,
+		store.isSuperLargeDirectory(dir), 0, name).Err()
+
+	if err != nil {
+		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
+	}
+
+	return nil
+}
+
+func (store *UniversalRedisLuaStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *UniversalRedisLuaStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+
+	data, err := store.Client.Get(ctx, store.getKey(string(fullpath))).Result()
+	if err == redis.Nil {
+		return nil, filer_pb.ErrNotFound
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("get %s : %v", fullpath, err)
+	}
+
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data)))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *UniversalRedisLuaStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
+
+	dir, name := fullpath.DirAndName()
+
+	err = stored_procedure.DeleteEntryScript.Run(ctx, store.Client,
+		[]string{store.getKey(string(fullpath)), store.getKey(genDirectoryListKey(string(fullpath))), store.getKey(genDirectoryListKey(dir))},
+		store.isSuperLargeDirectory(dir), name).Err()
+
+	if err != nil {
+		return fmt.Errorf("DeleteEntry %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *UniversalRedisLuaStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
+
+	if store.isSuperLargeDirectory(string(fullpath)) {
+		return nil
+	}
+
+	err = stored_procedure.DeleteFolderChildrenScript.Run(ctx, store.Client,
+		[]string{store.getKey(string(fullpath))}).Err()
+
+	if err != nil {
+		return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *UniversalRedisLuaStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
+}
+
+func (store *UniversalRedisLuaStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+
+	dirListKey := store.getKey(genDirectoryListKey(string(dirPath)))
+
+	min := "-"
+	if startFileName != "" {
+		if includeStartFile {
+			min = "[" + startFileName
+		} else {
+			min = "(" + startFileName
+		}
+	}
+
+	members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{
+		Min:    min,
+		Max:    "+",
+		Offset: 0,
+		Count:  limit,
+	}).Result()
+	if err != nil {
+		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
+	}
+
+	// fetch entry meta
+	for _, fileName := range members {
+		path := util.NewFullPath(string(dirPath), fileName)
+		entry, err := store.FindEntry(ctx, path)
+		lastFileName = fileName
+		if err != nil {
+			glog.V(0).InfofCtx(ctx, "list %s : %v", path, err)
+			if err == filer_pb.ErrNotFound {
+				continue
+			}
+		} else {
+			if entry.TtlSec > 0 {
+				if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+					store.DeleteEntry(ctx, path)
+					continue
+				}
+			}
+
+			resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(entry)
+			if resEachEntryFuncErr != nil {
+				err = fmt.Errorf("failed to process eachEntryFunc: %w", resEachEntryFuncErr)
+				break
+			}
+
+			if !resEachEntryFunc {
+				break
+			}
+		}
+	}
+
+	return lastFileName, err
+}
+
+func genDirectoryListKey(dir string) (dirList string) {
+	return dir + DIR_LIST_MARKER
+}
+
+func (store *UniversalRedisLuaStore) Shutdown() {
+	store.Client.Close()
+}

weed/filer/redis_lua/universal_redis_store_kv.go

@@ -0,0 +1,42 @@
+package redis_lua
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/redis/go-redis/v9"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+)
+
+func (store *UniversalRedisLuaStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+
+	_, err = store.Client.Set(ctx, string(key), value, 0).Result()
+
+	if err != nil {
+		return fmt.Errorf("kv put: %w", err)
+	}
+
+	return nil
+}
+
+func (store *UniversalRedisLuaStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+
+	data, err := store.Client.Get(ctx, string(key)).Result()
+
+	if err == redis.Nil {
+		return nil, filer.ErrKvNotFound
+	}
+
+	return []byte(data), err
+}
+
+func (store *UniversalRedisLuaStore) KvDelete(ctx context.Context, key []byte) (err error) {
+
+	_, err = store.Client.Del(ctx, string(key)).Result()
+
+	if err != nil {
+		return fmt.Errorf("kv delete: %w", err)
+	}
+
+	return nil
+}

weed/filer/stream.go

@@ -102,6 +102,10 @@ func PrepareStreamContent(masterClient wdclient.HasLookupFileIdFunction, jwtFunc
 
 type VolumeServerJwtFunction func(fileId string) string
 
+func noJwtFunc(string) string {
+	return ""
+}
+
 type CacheInvalidator interface {
 	InvalidateCache(fileId string)
 }
@@ -245,66 +249,6 @@ func PrepareStreamContentWithThrottler(ctx context.Context, masterClient wdclien
 	}, nil
 }
 
-// PrepareStreamContentWithPrefetch is like PrepareStreamContentWithThrottler but uses
-// concurrent chunk prefetching to overlap network I/O. When prefetchAhead > 1, fetch
-// goroutines establish HTTP connections to volume servers ahead of time, streaming data
-// through io.Pipe with minimal memory overhead.
-//
-// prefetchAhead controls the number of chunks fetched concurrently:
-//   - 0 or 1: falls back to sequential fetching (same as PrepareStreamContentWithThrottler)
-//   - 2+: uses pipe-based prefetch pipeline with that many concurrent fetches
-func PrepareStreamContentWithPrefetch(ctx context.Context, masterClient wdclient.HasLookupFileIdFunction, jwtFunc VolumeServerJwtFunction, chunks []*filer_pb.FileChunk, offset int64, size int64, downloadMaxBytesPs int64, prefetchAhead int) (DoStreamContent, error) {
-	if prefetchAhead <= 1 {
-		return PrepareStreamContentWithThrottler(ctx, masterClient, jwtFunc, chunks, offset, size, downloadMaxBytesPs)
-	}
-
-	glog.V(4).InfofCtx(ctx, "prepare to stream content with prefetch=%d for chunks: %d", prefetchAhead, len(chunks))
-	chunkViews := ViewFromChunks(ctx, masterClient.GetLookupFileIdFunction(), chunks, offset, size)
-
-	fileId2Url := make(map[string][]string)
-
-	for x := chunkViews.Front(); x != nil; x = x.Next {
-		chunkView := x.Value
-		var urlStrings []string
-		var err error
-		for _, backoff := range getLookupFileIdBackoffSchedule {
-			if err := ctx.Err(); err != nil {
-				return nil, err
-			}
-			urlStrings, err = masterClient.GetLookupFileIdFunction()(ctx, chunkView.FileId)
-			if err == nil && len(urlStrings) > 0 {
-				break
-			}
-			if err := ctx.Err(); err != nil {
-				return nil, err
-			}
-			glog.V(4).InfofCtx(ctx, "waiting for chunk: %s", chunkView.FileId)
-			timer := time.NewTimer(backoff)
-			select {
-			case <-ctx.Done():
-				if !timer.Stop() {
-					<-timer.C
-				}
-				return nil, ctx.Err()
-			case <-timer.C:
-			}
-		}
-		if err != nil {
-			glog.V(1).InfofCtx(ctx, "operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
-			return nil, err
-		} else if len(urlStrings) == 0 {
-			errUrlNotFound := fmt.Errorf("operation LookupFileId %s failed, err: urls not found", chunkView.FileId)
-			glog.ErrorCtx(ctx, errUrlNotFound)
-			return nil, errUrlNotFound
-		}
-		fileId2Url[chunkView.FileId] = urlStrings
-	}
-
-	return func(writer io.Writer) error {
-		return streamChunksPrefetched(ctx, writer, chunkViews, fileId2Url, jwtFunc, masterClient, offset, size, downloadMaxBytesPs, prefetchAhead)
-	}, nil
-}
-
 func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
 	streamFn, err := PrepareStreamContent(masterClient, JwtForVolumeServer, chunks, offset, size)
 	if err != nil {
@@ -332,6 +276,33 @@ func writeZero(w io.Writer, size int64) (err error) {
 	return
 }
 
+func ReadAll(ctx context.Context, buffer []byte, masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) error {
+
+	lookupFileIdFn := func(ctx context.Context, fileId string) (targetUrls []string, err error) {
+		return masterClient.LookupFileId(ctx, fileId)
+	}
+
+	chunkViews := ViewFromChunks(ctx, lookupFileIdFn, chunks, 0, int64(len(buffer)))
+
+	idx := 0
+
+	for x := chunkViews.Front(); x != nil; x = x.Next {
+		chunkView := x.Value
+		urlStrings, err := lookupFileIdFn(ctx, chunkView.FileId)
+		if err != nil {
+			glog.V(1).InfofCtx(ctx, "operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
+			return err
+		}
+
+		n, err := util_http.RetriedFetchChunkData(ctx, buffer[idx:idx+int(chunkView.ViewSize)], urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, chunkView.FileId)
+		if err != nil {
+			return err
+		}
+		idx += n
+	}
+	return nil
+}
+
 // ----------------  ChunkStreamReader ----------------------------------
 type ChunkStreamReader struct {
 	head         *Interval[*ChunkView]

weed/filer/stream_benchmark_test.go

@@ -1,317 +0,0 @@
-package filer
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"io"
-	"math/rand"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/wdclient"
-)
-
-func TestMain(m *testing.M) {
-	// Initialize the global HTTP client required by ReadUrlAsStream
-	util_http.InitGlobalHttpClient()
-	os.Exit(m.Run())
-}
-
-// mockMasterClientForBenchmark implements HasLookupFileIdFunction and CacheInvalidator
-type mockMasterClientForBenchmark struct {
-	urls map[string][]string
-}
-
-func (m *mockMasterClientForBenchmark) GetLookupFileIdFunction() wdclient.LookupFileIdFunctionType {
-	return func(ctx context.Context, fileId string) ([]string, error) {
-		if urls, ok := m.urls[fileId]; ok {
-			return urls, nil
-		}
-		return nil, fmt.Errorf("fileId %s not found", fileId)
-	}
-}
-
-func (m *mockMasterClientForBenchmark) InvalidateCache(fileId string) {}
-
-// noopJwtFunc returns empty JWT for testing
-func noopJwtFunc(fileId string) string {
-	return ""
-}
-
-// createMockVolumeServer creates an httptest server that serves chunk data
-// with configurable per-request latency to simulate network conditions.
-// The latency is applied once per request (simulating RTT), not per byte.
-func createMockVolumeServer(chunkData map[string][]byte, latency time.Duration) *httptest.Server {
-	var mu sync.RWMutex
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Simulate network latency (connection setup + RTT)
-		if latency > 0 {
-			time.Sleep(latency)
-		}
-
-		// Extract fileId from path (e.g., "/1,abc123")
-		path := r.URL.Path
-		if strings.HasPrefix(path, "/") {
-			path = path[1:]
-		}
-
-		mu.RLock()
-		data, ok := chunkData[path]
-		mu.RUnlock()
-		if !ok {
-			http.Error(w, "not found", http.StatusNotFound)
-			return
-		}
-
-		// Handle Range header
-		rangeHeader := r.Header.Get("Range")
-		if rangeHeader != "" {
-			var start, end int64
-			fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end)
-			if start >= 0 && end < int64(len(data)) && start <= end {
-				w.Header().Set("Content-Length", fmt.Sprintf("%d", end-start+1))
-				w.WriteHeader(http.StatusPartialContent)
-				w.Write(data[start : end+1])
-				return
-			}
-		}
-
-		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data)))
-		w.WriteHeader(http.StatusOK)
-		w.Write(data)
-	}))
-}
-
-// benchmarkConfig holds parameters for a single benchmark scenario
-type benchmarkConfig struct {
-	numChunks  int
-	chunkSize  int
-	latency    time.Duration
-	prefetch   int // 0 = sequential
-}
-
-func (c benchmarkConfig) name() string {
-	name := fmt.Sprintf("chunks=%d/size=%dKB/latency=%dms",
-		c.numChunks, c.chunkSize/1024, c.latency.Milliseconds())
-	if c.prefetch > 0 {
-		name += fmt.Sprintf("/prefetch=%d", c.prefetch)
-	}
-	return name
-}
-
-// setupBenchmark creates mock infrastructure and returns chunks, master client, and cleanup func
-func setupBenchmark(b *testing.B, cfg benchmarkConfig) ([]*filer_pb.FileChunk, *mockMasterClientForBenchmark, func()) {
-	b.Helper()
-
-	// Generate random chunk data
-	chunkData := make(map[string][]byte, cfg.numChunks)
-	chunks := make([]*filer_pb.FileChunk, cfg.numChunks)
-
-	for i := 0; i < cfg.numChunks; i++ {
-		fileId := fmt.Sprintf("1,%x", i)
-		data := make([]byte, cfg.chunkSize)
-		rand.Read(data)
-		chunkData[fileId] = data
-
-		chunks[i] = &filer_pb.FileChunk{
-			FileId:       fileId,
-			Offset:       int64(i * cfg.chunkSize),
-			Size:         uint64(cfg.chunkSize),
-			ModifiedTsNs: int64(i),
-			Fid:          &filer_pb.FileId{FileKey: uint64(i)},
-		}
-	}
-
-	// Start mock volume server
-	server := createMockVolumeServer(chunkData, cfg.latency)
-
-	// Build URL map
-	urls := make(map[string][]string, cfg.numChunks)
-	for i := 0; i < cfg.numChunks; i++ {
-		fileId := fmt.Sprintf("1,%x", i)
-		urls[fileId] = []string{server.URL + "/" + fileId}
-	}
-
-	masterClient := &mockMasterClientForBenchmark{urls: urls}
-	cleanup := func() { server.Close() }
-
-	return chunks, masterClient, cleanup
-}
-
-// runSequentialBenchmark runs the current sequential streaming path
-func runSequentialBenchmark(b *testing.B, cfg benchmarkConfig) {
-	chunks, masterClient, cleanup := setupBenchmark(b, cfg)
-	defer cleanup()
-
-	totalSize := int64(cfg.numChunks * cfg.chunkSize)
-
-	b.ResetTimer()
-	b.SetBytes(totalSize)
-
-	for i := 0; i < b.N; i++ {
-		streamFn, err := PrepareStreamContentWithThrottler(
-			context.Background(),
-			masterClient,
-			noopJwtFunc,
-			chunks,
-			0,
-			totalSize,
-			0, // no throttle
-		)
-		if err != nil {
-			b.Fatal(err)
-		}
-		if err := streamFn(io.Discard); err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-// BenchmarkStreamSequential benchmarks the current sequential streaming path.
-// This provides the BEFORE baseline for comparison.
-func BenchmarkStreamSequential(b *testing.B) {
-	configs := []benchmarkConfig{
-		// Pure throughput (no latency)
-		{numChunks: 16, chunkSize: 64 * 1024, latency: 0},
-		{numChunks: 64, chunkSize: 64 * 1024, latency: 0},
-		// Moderate latency — shows RTT gap overhead
-		{numChunks: 16, chunkSize: 64 * 1024, latency: 5 * time.Millisecond},
-		{numChunks: 64, chunkSize: 64 * 1024, latency: 5 * time.Millisecond},
-		// High latency — significant RTT overhead
-		{numChunks: 16, chunkSize: 64 * 1024, latency: 20 * time.Millisecond},
-		{numChunks: 64, chunkSize: 64 * 1024, latency: 10 * time.Millisecond},
-	}
-
-	for _, cfg := range configs {
-		b.Run(cfg.name(), func(b *testing.B) {
-			runSequentialBenchmark(b, cfg)
-		})
-	}
-}
-
-// BenchmarkStreamSequentialVerify is a quick functional test that the benchmark
-// infrastructure works correctly — ensures data integrity through the pipeline.
-func BenchmarkStreamSequentialVerify(b *testing.B) {
-	cfg := benchmarkConfig{numChunks: 4, chunkSize: 1024, latency: 0}
-	chunks, masterClient, cleanup := setupBenchmark(b, cfg)
-	defer cleanup()
-
-	totalSize := int64(cfg.numChunks * cfg.chunkSize)
-
-	streamFn, err := PrepareStreamContentWithThrottler(
-		context.Background(),
-		masterClient,
-		noopJwtFunc,
-		chunks,
-		0,
-		totalSize,
-		0,
-	)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	var buf bytes.Buffer
-	if err := streamFn(&buf); err != nil {
-		b.Fatal(err)
-	}
-
-	if buf.Len() != int(totalSize) {
-		b.Fatalf("expected %d bytes, got %d", totalSize, buf.Len())
-	}
-}
-
-// runPrefetchBenchmark runs the new prefetch streaming path
-func runPrefetchBenchmark(b *testing.B, cfg benchmarkConfig) {
-	chunks, masterClient, cleanup := setupBenchmark(b, cfg)
-	defer cleanup()
-
-	totalSize := int64(cfg.numChunks * cfg.chunkSize)
-
-	b.ResetTimer()
-	b.SetBytes(totalSize)
-
-	for i := 0; i < b.N; i++ {
-		streamFn, err := PrepareStreamContentWithPrefetch(
-			context.Background(),
-			masterClient,
-			noopJwtFunc,
-			chunks,
-			0,
-			totalSize,
-			0, // no throttle
-			cfg.prefetch,
-		)
-		if err != nil {
-			b.Fatal(err)
-		}
-		if err := streamFn(io.Discard); err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-// BenchmarkStreamPrefetch benchmarks the new prefetch streaming path.
-// Compare against BenchmarkStreamSequential for the AFTER measurement.
-func BenchmarkStreamPrefetch(b *testing.B) {
-	configs := []benchmarkConfig{
-		// Pure throughput (no latency) — should be similar to sequential
-		{numChunks: 16, chunkSize: 64 * 1024, latency: 0, prefetch: 4},
-		{numChunks: 64, chunkSize: 64 * 1024, latency: 0, prefetch: 4},
-		// Moderate latency — prefetch should eliminate most RTT overhead
-		{numChunks: 16, chunkSize: 64 * 1024, latency: 5 * time.Millisecond, prefetch: 4},
-		{numChunks: 64, chunkSize: 64 * 1024, latency: 5 * time.Millisecond, prefetch: 4},
-		// High latency — most benefit from prefetch
-		{numChunks: 16, chunkSize: 64 * 1024, latency: 20 * time.Millisecond, prefetch: 4},
-		{numChunks: 64, chunkSize: 64 * 1024, latency: 10 * time.Millisecond, prefetch: 4},
-		// Vary prefetch count with moderate latency
-		{numChunks: 64, chunkSize: 64 * 1024, latency: 5 * time.Millisecond, prefetch: 2},
-		{numChunks: 64, chunkSize: 64 * 1024, latency: 5 * time.Millisecond, prefetch: 8},
-	}
-
-	for _, cfg := range configs {
-		b.Run(cfg.name(), func(b *testing.B) {
-			runPrefetchBenchmark(b, cfg)
-		})
-	}
-}
-
-// BenchmarkStreamPrefetchVerify verifies data integrity through the prefetch pipeline.
-func BenchmarkStreamPrefetchVerify(b *testing.B) {
-	cfg := benchmarkConfig{numChunks: 4, chunkSize: 1024, latency: 0, prefetch: 4}
-	chunks, masterClient, cleanup := setupBenchmark(b, cfg)
-	defer cleanup()
-
-	totalSize := int64(cfg.numChunks * cfg.chunkSize)
-
-	streamFn, err := PrepareStreamContentWithPrefetch(
-		context.Background(),
-		masterClient,
-		noopJwtFunc,
-		chunks,
-		0,
-		totalSize,
-		0,
-		cfg.prefetch,
-	)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	var buf bytes.Buffer
-	if err := streamFn(&buf); err != nil {
-		b.Fatal(err)
-	}
-
-	if buf.Len() != int(totalSize) {
-		b.Fatalf("expected %d bytes, got %d", totalSize, buf.Len())
-	}
-}

weed/filer/stream_failover_test.go

@@ -0,0 +1,281 @@
+package filer
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+)
+
+// mockMasterClient implements HasLookupFileIdFunction and CacheInvalidator
+type mockMasterClient struct {
+	lookupFunc         func(ctx context.Context, fileId string) ([]string, error)
+	invalidatedFileIds []string
+}
+
+func (m *mockMasterClient) GetLookupFileIdFunction() wdclient.LookupFileIdFunctionType {
+	return m.lookupFunc
+}
+
+func (m *mockMasterClient) InvalidateCache(fileId string) {
+	m.invalidatedFileIds = append(m.invalidatedFileIds, fileId)
+}
+
+// Test urlSlicesEqual helper function
+func TestUrlSlicesEqual(t *testing.T) {
+	tests := []struct {
+		name     string
+		a        []string
+		b        []string
+		expected bool
+	}{
+		{
+			name:     "identical slices",
+			a:        []string{"http://server1", "http://server2"},
+			b:        []string{"http://server1", "http://server2"},
+			expected: true,
+		},
+		{
+			name:     "same URLs different order",
+			a:        []string{"http://server1", "http://server2"},
+			b:        []string{"http://server2", "http://server1"},
+			expected: true,
+		},
+		{
+			name:     "different URLs",
+			a:        []string{"http://server1", "http://server2"},
+			b:        []string{"http://server1", "http://server3"},
+			expected: false,
+		},
+		{
+			name:     "different lengths",
+			a:        []string{"http://server1"},
+			b:        []string{"http://server1", "http://server2"},
+			expected: false,
+		},
+		{
+			name:     "empty slices",
+			a:        []string{},
+			b:        []string{},
+			expected: true,
+		},
+		{
+			name:     "duplicates in both",
+			a:        []string{"http://server1", "http://server1"},
+			b:        []string{"http://server1", "http://server1"},
+			expected: true,
+		},
+		{
+			name:     "different duplicate counts",
+			a:        []string{"http://server1", "http://server1"},
+			b:        []string{"http://server1", "http://server2"},
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := urlSlicesEqual(tt.a, tt.b)
+			if result != tt.expected {
+				t.Errorf("urlSlicesEqual(%v, %v) = %v; want %v", tt.a, tt.b, result, tt.expected)
+			}
+		})
+	}
+}
+
+// Test cache invalidation when read fails
+func TestStreamContentWithCacheInvalidation(t *testing.T) {
+	ctx := context.Background()
+	fileId := "3,01234567890"
+
+	callCount := 0
+	oldUrls := []string{"http://failed-server:8080"}
+	newUrls := []string{"http://working-server:8080"}
+
+	mock := &mockMasterClient{
+		lookupFunc: func(ctx context.Context, fid string) ([]string, error) {
+			callCount++
+			if callCount == 1 {
+				// First call returns failing server
+				return oldUrls, nil
+			}
+			// After invalidation, return working server
+			return newUrls, nil
+		},
+	}
+
+	// Create a simple chunk
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: fileId,
+			Offset: 0,
+			Size:   10,
+		},
+	}
+
+	streamFn, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
+	if err != nil {
+		t.Fatalf("PrepareStreamContentWithThrottler failed: %v", err)
+	}
+
+	// Note: This test can't fully execute streamFn because it would require actual HTTP servers
+	// However, we can verify the setup was created correctly
+	if streamFn == nil {
+		t.Fatal("Expected non-nil stream function")
+	}
+
+	// Verify the lookup was called
+	if callCount != 1 {
+		t.Errorf("Expected 1 lookup call, got %d", callCount)
+	}
+}
+
+// Test that InvalidateCache is called on read failure
+func TestCacheInvalidationInterface(t *testing.T) {
+	mock := &mockMasterClient{
+		lookupFunc: func(ctx context.Context, fileId string) ([]string, error) {
+			return []string{"http://server:8080"}, nil
+		},
+	}
+
+	fileId := "3,test123"
+
+	// Simulate invalidation
+	if invalidator, ok := interface{}(mock).(CacheInvalidator); ok {
+		invalidator.InvalidateCache(fileId)
+	} else {
+		t.Fatal("mockMasterClient should implement CacheInvalidator")
+	}
+
+	// Check that the file ID was recorded as invalidated
+	if len(mock.invalidatedFileIds) != 1 {
+		t.Fatalf("Expected 1 invalidated file ID, got %d", len(mock.invalidatedFileIds))
+	}
+	if mock.invalidatedFileIds[0] != fileId {
+		t.Errorf("Expected invalidated file ID %s, got %s", fileId, mock.invalidatedFileIds[0])
+	}
+}
+
+// Test retry logic doesn't retry with same URLs
+func TestRetryLogicSkipsSameUrls(t *testing.T) {
+	// This test verifies that the urlSlicesEqual check prevents infinite retries
+	sameUrls := []string{"http://server1:8080", "http://server2:8080"}
+	differentUrls := []string{"http://server3:8080", "http://server4:8080"}
+
+	// Same URLs should return true (and thus skip retry)
+	if !urlSlicesEqual(sameUrls, sameUrls) {
+		t.Error("Expected same URLs to be equal")
+	}
+
+	// Different URLs should return false (and thus allow retry)
+	if urlSlicesEqual(sameUrls, differentUrls) {
+		t.Error("Expected different URLs to not be equal")
+	}
+}
+
+func TestCanceledStreamSkipsCacheInvalidation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	fileId := "3,canceled"
+
+	mock := &mockMasterClient{
+		lookupFunc: func(ctx context.Context, fid string) ([]string, error) {
+			return []string{"http://server:8080"}, nil
+		},
+	}
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: fileId,
+			Offset: 0,
+			Size:   10,
+		},
+	}
+
+	streamFn, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
+	if err != nil {
+		t.Fatalf("PrepareStreamContentWithThrottler failed: %v", err)
+	}
+
+	cancel()
+
+	err = streamFn(&bytes.Buffer{})
+	if err != context.Canceled {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+	if len(mock.invalidatedFileIds) != 0 {
+		t.Fatalf("expected no cache invalidation on cancellation, got %v", mock.invalidatedFileIds)
+	}
+}
+
+func TestPrepareStreamContentSkipsLookupWhenContextAlreadyCanceled(t *testing.T) {
+	oldSchedule := getLookupFileIdBackoffSchedule
+	getLookupFileIdBackoffSchedule = []time.Duration{time.Millisecond}
+	t.Cleanup(func() {
+		getLookupFileIdBackoffSchedule = oldSchedule
+	})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	lookupCalls := 0
+	mock := &mockMasterClient{
+		lookupFunc: func(ctx context.Context, fileId string) ([]string, error) {
+			lookupCalls++
+			return nil, errors.New("lookup should not run")
+		},
+	}
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: "3,precanceled",
+			Offset: 0,
+			Size:   10,
+		},
+	}
+
+	_, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+	if lookupCalls != 0 {
+		t.Fatalf("expected no lookup calls after cancellation, got %d", lookupCalls)
+	}
+}
+
+func TestPrepareStreamContentStopsLookupRetriesAfterContextCancellation(t *testing.T) {
+	oldSchedule := getLookupFileIdBackoffSchedule
+	getLookupFileIdBackoffSchedule = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond}
+	t.Cleanup(func() {
+		getLookupFileIdBackoffSchedule = oldSchedule
+	})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	lookupCalls := 0
+	mock := &mockMasterClient{
+		lookupFunc: func(ctx context.Context, fileId string) ([]string, error) {
+			lookupCalls++
+			cancel()
+			return nil, context.Canceled
+		},
+	}
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: "3,cancel-during-lookup",
+			Offset: 0,
+			Size:   10,
+		},
+	}
+
+	_, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+	if lookupCalls != 1 {
+		t.Fatalf("expected lookup retries to stop after cancellation, got %d calls", lookupCalls)
+	}
+}

weed/filer/stream_prefetch.go

@@ -1,274 +0,0 @@
-package filer
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"sync"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/stats"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-	"github.com/seaweedfs/seaweedfs/weed/util/mem"
-	"github.com/seaweedfs/seaweedfs/weed/wdclient"
-)
-
-// chunkPipeResult represents a prefetched chunk streaming through a pipe.
-// The fetch goroutine writes data into the pipeWriter; the consumer reads from pipeReader.
-type chunkPipeResult struct {
-	chunkView  *ChunkView
-	reader     *io.PipeReader
-	fetchErr   error         // final error from fetch goroutine
-	written    int64         // bytes written by fetch goroutine
-	done       chan struct{} // closed when fetch goroutine finishes
-	urlStrings []string      // snapshot of URLs at dispatch time (for retry logic)
-}
-
-// streamChunksPrefetched streams chunks with concurrent prefetch using io.Pipe.
-//
-// For each chunk in file order, a goroutine is launched (bounded by a semaphore)
-// that establishes an HTTP connection to the volume server and streams data through
-// an io.Pipe. The consumer reads from pipes in order, writing to the response.
-//
-// Memory usage is minimal: pipes are synchronous (no buffering), and only one
-// reusable copy buffer is allocated for the consumer.
-func streamChunksPrefetched(
-	ctx context.Context,
-	writer io.Writer,
-	chunkViews *IntervalList[*ChunkView],
-	fileId2Url map[string][]string,
-	jwtFunc VolumeServerJwtFunction,
-	masterClient wdclient.HasLookupFileIdFunction,
-	offset int64,
-	size int64,
-	downloadMaxBytesPs int64,
-	prefetchAhead int,
-) error {
-	downloadThrottler := util.NewWriteThrottler(downloadMaxBytesPs)
-
-	// Create a local cancellable context so the consumer can stop the producer
-	// and all in-flight fetch goroutines on error (e.g., client disconnect).
-	localCtx, localCancel := context.WithCancel(ctx)
-	defer localCancel()
-
-	// Ordered channel: one entry per chunk, in file order.
-	// Capacity = prefetchAhead so the producer can run ahead.
-	// Uses pointer to avoid copying the struct while fetch goroutines write to it.
-	results := make(chan *chunkPipeResult, prefetchAhead)
-
-	// Semaphore to limit concurrent fetch goroutines (and thus HTTP connections).
-	sem := make(chan struct{}, prefetchAhead)
-
-	// Producer: walks chunk list, launches fetch goroutines, sends results in order.
-	// The producer only reads from fileId2Url (populated before streaming starts),
-	// so there is no concurrent map access — the consumer never writes to it.
-	var producerWg sync.WaitGroup
-	producerWg.Add(1)
-	go func() {
-		defer producerWg.Done()
-		defer close(results)
-
-		for x := chunkViews.Front(); x != nil; x = x.Next {
-			chunkView := x.Value
-
-			// Check context before starting new fetch
-			select {
-			case <-localCtx.Done():
-				return
-			default:
-			}
-
-			// Acquire semaphore slot (bounds concurrent HTTP connections)
-			select {
-			case sem <- struct{}{}:
-			case <-localCtx.Done():
-				return
-			}
-
-			pr, pw := io.Pipe()
-			urlStrings := fileId2Url[chunkView.FileId]
-			jwt := jwtFunc(chunkView.FileId)
-
-			result := &chunkPipeResult{
-				chunkView:  chunkView,
-				reader:     pr,
-				done:       make(chan struct{}),
-				urlStrings: urlStrings,
-			}
-
-			// Launch fetch goroutine
-			go func(cv *ChunkView, urls []string, jwt string, pw *io.PipeWriter, res *chunkPipeResult) {
-				defer func() { <-sem }() // release semaphore
-				defer close(res.done)
-
-				written, err := retriedStreamFetchChunkData(
-					localCtx, pw, urls, jwt,
-					cv.CipherKey, cv.IsGzipped, cv.IsFullChunk(),
-					cv.OffsetInChunk, int(cv.ViewSize),
-				)
-				res.written = written
-				res.fetchErr = err
-
-				if err != nil {
-					pw.CloseWithError(err)
-				} else {
-					pw.Close()
-				}
-			}(chunkView, urlStrings, jwt, pw, result)
-
-			// Send result to consumer (blocks if channel full, back-pressuring producer)
-			select {
-			case results <- result:
-			case <-localCtx.Done():
-				// Consumer gone; close the pipe and wait for the fetch goroutine
-				// to finish so we don't leak it (this result was never sent to
-				// the channel, so the drain loop won't handle it).
-				pr.Close()
-				<-result.done
-				return
-			}
-		}
-	}()
-
-	// Consumer: reads from results channel in order, writes to response writer.
-	// Use the SeaweedFS memory pool for the copy buffer to reduce GC pressure.
-	copyBuf := mem.Allocate(256 * 1024)
-	defer mem.Free(copyBuf)
-	remaining := size
-
-	var consumeErr error
-	for result := range results {
-		chunkView := result.chunkView
-
-		// Handle gap before this chunk (zero-fill)
-		if offset < chunkView.ViewOffset {
-			gap := chunkView.ViewOffset - offset
-			remaining -= gap
-			glog.V(4).InfofCtx(ctx, "prefetch zero [%d,%d)", offset, chunkView.ViewOffset)
-			if err := writeZero(writer, gap); err != nil {
-				consumeErr = fmt.Errorf("write zero [%d,%d): %w", offset, chunkView.ViewOffset, err)
-				result.reader.Close()
-				break
-			}
-			offset = chunkView.ViewOffset
-		}
-
-		// Stream chunk data from pipe to response
-		start := time.Now()
-		_, copyErr := io.CopyBuffer(writer, result.reader, copyBuf)
-		result.reader.Close()
-
-		// Wait for fetch goroutine to finish to get final error
-		<-result.done
-
-		// Determine the effective error
-		err := copyErr
-		if err == nil && result.fetchErr != nil && result.written == 0 {
-			err = result.fetchErr
-		}
-
-		// If the fetcher itself failed before writing any data, try cache invalidation
-		// + re-fetch (same as sequential path stream.go:197). We check result.fetchErr
-		// and result.written (not copied) to avoid wrongly retrying when the fetch
-		// succeeded but the response writer failed on the first write.
-		if result.fetchErr != nil && result.written == 0 {
-			if err := localCtx.Err(); err != nil {
-				consumeErr = err
-				break
-			}
-			retryErr := retryWithCacheInvalidation(localCtx, writer, chunkView, result.urlStrings, jwtFunc, masterClient)
-			if retryErr != nil {
-				stats.FilerHandlerCounter.WithLabelValues("chunkDownloadError").Inc()
-				consumeErr = fmt.Errorf("read chunk: %w", retryErr)
-				break
-			}
-			// Retry succeeded
-			err = nil
-		} else if err != nil {
-			if localCtx.Err() != nil {
-				consumeErr = localCtx.Err()
-			} else {
-				stats.FilerHandlerCounter.WithLabelValues("chunkDownloadError").Inc()
-				consumeErr = fmt.Errorf("read chunk: %w", err)
-			}
-			break
-		}
-
-		offset += int64(chunkView.ViewSize)
-		remaining -= int64(chunkView.ViewSize)
-		stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
-		stats.FilerHandlerCounter.WithLabelValues("chunkDownload").Inc()
-		downloadThrottler.MaybeSlowdown(int64(chunkView.ViewSize))
-	}
-
-	// Cancel the local context to stop the producer and any in-flight fetchers early.
-	// This ensures goroutines don't linger after the consumer exits (e.g., on write error).
-	localCancel()
-
-	// Drain remaining results to close pipes and unblock fetch goroutines
-	for result := range results {
-		result.reader.Close()
-		<-result.done
-	}
-
-	// Wait for producer to finish
-	producerWg.Wait()
-
-	if consumeErr != nil {
-		return consumeErr
-	}
-
-	// Handle trailing zero-fill
-	if remaining > 0 {
-		glog.V(4).InfofCtx(ctx, "prefetch zero [%d,%d)", offset, offset+remaining)
-		if err := writeZero(writer, remaining); err != nil {
-			return fmt.Errorf("write zero [%d,%d): %w", offset, offset+remaining, err)
-		}
-	}
-
-	return nil
-}
-
-// retryWithCacheInvalidation attempts to re-fetch a chunk after invalidating the URL cache.
-// This mirrors the retry logic in PrepareStreamContentWithThrottler's sequential path.
-func retryWithCacheInvalidation(
-	ctx context.Context,
-	writer io.Writer,
-	chunkView *ChunkView,
-	oldUrlStrings []string,
-	jwtFunc VolumeServerJwtFunction,
-	masterClient wdclient.HasLookupFileIdFunction,
-) error {
-	invalidator, ok := masterClient.(CacheInvalidator)
-	if !ok {
-		return fmt.Errorf("read chunk %s failed and no cache invalidator available", chunkView.FileId)
-	}
-
-	glog.V(0).InfofCtx(ctx, "prefetch read chunk %s failed, invalidating cache and retrying", chunkView.FileId)
-	invalidator.InvalidateCache(chunkView.FileId)
-
-	newUrlStrings, lookupErr := masterClient.GetLookupFileIdFunction()(ctx, chunkView.FileId)
-	if lookupErr != nil {
-		glog.WarningfCtx(ctx, "failed to re-lookup chunk %s after cache invalidation: %v", chunkView.FileId, lookupErr)
-		return fmt.Errorf("re-lookup chunk %s: %w", chunkView.FileId, lookupErr)
-	}
-	if len(newUrlStrings) == 0 {
-		glog.WarningfCtx(ctx, "re-lookup for chunk %s returned no locations, skipping retry", chunkView.FileId)
-		return fmt.Errorf("re-lookup chunk %s: no locations", chunkView.FileId)
-	}
-
-	if urlSlicesEqual(oldUrlStrings, newUrlStrings) {
-		glog.V(0).InfofCtx(ctx, "re-lookup returned same locations for chunk %s, skipping retry", chunkView.FileId)
-		return fmt.Errorf("read chunk %s failed, same locations after cache invalidation", chunkView.FileId)
-	}
-
-	glog.V(0).InfofCtx(ctx, "retrying read chunk %s with new locations: %v", chunkView.FileId, newUrlStrings)
-	jwt := jwtFunc(chunkView.FileId)
-	_, err := retriedStreamFetchChunkData(
-		ctx, writer, newUrlStrings, jwt,
-		chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(),
-		chunkView.OffsetInChunk, int(chunkView.ViewSize),
-	)
-	return err
-}

weed/filer/stream_prefetch_test.go

@@ -1,365 +0,0 @@
-package filer
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"io"
-	"math/rand"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/wdclient"
-)
-
-// testMasterClient implements HasLookupFileIdFunction and CacheInvalidator for tests
-type testMasterClient struct {
-	urls             map[string][]string
-	invalidatedCount int32
-}
-
-func (m *testMasterClient) GetLookupFileIdFunction() wdclient.LookupFileIdFunctionType {
-	return func(ctx context.Context, fileId string) ([]string, error) {
-		if urls, ok := m.urls[fileId]; ok {
-			return urls, nil
-		}
-		return nil, fmt.Errorf("fileId %s not found", fileId)
-	}
-}
-
-func (m *testMasterClient) InvalidateCache(fileId string) {
-	atomic.AddInt32(&m.invalidatedCount, 1)
-}
-
-func noopJwt(fileId string) string { return "" }
-
-// createTestServer creates a mock volume server that serves chunk data.
-// Supports Range header for partial chunk reads (exercising OffsetInChunk paths).
-func createTestServer(chunkData map[string][]byte) *httptest.Server {
-	var mu sync.RWMutex
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		path := r.URL.Path
-		if strings.HasPrefix(path, "/") {
-			path = path[1:]
-		}
-		mu.RLock()
-		data, ok := chunkData[path]
-		mu.RUnlock()
-		if !ok {
-			http.Error(w, "not found", http.StatusNotFound)
-			return
-		}
-
-		// Handle Range header for partial chunk reads
-		rangeHeader := r.Header.Get("Range")
-		if rangeHeader != "" {
-			var start, end int64
-			if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end); err == nil {
-				if start < 0 || end >= int64(len(data)) || start > end {
-					w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", len(data)))
-					w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
-					return
-				}
-				rangeData := data[start : end+1]
-				w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
-				w.Header().Set("Content-Length", fmt.Sprintf("%d", len(rangeData)))
-				w.WriteHeader(http.StatusPartialContent)
-				w.Write(rangeData)
-				return
-			}
-		}
-
-		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data)))
-		w.WriteHeader(http.StatusOK)
-		w.Write(data)
-	}))
-}
-
-// makeChunksAndServer creates N chunks of given size, a mock server, and a master client
-func makeChunksAndServer(t *testing.T, numChunks, chunkSize int) ([]*filer_pb.FileChunk, *testMasterClient, map[string][]byte, func()) {
-	t.Helper()
-
-	chunkData := make(map[string][]byte, numChunks)
-	chunks := make([]*filer_pb.FileChunk, numChunks)
-
-	for i := 0; i < numChunks; i++ {
-		fileId := fmt.Sprintf("1,%x", i)
-		data := make([]byte, chunkSize)
-		rand.Read(data)
-		chunkData[fileId] = data
-
-		chunks[i] = &filer_pb.FileChunk{
-			FileId:       fileId,
-			Offset:       int64(i * chunkSize),
-			Size:         uint64(chunkSize),
-			ModifiedTsNs: int64(i),
-			Fid:          &filer_pb.FileId{FileKey: uint64(i)},
-		}
-	}
-
-	server := createTestServer(chunkData)
-	urls := make(map[string][]string, numChunks)
-	for i := 0; i < numChunks; i++ {
-		fileId := fmt.Sprintf("1,%x", i)
-		urls[fileId] = []string{server.URL + "/" + fileId}
-	}
-
-	masterClient := &testMasterClient{urls: urls}
-	return chunks, masterClient, chunkData, func() { server.Close() }
-}
-
-// TestPrefetchInOrderDelivery verifies chunks are written to the output in correct file order
-func TestPrefetchInOrderDelivery(t *testing.T) {
-	chunks, masterClient, chunkData, cleanup := makeChunksAndServer(t, 8, 4096)
-	defer cleanup()
-
-	totalSize := int64(8 * 4096)
-
-	streamFn, err := PrepareStreamContentWithPrefetch(
-		context.Background(), masterClient, noopJwt,
-		chunks, 0, totalSize, 0, 4,
-	)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var buf bytes.Buffer
-	if err := streamFn(&buf); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify total size
-	if buf.Len() != int(totalSize) {
-		t.Fatalf("expected %d bytes, got %d", totalSize, buf.Len())
-	}
-
-	// Verify data matches chunk-by-chunk in order
-	result := buf.Bytes()
-	for i := 0; i < 8; i++ {
-		fileId := fmt.Sprintf("1,%x", i)
-		expected := chunkData[fileId]
-		got := result[i*4096 : (i+1)*4096]
-		if !bytes.Equal(expected, got) {
-			t.Fatalf("chunk %d (%s) data mismatch at offset %d", i, fileId, i*4096)
-		}
-	}
-}
-
-// TestPrefetchSingleChunk verifies the pipeline works with just one chunk
-func TestPrefetchSingleChunk(t *testing.T) {
-	chunks, masterClient, chunkData, cleanup := makeChunksAndServer(t, 1, 8192)
-	defer cleanup()
-
-	streamFn, err := PrepareStreamContentWithPrefetch(
-		context.Background(), masterClient, noopJwt,
-		chunks, 0, 8192, 0, 4,
-	)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var buf bytes.Buffer
-	if err := streamFn(&buf); err != nil {
-		t.Fatal(err)
-	}
-
-	expected := chunkData["1,0"]
-	if !bytes.Equal(expected, buf.Bytes()) {
-		t.Fatal("single chunk data mismatch")
-	}
-}
-
-// TestPrefetchFallbackToSequential verifies prefetch=1 falls back to sequential path
-func TestPrefetchFallbackToSequential(t *testing.T) {
-	chunks, masterClient, chunkData, cleanup := makeChunksAndServer(t, 4, 1024)
-	defer cleanup()
-
-	totalSize := int64(4 * 1024)
-
-	streamFn, err := PrepareStreamContentWithPrefetch(
-		context.Background(), masterClient, noopJwt,
-		chunks, 0, totalSize, 0, 1, // prefetch=1 -> sequential
-	)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var buf bytes.Buffer
-	if err := streamFn(&buf); err != nil {
-		t.Fatal(err)
-	}
-
-	if buf.Len() != int(totalSize) {
-		t.Fatalf("expected %d bytes, got %d", totalSize, buf.Len())
-	}
-
-	// Verify data order
-	result := buf.Bytes()
-	for i := 0; i < 4; i++ {
-		fileId := fmt.Sprintf("1,%x", i)
-		expected := chunkData[fileId]
-		got := result[i*1024 : (i+1)*1024]
-		if !bytes.Equal(expected, got) {
-			t.Fatalf("chunk %d data mismatch", i)
-		}
-	}
-}
-
-// TestPrefetchContextCancellation verifies all goroutines clean up on cancellation
-func TestPrefetchContextCancellation(t *testing.T) {
-	// Use a slow server so cancellation happens mid-stream
-	var requestCount int32
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		atomic.AddInt32(&requestCount, 1)
-		// Slow response
-		time.Sleep(100 * time.Millisecond)
-		w.Header().Set("Content-Length", "1024")
-		w.WriteHeader(http.StatusOK)
-		w.Write(make([]byte, 1024))
-	}))
-	defer server.Close()
-
-	numChunks := 16
-	chunks := make([]*filer_pb.FileChunk, numChunks)
-	urls := make(map[string][]string, numChunks)
-	for i := 0; i < numChunks; i++ {
-		fileId := fmt.Sprintf("1,%x", i)
-		chunks[i] = &filer_pb.FileChunk{
-			FileId: fileId, Offset: int64(i * 1024), Size: 1024,
-			ModifiedTsNs: int64(i), Fid: &filer_pb.FileId{FileKey: uint64(i)},
-		}
-		urls[fileId] = []string{server.URL + "/" + fileId}
-	}
-	masterClient := &testMasterClient{urls: urls}
-
-	// Cancel after a short time
-	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
-	defer cancel()
-
-	streamFn, err := PrepareStreamContentWithPrefetch(
-		ctx, masterClient, noopJwt,
-		chunks, 0, int64(numChunks*1024), 0, 4,
-	)
-	if err != nil {
-		// URL resolution may fail due to cancellation — that's expected
-		return
-	}
-
-	err = streamFn(io.Discard)
-	if err == nil {
-		t.Fatal("expected error from cancelled context")
-	}
-
-	// Verify not all chunks were requested (cancellation stopped early)
-	reqs := atomic.LoadInt32(&requestCount)
-	if reqs >= int32(numChunks) {
-		t.Logf("warning: all %d chunks were requested despite cancellation (got %d)", numChunks, reqs)
-	}
-}
-
-// TestPrefetchRangeRequest verifies prefetch works with offset/size subset
-func TestPrefetchRangeRequest(t *testing.T) {
-	chunks, masterClient, chunkData, cleanup := makeChunksAndServer(t, 8, 4096)
-	defer cleanup()
-
-	// Request only chunks 2-5 (offset=8192, size=16384)
-	offset := int64(2 * 4096)
-	size := int64(4 * 4096)
-
-	streamFn, err := PrepareStreamContentWithPrefetch(
-		context.Background(), masterClient, noopJwt,
-		chunks, offset, size, 0, 4,
-	)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var buf bytes.Buffer
-	if err := streamFn(&buf); err != nil {
-		t.Fatal(err)
-	}
-
-	if buf.Len() != int(size) {
-		t.Fatalf("expected %d bytes, got %d", size, buf.Len())
-	}
-
-	// Verify data matches chunks 2-5
-	result := buf.Bytes()
-	for i := 2; i < 6; i++ {
-		fileId := fmt.Sprintf("1,%x", i)
-		expected := chunkData[fileId]
-		start := (i - 2) * 4096
-		got := result[start : start+4096]
-		if !bytes.Equal(expected, got) {
-			t.Fatalf("chunk %d data mismatch in range request", i)
-		}
-	}
-}
-
-// TestPrefetchLargePrefetchCount verifies prefetch > numChunks is handled gracefully
-func TestPrefetchLargePrefetchCount(t *testing.T) {
-	chunks, masterClient, _, cleanup := makeChunksAndServer(t, 3, 1024)
-	defer cleanup()
-
-	totalSize := int64(3 * 1024)
-
-	// prefetch=10 but only 3 chunks — should work fine
-	streamFn, err := PrepareStreamContentWithPrefetch(
-		context.Background(), masterClient, noopJwt,
-		chunks, 0, totalSize, 0, 10,
-	)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var buf bytes.Buffer
-	if err := streamFn(&buf); err != nil {
-		t.Fatal(err)
-	}
-
-	if buf.Len() != int(totalSize) {
-		t.Fatalf("expected %d bytes, got %d", totalSize, buf.Len())
-	}
-}
-
-// TestPrefetchConcurrentDownloads verifies multiple concurrent prefetch streams
-func TestPrefetchConcurrentDownloads(t *testing.T) {
-	chunks, masterClient, _, cleanup := makeChunksAndServer(t, 8, 2048)
-	defer cleanup()
-
-	totalSize := int64(8 * 2048)
-
-	var wg sync.WaitGroup
-	errors := make(chan error, 4)
-
-	for i := 0; i < 4; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			streamFn, err := PrepareStreamContentWithPrefetch(
-				context.Background(), masterClient, noopJwt,
-				chunks, 0, totalSize, 0, 4,
-			)
-			if err != nil {
-				errors <- err
-				return
-			}
-			if err := streamFn(io.Discard); err != nil {
-				errors <- err
-			}
-		}()
-	}
-
-	wg.Wait()
-	close(errors)
-
-	for err := range errors {
-		t.Fatalf("concurrent download error: %v", err)
-	}
-}

weed/iam/helpers.go

@@ -37,6 +37,11 @@ func GenerateRandomString(length int, charset string) (string, error) {
 	return string(b), nil
 }
 
+// GenerateAccessKeyId generates a new access key ID.
+func GenerateAccessKeyId() (string, error) {
+	return GenerateRandomString(AccessKeyIdLength, CharsetUpper)
+}
+
 // GenerateSecretAccessKey generates a new secret access key.
 func GenerateSecretAccessKey() (string, error) {
 	return GenerateRandomString(SecretAccessKeyLength, Charset)
@@ -174,3 +179,11 @@ func MapToIdentitiesAction(action string) string {
 		return ""
 	}
 }
+
+// MaskAccessKey masks an access key for logging, showing only the first 4 characters.
+func MaskAccessKey(accessKeyId string) string {
+	if len(accessKeyId) > 4 {
+		return accessKeyId[:4] + "***"
+	}
+	return accessKeyId
+}