refactor: remove unused gRPC connection age parameters

The GrpcMaxConnectionAge and GrpcMaxConnectionAgeGrace constants have a troubled history - they were removed in 2022 due to gRPC issues, reverted later, and recently re-added. However, they are not essential to the core worker reconnection fix which was solved through proper goroutine ordering. The Docker Swarm DNS handling mentioned in the comments is not critical, and these parameters have caused problems in the past. Removing them simplifies the configuration without losing functionality.
refactor: extract drain and close channel logic into helper function
2026-10-14 10:35:51 +00:00 · 2025-12-22 18:19:52 -08:00 · 2025-12-22 17:56:42 -08:00 · 2025-12-22 17:55:43 -08:00 · 2025-12-22 17:55:36 -08:00 · 2025-12-22 17:55:25 -08:00
195 changed files with 16521 additions and 2456 deletions
@@ -44,7 +44,7 @@ jobs:
        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}

      - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -60,7 +60,7 @@ jobs:
          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"

      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -93,7 +93,7 @@ jobs:
        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}

      - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -109,7 +109,7 @@ jobs:
          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"

      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -30,7 +30,7 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -30,7 +30,7 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -30,7 +30,7 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -30,7 +30,7 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -30,7 +30,7 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -45,7 +45,7 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -30,7 +30,7 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -224,3 +224,4 @@ jobs:
          echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"


+
@@ -36,7 +36,7 @@ jobs:
      uses: docker/setup-buildx-action@v3
      
    - name: Cache Docker layers
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: /tmp/.buildx-cache
        key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
@@ -134,7 +134,7 @@ jobs:

    - name: Archive logs
      if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
      with:
        name: output-logs
        path: docker/output.log
@@ -35,7 +35,7 @@ jobs:

    - name: Archive logs
      if: failure()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
      with:
        name: ec-integration-test-logs
        path: test/erasure_coding
@@ -183,7 +183,7 @@ jobs:
        
    - name: Upload Test Artifacts
      if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
      with:
        name: fuse-integration-test-results
        path: |
@@ -53,6 +53,7 @@ jobs:
          mkdir -p "$WEED_DATA_DIR"
          
          # Start SeaweedFS with optimized settings for CI
+          # Include S3 API for s3copier integration tests
          weed server -dir="$WEED_DATA_DIR" \
            -master.raftHashicorp \
            -master.electionTimeout=1s \
@@ -61,6 +62,9 @@ jobs:
            -volume.preStopSeconds=1 \
            -master.peers=none \
            -filer -filer.maxMB=64 \
+            -s3 -s3.port=8333 \
+            -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \
+            -s3.allowDeleteBucketNotEmpty=true \
            -master.port=9333 \
            -volume.port=8080 \
            -filer.port=8888 \
@@ -105,6 +109,16 @@ jobs:
            sleep 2
          done
          
+          # Wait for S3 API
+          for i in {1..30}; do
+            if curl -s http://localhost:8333/ > /dev/null 2>&1; then
+              echo "✓ S3 API is ready"
+              break
+            fi
+            echo "Waiting for S3 API... ($i/30)"
+            sleep 2
+          done
+          
          echo "✓ All SeaweedFS components are ready!"
          
          # Display cluster status
@@ -133,6 +147,20 @@ jobs:
        run: |
          mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
      
+      - name: Run S3 ETag Validation Tests (Issue #7768)
+        working-directory: other/java/s3copier
+        env:
+          S3_ENDPOINT: http://127.0.0.1:8333
+          S3_ACCESS_KEY: some_access_key1
+          S3_SECRET_KEY: some_secret_key1
+        run: |
+          echo "Running S3 ETag validation tests against $S3_ENDPOINT"
+          mvn test -Dtest=ETagValidationTest \
+            -DS3_ENDPOINT=$S3_ENDPOINT \
+            -DS3_ACCESS_KEY=$S3_ACCESS_KEY \
+            -DS3_SECRET_KEY=$S3_SECRET_KEY \
+            -Dmaven.javadoc.skip=true -Dgpg.skip=true
+      
      - name: Display logs on failure
        if: failure()
        run: |
@@ -49,7 +49,7 @@ jobs:
      
      - name: Upload Test Reports
        if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: test-reports-java-${{ matrix.java }}
          path: |
@@ -70,7 +70,7 @@ jobs:
        
    - name: Archive logs on failure
      if: failure()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
      with:
        name: metadata-subscribe-test-logs
        path: |
@@ -35,7 +35,7 @@ jobs:
      uses: docker/setup-buildx-action@v3

    - name: Cache Docker layers
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: /tmp/.buildx-cache
        key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
@@ -62,7 +62,7 @@ jobs:

    - name: Archive logs
      if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
      with:
        name: postgres-logs
        path: test/postgres/postgres-output.log
@@ -77,9 +77,10 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-filer-group-test-logs
          path: test/s3/filer_group/weed-test*.log
          retention-days: 3

+
@@ -76,7 +76,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-versioning-test-logs-${{ matrix.test-type }}
          path: test/s3/versioning/weed-test*.log
@@ -124,7 +124,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-versioning-compatibility-logs
          path: test/s3/versioning/weed-test*.log
@@ -172,7 +172,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-cors-compatibility-logs
          path: test/s3/cors/weed-test*.log
@@ -239,7 +239,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-retention-test-logs-${{ matrix.test-type }}
          path: test/s3/retention/weed-test*.log
@@ -306,7 +306,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-cors-test-logs-${{ matrix.test-type }}
          path: test/s3/cors/weed-test*.log
@@ -355,7 +355,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-retention-worm-logs
          path: test/s3/retention/weed-test*.log
@@ -365,8 +365,6 @@ jobs:
    name: S3 Versioning Stress Test
    runs-on: ubuntu-22.04
    timeout-minutes: 35
-    # Only run stress tests on master branch pushes to avoid overloading PR testing
-    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
    
    steps:
      - name: Check out code
@@ -391,6 +389,9 @@ jobs:
          uname -a
          free -h
          
+          # Start server for stress tests
+          make start-server
+          
          # Run stress tests (concurrent operations)
          make test-versioning-stress || {
            echo "❌ Stress test failed, checking logs..."
@@ -398,14 +399,30 @@ jobs:
              echo "=== Server logs ==="
              tail -200 weed-test.log
            fi
+            make stop-server
            make clean
            exit 1
          }
+          
+          # Run pagination stress tests (>1000 versions)
+          echo "=== Running pagination stress tests ==="
+          make test-versioning-pagination-stress || {
+            echo "❌ Pagination stress test failed, checking logs..."
+            if [ -f weed-test.log ]; then
+              echo "=== Server logs ==="
+              tail -200 weed-test.log
+            fi
+            make stop-server
+            make clean
+            exit 1
+          }
+          
+          make stop-server
          make clean

      - name: Upload stress test logs
        if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-versioning-stress-logs
          path: test/s3/versioning/weed-test*.log
@@ -459,10 +476,65 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-tagging-test-logs
          path: test/s3/tagging/weed-test*.log
          retention-days: 3

+  s3-remote-cache-tests:
+    name: S3 Remote Cache Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false
+
+      - name: Run S3 Remote Cache Tests
+        timeout-minutes: 15
+        working-directory: test/s3/remote_cache
+        run: |
+          set -x
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          
+          # Run the remote cache integration tests
+          # Tests singleflight deduplication for caching remote objects
+          make test-with-server || {
+            echo "❌ Test failed, checking logs..."
+            if [ -f primary-weed.log ]; then
+              echo "=== Primary server logs ==="
+              tail -100 primary-weed.log
+            fi
+            if [ -f remote-weed.log ]; then
+              echo "=== Remote server logs ==="
+              tail -100 remote-weed.log
+            fi
+            echo "=== Process information ==="
+            ps aux | grep -E "(weed|test)" || true
+            exit 1
+          }
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: s3-remote-cache-test-logs
+          path: |
+            test/s3/remote_cache/primary-weed.log
+            test/s3/remote_cache/remote-weed.log
+          retention-days: 3
+
  # Removed SSE-C integration tests and compatibility job
@@ -65,7 +65,7 @@ jobs:

      - name: Upload test results on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: iam-unit-test-results
          path: |
@@ -162,7 +162,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-iam-integration-logs-${{ matrix.test-type }}
          path: test/s3/iam/weed-*.log
@@ -222,7 +222,7 @@ jobs:

      - name: Upload distributed test logs
        if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-iam-distributed-logs
          path: test/s3/iam/weed-*.log
@@ -274,7 +274,7 @@ jobs:

      - name: Upload performance test results
        if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-iam-performance-results
          path: |
@@ -152,7 +152,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-keycloak-test-logs
          path: |
@@ -121,7 +121,7 @@ jobs:
      
      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: test-logs-python-${{ matrix.python-version }}
          path: |
@@ -95,7 +95,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-sse-test-logs-${{ matrix.test-type }}
          path: test/s3/sse/weed-test*.log
@@ -143,7 +143,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-sse-compatibility-logs
          path: test/s3/sse/weed-test*.log
@@ -192,7 +192,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-sse-metadata-persistence-logs
          path: test/s3/sse/weed-test*.log
@@ -241,7 +241,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-sse-copy-operations-logs
          path: test/s3/sse/weed-test*.log
@@ -290,7 +290,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-sse-multipart-logs
          path: test/s3/sse/weed-test*.log
@@ -340,7 +340,7 @@ jobs:

      - name: Upload performance test logs
        if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-sse-performance-logs
          path: test/s3/sse/weed-test*.log
@@ -91,3 +91,4 @@ jobs:
        echo "| readonly | /public | Read-only |" >> $GITHUB_STEP_SUMMARY


+
@@ -125,7 +125,7 @@ jobs:

    - name: Upload test results
      if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
      with:
        name: spark-test-results
        path: test/java/spark/target/surefire-reports/
@@ -150,7 +150,7 @@ jobs:
    - name: Cache Apache Spark
      if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
      id: cache-spark
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: spark-3.5.0-bin-hadoop3
        key: spark-3.5.0-hadoop3
@@ -18,7 +18,7 @@ defaults:
 jobs:
  awscli-tests:
    runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

@@ -103,7 +103,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: seaweedfs-logs
          # Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
@@ -106,7 +106,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: tus-test-logs
          path: |
@@ -126,3 +126,8 @@ weed-iam
 test/kafka/kafka-client-loadtest/weed-linux-arm64
 /test/tus/filerldb2
 coverage.out
+/test/s3/remote_cache/test-primary-data
+/test/s3/remote_cache/test-remote-data
+test/s3/remote_cache/remote-server.pid
+test/s3/remote_cache/primary-server.pid
+/test/erasure_coding/filerldb2
@@ -28,7 +28,7 @@ require (
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
-	github.com/go-redsync/redsync/v4 v4.14.0
+	github.com/go-redsync/redsync/v4 v4.15.0
 	github.com/go-sql-driver/mysql v1.9.3
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.7.0
@@ -71,7 +71,7 @@ require (
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
-	github.com/seaweedfs/raft v1.1.3
+	github.com/seaweedfs/raft v1.1.6
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect
@@ -96,14 +96,14 @@ require (
 	gocloud.dev v0.43.0
 	gocloud.dev/pubsub/natspubsub v0.43.0
 	gocloud.dev/pubsub/rabbitpubsub v0.43.0
-	golang.org/x/crypto v0.45.0
+	golang.org/x/crypto v0.46.0
 	golang.org/x/exp v0.0.0-20250811191247-51f88131bc50
-	golang.org/x/image v0.33.0
-	golang.org/x/net v0.47.0
+	golang.org/x/image v0.34.0
+	golang.org/x/net v0.48.0
 	golang.org/x/oauth2 v0.32.0 // indirect
-	golang.org/x/sys v0.38.0
-	golang.org/x/text v0.31.0 // indirect
-	golang.org/x/tools v0.38.0 // indirect
+	golang.org/x/sys v0.39.0
+	golang.org/x/text v0.32.0 // indirect
+	golang.org/x/tools v0.39.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.247.0
 	google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 // indirect
@@ -126,9 +126,9 @@ require (
 	github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
 	github.com/arangodb/go-driver v1.6.9
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.40.1
-	github.com/aws/aws-sdk-go-v2/config v1.31.3
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.3
+	github.com/aws/aws-sdk-go-v2 v1.41.0
+	github.com/aws/aws-sdk-go-v2/config v1.32.6
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.6
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
@@ -152,14 +152,14 @@ require (
 	github.com/rabbitmq/amqp091-go v1.10.0
 	github.com/rclone/rclone v1.71.2
 	github.com/rdleal/intervalst v1.5.0
-	github.com/redis/go-redis/v9 v9.14.1
+	github.com/redis/go-redis/v9 v9.17.2
 	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/shirou/gopsutil/v4 v4.25.11
 	github.com/tarantool/go-tarantool/v2 v2.4.1
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/xeipuuv/gojsonschema v1.2.0
-	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0
+	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0
 	go.etcd.io/etcd/client/pkg/v3 v3.6.6
 	go.uber.org/atomic v1.11.0
 	golang.org/x/sync v0.19.0
@@ -173,6 +173,7 @@ require (
 	cloud.google.com/go/longrunning v0.6.7 // indirect
 	cloud.google.com/go/pubsub/v2 v2.0.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
 	github.com/bazelbuild/rules_go v0.46.0 // indirect
 	github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
 	github.com/blevesearch/snowballstem v0.9.0 // indirect
@@ -220,7 +221,7 @@ require (
 	go.opentelemetry.io/proto/otlp v1.7.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/mod v0.29.0 // indirect
+	golang.org/x/mod v0.30.0 // indirect
 	gonum.org/v1/gonum v0.16.0 // indirect
 )

@@ -260,21 +261,21 @@ require (
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
 	github.com/aws/smithy-go v1.24.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
@@ -445,7 +446,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/arch v0.20.0 // indirect
-	golang.org/x/term v0.37.0 // indirect
+	golang.org/x/term v0.38.0 // indirect
 	golang.org/x/time v0.12.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
@@ -665,46 +665,48 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.40.1 h1:difXb4maDZkRH0x//Qkwcfpdg1XQVXEAEs2DdXldFFc=
-github.com/aws/aws-sdk-go-v2 v1.40.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
+github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
+github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
-github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.3 h1:01Ym72hK43hjwDeJUfi1l2oYLXBAOR8gNSZNmXmvuas=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.3/go.mod h1:55nWF/Sr9Zvls0bGnWkRxUdhzKqj9uRNlPvgV1vgxKc=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 h1:utxLraaifrSBkeyII9mIbVwXXWrZdlPO7FIKmyLCEcY=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15/go.mod h1:hW6zjYUDQwfz3icf4g2O41PHi77u10oAzJ84iSzR/lo=
+github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8=
+github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 h1:Y5YXgygXwDI5P4RkteB5yF7v35neH7LfJKBG+hzIons=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15/go.mod h1:K+/1EpG42dFSY7CBj+Fruzm8PsCGWTXJ3jdeJ659oGQ=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 h1:AvltKnW9ewxX2hFmQS0FyJH93aSvJVUEFvXfU+HWtSE=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15/go.mod h1:3I4oCdZdmgrREhU74qS1dK9yZ62yumob+58AbFR4cQA=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 h1:3/u/4yZOffg5jdNk1sDpOQ4Y+R6Xbh+GzpDrSZjuy3U=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15/go.mod h1:4Zkjq0FKjE78NKjabuM4tRXKFzUJWXgP0ItEZK8l7JU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
 github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE=
 github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 h1:8sTTiw+9yuNXcfWeqKF2x01GqCF49CpP4Z9nKrrk/ts=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.6/go.mod h1:8WYg+Y40Sn3X2hioaaWAAIngndR8n1XFdRPPX+7QBaM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 h1:E+KqWoVsSrj1tJ6I/fjDIu5xoS2Zacuu1zT+H7KtiIk=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11/go.mod h1:qyWHz+4lvkXcr3+PoGlGHEI+3DLLiU6/GdrFfMaAhB0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 h1:tzMkjh0yTChUqJDgGkcDdxvZDSrJ/WB6R6ymI5ehqJI=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.3/go.mod h1:T270C0R5sZNLbWUe8ueiAF42XSZxxPocTaGSgs5c/60=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
 github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
 github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
@@ -986,8 +988,8 @@ github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOr
 github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
-github.com/go-redsync/redsync/v4 v4.14.0 h1:zyxzFJsmQHIPBl8iBT7KFKohWsjsghgGLiP8TnFMLNc=
-github.com/go-redsync/redsync/v4 v4.14.0/go.mod h1:twMlVd19upZ/juvJyJGlQOSQxor1oeHtjs62l4pRFzo=
+github.com/go-redsync/redsync/v4 v4.15.0 h1:KH/XymuxSV7vyKs6z1Cxxj+N+N18JlPxgXeP6x4JY54=
+github.com/go-redsync/redsync/v4 v4.15.0/go.mod h1:qNp+lLs3vkfZbtA/aM/OjlZHfEr5YTAYhRktFPKHC7s=
 github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
 github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
 github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
@@ -1070,8 +1072,8 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
 github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s=
-github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
+github.com/gomodule/redigo v1.9.3 h1:dNPSXeXv6HCq2jdyWfjgmhBdqnR6PRO3m/G05nvpPC8=
+github.com/gomodule/redigo v1.9.3/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
@@ -1575,12 +1577,12 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
 github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
-github.com/redis/go-redis/v9 v9.14.1 h1:nDCrEiJmfOWhD76xlaw+HXT0c9hfNWeXgl0vIRYSDvQ=
-github.com/redis/go-redis/v9 v9.14.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
-github.com/redis/rueidis v1.0.64 h1:XqgbueDuNV3qFdVdQwAHJl1uNt90zUuAJuzqjH4cw6Y=
-github.com/redis/rueidis v1.0.64/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
-github.com/redis/rueidis/rueidiscompat v1.0.64 h1:M8JbLP4LyHQhBLBRsUQIzui8/LyTtdESNIMVveqm4RY=
-github.com/redis/rueidis/rueidiscompat v1.0.64/go.mod h1:8pJVPhEjpw0izZFSxYwDziUiEYEkEklTSw/nZzga61M=
+github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
+github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
+github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
+github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
+github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
+github.com/redis/rueidis/rueidiscompat v1.0.69/go.mod h1:iC4Y8DoN0Uth0Uezg9e2trvNRC7QAgGeuP2OPLb5ccI=
 github.com/rekby/fixenv v0.3.2/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
 github.com/rekby/fixenv v0.6.1 h1:jUFiSPpajT4WY2cYuc++7Y1zWrnCxnovGCIX72PZniM=
 github.com/rekby/fixenv v0.6.1/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
@@ -1618,8 +1620,8 @@ github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138 h1:bX
 github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
 github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
 github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
-github.com/seaweedfs/raft v1.1.3 h1:5B6hgneQ7IuU4Ceom/f6QUt8pEeqjcsRo+IxlyPZCws=
-github.com/seaweedfs/raft v1.1.3/go.mod h1:9cYlEBA+djJbnf/5tWsCybtbL7ICYpi+Uxcg3MxjuNs=
+github.com/seaweedfs/raft v1.1.6 h1:e83Xn0boscPnuSiBllUPeWRVS6JKrqJPYBozgFyBiC0=
+github.com/seaweedfs/raft v1.1.6/go.mod h1:9cYlEBA+djJbnf/5tWsCybtbL7ICYpi+Uxcg3MxjuNs=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
@@ -1777,12 +1779,12 @@ github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mo
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
-github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0 h1:/NyPd9KnCJgzrEXCArqk1ThqCH2Dh31uUwl88o/VkuM=
-github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
+github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaFdZ6zevvYe+TRxOR8ifa2s=
+github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0 h1:Ml4WMsaRY+/+Wh3DCp1A+/zjM1aMdQg3irjuYFwWVHQ=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0/go.mod h1:/LjMxb/rXmoGAAnImoqAFIlhO5ampHacbvDetQitCk4=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0 h1:uvqheUfoEJz0CLLLaYct69wpvoaosM/2joWFcbwoqfw=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0/go.mod h1:/LjMxb/rXmoGAAnImoqAFIlhO5ampHacbvDetQitCk4=
 github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
 github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
 github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
@@ -1917,8 +1919,8 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
+golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1949,8 +1951,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
 golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
-golang.org/x/image v0.33.0 h1:LXRZRnv1+zGd5XBUVRFmYEphyyKJjQjCRiOuAP3sZfQ=
-golang.org/x/image v0.33.0/go.mod h1:DD3OsTYT9chzuzTQt+zMcOlBHgfoKQb1gry8p76Y1sc=
+golang.org/x/image v0.34.0 h1:33gCkyw9hmwbZJeZkct8XyR11yH889EQt/QH4VmXMn8=
+golang.org/x/image v0.34.0/go.mod h1:2RNFBZRB+vnwwFil8GkMdRvrJOFd1AzdZI6vOY+eJVU=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1985,8 +1987,8 @@ golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
-golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
+golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
+golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2056,8 +2058,8 @@ golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
+golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2218,8 +2220,8 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2237,8 +2239,8 @@ golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
-golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
-golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
+golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2260,8 +2262,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
+golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2341,8 +2343,8 @@ golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58
 golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
-golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
+golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
 golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
 golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1595,6 +1595,101 @@
      "title": "S3 Bucket Traffic Sent",
      "type": "timeseries"
    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "reqps",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 41
+      },
+      "id": 86,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$NAMESPACE\"}[$__interval])) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}}",
+          "refId": "A"
+        }
+      ],
+      "title": "S3 API Calls per Bucket",
+      "type": "timeseries"
+    },
    {
      "datasource": {
        "type": "prometheus",
@@ -1659,8 +1754,8 @@
      },
      "gridPos": {
        "h": 7,
-        "w": 24,
-        "x": 0,
+        "w": 12,
+        "x": 12,
        "y": 41
      },
      "id": 72,
@@ -3266,6 +3361,209 @@
      ],
      "title": "Filer Go Routines",
      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 48
+      },
+      "id": 89,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}} (logical)",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}} (physical)",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "S3 Bucket Size",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 48
+      },
+      "id": 90,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_object_count{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "S3 Bucket Object Count",
+      "type": "timeseries"
    }
  ],
  "refresh": "",
@@ -3356,4 +3654,4 @@
  "uid": "a24009d7-cbda-4443-a132-1cc1c4677304",
  "version": 1,
  "weekStart": ""
-}
+}
@@ -203,6 +203,9 @@ spec:
            - name: client-cert
              readOnly: true
              mountPath: /usr/local/share/ca-certificates/client/
+            - name: admin-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/admin/
            {{- end }}
            {{ tpl .Values.admin.extraVolumeMounts . | nindent 12 | trim }}
          ports:
@@ -298,6 +301,9 @@ spec:
        - name: client-cert
          secret:
            secretName: {{ template "seaweedfs.name" . }}-client-cert
+        - name: admin-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-admin-cert
        {{- end }}
        {{ tpl .Values.admin.extraVolumes . | indent 8 | trim }}
      {{- if .Values.admin.nodeSelector }}
@@ -0,0 +1,43 @@
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
+apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
+kind: Certificate
+metadata:
+  name: {{ template "seaweedfs.name" . }}-admin-cert
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: admin
+  {{- if .Values.admin.annotations }}
+  annotations:
+    {{- toYaml .Values.admin.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  secretName: {{ template "seaweedfs.name" . }}-admin-cert
+  issuerRef:
+    name: {{ template "seaweedfs.name" . }}-ca-issuer
+    kind: Issuer
+  commonName: {{ .Values.certificates.commonName }}
+  subject:
+    organizations:
+    - "SeaweedFS CA"
+  dnsNames:
+    - '*.{{ template "seaweedfs.name" . }}-admin'
+    - '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}'
+    - '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}.svc'
+    - '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}.svc.cluster.local'
+{{- if .Values.certificates.ipAddresses }}
+  ipAddresses:
+    {{- range .Values.certificates.ipAddresses }}
+    - {{ . }}
+    {{- end }}
+{{- end }}
+  privateKey:
+    algorithm: {{ .Values.certificates.keyAlgorithm }}
+    size: {{ .Values.certificates.keySize }}
+  duration: {{ .Values.certificates.duration }}
+  renewBefore: {{ .Values.certificates.renewBefore }}
+{{- end }}
+
@@ -0,0 +1,43 @@
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
+apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
+kind: Certificate
+metadata:
+  name: {{ template "seaweedfs.name" . }}-worker-cert
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: worker
+  {{- if .Values.worker.annotations }}
+  annotations:
+    {{- toYaml .Values.worker.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  secretName: {{ template "seaweedfs.name" . }}-worker-cert
+  issuerRef:
+    name: {{ template "seaweedfs.name" . }}-ca-issuer
+    kind: Issuer
+  commonName: {{ .Values.certificates.commonName }}
+  subject:
+    organizations:
+    - "SeaweedFS CA"
+  dnsNames:
+    - '*.{{ template "seaweedfs.name" . }}-worker'
+    - '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}'
+    - '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}.svc'
+    - '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}.svc.cluster.local'
+{{- if .Values.certificates.ipAddresses }}
+  ipAddresses:
+    {{- range .Values.certificates.ipAddresses }}
+    - {{ . }}
+    {{- end }}
+{{- end }}
+  privateKey:
+    algorithm: {{ .Values.certificates.keyAlgorithm }}
+    size: {{ .Values.certificates.keySize }}
+  duration: {{ .Values.certificates.duration }}
+  renewBefore: {{ .Values.certificates.renewBefore }}
+{{- end }}
+
@@ -65,6 +65,14 @@ data:
    cert = "/usr/local/share/ca-certificates/filer/tls.crt"
    key  = "/usr/local/share/ca-certificates/filer/tls.key"

+    [grpc.admin]
+    cert = "/usr/local/share/ca-certificates/admin/tls.crt"
+    key  = "/usr/local/share/ca-certificates/admin/tls.key"
+
+    [grpc.worker]
+    cert = "/usr/local/share/ca-certificates/worker/tls.crt"
+    key  = "/usr/local/share/ca-certificates/worker/tls.key"
+
    # use this for any place needs a grpc client
    # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
    [grpc.client]
@@ -171,6 +171,9 @@ spec:
            - name: client-cert
              readOnly: true
              mountPath: /usr/local/share/ca-certificates/client/
+            - name: worker-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/worker/
            {{- end }}
            {{ tpl .Values.worker.extraVolumeMounts . | nindent 12 | trim }}
          ports:
@@ -262,6 +265,9 @@ spec:
        - name: client-cert
          secret:
            secretName: {{ template "seaweedfs.name" . }}-client-cert
+        - name: worker-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-worker-cert
        {{- end }}
        {{ tpl .Values.worker.extraVolumes . | indent 8 | trim }}
      {{- if .Values.worker.nodeSelector }}
@@ -0,0 +1,110 @@
+# SeaweedFS S3 Java SDK Compatibility Tests
+
+This project contains Java-based integration tests for SeaweedFS S3 API compatibility.
+
+## Overview
+
+Tests are provided for both AWS SDK v1 and v2 to ensure compatibility with the various SDK versions commonly used in production.
+
+## SDK Versions
+
+| SDK | Version | Notes |
+|-----|---------|-------|
+| AWS SDK v1 for Java | 1.12.600 | Legacy SDK, less strict ETag validation |
+| AWS SDK v2 for Java | 2.20.127 | Modern SDK with strict checksum validation |
+
+## Running Tests
+
+### Prerequisites
+
+1. SeaweedFS running with S3 API enabled:
+   ```bash
+   weed server -s3
+   ```
+
+2. Java 18+ and Maven
+
+### Run All Tests
+
+```bash
+mvn test
+```
+
+### Run Specific Tests
+
+```bash
+# Run only ETag validation tests (AWS SDK v2)
+mvn test -Dtest=ETagValidationTest
+
+# Run with custom endpoint
+mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
+```
+
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `S3_ENDPOINT` | `http://127.0.0.1:8333` | S3 API endpoint URL |
+| `S3_ACCESS_KEY` | `some_access_key1` | Access key ID |
+| `S3_SECRET_KEY` | `some_secret_key1` | Secret access key |
+| `S3_REGION` | `us-east-1` | AWS region |
+
+## Test Coverage
+
+### ETagValidationTest (AWS SDK v2)
+
+Tests for [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768) - ETag format validation.
+
+| Test | Description |
+|------|-------------|
+| `testSmallFilePutObject` | Verify small files return pure MD5 ETag |
+| `testLargeFilePutObject_Issue7768` | **Critical**: Verify large files (>8MB) return pure MD5 ETag |
+| `testExtraLargeFilePutObject` | Verify very large files (>24MB) return pure MD5 ETag |
+| `testMultipartUploadETag` | Verify multipart uploads return composite ETag |
+| `testETagConsistency` | Verify ETag consistency across PUT/HEAD/GET |
+| `testMultipleLargeFileUploads` | Stress test multiple large uploads |
+
+### Background: Issue #7768
+
+AWS SDK v2 for Java includes checksum validation that decodes the ETag as hexadecimal. When SeaweedFS returned composite ETags (`<md5>-<count>`) for regular `PutObject` with internally auto-chunked files, the SDK failed with:
+
+```
+java.lang.IllegalArgumentException: Invalid base 16 character: '-'
+```
+
+**Per AWS S3 specification:**
+- `PutObject`: ETag is always a pure MD5 hex string (32 chars)
+- `CompleteMultipartUpload`: ETag is composite format (`<md5>-<partcount>`)
+
+The fix ensures SeaweedFS follows this specification.
+
+## Project Structure
+
+```
+src/
+├── main/java/com/seaweedfs/s3/
+│   ├── PutObject.java           # Example PutObject with SDK v1
+│   └── HighLevelMultipartUpload.java
+└── test/java/com/seaweedfs/s3/
+    ├── PutObjectTest.java       # Basic SDK v1 test
+    └── ETagValidationTest.java  # Comprehensive SDK v2 ETag tests
+```
+
+## Validated SDK Versions
+
+This Java test project validates:
+
+- ✅ AWS SDK v2 for Java 2.20.127+
+- ✅ AWS SDK v1 for Java 1.12.600+
+
+Go SDK validation is performed by separate test suites:
+- See [Go ETag Tests](/test/s3/etag/) for AWS SDK v2 for Go tests
+- See [test/s3/SDK_COMPATIBILITY.md](/test/s3/SDK_COMPATIBILITY.md) for full SDK compatibility matrix
+
+## Related
+
+- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
+- [Go ETag Tests](/test/s3/etag/)
+- [SDK Compatibility Matrix](/test/s3/SDK_COMPATIBILITY.md)
+
@@ -6,18 +6,29 @@
  <packaging>jar</packaging>
  <version>1.0-SNAPSHOT</version>
  <properties>
-    <maven.compiler.source>18</maven.compiler.source>
-    <maven.compiler.target>18</maven.compiler.target>
+    <maven.compiler.source>11</maven.compiler.source>
+    <maven.compiler.target>11</maven.compiler.target>
+    <aws.sdk.v1.version>1.12.600</aws.sdk.v1.version>
+    <aws.sdk.v2.version>2.20.127</aws.sdk.v2.version>
  </properties>
  <name>copier</name>
  <url>http://maven.apache.org</url>

  <dependencyManagement>
    <dependencies>
+      <!-- AWS SDK v1 BOM -->
      <dependency>
        <groupId>com.amazonaws</groupId>
        <artifactId>aws-java-sdk-bom</artifactId>
-        <version>1.11.327</version>
+        <version>${aws.sdk.v1.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+      <!-- AWS SDK v2 BOM -->
+      <dependency>
+        <groupId>software.amazon.awssdk</groupId>
+        <artifactId>bom</artifactId>
+        <version>${aws.sdk.v2.version}</version>
        <type>pom</type>
        <scope>import</scope>
      </dependency>
@@ -25,15 +36,50 @@
  </dependencyManagement>

  <dependencies>
+    <!-- AWS SDK v1 (for backward compatibility with existing code) -->
    <dependency>
      <groupId>com.amazonaws</groupId>
      <artifactId>aws-java-sdk-s3</artifactId>
    </dependency>
+    
+    <!-- AWS SDK v2 (for modern tests with checksum validation) -->
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3-transfer-manager</artifactId>
+    </dependency>
+    
+    <!-- Test dependencies -->
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
-      <version>4.13.1</version>
+      <version>4.13.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>5.10.0</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
+      <version>3.24.2</version>
      <scope>test</scope>
    </dependency>
  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>3.2.2</version>
+      </plugin>
+    </plugins>
+  </build>
 </project>
@@ -0,0 +1,439 @@
+package com.seaweedfs.s3;
+
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.*;
+
+import java.net.URI;
+import java.security.MessageDigest;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.regex.Pattern;
+
+import static org.assertj.core.api.Assertions.*;
+
+/**
+ * AWS SDK v2 Integration Tests for S3 ETag Format Validation.
+ * 
+ * These tests verify that SeaweedFS returns correct ETag formats that are
+ * compatible with AWS SDK v2's checksum validation.
+ * 
+ * Background (GitHub Issue #7768):
+ * AWS SDK v2 for Java validates ETags as hexadecimal MD5 hashes for PutObject
+ * responses. If the ETag contains non-hex characters (like '-' in composite
+ * format), the SDK fails with "Invalid base 16 character: '-'".
+ * 
+ * Per AWS S3 specification:
+ * - Regular PutObject: ETag is always a pure MD5 hex string (32 chars)
+ * - CompleteMultipartUpload: ETag is composite format "<md5>-<partcount>"
+ * 
+ * To run these tests:
+ *   mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
+ * 
+ * Or set environment variable:
+ *   export S3_ENDPOINT=http://localhost:8333
+ *   mvn test -Dtest=ETagValidationTest
+ */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@DisplayName("S3 ETag Format Validation Tests (AWS SDK v2)")
+class ETagValidationTest {
+
+    // Configuration - can be overridden via system properties or environment variables
+    private static final String DEFAULT_ENDPOINT = "http://127.0.0.1:8333";
+    private static final String DEFAULT_ACCESS_KEY = "some_access_key1";
+    private static final String DEFAULT_SECRET_KEY = "some_secret_key1";
+    private static final String DEFAULT_REGION = "us-east-1";
+    
+    // Auto-chunking threshold in SeaweedFS (must match s3api_object_handlers_put.go)
+    private static final int AUTO_CHUNK_SIZE = 8 * 1024 * 1024; // 8MB
+    
+    // Test sizes
+    private static final int SMALL_FILE_SIZE = 1024; // 1KB
+    private static final int LARGE_FILE_SIZE = 10 * 1024 * 1024; // 10MB (triggers auto-chunking)
+    private static final int XL_FILE_SIZE = 25 * 1024 * 1024; // 25MB (multiple chunks)
+    private static final int MULTIPART_PART_SIZE = 5 * 1024 * 1024; // 5MB per part
+    
+    // ETag format patterns
+    private static final Pattern PURE_MD5_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}\"?$");
+    private static final Pattern COMPOSITE_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}-\\d+\"?$");
+    
+    private S3Client s3Client;
+    private String testBucketName;
+    private final SecureRandom random = new SecureRandom();
+
+    @BeforeAll
+    void setUp() {
+        String endpoint = getConfig("S3_ENDPOINT", DEFAULT_ENDPOINT);
+        String accessKey = getConfig("S3_ACCESS_KEY", DEFAULT_ACCESS_KEY);
+        String secretKey = getConfig("S3_SECRET_KEY", DEFAULT_SECRET_KEY);
+        String region = getConfig("S3_REGION", DEFAULT_REGION);
+        
+        System.out.println("Connecting to S3 endpoint: " + endpoint);
+        
+        s3Client = S3Client.builder()
+                .endpointOverride(URI.create(endpoint))
+                .credentialsProvider(StaticCredentialsProvider.create(
+                        AwsBasicCredentials.create(accessKey, secretKey)))
+                .region(Region.of(region))
+                .forcePathStyle(true) // Required for SeaweedFS
+                .build();
+        
+        // Create test bucket
+        testBucketName = "test-etag-" + UUID.randomUUID().toString().substring(0, 8);
+        s3Client.createBucket(CreateBucketRequest.builder()
+                .bucket(testBucketName)
+                .build());
+        
+        System.out.println("Created test bucket: " + testBucketName);
+    }
+
+    @AfterAll
+    void tearDown() {
+        if (s3Client != null && testBucketName != null) {
+            try {
+                // Delete all objects with pagination
+                String continuationToken = null;
+                do {
+                    ListObjectsV2Response listResp = s3Client.listObjectsV2(
+                            ListObjectsV2Request.builder()
+                                    .bucket(testBucketName)
+                                    .continuationToken(continuationToken)
+                                    .build());
+                    for (S3Object obj : listResp.contents()) {
+                        s3Client.deleteObject(DeleteObjectRequest.builder()
+                                .bucket(testBucketName)
+                                .key(obj.key())
+                                .build());
+                    }
+                    continuationToken = listResp.nextContinuationToken();
+                } while (continuationToken != null);
+                
+                // Abort any multipart uploads
+                ListMultipartUploadsResponse mpResp = s3Client.listMultipartUploads(
+                        ListMultipartUploadsRequest.builder().bucket(testBucketName).build());
+                for (MultipartUpload upload : mpResp.uploads()) {
+                    s3Client.abortMultipartUpload(AbortMultipartUploadRequest.builder()
+                            .bucket(testBucketName)
+                            .key(upload.key())
+                            .uploadId(upload.uploadId())
+                            .build());
+                }
+                
+                // Delete bucket
+                s3Client.deleteBucket(DeleteBucketRequest.builder()
+                        .bucket(testBucketName)
+                        .build());
+                
+                System.out.println("Cleaned up test bucket: " + testBucketName);
+            } catch (Exception e) {
+                System.err.println("Warning: Failed to cleanup test bucket: " + e.getMessage());
+            }
+            s3Client.close();
+        }
+    }
+
+    @Test
+    @DisplayName("Small file PutObject should return pure MD5 ETag")
+    void testSmallFilePutObject() throws Exception {
+        byte[] testData = generateRandomData(SMALL_FILE_SIZE);
+        String expectedMD5 = calculateMD5Hex(testData);
+        String objectKey = "small-file-" + UUID.randomUUID() + ".bin";
+        
+        PutObjectResponse response = s3Client.putObject(
+                PutObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build(),
+                RequestBody.fromBytes(testData));
+        
+        String etag = response.eTag();
+        System.out.println("Small file ETag: " + etag + " (expected MD5: " + expectedMD5 + ")");
+        
+        assertThat(etag)
+                .describedAs("Small file ETag should be pure MD5")
+                .matches(PURE_MD5_PATTERN);
+        assertThat(cleanETag(etag))
+                .describedAs("ETag should match calculated MD5")
+                .isEqualTo(expectedMD5);
+        assertThat(etag)
+                .describedAs("ETag should not contain hyphen")
+                .doesNotContain("-");
+    }
+
+    /**
+     * Critical test for GitHub Issue #7768.
+     * 
+     * This test uploads a file larger than the auto-chunking threshold (8MB),
+     * which triggers SeaweedFS to split the file into multiple internal chunks.
+     * 
+     * Previously, this caused SeaweedFS to return a composite ETag like
+     * "d41d8cd98f00b204e9800998ecf8427e-2", which AWS SDK v2 rejected because
+     * it validates the ETag as hexadecimal and '-' is not a valid hex character.
+     * 
+     * The fix ensures that regular PutObject always returns a pure MD5 ETag,
+     * regardless of internal chunking.
+     */
+    @Test
+    @DisplayName("Large file PutObject (>8MB) should return pure MD5 ETag - Issue #7768")
+    void testLargeFilePutObject_Issue7768() throws Exception {
+        byte[] testData = generateRandomData(LARGE_FILE_SIZE);
+        String expectedMD5 = calculateMD5Hex(testData);
+        String objectKey = "large-file-" + UUID.randomUUID() + ".bin";
+        
+        System.out.println("Uploading large file (" + LARGE_FILE_SIZE + " bytes, " +
+                "> " + AUTO_CHUNK_SIZE + " byte auto-chunk threshold)...");
+        
+        // This is where Issue #7768 would manifest - SDK v2 validates ETag
+        PutObjectResponse response = s3Client.putObject(
+                PutObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build(),
+                RequestBody.fromBytes(testData));
+        
+        String etag = response.eTag();
+        int expectedChunks = (LARGE_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
+        System.out.println("Large file ETag: " + etag + 
+                " (expected MD5: " + expectedMD5 + ", internal chunks: ~" + expectedChunks + ")");
+        
+        // These assertions would fail before the fix
+        assertThat(etag)
+                .describedAs("Large file PutObject ETag MUST be pure MD5 (not composite)")
+                .matches(PURE_MD5_PATTERN);
+        assertThat(etag)
+                .describedAs("Large file ETag should NOT be composite format")
+                .doesNotMatch(COMPOSITE_PATTERN);
+        assertThat(etag)
+                .describedAs("ETag should not contain hyphen for regular PutObject")
+                .doesNotContain("-");
+        assertThat(cleanETag(etag))
+                .describedAs("ETag should match calculated MD5")
+                .isEqualTo(expectedMD5);
+        
+        // Verify hex decoding works (this is what fails in Issue #7768)
+        assertThatCode(() -> hexToBytes(cleanETag(etag)))
+                .describedAs("ETag should be valid hexadecimal (AWS SDK v2 validation)")
+                .doesNotThrowAnyException();
+    }
+
+    @Test
+    @DisplayName("Extra large file PutObject (>24MB) should return pure MD5 ETag")
+    void testExtraLargeFilePutObject() throws Exception {
+        byte[] testData = generateRandomData(XL_FILE_SIZE);
+        String expectedMD5 = calculateMD5Hex(testData);
+        String objectKey = "xl-file-" + UUID.randomUUID() + ".bin";
+        
+        int expectedChunks = (XL_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
+        System.out.println("Uploading XL file (" + XL_FILE_SIZE + " bytes, ~" + 
+                expectedChunks + " internal chunks)...");
+        
+        PutObjectResponse response = s3Client.putObject(
+                PutObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build(),
+                RequestBody.fromBytes(testData));
+        
+        String etag = response.eTag();
+        System.out.println("XL file ETag: " + etag);
+        
+        assertThat(etag)
+                .describedAs("XL file PutObject ETag MUST be pure MD5")
+                .matches(PURE_MD5_PATTERN);
+        assertThat(cleanETag(etag))
+                .describedAs("ETag should match calculated MD5")
+                .isEqualTo(expectedMD5);
+    }
+
+    @Test
+    @DisplayName("Multipart upload should return composite ETag")
+    void testMultipartUploadETag() throws Exception {
+        int totalSize = 15 * 1024 * 1024; // 15MB = 3 parts
+        byte[] testData = generateRandomData(totalSize);
+        String objectKey = "multipart-file-" + UUID.randomUUID() + ".bin";
+        
+        System.out.println("Performing multipart upload (" + totalSize + " bytes)...");
+        
+        // Initiate multipart upload
+        CreateMultipartUploadResponse createResp = s3Client.createMultipartUpload(
+                CreateMultipartUploadRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build());
+        String uploadId = createResp.uploadId();
+        
+        List<CompletedPart> completedParts = new ArrayList<>();
+        int partNumber = 1;
+        
+        // Upload parts
+        for (int offset = 0; offset < totalSize; offset += MULTIPART_PART_SIZE) {
+            int end = Math.min(offset + MULTIPART_PART_SIZE, totalSize);
+            byte[] partData = new byte[end - offset];
+            System.arraycopy(testData, offset, partData, 0, partData.length);
+            
+            UploadPartResponse uploadResp = s3Client.uploadPart(
+                    UploadPartRequest.builder()
+                            .bucket(testBucketName)
+                            .key(objectKey)
+                            .uploadId(uploadId)
+                            .partNumber(partNumber)
+                            .build(),
+                    RequestBody.fromBytes(partData));
+            
+            completedParts.add(CompletedPart.builder()
+                    .partNumber(partNumber)
+                    .eTag(uploadResp.eTag())
+                    .build());
+            partNumber++;
+        }
+        
+        // Complete multipart upload
+        CompleteMultipartUploadResponse completeResp = s3Client.completeMultipartUpload(
+                CompleteMultipartUploadRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .uploadId(uploadId)
+                        .multipartUpload(CompletedMultipartUpload.builder()
+                                .parts(completedParts)
+                                .build())
+                        .build());
+        
+        String etag = completeResp.eTag();
+        System.out.println("Multipart upload ETag: " + etag + " (" + completedParts.size() + " parts)");
+        
+        // Multipart uploads SHOULD have composite ETag
+        assertThat(etag)
+                .describedAs("Multipart upload ETag SHOULD be composite format")
+                .matches(COMPOSITE_PATTERN);
+        assertThat(etag)
+                .describedAs("Multipart ETag should contain hyphen")
+                .contains("-");
+        
+        // Verify part count in ETag
+        String[] parts = cleanETag(etag).split("-");
+        assertThat(parts).hasSize(2);
+        assertThat(parts[1])
+                .describedAs("Part count in ETag should match uploaded parts")
+                .isEqualTo(String.valueOf(completedParts.size()));
+    }
+
+    @Test
+    @DisplayName("ETag should be consistent across PUT, HEAD, and GET")
+    void testETagConsistency() throws Exception {
+        byte[] testData = generateRandomData(LARGE_FILE_SIZE);
+        String objectKey = "consistency-test-" + UUID.randomUUID() + ".bin";
+        
+        // PUT
+        PutObjectResponse putResp = s3Client.putObject(
+                PutObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build(),
+                RequestBody.fromBytes(testData));
+        String putETag = putResp.eTag();
+        
+        // HEAD
+        HeadObjectResponse headResp = s3Client.headObject(
+                HeadObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build());
+        String headETag = headResp.eTag();
+        
+        // GET
+        GetObjectResponse getResp = s3Client.getObject(
+                GetObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build())
+                .response();
+        String getETag = getResp.eTag();
+        
+        System.out.println("PUT ETag: " + putETag + ", HEAD ETag: " + headETag + ", GET ETag: " + getETag);
+        
+        assertThat(putETag).isEqualTo(headETag);
+        assertThat(putETag).isEqualTo(getETag);
+    }
+
+    @Test
+    @DisplayName("Multiple large file uploads should all return pure MD5 ETags")
+    void testMultipleLargeFileUploads() throws Exception {
+        int numFiles = 3;
+        
+        for (int i = 0; i < numFiles; i++) {
+            byte[] testData = generateRandomData(LARGE_FILE_SIZE);
+            String expectedMD5 = calculateMD5Hex(testData);
+            String objectKey = "multi-large-" + i + "-" + UUID.randomUUID() + ".bin";
+            
+            PutObjectResponse response = s3Client.putObject(
+                    PutObjectRequest.builder()
+                            .bucket(testBucketName)
+                            .key(objectKey)
+                            .build(),
+                    RequestBody.fromBytes(testData));
+            
+            String etag = response.eTag();
+            System.out.println("File " + i + " ETag: " + etag);
+            
+            assertThat(etag)
+                    .describedAs("File " + i + " ETag should be pure MD5")
+                    .matches(PURE_MD5_PATTERN);
+            assertThat(cleanETag(etag))
+                    .describedAs("File " + i + " ETag should match MD5")
+                    .isEqualTo(expectedMD5);
+            
+            // Validate hex decoding
+            assertThatCode(() -> hexToBytes(cleanETag(etag)))
+                    .doesNotThrowAnyException();
+        }
+    }
+
+    // Helper methods
+    
+    private String getConfig(String key, String defaultValue) {
+        String value = System.getProperty(key);
+        if (value == null) {
+            value = System.getenv(key);
+        }
+        return value != null ? value : defaultValue;
+    }
+    
+    private byte[] generateRandomData(int size) {
+        byte[] data = new byte[size];
+        random.nextBytes(data);
+        return data;
+    }
+    
+    private String calculateMD5Hex(byte[] data) throws Exception {
+        MessageDigest md = MessageDigest.getInstance("MD5");
+        byte[] digest = md.digest(data);
+        StringBuilder sb = new StringBuilder();
+        for (byte b : digest) {
+            sb.append(String.format("%02x", b));
+        }
+        return sb.toString();
+    }
+    
+    private String cleanETag(String etag) {
+        if (etag == null) return null;
+        return etag.replace("\"", "");
+    }
+    
+    private byte[] hexToBytes(String hex) {
+        int len = hex.length();
+        byte[] data = new byte[len / 2];
+        for (int i = 0; i < len; i += 2) {
+            data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4)
+                    + Character.digit(hex.charAt(i + 1), 16));
+        }
+        return data;
+    }
+}
+
@@ -1319,6 +1319,99 @@
        "alignLevel": null
      }
    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "hiddenSeries": false,
+      "id": 90,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.1.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(SeaweedFS_s3_request_total[$__interval])) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 API Calls per Bucket",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "reqps",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
    {
      "aliasColors": {},
      "bars": false,
@@ -1332,8 +1425,8 @@
      "grid": {},
      "gridPos": {
        "h": 7,
-        "w": 24,
-        "x": 0,
+        "w": 12,
+        "x": 12,
        "y": 16
      },
      "hiddenSeries": false,
@@ -2960,6 +3053,200 @@
        "align": false,
        "alignLevel": null
      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 37
+      },
+      "hiddenSeries": false,
+      "id": 87,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.1.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "max(SeaweedFS_s3_bucket_size_bytes) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}} (logical)",
+          "refId": "A"
+        },
+        {
+          "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}} (physical)",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 Bucket Size",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "decbytes",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 37
+      },
+      "hiddenSeries": false,
+      "id": 88,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.1.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "max(SeaweedFS_s3_bucket_object_count) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 Bucket Object Count",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
    }
  ],
  "refresh": "30s",
@@ -875,6 +875,85 @@
            }
          ]
        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS-DEV}",
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 16
+          },
+          "hiddenSeries": false,
+          "id": 86,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(SeaweedFS_s3_request_total[$__interval])) by (bucket)",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{bucket}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "S3 API Calls per Bucket",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "reqps",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ]
+        },
        {
          "aliasColors": {},
          "bars": false,
@@ -1087,6 +1166,202 @@
              "show": false
            }
          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "fillGradient": 0,
+          "fieldConfig": {
+            "defaults": {
+              "unit": "decbytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 37
+          },
+          "hiddenSeries": false,
+          "id": 87,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.1.2",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(SeaweedFS_s3_bucket_size_bytes) by (bucket)",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{bucket}} (logical)",
+              "refId": "A"
+            },
+            {
+              "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes) by (bucket)",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{bucket}} (physical)",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "S3 Bucket Size",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "decbytes",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "fillGradient": 0,
+          "fieldConfig": {
+            "defaults": {
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 37
+          },
+          "hiddenSeries": false,
+          "id": 88,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.1.2",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(SeaweedFS_s3_bucket_object_count) by (bucket)",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{bucket}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "S3 Bucket Object Count",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
        }
      ],
      "repeat": null,
@@ -2111,4 +2386,4 @@
  "timezone": "browser",
  "title": "SeaweedFS ks8 heartbeat",
  "version": 1
-}
+}
@@ -447,6 +447,99 @@
        "alignLevel": null
      }
    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fill": 1,
+      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 6,
+        "x": 0,
+        "y": 15
+      },
+      "hiddenSeries": false,
+      "id": 86,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.1.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$namespace\",service=~\"$service-api\"}[$__interval])) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 API Calls per Bucket",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "reqps",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
    {
      "aliasColors": {},
      "bars": false,
@@ -466,8 +559,8 @@
      "grid": {},
      "gridPos": {
        "h": 7,
-        "w": 8,
-        "x": 0,
+        "w": 6,
+        "x": 12,
        "y": 15
      },
      "hiddenSeries": false,
@@ -577,8 +670,8 @@
      "grid": {},
      "gridPos": {
        "h": 7,
-        "w": 8,
-        "x": 8,
+        "w": 6,
+        "x": 6,
        "y": 15
      },
      "hiddenSeries": false,
@@ -688,8 +781,8 @@
      "grid": {},
      "gridPos": {
        "h": 7,
-        "w": 8,
-        "x": 16,
+        "w": 6,
+        "x": 18,
        "y": 15
      },
      "hiddenSeries": false,
@@ -2354,6 +2447,209 @@
        "align": false,
        "alignLevel": null
      }
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 41
+      },
+      "id": 89,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}} (logical)",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}} (physical)",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "S3 Bucket Size",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 41
+      },
+      "id": 90,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_object_count{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "S3 Bucket Object Count",
+      "type": "timeseries"
    }
  ],
  "refresh": "30s",
@@ -2545,4 +2841,4 @@
  "timezone": "browser",
  "title": "SeaweedFS",
  "version": 2
-}
+}
@@ -0,0 +1,187 @@
+# Makefile for EC integration testing
+# Usage:
+#   make start      - Start the test cluster (master + 6 volume servers + filer)
+#   make stop       - Stop the test cluster
+#   make populate   - Populate test data (~300MB across 7 volumes)
+#   make shell      - Open weed shell connected to the test cluster
+#   make clean      - Stop cluster and remove all test data
+#   make setup      - Start cluster and populate data (one command)
+#
+# Requirements: curl, jq
+
+WEED_BINARY := $(shell pwd)/../../weed/weed
+TEST_DIR := /tmp/ec_manual_test
+# Use non-standard ports to avoid conflicts with existing SeaweedFS servers
+MASTER_PORT := 29333
+FILER_PORT := 28888
+VOLUME_BASE_PORT := 28080
+NUM_VOLUME_SERVERS := 6
+VOLUME_SIZE_LIMIT_MB := 30
+MAX_VOLUMES_PER_SERVER := 10
+
+# Build weed binary if it doesn't exist
+$(WEED_BINARY):
+	cd ../../weed && go build -o weed .
+
+.PHONY: build
+build: $(WEED_BINARY)
+
+.PHONY: start
+start: build
+	@echo "=== Starting SeaweedFS test cluster ==="
+	@mkdir -p $(TEST_DIR)/master $(TEST_DIR)/filer
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do mkdir -p $(TEST_DIR)/volume$$i; done
+	@# Create security.toml with JWT disabled
+	@echo "# Disable JWT for testing" > $(TEST_DIR)/security.toml
+	@echo '[jwt.signing]' >> $(TEST_DIR)/security.toml
+	@echo 'key = ""' >> $(TEST_DIR)/security.toml
+	@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
+	@echo '' >> $(TEST_DIR)/security.toml
+	@echo '[jwt.signing.read]' >> $(TEST_DIR)/security.toml
+	@echo 'key = ""' >> $(TEST_DIR)/security.toml
+	@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
+	@# Create filer.toml with leveldb2
+	@echo '[leveldb2]' > $(TEST_DIR)/filer.toml
+	@echo 'enabled = true' >> $(TEST_DIR)/filer.toml
+	@echo 'dir = "$(TEST_DIR)/filer/filerldb2"' >> $(TEST_DIR)/filer.toml
+	@# Start master
+	@echo "Starting master on port $(MASTER_PORT)..."
+	@cd $(TEST_DIR) && $(WEED_BINARY) master \
+		-port=$(MASTER_PORT) \
+		-mdir=$(TEST_DIR)/master \
+		-volumeSizeLimitMB=$(VOLUME_SIZE_LIMIT_MB) \
+		-ip=127.0.0.1 \
+		> $(TEST_DIR)/master/master.log 2>&1 & echo $$! > $(TEST_DIR)/master.pid
+	@sleep 3
+	@# Start volume servers (run from TEST_DIR to find security.toml)
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
+		port=$$(($(VOLUME_BASE_PORT) + $$i)); \
+		echo "Starting volume server $$i on port $$port (rack$$i)..."; \
+		cd $(TEST_DIR) && $(WEED_BINARY) volume \
+			-port=$$port \
+			-dir=$(TEST_DIR)/volume$$i \
+			-max=$(MAX_VOLUMES_PER_SERVER) \
+			-master=127.0.0.1:$(MASTER_PORT) \
+			-ip=127.0.0.1 \
+			-dataCenter=dc1 \
+			-rack=rack$$i \
+			> $(TEST_DIR)/volume$$i/volume.log 2>&1 & echo $$! > $(TEST_DIR)/volume$$i.pid; \
+	done
+	@sleep 3
+	@# Start filer (run from TEST_DIR to find security.toml)
+	@echo "Starting filer on port $(FILER_PORT)..."
+	@cd $(TEST_DIR) && $(WEED_BINARY) filer \
+		-port=$(FILER_PORT) \
+		-master=127.0.0.1:$(MASTER_PORT) \
+		-ip=127.0.0.1 \
+		> $(TEST_DIR)/filer/filer.log 2>&1 & echo $$! > $(TEST_DIR)/filer.pid
+	@sleep 3
+	@echo ""
+	@echo "=== Cluster started ==="
+	@echo "Master: http://127.0.0.1:$(MASTER_PORT)"
+	@echo "Filer:  http://127.0.0.1:$(FILER_PORT)"
+	@echo "Volume servers: http://127.0.0.1:$(VOLUME_BASE_PORT) - http://127.0.0.1:$$(($(VOLUME_BASE_PORT) + $(NUM_VOLUME_SERVERS) - 1))"
+	@echo ""
+	@echo "Run 'make shell' to open weed shell"
+	@echo "Run 'make populate' to add test data"
+
+.PHONY: stop
+stop:
+	@echo "=== Stopping SeaweedFS test cluster ==="
+	@# Stop filer by PID
+	@-[ -f $(TEST_DIR)/filer.pid ] && kill $$(cat $(TEST_DIR)/filer.pid) 2>/dev/null && rm -f $(TEST_DIR)/filer.pid || true
+	@# Stop volume servers by PID
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
+		[ -f $(TEST_DIR)/volume$$i.pid ] && kill $$(cat $(TEST_DIR)/volume$$i.pid) 2>/dev/null && rm -f $(TEST_DIR)/volume$$i.pid || true; \
+	done
+	@# Stop master by PID
+	@-[ -f $(TEST_DIR)/master.pid ] && kill $$(cat $(TEST_DIR)/master.pid) 2>/dev/null && rm -f $(TEST_DIR)/master.pid || true
+	@# Fallback: use pkill with specific patterns to ensure cleanup
+	@-pkill -f "weed filer.*-master=127.0.0.1:$(MASTER_PORT)" 2>/dev/null || true
+	@-pkill -f "weed volume.*-dir=$(TEST_DIR)/volume" 2>/dev/null || true
+	@-pkill -f "weed master.*-mdir=$(TEST_DIR)/master" 2>/dev/null || true
+	@echo "Cluster stopped."
+
+.PHONY: clean
+clean: stop
+	@echo "Removing test data..."
+	@rm -rf $(TEST_DIR)
+	@echo "Clean complete."
+
+.PHONY: populate
+populate:
+	@echo "=== Populating test data (~300MB) ==="
+	@# Create a 500KB test file template using mktemp for isolation
+	@tmpfile=$$(mktemp) && \
+	dd if=/dev/urandom bs=1024 count=500 of=$$tmpfile 2>/dev/null && \
+	uploaded=0; \
+	for i in $$(seq 1 600); do \
+		response=$$(curl -s "http://127.0.0.1:$(MASTER_PORT)/dir/assign?collection=ectest&replication=000"); \
+		fid=$$(echo $$response | jq -r '.fid'); \
+		url=$$(echo $$response | jq -r '.url'); \
+		if [ "$$fid" != "null" ] && [ -n "$$fid" ]; then \
+			curl -s -F "file=@$$tmpfile;filename=file_$$i.bin" "http://$$url/$$fid" > /dev/null; \
+			uploaded=$$((uploaded + 1)); \
+		fi; \
+		if [ $$((i % 100)) -eq 0 ]; then \
+			echo "Uploaded $$uploaded files..."; \
+		fi; \
+	done; \
+	rm -f $$tmpfile; \
+	echo ""; \
+	echo "=== Data population complete ==="; \
+	echo "Uploaded $$uploaded files (~$$((uploaded * 500 / 1024))MB)"
+	@echo ""
+	@echo "Volume status:"
+	@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
+		'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | "  Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack)"' 2>/dev/null || true
+
+.PHONY: shell
+shell: build
+	@echo "Opening weed shell..."
+	@echo "Commands to try:"
+	@echo "  lock"
+	@echo "  volume.list"
+	@echo "  ec.encode -collection=ectest -quietFor=1s -force"
+	@echo "  ec.balance -collection=ectest"
+	@echo "  unlock"
+	@echo ""
+	@$(WEED_BINARY) shell -master=127.0.0.1:$(MASTER_PORT) -filer=127.0.0.1:$(FILER_PORT)
+
+.PHONY: setup
+setup: clean start
+	@sleep 2
+	@$(MAKE) populate
+
+.PHONY: status
+status:
+	@echo "=== Cluster Status ==="
+	@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
+		'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | "Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack) (\($$server))"' 2>/dev/null | sort -t: -k1 -n || echo "Cluster not running"
+	@echo ""
+	@echo "=== EC Shards ==="
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
+		count=$$(ls $(TEST_DIR)/volume$$i/*.ec[0-9]* 2>/dev/null | wc -l | tr -d ' '); \
+		if [ "$$count" != "0" ]; then \
+			echo "  volume$$i (port $$(($(VOLUME_BASE_PORT) + $$i))): $$count EC shard files"; \
+		fi; \
+	done
+
+.PHONY: help
+help:
+	@echo "EC Integration Test Makefile"
+	@echo ""
+	@echo "Targets:"
+	@echo "  make start     - Start test cluster (master + 6 volume servers + filer)"
+	@echo "  make stop      - Stop test cluster"
+	@echo "  make populate  - Populate ~300MB of test data"
+	@echo "  make shell     - Open weed shell"
+	@echo "  make setup     - Clean, start, and populate (all-in-one)"
+	@echo "  make status    - Show cluster and EC shard status"
+	@echo "  make clean     - Stop cluster and remove all test data"
+	@echo "  make help      - Show this help"
+	@echo ""
+	@echo "Quick start:"
+	@echo "  make setup     # Start cluster and populate data"
+	@echo "  make shell     # Open shell to run EC commands"
+
@@ -78,6 +78,43 @@ go test -v -run TestECEncodingMasterTimingRaceCondition
 go test -v -short
 ```

+## Manual Testing with Makefile
+
+A Makefile is provided for manual EC testing.
+
+**Requirements:** `curl`, `jq` (command-line JSON processor)
+
+```bash
+# Quick start: start cluster and populate data
+make setup
+
+# Open weed shell to run EC commands
+make shell
+
+# Individual targets
+make start      # Start test cluster (master + 6 volume servers + filer)
+make stop       # Stop test cluster
+make populate   # Populate ~300MB of test data
+make status     # Show cluster and EC shard status
+make clean      # Stop cluster and remove all test data
+make help       # Show all targets
+```
+
+### EC Rebalance Limited Slots (Unit Test)
+
+The "no free ec shard slots" issue is tested with a **unit test** that works directly on
+topology data structures without requiring a running cluster.
+
+**Location**: `weed/shell/ec_rebalance_slots_test.go`
+
+Tests included:
+- `TestECRebalanceWithLimitedSlots`: Tests a topology with 6 servers, 7 EC volumes (98 shards)
+- `TestECRebalanceZeroFreeSlots`: Reproduces the exact 0 free slots scenario
+
+**Known Issue**: When volume servers are at capacity (`volumeCount == maxVolumeCount`),
+the rebalance step fails with "no free ec shard slots" instead of recognizing that
+moving shards frees slots on source servers.
+
 ## Test Results

 **With the fix**: Shows "Collecting volume locations for N volumes before EC encoding..." message
@@ -15,8 +15,10 @@ import (

 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/seaweedfs/seaweedfs/weed/shell"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc"
@@ -386,7 +388,7 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
 	// Find weed binary
 	weedBinary := findWeedBinary()
 	if weedBinary == "" {
-		return nil, fmt.Errorf("weed binary not found")
+		return nil, fmt.Errorf("weed binary not found - build with 'go build' or 'make' first")
 	}

 	cluster := &TestCluster{}
@@ -395,6 +397,11 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
 	masterDir := filepath.Join(dataDir, "master")
 	os.MkdirAll(masterDir, 0755)

+	// Create an empty security.toml to disable JWT authentication in tests
+	// This prevents the test from picking up ~/.seaweedfs/security.toml
+	securityToml := filepath.Join(dataDir, "security.toml")
+	os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
+
 	// Start master server
 	masterCmd := exec.CommandContext(ctx, weedBinary, "master",
 		"-port", "9333",
@@ -403,6 +410,7 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
 		"-ip", "127.0.0.1",
 		"-peers", "none", // Faster startup when no multiple masters needed
 	)
+	masterCmd.Dir = dataDir // Run from test dir so it picks up our security.toml

 	masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
 	if err != nil {
@@ -435,6 +443,7 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
 			"-dataCenter", "dc1",
 			"-rack", rack,
 		)
+		volumeCmd.Dir = dataDir // Run from test dir so it picks up our security.toml

 		volumeLogFile, err := os.Create(filepath.Join(volumeDir, "volume.log"))
 		if err != nil {
@@ -469,6 +478,10 @@ func findWeedBinary() string {

 	for _, candidate := range candidates {
 		if _, err := os.Stat(candidate); err == nil {
+			// Convert to absolute path so it works when command's Dir is changed
+			if absPath, err := filepath.Abs(candidate); err == nil {
+				return absPath
+			}
 			return candidate
 		}
 	}
@@ -2174,3 +2187,249 @@ func countShardsPerRack(testDir string, volumeId uint32) map[string]int {

 	return rackDistribution
 }
+
+// TestECEncodeReplicatedVolumeSync tests that ec.encode properly syncs missing entries
+// between replicas before encoding. This addresses issue #7797.
+func TestECEncodeReplicatedVolumeSync(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping replicated volume sync integration test in short mode")
+	}
+
+	// Create temporary directory for test data
+	testDir, err := os.MkdirTemp("", "seaweedfs_ec_replica_sync_test_")
+	require.NoError(t, err)
+	defer os.RemoveAll(testDir)
+
+	// Start SeaweedFS cluster
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	cluster, err := startSeaweedFSCluster(ctx, testDir)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	// Wait for servers to be ready
+	require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
+	for i := 0; i < 6; i++ {
+		require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:%d", 8080+i), 30*time.Second))
+	}
+
+	// Create command environment
+	options := &shell.ShellOptions{
+		Masters:        stringPtr("127.0.0.1:9333"),
+		GrpcDialOption: grpc.WithInsecure(),
+		FilerGroup:     stringPtr("default"),
+	}
+	commandEnv := shell.NewCommandEnv(options)
+
+	// Connect to master with longer timeout
+	ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel2()
+	go commandEnv.MasterClient.KeepConnectedToMaster(ctx2)
+	commandEnv.MasterClient.WaitUntilConnected(ctx2)
+
+	// Wait for volume servers to register with master
+	time.Sleep(5 * time.Second)
+
+	// Test: Create replicated volume and verify sync behavior with consistent replicas
+	t.Run("sync_replicated_volume_consistent", func(t *testing.T) {
+		const testCollection = "replicated_consistent"
+		const testReplication = "010" // 2 copies on different servers on the same rack
+
+		// Retry a few times as volume servers may still be registering
+		var volumeId needle.VolumeId
+		var uploadErr error
+		for retry := 0; retry < 5; retry++ {
+			volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
+			if uploadErr == nil {
+				break
+			}
+			t.Logf("Upload attempt %d failed: %v, retrying...", retry+1, uploadErr)
+			time.Sleep(3 * time.Second)
+		}
+		if uploadErr != nil {
+			t.Skipf("Could not create replicated volume: %v", uploadErr)
+			return
+		}
+		t.Logf("Created replicated volume %d with collection %s, replication %s", volumeId, testCollection, testReplication)
+
+		// Acquire lock
+		locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
+		if !locked {
+			t.Skip("Could not acquire lock within timeout")
+		}
+		defer unlock()
+
+		// Execute EC encoding with the same collection
+		ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
+		args := []string{
+			"-volumeId", fmt.Sprintf("%d", volumeId),
+			"-collection", testCollection,
+			"-force",
+		}
+
+		outputStr, encodeErr := captureCommandOutput(t, ecEncodeCmd, args, commandEnv)
+		t.Logf("EC encode output:\n%s", outputStr)
+
+		// For consistent replicas, should see "all X replicas are consistent"
+		assert.Contains(t, outputStr, "replicas are consistent", "Should detect replicas are consistent")
+
+		if encodeErr != nil {
+			t.Logf("EC encoding result: %v", encodeErr)
+		} else {
+			t.Log("EC encoding completed successfully")
+		}
+	})
+
+	// Test: Create divergent replicas and verify sync/union code is exercised
+	t.Run("sync_replicated_volume_divergent", func(t *testing.T) {
+		const testCollection = "replicated_divergent"
+		const testReplication = "010" // 2 copies on different servers on the same rack
+
+		// Create replicated volume with initial data
+		var volumeId needle.VolumeId
+		var uploadErr error
+		for retry := 0; retry < 5; retry++ {
+			volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
+			if uploadErr == nil {
+				break
+			}
+			t.Logf("Upload attempt %d failed: %v, retrying...", retry+1, uploadErr)
+			time.Sleep(3 * time.Second)
+		}
+		if uploadErr != nil {
+			t.Skipf("Could not create replicated volume: %v", uploadErr)
+			return
+		}
+		t.Logf("Created replicated volume %d with collection %s, replication %s", volumeId, testCollection, testReplication)
+
+		// Get volume locations to identify the two replicas
+		locations, found := commandEnv.MasterClient.GetLocationsClone(uint32(volumeId))
+		if !found || len(locations) < 2 {
+			t.Skipf("Could not get 2 replica locations for volume %d (got %d)", volumeId, len(locations))
+			return
+		}
+		t.Logf("Volume %d has replicas at: %v and %v", volumeId, locations[0].Url, locations[1].Url)
+
+		// Write an extra entry to ONLY one replica to create divergence
+		// We'll use the WriteNeedleBlob gRPC call to inject data directly
+		extraNeedleId := uint64(999999) // Use a high needle ID unlikely to conflict
+		extraData := []byte("extra data written to only one replica to create divergence")
+
+		err := injectNeedleToOneReplica(grpc.WithInsecure(), volumeId, locations[0], extraNeedleId, extraData)
+		if err != nil {
+			t.Logf("Could not inject divergent needle (may not be supported): %v", err)
+			// Fall back to testing consistent path
+		} else {
+			t.Logf("Injected extra needle %d to replica %s to create divergence", extraNeedleId, locations[0].Url)
+		}
+
+		// Acquire lock
+		locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
+		if !locked {
+			t.Skip("Could not acquire lock within timeout")
+		}
+		defer unlock()
+
+		// Execute EC encoding - should detect divergence and build union
+		ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
+		args := []string{
+			"-volumeId", fmt.Sprintf("%d", volumeId),
+			"-collection", testCollection,
+			"-force",
+		}
+
+		outputStr, encodeErr := captureCommandOutput(t, ecEncodeCmd, args, commandEnv)
+		t.Logf("EC encode output:\n%s", outputStr)
+
+		// Check if divergence was detected
+		if strings.Contains(outputStr, "building union") {
+			t.Log("SUCCESS: Divergent replicas detected - sync/union code was exercised")
+			assert.Contains(t, outputStr, "selected", "Should show which replica was selected as best")
+			// The "copied" message only appears if the best replica is missing entries from other replicas
+			// In our case, we injected into what becomes the best replica (higher file count),
+			// so copying may not be needed. The key is that "building union" was triggered.
+			if strings.Contains(outputStr, "copied") {
+				t.Log("Entries were copied between replicas")
+			} else {
+				t.Log("Best replica already had all entries (injected entry was on best replica)")
+			}
+		} else if strings.Contains(outputStr, "replicas are consistent") {
+			t.Log("Replicas were consistent (injection may not have worked)")
+		} else {
+			t.Log("Single replica or sync not triggered")
+		}
+
+		if encodeErr != nil {
+			t.Logf("EC encoding result: %v", encodeErr)
+		} else {
+			t.Log("EC encoding completed successfully")
+		}
+	})
+}
+
+// injectNeedleToOneReplica writes a needle directly to one replica to create divergence
+func injectNeedleToOneReplica(grpcDialOption grpc.DialOption, vid needle.VolumeId, location wdclient.Location, needleId uint64, data []byte) error {
+	return operation.WithVolumeServerClient(false, location.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+		_, err := client.WriteNeedleBlob(context.Background(), &volume_server_pb.WriteNeedleBlobRequest{
+			VolumeId:   uint32(vid),
+			NeedleId:   needleId,
+			Size:       int32(len(data)),
+			NeedleBlob: data,
+		})
+		return err
+	})
+}
+
+// uploadTestDataWithReplication uploads test data and returns the volume ID
+// using the specified collection and replication level. All files are uploaded to the same volume.
+func uploadTestDataWithReplication(t *testing.T, masterAddr string, collection string, replication string) (needle.VolumeId, error) {
+	const numFiles = 20 // Reduced count since we're uploading to same volume
+
+	// Assign multiple file IDs from the same volume
+	assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
+		return pb.ServerAddress(masterAddr)
+	}, grpc.WithInsecure(), &operation.VolumeAssignRequest{
+		Count:       uint64(numFiles),
+		Collection:  collection,
+		Replication: replication,
+	})
+	if err != nil {
+		return 0, fmt.Errorf("failed to assign volume: %v", err)
+	}
+
+	// Parse volume ID from file ID
+	fid, err := needle.ParseFileIdFromString(assignResult.Fid)
+	if err != nil {
+		return 0, fmt.Errorf("failed to parse file ID: %v", err)
+	}
+	volumeId := fid.VolumeId
+
+	// Create uploader for upload
+	uploader, err := operation.NewUploader()
+	if err != nil {
+		return 0, fmt.Errorf("failed to create uploader: %v", err)
+	}
+
+	// Upload test data to the same volume using the assigned file IDs
+	// The first FID is assignResult.Fid, subsequent ones increment the needle key
+	baseNeedleKey := uint64(fid.Key)
+	for i := 0; i < numFiles; i++ {
+		data := []byte(fmt.Sprintf("test data for replicated volume sync test - entry %d - padding to make it larger %s",
+			i, strings.Repeat("x", 1000)))
+
+		// Construct FID for this file (same volume, incrementing needle key)
+		currentFid := fmt.Sprintf("%d,%x%08x", volumeId, baseNeedleKey+uint64(i), fid.Cookie)
+		uploadUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, currentFid)
+
+		_, _, uploadErr := uploader.Upload(context.Background(), bytes.NewReader(data), &operation.UploadOption{
+			UploadUrl: uploadUrl,
+			Filename:  fmt.Sprintf("file%d.txt", i),
+		})
+		if uploadErr != nil {
+			t.Logf("Upload %d failed: %v", i, uploadErr)
+		}
+	}
+
+	return volumeId, nil
+}
@@ -5,13 +5,14 @@ ARG FOUNDATIONDB_VERSION=7.4.5
 ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}

 # Install build dependencies and download prebuilt FoundationDB clients
-RUN apt-get update && apt-get install -y \
+SHELL ["/bin/bash", "-c"]
+RUN set -euo pipefail && \
+    apt-get update && apt-get install -y \
    build-essential \
    git \
    wget \
    ca-certificates \
    && rm -rf /var/lib/apt/lists/* && \
-    set -euo pipefail && \
    case "${FOUNDATIONDB_VERSION}" in \
        "7.4.5") EXPECTED_SHA256="f2176b86b7e1b561c3632b4e6e7efb82e3b8f57c2ff0d0ac4671e742867508aa" ;; \
        *) echo "ERROR: No known ARM64 client checksum for FoundationDB ${FOUNDATIONDB_VERSION}. Please update this Dockerfile." >&2; exit 1 ;; \
@@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y \
    && rm -rf /var/lib/apt/lists/*

 # Install FoundationDB server + client debs with checksum verification
+SHELL ["/bin/bash", "-c"]
 RUN set -euo pipefail && \
    apt-get update && \
    case "${FOUNDATIONDB_VERSION}" in \
@@ -50,6 +50,24 @@ test-benchmark: ## Run performance benchmarks
 	@echo "$(YELLOW)Running FoundationDB performance benchmarks...$(NC)"
 	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb -bench=. ./test/foundationdb/...

+test-benchmark-filer: ## Run filer store benchmarks (batch vs no-batch comparison)
+	@echo "$(YELLOW)Running FoundationDB filer store benchmarks...$(NC)"
+	@echo "$(BLUE)Comparing batched vs non-batched write performance$(NC)"
+	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb \
+		-bench='BenchmarkFoundationDBStore_.*' \
+		-benchmem \
+		-benchtime=5s \
+		./weed/filer/foundationdb/...
+
+test-benchmark-concurrent: ## Run concurrent operation benchmarks
+	@echo "$(YELLOW)Running concurrent operation benchmarks...$(NC)"
+	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb \
+		-bench='BenchmarkFoundationDBStore_Concurrent.*' \
+		-benchmem \
+		-benchtime=10s \
+		-cpu=1,2,4,8 \
+		./weed/filer/foundationdb/...
+
 # ARM64 specific targets (Apple Silicon / M1/M2/M3 Macs)
 setup-arm64: ## Set up ARM64-native FoundationDB cluster (builds from source)
 	@echo "$(YELLOW)Setting up ARM64-native FoundationDB cluster...$(NC)"
@@ -353,12 +353,31 @@ The tests are designed to be reliable in CI environments with:
 Run performance benchmarks:

 ```bash
-make test-benchmark
+make test-benchmark              # Run all benchmarks
+make test-benchmark-filer        # Filer store benchmarks with batch comparison
+make test-benchmark-concurrent   # Concurrent operation benchmarks (varies CPU count)

 # Sample expected results:
-# BenchmarkFoundationDBStore_InsertEntry-8    1000    1.2ms per op
-# BenchmarkFoundationDBStore_FindEntry-8      5000    0.5ms per op  
-# BenchmarkFoundationDBStore_KvOperations-8   2000    0.8ms per op
+# BenchmarkFoundationDBStore_InsertEntry-8           1000    1.2ms per op
+# BenchmarkFoundationDBStore_FindEntry-8             5000    0.5ms per op  
+# BenchmarkFoundationDBStore_KvOperations-8          2000    0.8ms per op
+# BenchmarkFoundationDBStore_InsertEntry_NoBatch-8   1000    1.0ms per op  (optimal for S3)
+# BenchmarkFoundationDBStore_InsertEntry_WithBatch-8 1200    0.9ms per op  (bulk ingestion)
+# BenchmarkFoundationDBStore_ConcurrentInsert_*-8    5000    0.3ms per op  (parallel writes)
+```
+
+### Batch vs Non-Batch Performance
+
+The FoundationDB filer store supports two write modes:
+
+| Mode | Config | Best For | Latency | Throughput |
+|------|--------|----------|---------|------------|
+| **Direct Commit** (default) | `batch_enabled = false` | S3 API, low-latency workloads | ~1-5ms per op | Good |
+| **Batched** | `batch_enabled = true` | Bulk ingestion, high throughput | Variable | Higher |
+
+Run the comparison benchmark:
+```bash
+make test-benchmark-filer
 ```

 ## Contributing
@@ -28,8 +28,8 @@ services:
          echo 'testing:testing@fdb1:4500,fdb2:4502,fdb3:4504' > /var/fdb/config/fdb.cluster
        fi
        # Start FDB processes
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4501 --listen_address=0.0.0.0:4501 --coordination=fdb1:4500 &
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4500 --listen_address=0.0.0.0:4500 --coordination=fdb1:4500 --class=coordination &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4501 --listen_address=0.0.0.0:4501 --coordination=fdb1:4500 &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4500 --listen_address=0.0.0.0:4500 --coordination=fdb1:4500 --class=coordination &
        wait
      "

@@ -59,8 +59,8 @@ services:
        # Wait for cluster file from fdb1
        while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
        # Start FDB processes
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4503 --listen_address=0.0.0.0:4503 --coordination=fdb1:4500 &
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4502 --listen_address=0.0.0.0:4502 --coordination=fdb1:4500 --class=coordination &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4503 --listen_address=0.0.0.0:4503 --coordination=fdb1:4500 &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4502 --listen_address=0.0.0.0:4502 --coordination=fdb1:4500 --class=coordination &
        wait
      "

@@ -90,8 +90,8 @@ services:
        # Wait for cluster file from fdb1
        while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
        # Start FDB processes
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4505 --listen_address=0.0.0.0:4505 --coordination=fdb1:4500 &
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4504 --listen_address=0.0.0.0:4504 --coordination=fdb1:4500 --class=coordination &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4505 --listen_address=0.0.0.0:4505 --coordination=fdb1:4500 &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4504 --listen_address=0.0.0.0:4504 --coordination=fdb1:4500 --class=coordination &
        wait
      "

@@ -1,19 +1,8 @@
-# FoundationDB Filer Configuration
+# FoundationDB Filer Configuration for Testing

 [foundationdb]
 enabled = true
 cluster_file = "/var/fdb/config/fdb.cluster"
-api_version = 740
-timeout = "5s"
-max_retry_delay = "1s"
-directory_prefix = "seaweedfs"
-
-# For testing different configurations
-[foundationdb.test]
-enabled = false
-cluster_file = "/var/fdb/config/fdb.cluster"
-api_version = 740
-timeout = "10s"
-max_retry_delay = "2s"
-directory_prefix = "seaweedfs_test"
-location = "/test"
+# api_version = 740
+# timeout = "5s"
+# directory_prefix = "seaweedfs"
@@ -22,7 +22,7 @@
        <junit.version>4.13.2</junit.version>
        <seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
        <jackson.version>2.18.2</jackson.version> <!-- Upgraded from 2.15.3 -->
-        <netty.version>4.1.125.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
+        <netty.version>4.1.129.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
        <parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
        <parquet.format.version>2.12.0</parquet.format.version>
        <surefire.jvm.args>
@@ -172,7 +172,8 @@ services:
      - -master=seaweedfs-master:9333
      - -ip=seaweedfs-mq-broker
      - -port=17777
-      - -port.pprof=18777
+      - -debug
+      - -debug.port=18777
    depends_on:
      seaweedfs-filer:
        condition: service_healthy
@@ -43,25 +43,25 @@ require (
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go v1.55.8 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.40.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.41.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.31.3 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.18.20 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.5 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect
-	github.com/aws/smithy-go v1.23.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
+	github.com/aws/smithy-go v1.24.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
@@ -229,15 +229,15 @@ require (
 	go.opentelemetry.io/otel/trace v1.38.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/crypto v0.46.0 // indirect
 	golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 // indirect
-	golang.org/x/image v0.33.0 // indirect
-	golang.org/x/net v0.47.0 // indirect
+	golang.org/x/image v0.34.0 // indirect
+	golang.org/x/net v0.48.0 // indirect
 	golang.org/x/oauth2 v0.32.0 // indirect
 	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.38.0 // indirect
-	golang.org/x/term v0.37.0 // indirect
-	golang.org/x/text v0.31.0 // indirect
+	golang.org/x/sys v0.39.0 // indirect
+	golang.org/x/term v0.38.0 // indirect
+	golang.org/x/text v0.32.0 // indirect
 	golang.org/x/time v0.12.0 // indirect
 	google.golang.org/api v0.247.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
@@ -102,44 +102,44 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
-github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
+github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
+github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
 github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
 github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.5 h1:xMo63RlqP3ZZydpJDMBsH9uJ10hgHYfQFIk1cHDXrR4=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.5/go.mod h1:hhbH6oRcou+LpXfA/0vPElh/e0M3aFeOblE1sssAAEk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY=
-github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs=
-github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
-github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 h1:eYnlt6QxnFINKzwxP5/Ucs1vkG7VT3Iezmvfgc2waUw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.7/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
+github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
+github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo=
@@ -536,10 +536,10 @@ github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4
 github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
-github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
-github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
-github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg=
-github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
+github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
+github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
+github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
+github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
 github.com/rclone/rclone v1.71.2 h1:3Jk5xNPFrZhVABRuN/OPvApuZQddpE2tkhYMuEn1Ud4=
 github.com/rclone/rclone v1.71.2/go.mod h1:dCK9FzPDlpkbQJ9M7MmWsmv3X5nibfWe+ogJXu6gSgM=
 github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg=
@@ -708,8 +708,6 @@ go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJr
 go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
-go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
@@ -732,8 +730,8 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
+golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -748,8 +746,8 @@ golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 h1:3yiSh9fhy5/RhCSntf4Sy0Tnx
 golang.org/x/exp v0.0.0-20250811191247-51f88131bc50/go.mod h1:rT6SFzZ7oxADUDx58pcaKFTcZ+inxAa9fTrYx/uVYwg=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.33.0 h1:LXRZRnv1+zGd5XBUVRFmYEphyyKJjQjCRiOuAP3sZfQ=
-golang.org/x/image v0.33.0/go.mod h1:DD3OsTYT9chzuzTQt+zMcOlBHgfoKQb1gry8p76Y1sc=
+golang.org/x/image v0.34.0 h1:33gCkyw9hmwbZJeZkct8XyR11yH889EQt/QH4VmXMn8=
+golang.org/x/image v0.34.0/go.mod h1:2RNFBZRB+vnwwFil8GkMdRvrJOFd1AzdZI6vOY+eJVU=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -774,8 +772,6 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
-golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -815,8 +811,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
+golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -894,8 +890,8 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -908,8 +904,8 @@ golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
-golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
-golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
+golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -925,8 +921,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
+golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -979,8 +975,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
-golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
+golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -181,7 +181,8 @@ services:
      - -ip=seaweedfs-mq-broker
      - -port=17777
      - -logFlushInterval=0
-      - -port.pprof=18777
+      - -debug
+      - -debug.port=18777
    depends_on:
      seaweedfs-filer:
        condition: service_healthy
@@ -25,3 +25,4 @@ test-resume: build-weed
 clean:
 	rm -f ../../weed/weed

+
@@ -47,3 +47,4 @@ go test -v -timeout 5m ./test/metadata_subscribe/...
 - Tests create temporary directories that are cleaned up after completion
 - Tests use ports 9333 (master), 8080 (volume), 8888 (filer)

+
@@ -0,0 +1,126 @@
+# S3 SDK Compatibility Testing
+
+This document describes the SDK versions tested against SeaweedFS S3 API and known compatibility considerations.
+
+## Validated SDK Versions
+
+### Go SDKs
+
+| SDK | Version | Test Location | Status |
+|-----|---------|---------------|--------|
+| AWS SDK v2 for Go | 1.20+ | `test/s3/etag/`, `test/s3/copying/` | ✅ Tested |
+| AWS SDK v1 for Go | 1.x | `test/s3/basic/` | ✅ Tested |
+
+### Java SDKs
+
+| SDK | Version | Test Location | Status |
+|-----|---------|---------------|--------|
+| AWS SDK v2 for Java | 2.20.127+ | `other/java/s3copier/` | ✅ Tested |
+| AWS SDK v1 for Java | 1.12.600+ | `other/java/s3copier/` | ✅ Tested |
+
+### Python SDKs
+
+| SDK | Version | Test Location | Status |
+|-----|---------|---------------|--------|
+| boto3 | 1.x | `test/s3/parquet/` | ✅ Tested |
+| PyArrow S3 | 14+ | `test/s3/parquet/` | ✅ Tested |
+
+## SDK-Specific Considerations
+
+### AWS SDK v2 for Java - ETag Validation
+
+**Issue**: [GitHub #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+
+AWS SDK v2 for Java includes strict ETag validation in `ChecksumsEnabledValidator.validatePutObjectChecksum`. It decodes the ETag as a hexadecimal MD5 hash using `Base16Codec.decode()`.
+
+**Impact**: If the ETag contains non-hexadecimal characters (like `-` in composite format), the SDK fails with:
+```text
+java.lang.IllegalArgumentException: Invalid base 16 character: '-'
+```
+
+**Resolution**: SeaweedFS now correctly returns:
+- **PutObject**: Pure MD5 hex ETag (32 characters) regardless of internal chunking
+- **CompleteMultipartUpload**: Composite ETag (`<md5>-<partcount>`)
+
+**Test Coverage**: `test/s3/etag/` and `other/java/s3copier/ETagValidationTest.java`
+
+### AWS SDK v1 vs v2 Differences
+
+| Feature | SDK v1 | SDK v2 |
+|---------|--------|--------|
+| ETag hex validation | No | Yes (strict) |
+| Checksum validation | Basic | Enhanced |
+| Async support | Limited | Full |
+| Default retry behavior | Lenient | Stricter |
+
+### Large File Handling
+
+SeaweedFS auto-chunks files larger than **8MB** for efficient storage. This is transparent to clients, but previously affected ETag format. The current implementation ensures:
+
+1. Regular `PutObject` (any size): Returns pure MD5 ETag
+2. Multipart upload: Returns composite ETag per AWS S3 specification
+
+## Test Categories by File Size
+
+| Category | Size | Chunks | ETag Format |
+|----------|------|--------|-------------|
+| Small | < 256KB | 1 (inline) | Pure MD5 |
+| Medium | 256KB - 8MB | 1 | Pure MD5 |
+| Large | 8MB - 24MB | 2-3 | Pure MD5 |
+| Extra Large | > 24MB | 4+ | Pure MD5 |
+| Multipart | N/A | Per part | Composite |
+
+## Running SDK Compatibility Tests
+
+### Go Tests
+
+```bash
+# Run all ETag tests
+cd test/s3/etag && make test
+
+# Run large file tests only
+cd test/s3/etag && make test-large
+```
+
+### Java Tests
+
+```bash
+# Run all Java SDK tests
+cd other/java/s3copier && mvn test
+
+# Run only ETag validation tests
+cd other/java/s3copier && mvn test -Dtest=ETagValidationTest
+```
+
+### Python Tests
+
+```bash
+# Run PyArrow S3 tests
+cd test/s3/parquet && make test
+```
+
+## Adding New SDK Tests
+
+When adding tests for new SDKs, ensure:
+
+1. **Large file tests (>8MB)**: Critical for verifying ETag format with auto-chunking
+2. **Multipart upload tests**: Verify composite ETag format
+3. **Checksum validation**: Test SDK-specific checksum validation if applicable
+4. **Document SDK version**: Add to this compatibility matrix
+
+## Known Issues and Workarounds
+
+### Issue: Older SDK Versions
+
+Some very old SDK versions (e.g., AWS SDK v1 for Java < 1.11.x) may have different behavior. Testing with the versions listed above is recommended.
+
+### Issue: Custom Checksum Algorithms
+
+AWS SDK v2 supports SHA-256 and CRC32 checksums in addition to MD5. SeaweedFS currently returns MD5-based ETags. For checksums other than MD5, use the `x-amz-checksum-*` headers.
+
+## References
+
+- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
+- [AWS SDK v2 Migration Guide](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/migration.html)
+- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+
@@ -0,0 +1,50 @@
+# ETag Format Integration Tests
+#
+# These tests verify S3 ETag format compatibility, particularly for large files
+# that trigger SeaweedFS auto-chunking. This addresses GitHub Issue #7768.
+#
+# Prerequisites:
+#   - SeaweedFS running with S3 API enabled on port 8333
+#   - Go 1.21+
+#
+# Usage:
+#   make test          - Run all tests
+#   make test-large    - Run only large file tests
+#   make test-verbose  - Run with verbose output
+#   make clean         - Clean test artifacts
+
+.PHONY: all test test-large test-verbose test-quick clean help
+
+# Default S3 endpoint
+S3_ENDPOINT ?= http://127.0.0.1:8333
+
+all: test
+
+test:
+	@echo "Running ETag format tests against $(S3_ENDPOINT)..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m ./...
+
+test-large:
+	@echo "Running large file ETag tests..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m -run "LargeFile|ExtraLarge" ./...
+
+test-verbose:
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m -count=1 ./...
+
+test-quick:
+	@echo "Running quick ETag tests (small files only)..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 1m -run "SmallFile|Consistency" ./...
+
+clean:
+	go clean -testcache
+
+help:
+	@echo "ETag Format Integration Tests"
+	@echo "Targets:"
+	@echo "  test          Run all ETag format tests"
+	@echo "  test-large    Run only large file tests (>8MB)"
+	@echo "  test-quick    Run quick tests (small files only)"
+	@echo "  test-verbose  Run with verbose output"
+	@echo "  clean         Clean test cache"
+	@echo "Environment Variables:"
+	@echo "  S3_ENDPOINT   S3 endpoint URL (default: http://127.0.0.1:8333)"
@@ -0,0 +1,93 @@
+# S3 ETag Format Integration Tests
+
+This test suite verifies that SeaweedFS returns correct ETag formats for S3 operations, ensuring compatibility with AWS S3 SDKs.
+
+## Background
+
+**GitHub Issue #7768**: AWS S3 SDK for Java v2 was failing with `Invalid base 16 character: '-'` when performing `PutObject` on large files.
+
+### Root Cause
+
+SeaweedFS internally auto-chunks large files (>8MB) for efficient storage. Previously, when a regular `PutObject` request resulted in multiple internal chunks, SeaweedFS returned a composite ETag format (`<md5>-<count>`) instead of a pure MD5 hash.
+
+### AWS S3 Specification
+
+| Operation | ETag Format | Example |
+|-----------|-------------|---------|
+| PutObject (any size) | Pure MD5 hex (32 chars) | `d41d8cd98f00b204e9800998ecf8427e` |
+| CompleteMultipartUpload | Composite (`<md5>-<partcount>`) | `d41d8cd98f00b204e9800998ecf8427e-3` |
+
+AWS S3 SDK v2 for Java validates `PutObject` ETags as hexadecimal, which fails when the ETag contains a hyphen.
+
+## Test Coverage
+
+| Test | File Size | Purpose |
+|------|-----------|---------|
+| `TestPutObjectETagFormat_SmallFile` | 1KB | Verify single-chunk uploads return pure MD5 |
+| `TestPutObjectETagFormat_LargeFile` | 10MB | **Critical**: Verify auto-chunked uploads return pure MD5 |
+| `TestPutObjectETagFormat_ExtraLargeFile` | 25MB | Verify multi-chunk auto-chunked uploads return pure MD5 |
+| `TestMultipartUploadETagFormat` | 15MB | Verify multipart uploads correctly return composite ETag |
+| `TestPutObjectETagConsistency` | Various | Verify ETag consistency across PUT/HEAD/GET |
+| `TestETagHexValidation` | 10MB | Simulate AWS SDK v2 hex validation |
+| `TestMultipleLargeFileUploads` | 10MB x5 | Stress test multiple large uploads |
+
+## Prerequisites
+
+1. SeaweedFS running with S3 API enabled:
+   ```bash
+   weed server -s3
+   ```
+
+2. Go 1.21 or later
+
+3. AWS SDK v2 for Go (installed via go modules)
+
+## Running Tests
+
+```bash
+# Run all tests
+make test
+
+# Run only large file tests (the critical ones for issue #7768)
+make test-large
+
+# Run quick tests (small files only)
+make test-quick
+
+# Run with verbose output
+make test-verbose
+```
+
+## Configuration
+
+By default, tests connect to `http://127.0.0.1:8333`. To use a different endpoint:
+
+```bash
+S3_ENDPOINT=http://localhost:8333 make test
+```
+
+Or modify `defaultConfig` in `s3_etag_test.go`.
+
+## SDK Compatibility
+
+These tests use **AWS SDK v2 for Go**, which has the same ETag validation behavior as AWS SDK v2 for Java. The tests include:
+
+- ETag format validation (pure MD5 vs composite)
+- Hex decoding validation (simulates `Base16Codec.decode`)
+- Content integrity verification
+
+## Validated SDK Versions
+
+| SDK | Version | Status |
+|-----|---------|--------|
+| AWS SDK v2 for Go | 1.20+ | ✅ Tested |
+| AWS SDK v2 for Java | 2.20+ | ✅ Compatible (issue #7768 fixed) |
+| AWS SDK v1 for Go | 1.x | ✅ Compatible (less strict validation) |
+| AWS SDK v1 for Java | 1.x | ✅ Compatible (less strict validation) |
+
+## Related
+
+- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
+
+
@@ -0,0 +1,543 @@
+// Package etag_test provides integration tests for S3 ETag format validation.
+//
+// These tests verify that SeaweedFS returns correct ETag formats for different
+// upload scenarios, ensuring compatibility with AWS S3 SDKs that validate ETags.
+//
+// Background (GitHub Issue #7768):
+// AWS S3 SDK for Java v2 validates ETags as hexadecimal MD5 hashes for PutObject
+// responses. SeaweedFS was incorrectly returning composite ETags ("<md5>-<count>")
+// for regular PutObject when files were internally auto-chunked (>8MB), causing
+// the SDK to fail with "Invalid base 16 character: '-'".
+//
+// Per AWS S3 specification:
+// - Regular PutObject: ETag is always a pure MD5 hex string (32 chars)
+// - Multipart Upload (CompleteMultipartUpload): ETag is "<md5>-<partcount>"
+//
+// These tests ensure this behavior is maintained.
+package etag_test
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+	mathrand "math/rand"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// S3TestConfig holds configuration for S3 tests
+type S3TestConfig struct {
+	Endpoint     string
+	AccessKey    string
+	SecretKey    string
+	Region       string
+	BucketPrefix string
+}
+
+// Default test configuration
+var defaultConfig = &S3TestConfig{
+	Endpoint:     "http://127.0.0.1:8333",
+	AccessKey:    "some_access_key1",
+	SecretKey:    "some_secret_key1",
+	Region:       "us-east-1",
+	BucketPrefix: "test-etag-",
+}
+
+// Constants for auto-chunking thresholds (must match s3api_object_handlers_put.go)
+const (
+	// SeaweedFS auto-chunks files larger than 8MB
+	autoChunkSize = 8 * 1024 * 1024
+
+	// Test sizes
+	smallFileSize  = 1 * 1024              // 1KB - single chunk
+	mediumFileSize = 256 * 1024            // 256KB - single chunk (at threshold)
+	largeFileSize  = 10 * 1024 * 1024      // 10MB - triggers auto-chunking (2 chunks)
+	xlFileSize     = 25 * 1024 * 1024      // 25MB - triggers auto-chunking (4 chunks)
+	multipartSize  = 5 * 1024 * 1024       // 5MB per part for multipart uploads
+)
+
+// ETag format patterns
+var (
+	// Pure MD5 ETag: 32 hex characters (with or without quotes)
+	pureMD5Pattern = regexp.MustCompile(`^"?[a-f0-9]{32}"?$`)
+
+	// Composite ETag for multipart: 32 hex chars, hyphen, part count (with or without quotes)
+	compositePattern = regexp.MustCompile(`^"?[a-f0-9]{32}-\d+"?$`)
+)
+
+func init() {
+	mathrand.Seed(time.Now().UnixNano())
+}
+
+// getS3Client creates an AWS S3 v2 client for testing
+func getS3Client(t *testing.T) *s3.Client {
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(defaultConfig.Region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			defaultConfig.AccessKey,
+			defaultConfig.SecretKey,
+			"",
+		)),
+		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
+			func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+				return aws.Endpoint{
+					URL:               defaultConfig.Endpoint,
+					SigningRegion:     defaultConfig.Region,
+					HostnameImmutable: true,
+				}, nil
+			})),
+	)
+	require.NoError(t, err)
+
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+}
+
+// getNewBucketName generates a unique bucket name
+func getNewBucketName() string {
+	timestamp := time.Now().UnixNano()
+	randomSuffix := mathrand.Intn(100000)
+	return fmt.Sprintf("%s%d-%d", defaultConfig.BucketPrefix, timestamp, randomSuffix)
+}
+
+// generateRandomData generates random test data of specified size
+func generateRandomData(size int) []byte {
+	data := make([]byte, size)
+	if _, err := rand.Read(data); err != nil {
+		panic(fmt.Sprintf("failed to generate random test data: %v", err))
+	}
+	return data
+}
+
+// calculateMD5 calculates the MD5 hash of data and returns hex string
+func calculateMD5(data []byte) string {
+	hash := md5.Sum(data)
+	return hex.EncodeToString(hash[:])
+}
+
+// cleanETag removes quotes from ETag
+func cleanETag(etag string) string {
+	return strings.Trim(etag, `"`)
+}
+
+// isPureMD5ETag checks if ETag is a pure MD5 hex string (no composite format)
+func isPureMD5ETag(etag string) bool {
+	return pureMD5Pattern.MatchString(etag)
+}
+
+// isCompositeETag checks if ETag is in composite format (md5-count)
+func isCompositeETag(etag string) bool {
+	return compositePattern.MatchString(etag)
+}
+
+// createTestBucket creates a new bucket for testing
+func createTestBucket(ctx context.Context, client *s3.Client, bucketName string) error {
+	_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	return err
+}
+
+// cleanupTestBucket deletes all objects and the bucket
+func cleanupTestBucket(ctx context.Context, client *s3.Client, bucketName string) {
+	// Delete all objects
+	paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+	})
+
+	for paginator.HasMorePages() {
+		page, err := paginator.NextPage(ctx)
+		if err != nil {
+			break
+		}
+		for _, obj := range page.Contents {
+			client.DeleteObject(ctx, &s3.DeleteObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    obj.Key,
+			})
+		}
+	}
+
+	// Abort any in-progress multipart uploads
+	mpPaginator := s3.NewListMultipartUploadsPaginator(client, &s3.ListMultipartUploadsInput{
+		Bucket: aws.String(bucketName),
+	})
+	for mpPaginator.HasMorePages() {
+		page, err := mpPaginator.NextPage(ctx)
+		if err != nil {
+			break
+		}
+		for _, upload := range page.Uploads {
+			client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
+				Bucket:   aws.String(bucketName),
+				Key:      upload.Key,
+				UploadId: upload.UploadId,
+			})
+		}
+	}
+
+	// Delete bucket
+	client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+}
+
+// TestPutObjectETagFormat_SmallFile verifies ETag format for small files (single chunk)
+func TestPutObjectETagFormat_SmallFile(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testData := generateRandomData(smallFileSize)
+	expectedMD5 := calculateMD5(testData)
+	objectKey := "small-file.bin"
+
+	// Upload small file
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload small file")
+
+	// Verify ETag format
+	etag := aws.ToString(putResp.ETag)
+	t.Logf("Small file (%d bytes) ETag: %s", smallFileSize, etag)
+
+	assert.True(t, isPureMD5ETag(etag),
+		"Small file ETag should be pure MD5, got: %s", etag)
+	assert.False(t, isCompositeETag(etag),
+		"Small file ETag should NOT be composite format, got: %s", etag)
+	assert.Equal(t, expectedMD5, cleanETag(etag),
+		"ETag should match calculated MD5")
+}
+
+// TestPutObjectETagFormat_LargeFile verifies ETag format for large files that trigger auto-chunking
+// This is the critical test for GitHub Issue #7768
+func TestPutObjectETagFormat_LargeFile(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testData := generateRandomData(largeFileSize)
+	expectedMD5 := calculateMD5(testData)
+	objectKey := "large-file.bin"
+
+	t.Logf("Uploading large file (%d bytes, > %d byte auto-chunk threshold)...",
+		largeFileSize, autoChunkSize)
+
+	// Upload large file (triggers auto-chunking internally)
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload large file")
+
+	// Verify ETag format - MUST be pure MD5, NOT composite
+	etag := aws.ToString(putResp.ETag)
+	t.Logf("Large file (%d bytes, ~%d internal chunks) ETag: %s",
+		largeFileSize, (largeFileSize/autoChunkSize)+1, etag)
+
+	assert.True(t, isPureMD5ETag(etag),
+		"Large file PutObject ETag MUST be pure MD5 (not composite), got: %s", etag)
+	assert.False(t, isCompositeETag(etag),
+		"Large file PutObject ETag should NOT contain '-' (composite format), got: %s", etag)
+	assert.False(t, strings.Contains(cleanETag(etag), "-"),
+		"ETag should not contain hyphen for regular PutObject, got: %s", etag)
+	assert.Equal(t, expectedMD5, cleanETag(etag),
+		"ETag should match calculated MD5 of entire content")
+
+	// Verify we can read back the object correctly
+	getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to get large file")
+	defer getResp.Body.Close()
+
+	downloadedData, err := io.ReadAll(getResp.Body)
+	require.NoError(t, err, "Failed to read large file content")
+	assert.Equal(t, testData, downloadedData, "Downloaded content should match uploaded content")
+}
+
+// TestPutObjectETagFormat_ExtraLargeFile tests even larger files with multiple internal chunks
+func TestPutObjectETagFormat_ExtraLargeFile(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testData := generateRandomData(xlFileSize)
+	expectedMD5 := calculateMD5(testData)
+	objectKey := "xl-file.bin"
+
+	expectedChunks := (xlFileSize / autoChunkSize) + 1
+	t.Logf("Uploading XL file (%d bytes, expected ~%d internal chunks)...",
+		xlFileSize, expectedChunks)
+
+	// Upload extra large file
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload XL file")
+
+	// Verify ETag format
+	etag := aws.ToString(putResp.ETag)
+	t.Logf("XL file (%d bytes) ETag: %s", xlFileSize, etag)
+
+	assert.True(t, isPureMD5ETag(etag),
+		"XL file PutObject ETag MUST be pure MD5, got: %s", etag)
+	assert.False(t, isCompositeETag(etag),
+		"XL file PutObject ETag should NOT be composite, got: %s", etag)
+	assert.Equal(t, expectedMD5, cleanETag(etag),
+		"ETag should match calculated MD5")
+}
+
+// TestMultipartUploadETagFormat verifies that ONLY multipart uploads get composite ETags
+func TestMultipartUploadETagFormat(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	// Create test data for multipart upload (15MB = 3 parts of 5MB each)
+	totalSize := 15 * 1024 * 1024
+	testData := generateRandomData(totalSize)
+	objectKey := "multipart-file.bin"
+
+	expectedPartCount := (totalSize + multipartSize - 1) / multipartSize // ceiling division
+	t.Logf("Performing multipart upload (%d bytes, %d parts)...",
+		totalSize, expectedPartCount)
+
+	// Initiate multipart upload
+	createResp, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	uploadId := createResp.UploadId
+	var completedParts []types.CompletedPart
+	partNumber := int32(1)
+
+	// Upload parts
+	for offset := 0; offset < totalSize; offset += multipartSize {
+		end := offset + multipartSize
+		if end > totalSize {
+			end = totalSize
+		}
+		partData := testData[offset:end]
+
+		uploadResp, err := client.UploadPart(ctx, &s3.UploadPartInput{
+			Bucket:     aws.String(bucketName),
+			Key:        aws.String(objectKey),
+			UploadId:   uploadId,
+			PartNumber: aws.Int32(partNumber),
+			Body:       bytes.NewReader(partData),
+		})
+		require.NoError(t, err, "Failed to upload part %d", partNumber)
+
+		completedParts = append(completedParts, types.CompletedPart{
+			ETag:       uploadResp.ETag,
+			PartNumber: aws.Int32(partNumber),
+		})
+		partNumber++
+	}
+
+	// Complete multipart upload
+	completeResp, err := client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: uploadId,
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: completedParts,
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+
+	// Verify ETag format - SHOULD be composite for multipart
+	etag := aws.ToString(completeResp.ETag)
+	t.Logf("Multipart upload ETag: %s", etag)
+
+	assert.True(t, isCompositeETag(etag),
+		"Multipart upload ETag SHOULD be composite format (md5-count), got: %s", etag)
+	assert.True(t, strings.Contains(cleanETag(etag), "-"),
+		"Multipart ETag should contain hyphen, got: %s", etag)
+
+	// Verify the part count in the ETag matches
+	parts := strings.Split(cleanETag(etag), "-")
+	require.Len(t, parts, 2, "Composite ETag should have format 'hash-count'")
+	assert.Equal(t, fmt.Sprintf("%d", len(completedParts)), parts[1],
+		"Part count in ETag should match number of parts uploaded")
+}
+
+// TestPutObjectETagConsistency verifies ETag consistency between PUT and GET
+func TestPutObjectETagConsistency(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testCases := []struct {
+		name string
+		size int
+	}{
+		{"tiny", 100},
+		{"small", smallFileSize},
+		{"medium", mediumFileSize},
+		{"large", largeFileSize},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			testData := generateRandomData(tc.size)
+			objectKey := fmt.Sprintf("consistency-test-%s.bin", tc.name)
+
+			// PUT object
+			putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+				Body:   bytes.NewReader(testData),
+			})
+			require.NoError(t, err)
+			putETag := aws.ToString(putResp.ETag)
+
+			// HEAD object
+			headResp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+			})
+			require.NoError(t, err)
+			headETag := aws.ToString(headResp.ETag)
+
+			// GET object
+			getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+			})
+			require.NoError(t, err)
+			getETag := aws.ToString(getResp.ETag)
+			getResp.Body.Close()
+
+			// All ETags should match
+			t.Logf("%s (%d bytes): PUT=%s, HEAD=%s, GET=%s",
+				tc.name, tc.size, putETag, headETag, getETag)
+
+			assert.Equal(t, putETag, headETag,
+				"PUT and HEAD ETags should match")
+			assert.Equal(t, putETag, getETag,
+				"PUT and GET ETags should match")
+
+			// All should be pure MD5 (not composite) for regular PutObject
+			assert.True(t, isPureMD5ETag(putETag),
+				"PutObject ETag should be pure MD5, got: %s", putETag)
+		})
+	}
+}
+
+// TestETagHexValidation simulates the AWS SDK v2 validation that caused issue #7768
+func TestETagHexValidation(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	// Test with a file large enough to trigger auto-chunking
+	testData := generateRandomData(largeFileSize)
+	objectKey := "hex-validation-test.bin"
+
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err)
+
+	etag := cleanETag(aws.ToString(putResp.ETag))
+
+	// Simulate AWS SDK v2's hex validation (Base16Codec.decode)
+	// This is what fails in issue #7768 when ETag contains '-'
+	t.Logf("Validating ETag as hex: %s", etag)
+
+	_, err = hex.DecodeString(etag)
+	assert.NoError(t, err,
+		"ETag should be valid hexadecimal (AWS SDK v2 validation). "+
+			"Got ETag: %s. If this fails with 'invalid byte', the ETag contains non-hex chars like '-'",
+		etag)
+}
+
+// TestMultipleLargeFileUploads verifies ETag format across multiple large uploads
+func TestMultipleLargeFileUploads(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	numFiles := 5
+	for i := 0; i < numFiles; i++ {
+		testData := generateRandomData(largeFileSize)
+		expectedMD5 := calculateMD5(testData)
+		objectKey := fmt.Sprintf("large-file-%d.bin", i)
+
+		putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(objectKey),
+			Body:   bytes.NewReader(testData),
+		})
+		require.NoError(t, err, "Failed to upload file %d", i)
+
+		etag := aws.ToString(putResp.ETag)
+		t.Logf("File %d ETag: %s (expected MD5: %s)", i, etag, expectedMD5)
+
+		assert.True(t, isPureMD5ETag(etag),
+			"File %d ETag should be pure MD5, got: %s", i, etag)
+		assert.Equal(t, expectedMD5, cleanETag(etag),
+			"File %d ETag should match MD5", i)
+
+		// Validate as hex (AWS SDK v2 check)
+		_, err = hex.DecodeString(cleanETag(etag))
+		assert.NoError(t, err, "File %d ETag should be valid hex", i)
+	}
+}
+
@@ -0,0 +1,20 @@
+{
+  "endpoint": "http://127.0.0.1:8333",
+  "access_key": "some_access_key1",
+  "secret_key": "some_secret_key1",
+  "region": "us-east-1",
+  "bucket_prefix": "test-etag-",
+  "notes": {
+    "description": "S3 ETag format integration tests",
+    "issue": "https://github.com/seaweedfs/seaweedfs/issues/7768",
+    "auto_chunk_size": "8MB - files larger than this trigger auto-chunking",
+    "test_sizes": {
+      "small": "1KB - single chunk",
+      "medium": "256KB - at inline threshold", 
+      "large": "10MB - 2 internal chunks",
+      "xl": "25MB - 4 internal chunks"
+    }
+  }
+}
+
+
@@ -5,4 +5,4 @@
  "secret_key": "some_secret_key1",
  "region": "us-east-1",
  "filer_group": "testgroup"
-}
+}
@@ -558,13 +558,14 @@ func TestS3IAMPresignedURLIntegration(t *testing.T) {
 	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
 	require.NoError(t, err)

-	// Use static bucket name but with cleanup to handle conflicts
-	err = framework.CreateBucketWithCleanup(adminClient, testBucket)
+	// Use unique bucket name to avoid conflicts with other tests
+	bucketName := framework.GenerateUniqueBucketName("test-iam-presigned")
+	err = framework.CreateBucket(adminClient, bucketName)
 	require.NoError(t, err)

 	// Put test object
 	_, err = adminClient.PutObject(&s3.PutObjectInput{
-		Bucket: aws.String(testBucket),
+		Bucket: aws.String(bucketName),
 		Key:    aws.String(testObjectKey),
 		Body:   strings.NewReader(testObjectData),
 	})
@@ -586,7 +587,7 @@ func TestS3IAMPresignedURLIntegration(t *testing.T) {

 		// Test direct object access with JWT Bearer token (recommended approach)
 		_, err := adminClient.GetObject(&s3.GetObjectInput{
-			Bucket: aws.String(testBucket),
+			Bucket: aws.String(bucketName),
 			Key:    aws.String(testObjectKey),
 		})
 		require.NoError(t, err, "Direct object access with JWT Bearer token works correctly")
@@ -597,13 +598,13 @@ func TestS3IAMPresignedURLIntegration(t *testing.T) {

 	// Cleanup
 	_, err = adminClient.DeleteObject(&s3.DeleteObjectInput{
-		Bucket: aws.String(testBucket),
+		Bucket: aws.String(bucketName),
 		Key:    aws.String(testObjectKey),
 	})
 	require.NoError(t, err)

 	_, err = adminClient.DeleteBucket(&s3.DeleteBucketInput{
-		Bucket: aws.String(testBucket),
+		Bucket: aws.String(bucketName),
 	})
 	require.NoError(t, err)
 }
@@ -36,7 +36,7 @@
    "sts": {
      "tokenDuration": "15m",
      "issuer": "seaweedfs-sts",
-      "signingKey": "test-sts-signing-key-for-integration-tests"
+      "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc="
    },
    "policy": {
      "defaultEffect": "Deny"
@@ -0,0 +1,210 @@
+# Remote Storage Cache Integration Tests
+# Tests the remote object caching functionality with singleflight deduplication
+# Uses two SeaweedFS instances: primary (with caching) and secondary (as remote storage)
+
+.PHONY: all help build-weed check-deps start-remote stop-remote start-primary stop-primary \
+        setup-remote test test-with-server clean logs logs-primary logs-remote health
+
+all: test-with-server
+
+# Configuration
+WEED_BINARY := ../../../weed/weed_binary
+
+# Primary SeaweedFS (the one being tested - has remote caching)
+PRIMARY_S3_PORT := 8333
+PRIMARY_FILER_PORT := 8888
+PRIMARY_MASTER_PORT := 9333
+PRIMARY_VOLUME_PORT := 8080
+PRIMARY_METRICS_PORT := 9324
+PRIMARY_DIR := ./test-primary-data
+
+# Secondary SeaweedFS (acts as "remote" S3 storage)
+REMOTE_S3_PORT := 8334
+REMOTE_FILER_PORT := 8889
+REMOTE_MASTER_PORT := 9334
+REMOTE_VOLUME_PORT := 8081
+REMOTE_METRICS_PORT := 9325
+REMOTE_DIR := ./test-remote-data
+
+# Test configuration
+TEST_TIMEOUT := 10m
+TEST_PATTERN := TestRemoteCache
+
+# Buckets
+REMOTE_BUCKET := remotesourcebucket
+
+# Default target
+help:
+	@echo "Remote Storage Cache Integration Tests"
+	@echo ""
+	@echo "Uses two SeaweedFS instances:"
+	@echo "  - Primary (port $(PRIMARY_S3_PORT)): Being tested, has remote caching"
+	@echo "  - Remote  (port $(REMOTE_S3_PORT)): Acts as remote S3 storage"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  help              - Show this help message"
+	@echo "  build-weed        - Build the SeaweedFS binary"
+	@echo "  check-deps        - Check dependencies"
+	@echo "  start-remote      - Start remote SeaweedFS (secondary)"
+	@echo "  stop-remote       - Stop remote SeaweedFS"
+	@echo "  start-primary     - Start primary SeaweedFS"
+	@echo "  stop-primary      - Stop primary SeaweedFS"
+	@echo "  setup-remote      - Configure remote storage mount"
+	@echo "  test              - Run tests (assumes servers are running)"
+	@echo "  test-with-server  - Start servers, run tests, stop servers"
+	@echo "  clean             - Clean up all resources"
+	@echo "  logs              - Show server logs"
+
+# Build the SeaweedFS binary
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd ../../../weed && go build -o weed_binary .
+	@chmod +x $(WEED_BINARY)
+	@echo "SeaweedFS binary built"
+
+check-deps: build-weed
+	@echo "Checking dependencies..."
+	@command -v go >/dev/null 2>&1 || (echo "Go is required" && exit 1)
+	@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found" && exit 1)
+	@echo "All dependencies available"
+
+# Start remote SeaweedFS (acts as the "remote" S3 storage)
+start-remote: check-deps
+	@echo "Starting remote SeaweedFS (secondary instance)..."
+	@rm -f remote-server.pid
+	@mkdir -p $(REMOTE_DIR)
+	@$(WEED_BINARY) server \
+		-s3 \
+		-s3.port=$(REMOTE_S3_PORT) \
+		-s3.allowDeleteBucketNotEmpty=true \
+		-filer \
+		-filer.port=$(REMOTE_FILER_PORT) \
+		-master.port=$(REMOTE_MASTER_PORT) \
+		-volume.port=$(REMOTE_VOLUME_PORT) \
+		-master.volumeSizeLimitMB=50 \
+		-volume.max=100 \
+		-dir=$(REMOTE_DIR) \
+		-volume.preStopSeconds=1 \
+		-metricsPort=$(REMOTE_METRICS_PORT) \
+		> remote-weed.log 2>&1 & echo $$! > remote-server.pid
+	@echo "Waiting for remote SeaweedFS to start..."
+	@for i in $$(seq 1 60); do \
+		if curl -s http://localhost:$(REMOTE_S3_PORT) >/dev/null 2>&1; then \
+			echo "Remote SeaweedFS started on port $(REMOTE_S3_PORT)"; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "ERROR: Remote SeaweedFS failed to start"; \
+	cat remote-weed.log; \
+	exit 1
+
+stop-remote:
+	@echo "Stopping remote SeaweedFS..."
+	@if [ -f remote-server.pid ]; then \
+		kill -TERM $$(cat remote-server.pid) 2>/dev/null || true; \
+		sleep 2; \
+		kill -KILL $$(cat remote-server.pid) 2>/dev/null || true; \
+		rm -f remote-server.pid; \
+	fi
+	@echo "Remote SeaweedFS stopped"
+
+# Start primary SeaweedFS (the one being tested)
+start-primary: check-deps
+	@echo "Starting primary SeaweedFS..."
+	@rm -f primary-server.pid
+	@mkdir -p $(PRIMARY_DIR)
+	@$(WEED_BINARY) server \
+		-s3 \
+		-s3.port=$(PRIMARY_S3_PORT) \
+		-s3.allowDeleteBucketNotEmpty=true \
+		-filer \
+		-filer.port=$(PRIMARY_FILER_PORT) \
+		-master.port=$(PRIMARY_MASTER_PORT) \
+		-volume.port=$(PRIMARY_VOLUME_PORT) \
+		-master.volumeSizeLimitMB=50 \
+		-volume.max=100 \
+		-dir=$(PRIMARY_DIR) \
+		-volume.preStopSeconds=1 \
+		-metricsPort=$(PRIMARY_METRICS_PORT) \
+		> primary-weed.log 2>&1 & echo $$! > primary-server.pid
+	@echo "Waiting for primary SeaweedFS to start..."
+	@for i in $$(seq 1 60); do \
+		if curl -s http://localhost:$(PRIMARY_S3_PORT) >/dev/null 2>&1; then \
+			echo "Primary SeaweedFS started on port $(PRIMARY_S3_PORT)"; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "ERROR: Primary SeaweedFS failed to start"; \
+	cat primary-weed.log; \
+	exit 1
+
+stop-primary:
+	@echo "Stopping primary SeaweedFS..."
+	@if [ -f primary-server.pid ]; then \
+		kill -TERM $$(cat primary-server.pid) 2>/dev/null || true; \
+		sleep 2; \
+		kill -KILL $$(cat primary-server.pid) 2>/dev/null || true; \
+		rm -f primary-server.pid; \
+	fi
+	@echo "Primary SeaweedFS stopped"
+
+# Create bucket on remote and configure remote storage mount on primary
+setup-remote:
+	@echo "Creating bucket on remote SeaweedFS..."
+	@curl -s -X PUT "http://localhost:$(REMOTE_S3_PORT)/$(REMOTE_BUCKET)" || echo "Bucket may already exist"
+	@sleep 1
+	@echo "Configuring remote storage on primary..."
+	@printf 'remote.configure -name=seaweedremote -type=s3 -s3.access_key=any -s3.secret_key=any -s3.endpoint=http://localhost:$(REMOTE_S3_PORT) -s3.region=us-east-1\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.configure done"
+	@sleep 2
+	@echo "Mounting remote bucket on primary..."
+	@printf 'remote.mount -dir=/buckets/remotemounted -remote=seaweedremote/$(REMOTE_BUCKET) -nonempty\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.mount done"
+	@sleep 1
+	@echo "Remote storage configured"
+
+# Run tests
+test: check-deps
+	@echo "Running remote cache tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" .
+	@echo "Tests completed"
+
+# Full test workflow
+test-with-server: start-remote start-primary
+	@sleep 3
+	@$(MAKE) setup-remote || (echo "Remote setup failed" && $(MAKE) stop-primary stop-remote && exit 1)
+	@sleep 2
+	@echo "Running remote cache tests..."
+	@$(MAKE) test || (echo "Tests failed" && tail -50 primary-weed.log && $(MAKE) stop-primary stop-remote && exit 1)
+	@$(MAKE) stop-primary stop-remote
+	@echo "All tests passed"
+
+# Show logs
+logs:
+	@echo "=== Primary SeaweedFS Logs ==="
+	@if [ -f primary-weed.log ]; then tail -50 primary-weed.log; else echo "No log file"; fi
+	@echo ""
+	@echo "=== Remote SeaweedFS Logs ==="
+	@if [ -f remote-weed.log ]; then tail -50 remote-weed.log; else echo "No log file"; fi
+
+logs-primary:
+	@if [ -f primary-weed.log ]; then tail -f primary-weed.log; else echo "No log file"; fi
+
+logs-remote:
+	@if [ -f remote-weed.log ]; then tail -f remote-weed.log; else echo "No log file"; fi
+
+# Clean up
+clean:
+	@$(MAKE) stop-primary
+	@$(MAKE) stop-remote
+	@rm -f primary-weed.log remote-weed.log primary-server.pid remote-server.pid
+	@rm -rf $(PRIMARY_DIR) $(REMOTE_DIR)
+	@rm -f remote_cache.test
+	@go clean -testcache
+	@echo "Cleanup completed"
+
+# Health check
+health:
+	@echo "Checking server status..."
+	@curl -s http://localhost:$(PRIMARY_S3_PORT) >/dev/null 2>&1 && echo "Primary S3 ($(PRIMARY_S3_PORT)): UP" || echo "Primary S3 ($(PRIMARY_S3_PORT)): DOWN"
+	@curl -s http://localhost:$(REMOTE_S3_PORT) >/dev/null 2>&1 && echo "Remote S3 ($(REMOTE_S3_PORT)): UP" || echo "Remote S3 ($(REMOTE_S3_PORT)): DOWN"
@@ -0,0 +1,157 @@
+# Remote Object Cache Integration Tests
+
+This directory contains integration tests for the remote object caching feature with singleflight deduplication.
+
+## Test Flow
+
+Each test follows this pattern:
+1. **Write to local** - Upload data to primary SeaweedFS (local storage)
+2. **Uncache** - Push data to remote storage and remove local chunks
+3. **Read** - Read data (triggers caching from remote back to local)
+
+This tests the full remote caching workflow including singleflight deduplication.
+
+## Architecture
+
+```text
+┌─────────────────────────────────────────────────────────────────┐
+│                        Test Client                               │
+│                                                                  │
+│    1. PUT data to primary SeaweedFS                             │
+│    2. remote.cache.uncache (push to remote, purge local)        │
+│    3. GET data (triggers caching from remote)                   │
+│    4. Verify singleflight deduplication                         │
+└──────────────────────────────────┬──────────────────────────────┘
+                                   │
+                 ┌─────────────────┴─────────────────┐
+                 ▼                                   ▼
+┌────────────────────────────────────┐   ┌────────────────────────────────┐
+│     Primary SeaweedFS              │   │     Remote SeaweedFS           │
+│        (port 8333)                 │   │        (port 8334)             │
+│                                    │   │                                │
+│  - Being tested                    │   │  - Acts as "remote" S3         │
+│  - Has remote storage mounted      │──▶│  - Receives uncached data      │
+│  - Caches remote objects           │   │  - Serves data for caching     │
+│  - Singleflight deduplication      │   │                                │
+└────────────────────────────────────┘   └────────────────────────────────┘
+```
+
+## What's Being Tested
+
+1. **Basic Remote Caching**: Write → Uncache → Read workflow
+2. **Singleflight Deduplication**: Concurrent reads only trigger ONE caching operation
+3. **Large Object Caching**: 5MB files cache correctly
+4. **Range Requests**: Partial reads work with cached objects
+5. **Not Found Handling**: Proper error for non-existent objects
+
+## Quick Start
+
+### Run Full Test Suite (Recommended)
+
+```bash
+# Build SeaweedFS, start both servers, run tests, stop servers
+make test-with-server
+```
+
+### Manual Steps
+
+```bash
+# 1. Build SeaweedFS binary
+make build-weed
+
+# 2. Start remote SeaweedFS (acts as "remote" storage)
+make start-remote
+
+# 3. Start primary SeaweedFS (the one being tested)
+make start-primary
+
+# 4. Configure remote storage mount
+make setup-remote
+
+# 5. Run tests
+make test
+
+# 6. Clean up
+make clean
+```
+
+## Configuration
+
+### Primary SeaweedFS (Being Tested)
+
+| Service | Port |
+|---------|------|
+| S3 API | 8333 |
+| Filer | 8888 |
+| Master | 9333 |
+| Volume | 8080 |
+
+### Remote SeaweedFS (Remote Storage)
+
+| Service | Port |
+|---------|------|
+| S3 API | 8334 |
+| Filer | 8889 |
+| Master | 9334 |
+| Volume | 8081 |
+
+## Makefile Targets
+
+```bash
+make help           # Show all available targets
+make build-weed     # Build SeaweedFS binary
+make start-remote   # Start remote SeaweedFS
+make start-primary  # Start primary SeaweedFS
+make setup-remote   # Configure remote storage mount
+make test           # Run tests
+make test-with-server  # Full automated test workflow
+make logs           # Show server logs
+make health         # Check server status
+make clean          # Stop servers and clean up
+```
+
+## Test Details
+
+### TestRemoteCacheBasic
+Basic workflow test:
+1. Write object to primary (local)
+2. Uncache (push to remote, remove local chunks)
+3. Read (triggers caching from remote)
+4. Read again (from local cache - should be faster)
+
+### TestRemoteCacheConcurrent
+Singleflight deduplication test:
+1. Write 1MB object
+2. Uncache to remote
+3. Launch 10 concurrent reads
+4. All should succeed with correct data
+5. Only ONE caching operation should run (singleflight)
+
+### TestRemoteCacheLargeObject
+Large file test (5MB) to verify chunked transfer works correctly.
+
+### TestRemoteCacheRangeRequest
+Tests HTTP range requests work correctly after caching.
+
+### TestRemoteCacheNotFound
+Tests proper error handling for non-existent objects.
+
+## Troubleshooting
+
+### View logs
+```bash
+make logs           # Show recent logs from both servers
+make logs-primary   # Follow primary logs in real-time
+make logs-remote    # Follow remote logs in real-time
+```
+
+### Check server health
+```bash
+make health
+```
+
+### Clean up and retry
+```bash
+make clean
+make test-with-server
+```
@@ -0,0 +1,375 @@
+package remote_cache
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Test configuration
+// Uses two SeaweedFS instances:
+// - Primary: The one being tested (has remote caching)
+// - Remote: Acts as the "remote" S3 storage
+const (
+	// Primary SeaweedFS
+	primaryEndpoint   = "http://localhost:8333"
+	primaryMasterPort = "9333"
+
+	// Remote SeaweedFS (acts as remote storage)
+	remoteEndpoint = "http://localhost:8334"
+
+	// Credentials (anonymous access for testing)
+	accessKey = "any"
+	secretKey = "any"
+
+	// Bucket name - mounted on primary as remote storage
+	testBucket = "remotemounted"
+
+	// Path to weed binary
+	weedBinary = "../../../weed/weed_binary"
+)
+
+var (
+	primaryClient     *s3.S3
+	primaryClientOnce sync.Once
+)
+
+func getPrimaryClient() *s3.S3 {
+	primaryClientOnce.Do(func() {
+		primaryClient = createS3Client(primaryEndpoint)
+	})
+	return primaryClient
+}
+
+func createS3Client(endpoint string) *s3.S3 {
+	sess, err := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String(endpoint),
+		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
+		DisableSSL:       aws.Bool(!strings.HasPrefix(endpoint, "https")),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	if err != nil {
+		panic(fmt.Sprintf("failed to create session: %v", err))
+	}
+	return s3.New(sess)
+}
+
+// skipIfNotRunning skips the test if the servers aren't running
+func skipIfNotRunning(t *testing.T) {
+	resp, err := http.Get(primaryEndpoint)
+	if err != nil {
+		t.Skipf("Primary SeaweedFS not running at %s: %v", primaryEndpoint, err)
+	}
+	resp.Body.Close()
+
+	resp, err = http.Get(remoteEndpoint)
+	if err != nil {
+		t.Skipf("Remote SeaweedFS not running at %s: %v", remoteEndpoint, err)
+	}
+	resp.Body.Close()
+}
+
+// runWeedShell executes a weed shell command
+func runWeedShell(t *testing.T, command string) (string, error) {
+	cmd := exec.Command(weedBinary, "shell", "-master=localhost:"+primaryMasterPort)
+	cmd.Stdin = strings.NewReader(command + "\nexit\n")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Logf("weed shell command '%s' failed: %v, output: %s", command, err, string(output))
+		return string(output), err
+	}
+	return string(output), nil
+}
+
+// uploadToPrimary uploads an object to the primary SeaweedFS (local write)
+func uploadToPrimary(t *testing.T, key string, data []byte) {
+	_, err := getPrimaryClient().PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(key),
+		Body:   bytes.NewReader(data),
+	})
+	require.NoError(t, err, "failed to upload to primary SeaweedFS")
+}
+
+// getFromPrimary gets an object from primary SeaweedFS
+func getFromPrimary(t *testing.T, key string) []byte {
+	resp, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(key),
+	})
+	require.NoError(t, err, "failed to get from primary SeaweedFS")
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
+	require.NoError(t, err, "failed to read response body")
+	return data
+}
+
+// syncToRemote syncs local data to remote storage
+func syncToRemote(t *testing.T) {
+	t.Log("Syncing to remote storage...")
+	output, err := runWeedShell(t, "remote.cache.uncache -dir=/buckets/"+testBucket+" -include=*")
+	if err != nil {
+		t.Logf("syncToRemote warning: %v", err)
+	}
+	t.Log(output)
+	time.Sleep(1 * time.Second)
+}
+
+// uncacheLocal purges the local cache, forcing data to be fetched from remote
+func uncacheLocal(t *testing.T, pattern string) {
+	t.Logf("Purging local cache for pattern: %s", pattern)
+	output, err := runWeedShell(t, fmt.Sprintf("remote.uncache -dir=/buckets/%s -include=%s", testBucket, pattern))
+	if err != nil {
+		t.Logf("uncacheLocal warning: %v", err)
+	}
+	t.Log(output)
+	time.Sleep(500 * time.Millisecond)
+}
+
+// TestRemoteCacheBasic tests the basic caching workflow:
+// 1. Write to local
+// 2. Uncache (push to remote, remove local chunks)
+// 3. Read (triggers caching from remote)
+func TestRemoteCacheBasic(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("test-basic-%d.txt", time.Now().UnixNano())
+	testData := []byte("Hello, this is test data for remote caching!")
+
+	// Step 1: Write to local
+	t.Log("Step 1: Writing object to primary SeaweedFS (local)...")
+	uploadToPrimary(t, testKey, testData)
+
+	// Verify it's readable
+	result := getFromPrimary(t, testKey)
+	assert.Equal(t, testData, result, "initial read mismatch")
+
+	// Step 2: Uncache - push to remote and remove local chunks
+	t.Log("Step 2: Uncaching (pushing to remote, removing local chunks)...")
+	uncacheLocal(t, testKey)
+
+	// Step 3: Read - this should trigger caching from remote
+	t.Log("Step 3: Reading object (should trigger caching from remote)...")
+	start := time.Now()
+	result = getFromPrimary(t, testKey)
+	firstReadDuration := time.Since(start)
+
+	assert.Equal(t, testData, result, "data mismatch after cache")
+	t.Logf("First read (from remote) took %v", firstReadDuration)
+
+	// Step 4: Read again - should be from local cache
+	t.Log("Step 4: Reading again (should be from local cache)...")
+	start = time.Now()
+	result = getFromPrimary(t, testKey)
+	secondReadDuration := time.Since(start)
+
+	assert.Equal(t, testData, result, "data mismatch on cached read")
+	t.Logf("Second read (from cache) took %v", secondReadDuration)
+
+	t.Log("Basic caching test passed")
+}
+
+// TestRemoteCacheConcurrent tests that concurrent reads of the same
+// remote object only trigger ONE caching operation (singleflight deduplication)
+func TestRemoteCacheConcurrent(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("test-concurrent-%d.txt", time.Now().UnixNano())
+	// Use larger data to make caching take measurable time
+	testData := make([]byte, 1024*1024) // 1MB
+	for i := range testData {
+		testData[i] = byte(i % 256)
+	}
+
+	// Step 1: Write to local
+	t.Log("Step 1: Writing 1MB object to primary SeaweedFS...")
+	uploadToPrimary(t, testKey, testData)
+
+	// Verify it's readable
+	result := getFromPrimary(t, testKey)
+	assert.Equal(t, len(testData), len(result), "initial size mismatch")
+
+	// Step 2: Uncache
+	t.Log("Step 2: Uncaching (pushing to remote)...")
+	uncacheLocal(t, testKey)
+
+	// Step 3: Launch many concurrent reads - singleflight should deduplicate
+	numRequests := 10
+	var wg sync.WaitGroup
+	var successCount atomic.Int32
+	var errorCount atomic.Int32
+	results := make(chan []byte, numRequests)
+
+	t.Logf("Step 3: Launching %d concurrent requests...", numRequests)
+	startTime := time.Now()
+
+	for i := 0; i < numRequests; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			resp, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
+				Bucket: aws.String(testBucket),
+				Key:    aws.String(testKey),
+			})
+			if err != nil {
+				t.Logf("Request %d failed: %v", idx, err)
+				errorCount.Add(1)
+				return
+			}
+			defer resp.Body.Close()
+
+			data, err := io.ReadAll(resp.Body)
+			if err != nil {
+				t.Logf("Request %d read failed: %v", idx, err)
+				errorCount.Add(1)
+				return
+			}
+
+			results <- data
+			successCount.Add(1)
+		}(i)
+	}
+
+	wg.Wait()
+	close(results)
+	totalDuration := time.Since(startTime)
+
+	t.Logf("All %d requests completed in %v", numRequests, totalDuration)
+	t.Logf("Successful: %d, Failed: %d", successCount.Load(), errorCount.Load())
+
+	// Verify all successful requests returned correct data
+	for data := range results {
+		assert.Equal(t, len(testData), len(data), "data length mismatch")
+	}
+
+	// All requests should succeed
+	assert.Equal(t, int32(numRequests), successCount.Load(), "some requests failed")
+	assert.Equal(t, int32(0), errorCount.Load(), "no requests should fail")
+
+	t.Log("Concurrent caching test passed")
+}
+
+// TestRemoteCacheLargeObject tests caching of larger objects
+func TestRemoteCacheLargeObject(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("test-large-%d.bin", time.Now().UnixNano())
+	// 5MB object
+	testData := make([]byte, 5*1024*1024)
+	for i := range testData {
+		testData[i] = byte(i % 256)
+	}
+
+	// Step 1: Write to local
+	t.Log("Step 1: Writing 5MB object to primary SeaweedFS...")
+	uploadToPrimary(t, testKey, testData)
+
+	// Verify it's readable
+	result := getFromPrimary(t, testKey)
+	assert.Equal(t, len(testData), len(result), "initial size mismatch")
+
+	// Step 2: Uncache
+	t.Log("Step 2: Uncaching...")
+	uncacheLocal(t, testKey)
+
+	// Step 3: Read from remote
+	t.Log("Step 3: Reading 5MB object (should cache from remote)...")
+	start := time.Now()
+	result = getFromPrimary(t, testKey)
+	duration := time.Since(start)
+
+	assert.Equal(t, len(testData), len(result), "size mismatch")
+	assert.Equal(t, testData, result, "data mismatch")
+	t.Logf("Large object cached in %v", duration)
+
+	t.Log("Large object caching test passed")
+}
+
+// TestRemoteCacheRangeRequest tests that range requests work after caching
+func TestRemoteCacheRangeRequest(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("test-range-%d.txt", time.Now().UnixNano())
+	testData := []byte("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+	// Write, uncache, then test range request
+	t.Log("Writing and uncaching object...")
+	uploadToPrimary(t, testKey, testData)
+	uncacheLocal(t, testKey)
+
+	// Range request should work and trigger caching
+	t.Log("Testing range request (bytes 10-19)...")
+	resp, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(testKey),
+		Range:  aws.String("bytes=10-19"),
+	})
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	rangeData, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	expected := testData[10:20] // "ABCDEFGHIJ"
+	assert.Equal(t, expected, rangeData, "range data mismatch")
+	t.Logf("Range request returned: %s", string(rangeData))
+
+	t.Log("Range request test passed")
+}
+
+// TestRemoteCacheNotFound tests that non-existent objects return proper errors
+func TestRemoteCacheNotFound(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("non-existent-object-%d", time.Now().UnixNano())
+
+	_, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(testKey),
+	})
+
+	assert.Error(t, err, "should get error for non-existent object")
+	t.Logf("Got expected error: %v", err)
+
+	t.Log("Not found test passed")
+}
+
+// TestMain sets up and tears down the test environment
+func TestMain(m *testing.M) {
+	if !isServerRunning(primaryEndpoint) {
+		fmt.Println("WARNING: Primary SeaweedFS not running at", primaryEndpoint)
+		fmt.Println("   Run 'make test-with-server' to start servers automatically")
+	}
+	if !isServerRunning(remoteEndpoint) {
+		fmt.Println("WARNING: Remote SeaweedFS not running at", remoteEndpoint)
+		fmt.Println("   Run 'make test-with-server' to start servers automatically")
+	}
+
+	os.Exit(m.Run())
+}
+
+func isServerRunning(url string) bool {
+	resp, err := http.Get(url)
+	if err != nil {
+		return false
+	}
+	resp.Body.Close()
+	return true
+}
@@ -81,7 +81,7 @@ start-server: check-deps
 	@echo "🔍 DEBUG: Creating volume directory..."
 	@mkdir -p ./test-volume-data
 	@echo "🔍 DEBUG: Launching SeaweedFS server in background..."
-	@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324"
+	@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -master.peers=none -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324"
 	@$(WEED_BINARY) server \
 		-debug \
 		-s3 \
@@ -91,6 +91,7 @@ start-server: check-deps
 		-filer \
 		-filer.maxMB=64 \
 		-master.volumeSizeLimitMB=50 \
+		-master.peers=none \
 		-volume.max=100 \
 		-dir=./test-volume-data \
 		-volume.preStopSeconds=1 \
@@ -221,13 +222,13 @@ test-with-server: start-server
 test-versioning-with-configs: check-deps
 	@echo "Testing with different S3 configurations..."
 	@echo "Testing with empty folder allowed..."
-	@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
+	@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -filer -master.volumeSizeLimitMB=100 -master.peers=none -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
 	@sleep 5
 	@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
 	@if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi
 	@sleep 2
 	@echo "Testing with delete bucket not empty disabled..."
-	@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
+	@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -master.peers=none -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
 	@sleep 5
 	@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
 	@if [ -f weed-config2.pid ]; then kill -TERM $$(cat weed-config2.pid) 2>/dev/null || true; rm -f weed-config2.pid; fi
@@ -239,6 +240,13 @@ test-versioning-stress: check-deps
 	@go test -v -timeout=20m -run "TestVersioningConcurrentOperations" . -count=5
 	@echo "✅ Stress tests completed"

+# Pagination stress testing (tests >1000 versions)
+test-versioning-pagination-stress: check-deps
+	@echo "Running pagination stress tests (>1000 versions)..."
+	@echo "This test creates 1500+ versions and may take several minutes..."
+	@ENABLE_STRESS_TESTS=true go test -v -timeout=30m -run "TestVersioningPagination" .
+	@echo "✅ Pagination stress tests completed"
+
 # Generate test reports
 test-report: check-deps
 	@echo "Generating test reports..."
@@ -272,6 +280,7 @@ debug-server:
 		-filer \
 		-filer.maxMB=16 \
 		-master.volumeSizeLimitMB=50 \
+		-master.peers=none \
 		-volume.max=100 \
 		-dir=./test-volume-data \
 		-volume.preStopSeconds=1 \
@@ -320,6 +329,7 @@ start-server-simple: check-deps
 		-filer \
 		-filer.maxMB=64 \
 		-master.volumeSizeLimitMB=50 \
+		-master.peers=none \
 		-volume.max=100 \
 		-volume.preStopSeconds=1 \
 		-metricsPort=9324 \
@@ -0,0 +1,521 @@
+package s3api
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestMultipartUploadVersioningListETag tests that multipart uploaded objects
+// in versioned buckets have correct ETags when listed.
+// This covers a bug where synthetic entries for versioned objects didn't include
+// proper ETag handling for multipart uploads (ETags with format "<md5>-<parts>").
+func TestMultipartUploadVersioningListETag(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err, "Failed to enable versioning")
+
+	// Create multipart upload
+	objectKey := "multipart-test-object"
+	createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	uploadId := *createResp.UploadId
+
+	// Upload 2 parts (minimum 5MB per part except last)
+	partSize := 5 * 1024 * 1024 // 5MB
+	part1Data := bytes.Repeat([]byte("a"), partSize)
+	part2Data := bytes.Repeat([]byte("b"), partSize)
+
+	// Calculate MD5 for each part
+	part1MD5 := md5.Sum(part1Data)
+	part2MD5 := md5.Sum(part2Data)
+
+	// Upload part 1
+	uploadPart1Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   aws.String(uploadId),
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(part1Data),
+	})
+	require.NoError(t, err, "Failed to upload part 1")
+
+	// Upload part 2
+	uploadPart2Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   aws.String(uploadId),
+		PartNumber: aws.Int32(2),
+		Body:       bytes.NewReader(part2Data),
+	})
+	require.NoError(t, err, "Failed to upload part 2")
+
+	// Complete multipart upload
+	completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: aws.String(uploadId),
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: []types.CompletedPart{
+				{
+					ETag:       uploadPart1Resp.ETag,
+					PartNumber: aws.Int32(1),
+				},
+				{
+					ETag:       uploadPart2Resp.ETag,
+					PartNumber: aws.Int32(2),
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+
+	// Verify the ETag from CompleteMultipartUpload has the multipart format (md5-parts)
+	completeETag := strings.Trim(*completeResp.ETag, "\"")
+	assert.Contains(t, completeETag, "-", "Multipart ETag should contain '-' (format: md5-parts)")
+	assert.True(t, strings.HasSuffix(completeETag, "-2"), "Multipart ETag should end with '-2' for 2 parts")
+
+	t.Logf("CompleteMultipartUpload ETag: %s", completeETag)
+	t.Logf("Part 1 MD5: %x", part1MD5)
+	t.Logf("Part 2 MD5: %x", part2MD5)
+
+	// HeadObject should return the same ETag
+	headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to head object")
+
+	headETag := strings.Trim(*headResp.ETag, "\"")
+	assert.Equal(t, completeETag, headETag, "HeadObject ETag should match CompleteMultipartUpload ETag")
+
+	// ListObjectsV2 should return the same ETag
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects")
+	require.Len(t, listResp.Contents, 1, "Should have exactly one object")
+
+	listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
+	assert.Equal(t, completeETag, listETag, "ListObjectsV2 ETag should match CompleteMultipartUpload ETag")
+	assert.NotEmpty(t, listETag, "ListObjectsV2 ETag should not be empty")
+
+	t.Logf("ListObjectsV2 ETag: %s", listETag)
+
+	// ListObjectVersions should also return the correct ETag
+	versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list object versions")
+	require.Len(t, versionsResp.Versions, 1, "Should have exactly one version")
+
+	versionETag := strings.Trim(*versionsResp.Versions[0].ETag, "\"")
+	assert.Equal(t, completeETag, versionETag, "ListObjectVersions ETag should match CompleteMultipartUpload ETag")
+	assert.NotEmpty(t, versionETag, "ListObjectVersions ETag should not be empty")
+
+	t.Logf("ListObjectVersions ETag: %s", versionETag)
+}
+
+// TestMultipartUploadMultipleVersionsListETag tests that multiple versions
+// of multipart uploaded objects all have correct ETags when listed.
+func TestMultipartUploadMultipleVersionsListETag(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err, "Failed to enable versioning")
+
+	objectKey := "multipart-multi-version-object"
+	partSize := 5 * 1024 * 1024 // 5MB
+	var expectedETags []string
+
+	// Create 3 versions using multipart upload
+	for version := 1; version <= 3; version++ {
+		// Create multipart upload
+		createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(objectKey),
+		})
+		require.NoError(t, err, "Failed to create multipart upload for version %d", version)
+
+		uploadId := *createResp.UploadId
+
+		// Create unique data for each version
+		partData := bytes.Repeat([]byte(fmt.Sprintf("%d", version)), partSize)
+
+		// Upload single part (still results in multipart ETag format)
+		uploadPartResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+			Bucket:     aws.String(bucketName),
+			Key:        aws.String(objectKey),
+			UploadId:   aws.String(uploadId),
+			PartNumber: aws.Int32(1),
+			Body:       bytes.NewReader(partData),
+		})
+		require.NoError(t, err, "Failed to upload part for version %d", version)
+
+		// Complete multipart upload
+		completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+			Bucket:   aws.String(bucketName),
+			Key:      aws.String(objectKey),
+			UploadId: aws.String(uploadId),
+			MultipartUpload: &types.CompletedMultipartUpload{
+				Parts: []types.CompletedPart{
+					{
+						ETag:       uploadPartResp.ETag,
+						PartNumber: aws.Int32(1),
+					},
+				},
+			},
+		})
+		require.NoError(t, err, "Failed to complete multipart upload for version %d", version)
+
+		etag := strings.Trim(*completeResp.ETag, "\"")
+		expectedETags = append(expectedETags, etag)
+		t.Logf("Version %d ETag: %s", version, etag)
+	}
+
+	// ListObjectVersions should return all versions with correct ETags
+	versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list object versions")
+	require.Len(t, versionsResp.Versions, 3, "Should have exactly 3 versions")
+
+	// Collect ETags from the listing
+	var listedETags []string
+	for _, v := range versionsResp.Versions {
+		etag := strings.Trim(*v.ETag, "\"")
+		listedETags = append(listedETags, etag)
+		assert.NotEmpty(t, etag, "Version ETag should not be empty")
+		assert.Contains(t, etag, "-", "Multipart ETag should contain '-'")
+	}
+
+	t.Logf("Expected ETags: %v", expectedETags)
+	t.Logf("Listed ETags: %v", listedETags)
+
+	// Verify all expected ETags are present (order may differ due to version ordering)
+	assert.ElementsMatch(t, expectedETags, listedETags, "Listed ETags should match all expected ETags")
+
+	// Regular ListObjectsV2 should return only the latest version with correct ETag
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects")
+	require.Len(t, listResp.Contents, 1, "Should have exactly one object in regular listing")
+
+	listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
+	// The latest version (version 3) should be the one shown
+	assert.Equal(t, expectedETags[2], listETag, "ListObjectsV2 should show latest version's ETag")
+}
+
+// TestMixedSingleAndMultipartVersionsListETag tests that a mix of
+// single-part and multipart uploaded versions all have correct ETags.
+func TestMixedSingleAndMultipartVersionsListETag(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err, "Failed to enable versioning")
+
+	objectKey := "mixed-upload-versions"
+
+	// Version 1: Regular PutObject (single-part, pure MD5 ETag)
+	content1 := []byte("This is version 1 content - single part upload")
+	putResp1, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(content1),
+	})
+	require.NoError(t, err, "Failed to put version 1")
+	etag1 := strings.Trim(*putResp1.ETag, "\"")
+	assert.NotContains(t, etag1, "-", "Single-part ETag should not contain '-'")
+	t.Logf("Version 1 (PutObject) ETag: %s", etag1)
+
+	// Version 2: Multipart upload
+	partSize := 5 * 1024 * 1024
+	partData := bytes.Repeat([]byte("x"), partSize)
+
+	createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	uploadPartResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   createResp.UploadId,
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(partData),
+	})
+	require.NoError(t, err, "Failed to upload part")
+
+	completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: createResp.UploadId,
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: []types.CompletedPart{
+				{
+					ETag:       uploadPartResp.ETag,
+					PartNumber: aws.Int32(1),
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+	etag2 := strings.Trim(*completeResp.ETag, "\"")
+	assert.Contains(t, etag2, "-", "Multipart ETag should contain '-'")
+	t.Logf("Version 2 (Multipart) ETag: %s", etag2)
+
+	// Version 3: Another regular PutObject
+	content3 := []byte("This is version 3 content - another single part upload")
+	putResp3, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(content3),
+	})
+	require.NoError(t, err, "Failed to put version 3")
+	etag3 := strings.Trim(*putResp3.ETag, "\"")
+	assert.NotContains(t, etag3, "-", "Single-part ETag should not contain '-'")
+	t.Logf("Version 3 (PutObject) ETag: %s", etag3)
+
+	// ListObjectVersions should return all 3 versions with correct ETags
+	versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list object versions")
+	require.Len(t, versionsResp.Versions, 3, "Should have exactly 3 versions")
+
+	var listedETags []string
+	for _, v := range versionsResp.Versions {
+		etag := strings.Trim(*v.ETag, "\"")
+		assert.NotEmpty(t, etag, "Version ETag should not be empty")
+		listedETags = append(listedETags, etag)
+		t.Logf("Listed version %s ETag: %s, IsLatest: %v", *v.VersionId, etag, *v.IsLatest)
+	}
+
+	// Verify all ETags were found
+	assert.ElementsMatch(t, []string{etag1, etag2, etag3}, listedETags, "Listed ETags should match all expected ETags")
+
+	// Regular ListObjectsV2 should return only the latest (version 3)
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects")
+	require.Len(t, listResp.Contents, 1, "Should have exactly one object")
+
+	listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
+	assert.Equal(t, etag3, listETag, "ListObjectsV2 should show latest version's ETag (version 3)")
+}
+
+// TestMultipartUploadDeleteMarkerListBehavior tests that delete markers work correctly
+// with multipart uploaded objects in versioned buckets.
+func TestMultipartUploadDeleteMarkerListBehavior(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err, "Failed to enable versioning")
+
+	objectKey := "multipart-delete-marker-test"
+	partSize := 5 * 1024 * 1024 // 5MB
+
+	// Create multipart upload
+	createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	// Upload 2 parts
+	part1Data := bytes.Repeat([]byte("a"), partSize)
+	part2Data := bytes.Repeat([]byte("b"), partSize)
+
+	uploadPart1Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   createResp.UploadId,
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(part1Data),
+	})
+	require.NoError(t, err, "Failed to upload part 1")
+
+	uploadPart2Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   createResp.UploadId,
+		PartNumber: aws.Int32(2),
+		Body:       bytes.NewReader(part2Data),
+	})
+	require.NoError(t, err, "Failed to upload part 2")
+
+	// Complete multipart upload
+	completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: createResp.UploadId,
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: []types.CompletedPart{
+				{ETag: uploadPart1Resp.ETag, PartNumber: aws.Int32(1)},
+				{ETag: uploadPart2Resp.ETag, PartNumber: aws.Int32(2)},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+
+	multipartETag := strings.Trim(*completeResp.ETag, "\"")
+	multipartVersionId := *completeResp.VersionId
+	t.Logf("Multipart upload completed: ETag=%s, VersionId=%s", multipartETag, multipartVersionId)
+
+	// Verify object is visible in ListObjectsV2
+	listBeforeDelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects before delete")
+	require.Len(t, listBeforeDelete.Contents, 1, "Object should be visible before delete")
+	assert.Equal(t, multipartETag, strings.Trim(*listBeforeDelete.Contents[0].ETag, "\""),
+		"Listed ETag should match multipart ETag before delete")
+
+	// Delete object (creates delete marker)
+	deleteResp, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to delete object")
+	require.NotNil(t, deleteResp.DeleteMarker, "Should create delete marker")
+	assert.True(t, *deleteResp.DeleteMarker, "DeleteMarker should be true")
+	require.NotNil(t, deleteResp.VersionId, "Delete marker should have version ID")
+
+	deleteMarkerVersionId := *deleteResp.VersionId
+	t.Logf("Delete marker created: VersionId=%s", deleteMarkerVersionId)
+
+	// ListObjectsV2 should NOT show the object anymore
+	listAfterDelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects after delete")
+	assert.Empty(t, listAfterDelete.Contents, "Object should NOT be visible after delete marker")
+
+	// ListObjectVersions should show both the original version AND the delete marker
+	versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list object versions")
+
+	// Should have 1 version (the multipart object)
+	require.Len(t, versionsResp.Versions, 1, "Should have exactly 1 version (the multipart object)")
+	version := versionsResp.Versions[0]
+	assert.Equal(t, multipartVersionId, *version.VersionId, "Version ID should match")
+	assert.Equal(t, multipartETag, strings.Trim(*version.ETag, "\""), "Version ETag should match multipart ETag")
+	assert.False(t, *version.IsLatest, "Multipart version should NOT be latest (delete marker is latest)")
+
+	// Should have 1 delete marker
+	require.Len(t, versionsResp.DeleteMarkers, 1, "Should have exactly 1 delete marker")
+	deleteMarker := versionsResp.DeleteMarkers[0]
+	assert.Equal(t, deleteMarkerVersionId, *deleteMarker.VersionId, "Delete marker version ID should match")
+	assert.True(t, *deleteMarker.IsLatest, "Delete marker should be latest")
+
+	t.Logf("ListObjectVersions: 1 version (ETag=%s), 1 delete marker", multipartETag)
+
+	// Access the specific version by version ID - should still work
+	getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(multipartVersionId),
+	})
+	require.NoError(t, err, "Should be able to get object by version ID after delete marker")
+	defer getResp.Body.Close()
+
+	assert.Equal(t, multipartETag, strings.Trim(*getResp.ETag, "\""),
+		"GetObject with version ID should return correct ETag")
+	assert.Equal(t, int64(partSize*2), *getResp.ContentLength,
+		"GetObject with version ID should return correct size")
+
+	t.Logf("Successfully retrieved version %s after delete marker", multipartVersionId)
+
+	// Delete the delete marker to "undelete" the object
+	_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(deleteMarkerVersionId),
+	})
+	require.NoError(t, err, "Failed to delete the delete marker")
+
+	// ListObjectsV2 should show the object again
+	listAfterUndelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects after undelete")
+	require.Len(t, listAfterUndelete.Contents, 1, "Object should be visible again after removing delete marker")
+	assert.Equal(t, multipartETag, strings.Trim(*listAfterUndelete.Contents[0].ETag, "\""),
+		"Undeleted object should have correct multipart ETag")
+
+	t.Logf("Object restored after delete marker removal, ETag=%s", multipartETag)
+}
+
@@ -0,0 +1,322 @@
+package s3api
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestVersioningPaginationOver1000Versions tests that ListObjectVersions correctly
+// handles objects with more than 1000 versions using pagination.
+// This is a stress test that creates 1500 versions and verifies all are listed.
+//
+// Run with: ENABLE_STRESS_TESTS=true go test -v -run TestVersioningPaginationOver1000Versions -timeout 30m
+func TestVersioningPaginationOver1000Versions(t *testing.T) {
+	if os.Getenv("ENABLE_STRESS_TESTS") != "true" {
+		t.Skip("Skipping stress test. Set ENABLE_STRESS_TESTS=true to run.")
+	}
+
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	enableVersioning(t, client, bucketName)
+	checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
+
+	objectKey := "test-many-versions"
+	// Use 1100 for CI (tests pagination boundary), can override with env var for quick local testing
+	numVersions := 1100
+	if quickTest := os.Getenv("QUICK_TEST"); quickTest == "true" {
+		numVersions = 50
+	}
+	versionIds := make([]string, 0, numVersions)
+
+	t.Logf("Creating %d versions of object %s...", numVersions, objectKey)
+	startTime := time.Now()
+
+	// Create many versions of the same object
+	for i := 0; i < numVersions; i++ {
+		content := fmt.Sprintf("content-version-%d", i+1)
+		putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(objectKey),
+			Body:   strings.NewReader(content),
+		})
+		require.NoError(t, err, "Failed to create version %d", i+1)
+		require.NotNil(t, putResp.VersionId)
+		versionIds = append(versionIds, *putResp.VersionId)
+
+		if (i+1)%100 == 0 {
+			t.Logf("Created %d/%d versions...", i+1, numVersions)
+		}
+	}
+
+	createDuration := time.Since(startTime)
+	t.Logf("Created %d versions in %v", numVersions, createDuration)
+	t.Logf("Version IDs collected: %d", len(versionIds))
+
+	// Quick sanity check - list versions to see what we have
+	quickResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	t.Logf("Quick check: got %d versions, %d delete markers, truncated=%v",
+		len(quickResp.Versions), len(quickResp.DeleteMarkers),
+		quickResp.IsTruncated != nil && *quickResp.IsTruncated)
+
+	// Test 1: List all versions without pagination limit
+	t.Run("ListAllVersionsNoPagination", func(t *testing.T) {
+		allVersions := listAllVersions(t, client, bucketName, objectKey)
+		t.Logf("Got %d versions total", len(allVersions))
+		assert.Len(t, allVersions, numVersions, "Should have all %d versions", numVersions)
+
+		// Verify all version IDs are unique
+		versionIdSet := make(map[string]bool)
+		for _, v := range allVersions {
+			assert.False(t, versionIdSet[*v.VersionId], "Duplicate version ID: %s", *v.VersionId)
+			versionIdSet[*v.VersionId] = true
+		}
+
+		// Verify only one version is marked as latest
+		latestCount := 0
+		for _, v := range allVersions {
+			if *v.IsLatest {
+				latestCount++
+				assert.Equal(t, versionIds[numVersions-1], *v.VersionId, "Latest should be the last created")
+			}
+		}
+		assert.Equal(t, 1, latestCount, "Only one version should be marked as latest")
+	})
+
+	// Test 2: List with explicit small max-keys to force multiple pages
+	t.Run("ListWithSmallMaxKeys", func(t *testing.T) {
+		maxKeys := int32(100)
+		allVersions := make([]types.ObjectVersion, 0, numVersions)
+		var keyMarker, versionIdMarker *string
+		pageCount := 0
+
+		for {
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:          aws.String(bucketName),
+				Prefix:          aws.String(objectKey),
+				MaxKeys:         aws.Int32(maxKeys),
+				KeyMarker:       keyMarker,
+				VersionIdMarker: versionIdMarker,
+			})
+			require.NoError(t, err)
+			pageCount++
+
+			allVersions = append(allVersions, resp.Versions...)
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+			versionIdMarker = resp.NextVersionIdMarker
+		}
+
+		t.Logf("Listed %d versions in %d pages (max-keys=%d)", len(allVersions), pageCount, maxKeys)
+		assert.Len(t, allVersions, numVersions, "Should have all %d versions", numVersions)
+		if numVersions > int(maxKeys) {
+			assert.Greater(t, pageCount, 1, "Should require multiple pages when versions > max-keys")
+		}
+	})
+
+	// Test 3: List with max-keys=1000 (S3 default) to test the boundary
+	t.Run("ListWithDefaultMaxKeys", func(t *testing.T) {
+		maxKeys := int32(1000)
+		allVersions := make([]types.ObjectVersion, 0, numVersions)
+		var keyMarker, versionIdMarker *string
+		pageCount := 0
+
+		for {
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:          aws.String(bucketName),
+				Prefix:          aws.String(objectKey),
+				MaxKeys:         aws.Int32(maxKeys),
+				KeyMarker:       keyMarker,
+				VersionIdMarker: versionIdMarker,
+			})
+			require.NoError(t, err)
+			pageCount++
+
+			allVersions = append(allVersions, resp.Versions...)
+			t.Logf("Page %d: got %d versions, truncated=%v", pageCount, len(resp.Versions), resp.IsTruncated != nil && *resp.IsTruncated)
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+			versionIdMarker = resp.NextVersionIdMarker
+		}
+
+		t.Logf("Listed %d versions in %d pages (max-keys=%d)", len(allVersions), pageCount, maxKeys)
+		assert.Len(t, allVersions, numVersions, "Should have all %d versions", numVersions)
+		if numVersions > 1000 {
+			assert.GreaterOrEqual(t, pageCount, 2, "Should require at least 2 pages for >1000 versions with max-keys=1000")
+		}
+	})
+
+	// Test 4: Verify we can retrieve specific versions
+	t.Run("RetrieveSpecificVersions", func(t *testing.T) {
+		// Test first, middle, and last versions
+		testIndices := []int{0, numVersions / 2, numVersions - 1}
+		for _, idx := range testIndices {
+			expectedContent := fmt.Sprintf("content-version-%d", idx+1)
+			getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+				Bucket:    aws.String(bucketName),
+				Key:       aws.String(objectKey),
+				VersionId: aws.String(versionIds[idx]),
+			})
+			require.NoError(t, err, "Failed to get version at index %d", idx)
+
+			buf := make([]byte, len(expectedContent))
+			_, err = getResp.Body.Read(buf)
+			getResp.Body.Close()
+
+			assert.Equal(t, expectedContent, string(buf), "Content mismatch for version %d", idx+1)
+		}
+	})
+}
+
+// TestVersioningPaginationMultipleObjectsManyVersions tests pagination with multiple objects
+// each having many versions, ensuring correct handling of key-marker and version-id-marker.
+//
+// Run with: ENABLE_STRESS_TESTS=true go test -v -run TestVersioningPaginationMultipleObjectsManyVersions -timeout 30m
+func TestVersioningPaginationMultipleObjectsManyVersions(t *testing.T) {
+	if os.Getenv("ENABLE_STRESS_TESTS") != "true" {
+		t.Skip("Skipping stress test. Set ENABLE_STRESS_TESTS=true to run.")
+	}
+
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	enableVersioning(t, client, bucketName)
+	checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
+
+	numObjects := 5
+	versionsPerObject := 300
+	totalExpectedVersions := numObjects * versionsPerObject
+
+	t.Logf("Creating %d objects with %d versions each (%d total)...", numObjects, versionsPerObject, totalExpectedVersions)
+	startTime := time.Now()
+
+	// Create multiple objects with many versions each
+	for i := 0; i < numObjects; i++ {
+		objectKey := fmt.Sprintf("object-%03d", i)
+		for j := 0; j < versionsPerObject; j++ {
+			content := fmt.Sprintf("obj%d-ver%d", i, j+1)
+			_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+				Body:   strings.NewReader(content),
+			})
+			require.NoError(t, err)
+		}
+		t.Logf("Created object %s with %d versions", objectKey, versionsPerObject)
+	}
+
+	createDuration := time.Since(startTime)
+	t.Logf("Created %d total versions in %v", totalExpectedVersions, createDuration)
+
+	// List all versions using pagination
+	t.Run("ListAllWithPagination", func(t *testing.T) {
+		maxKeys := int32(500)
+		allVersions := make([]types.ObjectVersion, 0, totalExpectedVersions)
+		var keyMarker, versionIdMarker *string
+		pageCount := 0
+
+		for {
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:          aws.String(bucketName),
+				MaxKeys:         aws.Int32(maxKeys),
+				KeyMarker:       keyMarker,
+				VersionIdMarker: versionIdMarker,
+			})
+			require.NoError(t, err)
+			pageCount++
+
+			allVersions = append(allVersions, resp.Versions...)
+			t.Logf("Page %d: got %d versions, truncated=%v", pageCount, len(resp.Versions), resp.IsTruncated != nil && *resp.IsTruncated)
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+			versionIdMarker = resp.NextVersionIdMarker
+		}
+
+		t.Logf("Listed %d versions in %d pages", len(allVersions), pageCount)
+		assert.Len(t, allVersions, totalExpectedVersions, "Should have all versions")
+
+		// Verify each object has correct number of versions
+		versionsByKey := make(map[string]int)
+		for _, v := range allVersions {
+			versionsByKey[*v.Key]++
+		}
+		assert.Len(t, versionsByKey, numObjects, "Should have %d unique objects", numObjects)
+		for key, count := range versionsByKey {
+			assert.Equal(t, versionsPerObject, count, "Object %s should have %d versions", key, versionsPerObject)
+		}
+
+		// Verify each object has exactly one latest version
+		latestByKey := make(map[string]int)
+		for _, v := range allVersions {
+			if *v.IsLatest {
+				latestByKey[*v.Key]++
+			}
+		}
+		for key, count := range latestByKey {
+			assert.Equal(t, 1, count, "Object %s should have exactly one latest version", key)
+		}
+	})
+}
+
+// listAllVersions is a helper to list all versions of a specific object using pagination
+func listAllVersions(t *testing.T, client *s3.Client, bucketName, objectKey string) []types.ObjectVersion {
+	var allVersions []types.ObjectVersion
+	var keyMarker, versionIdMarker *string
+	pageCount := 0
+
+	for {
+		resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+			Bucket:          aws.String(bucketName),
+			Prefix:          aws.String(objectKey),
+			KeyMarker:       keyMarker,
+			VersionIdMarker: versionIdMarker,
+		})
+		require.NoError(t, err)
+		pageCount++
+
+		t.Logf("Page %d: got %d versions, truncated=%v", pageCount, len(resp.Versions), resp.IsTruncated != nil && *resp.IsTruncated)
+		allVersions = append(allVersions, resp.Versions...)
+
+		// Check if truncated (handle nil pointer)
+		if resp.IsTruncated == nil || !*resp.IsTruncated {
+			break
+		}
+		keyMarker = resp.NextKeyMarker
+		versionIdMarker = resp.NextVersionIdMarker
+	}
+
+	t.Logf("Total: %d versions in %d pages", len(allVersions), pageCount)
+	return allVersions
+}
+
@@ -90,3 +90,4 @@ To debug test failures:
   ```


+
@@ -35,3 +35,4 @@
 ]


+
@@ -109,13 +109,6 @@ func (s *AdminServer) GetAdminData(username string) (AdminData, error) {
 		glog.Errorf("Failed to get cluster volume servers: %v", err)
 		return AdminData{}, err
 	}
-	// Sort the servers so they show up in consistent order after each reload
-	sort.Slice(volumeServersData.VolumeServers, func(i, j int) bool {
-		s1Name := volumeServersData.VolumeServers[i].GetDisplayAddress()
-		s2Name := volumeServersData.VolumeServers[j].GetDisplayAddress()
-
-		return s1Name < s2Name
-	})

 	// Get master nodes status
 	masterNodes := s.getMasterNodesStatus()
@@ -265,6 +258,11 @@ func (s *AdminServer) getFilerNodesStatus() []FilerNode {
 		return []FilerNode{}
 	}

+	// Sort filer nodes by address for consistent ordering on page refresh
+	sort.Slice(filerNodes, func(i, j int) bool {
+		return filerNodes[i].Address < filerNodes[j].Address
+	})
+
 	return filerNodes
 }

@@ -301,5 +299,10 @@ func (s *AdminServer) getMessageBrokerNodesStatus() []MessageBrokerNode {
 		return []MessageBrokerNode{}
 	}

+	// Sort message broker nodes by address for consistent ordering on page refresh
+	sort.Slice(messageBrokers, func(i, j int) bool {
+		return messageBrokers[i].Address < messageBrokers[j].Address
+	})
+
 	return messageBrokers
 }
@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"sort"
 	"strconv"
 	"time"

@@ -720,6 +721,11 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 		masters = append(masters, *masterInfo)
 	}

+	// Sort masters by address for consistent ordering on page refresh
+	sort.Slice(masters, func(i, j int) bool {
+		return masters[i].Address < masters[j].Address
+	})
+
 	// If no masters found at all, add the current master as fallback
 	if len(masters) == 0 {
 		currentMaster := s.masterClient.GetMaster(context.Background())
@@ -776,6 +782,11 @@ func (s *AdminServer) GetClusterFilers() (*ClusterFilersData, error) {
 		return nil, fmt.Errorf("failed to get filer nodes from master: %w", err)
 	}

+	// Sort filers by address for consistent ordering on page refresh
+	sort.Slice(filers, func(i, j int) bool {
+		return filers[i].Address < filers[j].Address
+	})
+
 	return &ClusterFilersData{
 		Filers:      filers,
 		TotalFilers: len(filers),
@@ -818,6 +829,11 @@ func (s *AdminServer) GetClusterBrokers() (*ClusterBrokersData, error) {
 		return nil, fmt.Errorf("failed to get broker nodes from master: %w", err)
 	}

+	// Sort brokers by address for consistent ordering on page refresh
+	sort.Slice(brokers, func(i, j int) bool {
+		return brokers[i].Address < brokers[j].Address
+	})
+
 	return &ClusterBrokersData{
 		Brokers:      brokers,
 		TotalBrokers: len(brokers),
@@ -544,6 +544,11 @@ func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, erro
 		volumeServers = append(volumeServers, *vs)
 	}

+	// Sort volume servers by address for consistent ordering on page refresh
+	sort.Slice(volumeServers, func(i, j int) bool {
+		return volumeServers[i].GetDisplayAddress() < volumeServers[j].GetDisplayAddress()
+	})
+
 	var totalCapacity int64
 	var totalVolumes int
 	for _, vs := range volumeServers {
@@ -117,7 +117,7 @@ func (s *WorkerGrpcServer) Stop() error {
 	s.connMutex.Lock()
 	for _, conn := range s.connections {
 		conn.cancel()
-		close(conn.outgoing)
+		s.safeCloseOutgoingChannel(conn, "Stop")
 	}
 	s.connections = make(map[string]*WorkerConnection)
 	s.connMutex.Unlock()
@@ -182,15 +182,27 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
 	}
 	conn.capabilities = capabilities

-	// Register connection
+	// Register connection - clean up old connection if worker is reconnecting
 	s.connMutex.Lock()
+	if oldConn, exists := s.connections[workerID]; exists {
+		glog.Infof("Worker %s reconnected, cleaning up old connection", workerID)
+		// Cancel old connection to stop its goroutines
+		oldConn.cancel()
+		// Don't close oldConn.outgoing here as it may cause panic in handleOutgoingMessages
+		// Let the goroutine exit naturally when it detects context cancellation
+	}
 	s.connections[workerID] = conn
 	s.connMutex.Unlock()

 	// Register worker with maintenance manager
 	s.registerWorkerWithManager(conn)

-	// Send registration response
+	// IMPORTANT: Start outgoing message handler BEFORE sending registration response
+	// This ensures the handler is ready to process messages and prevents race conditions
+	// where the worker might send requests before we're ready to respond
+	go s.handleOutgoingMessages(conn)
+
+	// Send registration response (after handler is started)
 	regResponse := &worker_pb.AdminMessage{
 		Timestamp: time.Now().Unix(),
 		Message: &worker_pb.AdminMessage_RegistrationResponse{
@@ -203,13 +215,11 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr

 	select {
 	case conn.outgoing <- regResponse:
+		glog.V(1).Infof("Registration response sent to worker %s", workerID)
 	case <-time.After(5 * time.Second):
 		glog.Errorf("Failed to send registration response to worker %s", workerID)
 	}

-	// Start outgoing message handler
-	go s.handleOutgoingMessages(conn)
-
 	// Handle incoming messages
 	for {
 		select {
@@ -235,7 +245,9 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
 			return err
 		}

+		s.connMutex.Lock()
 		conn.lastSeen = time.Now()
+		s.connMutex.Unlock()
 		s.handleWorkerMessage(conn, msg)
 	}
 }
@@ -440,16 +452,36 @@ func (s *WorkerGrpcServer) handleTaskLogResponse(conn *WorkerConnection, respons
 	s.logRequestsMutex.Unlock()
 }

+// safeCloseOutgoingChannel safely closes the outgoing channel for a worker connection.
+func (s *WorkerGrpcServer) safeCloseOutgoingChannel(conn *WorkerConnection, source string) {
+	defer func() {
+		if r := recover(); r != nil {
+			glog.V(1).Infof("%s: recovered from panic closing outgoing channel for worker %s: %v", source, conn.workerID, r)
+		}
+	}()
+	close(conn.outgoing)
+}
+
 // unregisterWorker removes a worker connection
 func (s *WorkerGrpcServer) unregisterWorker(workerID string) {
 	s.connMutex.Lock()
-	if conn, exists := s.connections[workerID]; exists {
-		conn.cancel()
-		close(conn.outgoing)
-		delete(s.connections, workerID)
+	conn, exists := s.connections[workerID]
+	if !exists {
+		s.connMutex.Unlock()
+		glog.V(2).Infof("unregisterWorker: worker %s not found in connections map (already unregistered)", workerID)
+		return
 	}
+
+	// Remove from map first to prevent duplicate cleanup attempts
+	delete(s.connections, workerID)
 	s.connMutex.Unlock()

+	// Cancel context to signal goroutines to stop
+	conn.cancel()
+
+	// Safely close the outgoing channel with recover to handle potential double-close
+	s.safeCloseOutgoingChannel(conn, "unregisterWorker")
+
 	glog.V(1).Infof("Unregistered worker %s", workerID)
 }

@@ -479,7 +511,7 @@ func (s *WorkerGrpcServer) cleanupStaleConnections() {
 		if conn.lastSeen.Before(cutoff) {
 			glog.Warningf("Cleaning up stale worker connection: %s", workerID)
 			conn.cancel()
-			close(conn.outgoing)
+			s.safeCloseOutgoingChannel(conn, "cleanupStaleConnections")
 			delete(s.connections, workerID)
 		}
 	}
@@ -266,7 +266,26 @@ func (s *MaintenanceIntegration) ScanWithTaskDetectors(volumeMetrics []*types.Vo

 // UpdateTopologyInfo updates the volume shard tracker with topology information for empty servers
 func (s *MaintenanceIntegration) UpdateTopologyInfo(topologyInfo *master_pb.TopologyInfo) error {
-	return s.activeTopology.UpdateTopology(topologyInfo)
+	// Log topology details before update for diagnostics
+	if topologyInfo != nil {
+		dcCount, nodeCount, diskCount := topology.CountTopologyResources(topologyInfo)
+		glog.V(2).Infof("UpdateTopologyInfo: received topology with %d datacenters, %d nodes, %d disks",
+			dcCount, nodeCount, diskCount)
+	} else {
+		glog.Warningf("UpdateTopologyInfo: received nil topologyInfo")
+	}
+
+	err := s.activeTopology.UpdateTopology(topologyInfo)
+
+	if err != nil {
+		glog.Errorf("UpdateTopologyInfo: topology update failed: %v", err)
+	} else {
+		// Log success with current disk count
+		currentDiskCount := s.activeTopology.GetDiskCount()
+		glog.V(1).Infof("UpdateTopologyInfo: topology update successful, active topology now has %d disks", currentDiskCount)
+	}
+
+	return err
 }

 // convertToExistingFormat converts task results to existing system format using dynamic mapping
@@ -137,6 +137,7 @@ func (mm *MaintenanceManager) Start() error {
 	// Start background processes
 	go mm.scanLoop()
 	go mm.cleanupLoop()
+	go mm.topologyStatusLoop() // Periodic diagnostic logging

 	glog.Infof("Maintenance manager started with scan interval %ds", mm.config.ScanIntervalSeconds)
 	return nil
@@ -255,6 +256,54 @@ func (mm *MaintenanceManager) cleanupLoop() {
 	}
 }

+// topologyStatusLoop periodically logs topology status for diagnostics
+func (mm *MaintenanceManager) topologyStatusLoop() {
+	// Log topology status every 5 minutes for diagnostic purposes
+	statusInterval := 5 * time.Minute
+	ticker := time.NewTicker(statusInterval)
+	defer ticker.Stop()
+
+	for mm.running {
+		select {
+		case <-mm.stopChan:
+			return
+		case <-ticker.C:
+			mm.logTopologyStatus()
+		}
+	}
+}
+
+// logTopologyStatus logs current topology and worker status for diagnostics
+func (mm *MaintenanceManager) logTopologyStatus() {
+	if mm.scanner == nil || mm.scanner.integration == nil {
+		glog.V(2).Infof("Topology status: scanner/integration not available")
+		return
+	}
+
+	activeTopology := mm.scanner.integration.GetActiveTopology()
+	if activeTopology == nil {
+		glog.V(1).Infof("Topology status: ActiveTopology is nil")
+		return
+	}
+
+	diskCount := activeTopology.GetDiskCount()
+	nodeCount := len(activeTopology.GetAllNodes())
+
+	// Get queue stats
+	stats := mm.queue.GetStats()
+	workerCount := len(mm.queue.GetWorkers())
+
+	mm.mutex.RLock()
+	errorCount := mm.errorCount
+	mm.mutex.RUnlock()
+
+	glog.V(0).Infof("Topology status: %d nodes, %d disks, %d workers, %d pending tasks, %d running tasks, errors: %d",
+		nodeCount, diskCount, workerCount,
+		stats.TasksByStatus[TaskStatusPending],
+		stats.TasksByStatus[TaskStatusInProgress]+stats.TasksByStatus[TaskStatusAssigned],
+		errorCount)
+}
+
 // performScan executes a maintenance scan with error handling and backoff
 func (mm *MaintenanceManager) performScan() {
 	defer func() {
@@ -8,11 +8,53 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 )

+// CountTopologyResources counts datacenters, nodes, and disks in topology info
+func CountTopologyResources(topologyInfo *master_pb.TopologyInfo) (dcCount, nodeCount, diskCount int) {
+	if topologyInfo == nil {
+		return 0, 0, 0
+	}
+	dcCount = len(topologyInfo.DataCenterInfos)
+	for _, dc := range topologyInfo.DataCenterInfos {
+		for _, rack := range dc.RackInfos {
+			nodeCount += len(rack.DataNodeInfos)
+			for _, node := range rack.DataNodeInfos {
+				diskCount += len(node.DiskInfos)
+			}
+		}
+	}
+	return
+}
+
 // UpdateTopology updates the topology information from master
 func (at *ActiveTopology) UpdateTopology(topologyInfo *master_pb.TopologyInfo) error {
 	at.mutex.Lock()
 	defer at.mutex.Unlock()

+	// Validate topology updates to prevent clearing disk maps with invalid data
+	if topologyInfo == nil {
+		glog.Warningf("UpdateTopology received nil topologyInfo, preserving last-known-good topology")
+		return fmt.Errorf("rejected invalid topology update: nil topologyInfo")
+	}
+
+	if len(topologyInfo.DataCenterInfos) == 0 {
+		glog.Warningf("UpdateTopology received empty DataCenterInfos, preserving last-known-good topology (had %d nodes, %d disks)",
+			len(at.nodes), len(at.disks))
+		return fmt.Errorf("rejected invalid topology update: empty DataCenterInfos (had %d nodes, %d disks)", len(at.nodes), len(at.disks))
+	}
+
+	// Count incoming topology for validation logging
+	dcCount, incomingNodes, incomingDisks := CountTopologyResources(topologyInfo)
+
+	// Reject updates that would wipe out a valid topology with an empty one (e.g. during master restart)
+	if incomingNodes == 0 && len(at.nodes) > 0 {
+		glog.Warningf("UpdateTopology received topology with 0 nodes, preserving last-known-good topology (had %d nodes, %d disks)",
+			len(at.nodes), len(at.disks))
+		return fmt.Errorf("rejected invalid topology update: 0 nodes (had %d nodes, %d disks)", len(at.nodes), len(at.disks))
+	}
+
+	glog.V(2).Infof("UpdateTopology: validating update with %d datacenters, %d nodes, %d disks (current: %d nodes, %d disks)",
+		dcCount, incomingNodes, incomingDisks, len(at.nodes), len(at.disks))
+
 	at.topologyInfo = topologyInfo
 	at.lastUpdated = time.Now()

@@ -142,8 +184,21 @@ func (at *ActiveTopology) GetNodeDisks(nodeID string) []*DiskInfo {
 	return disks
 }

+// GetDiskCount returns the total number of disks in the active topology
+func (at *ActiveTopology) GetDiskCount() int {
+	at.mutex.RLock()
+	defer at.mutex.RUnlock()
+	return len(at.disks)
+}
+
 // rebuildIndexes rebuilds the volume and EC shard indexes for O(1) lookups
 func (at *ActiveTopology) rebuildIndexes() {
+	// Nil-safety guard: return early if topology is not valid
+	if at.topologyInfo == nil || at.topologyInfo.DataCenterInfos == nil {
+		glog.V(1).Infof("rebuildIndexes: skipping rebuild due to nil topology or DataCenterInfos")
+		return
+	}
+
 	// Clear existing indexes
 	at.volumeIndex = make(map[uint32][]string)
 	at.ecShardIndex = make(map[uint32][]string)
@@ -2,330 +2,337 @@ package app

 import (
 	"fmt"
+	"sort"
 	"strings"
 	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 )

 templ ClusterEcVolumes(data dash.ClusterEcVolumesData) {
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <title>EC Volumes - SeaweedFS</title>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
-    <link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css" rel="stylesheet">
-</head>
-<body>
-    <div class="container-fluid">
-        <div class="row">
-            <div class="col-12">
-                <h2 class="mb-4">
-                    <i class="fas fa-database me-2"></i>EC Volumes
-                    <small class="text-muted">({fmt.Sprintf("%d", data.TotalVolumes)} volumes)</small>
-                </h2>
-            </div>
-        </div>
-
-        <!-- Statistics Cards -->
-        <div class="row mb-4">
-            <div class="col-md-3">
-                <div class="card text-bg-primary">
-                    <div class="card-body">
-                        <div class="d-flex justify-content-between">
-                            <div>
-                                <h6 class="card-title">Total Volumes</h6>
-                                <h4 class="mb-0">{fmt.Sprintf("%d", data.TotalVolumes)}</h4>
-                                <small>EC encoded volumes</small>
-                            </div>
-                            <div class="align-self-center">
-                                <i class="fas fa-cubes fa-2x"></i>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            <div class="col-md-3">
-                <div class="card text-bg-info">
-                    <div class="card-body">
-                        <div class="d-flex justify-content-between">
-                            <div>
-                                <h6 class="card-title">Total Shards</h6>
-                                <h4 class="mb-0">{fmt.Sprintf("%d", data.TotalShards)}</h4>
-                                <small>Distributed shards</small>
-                            </div>
-                            <div class="align-self-center">
-                                <i class="fas fa-puzzle-piece fa-2x"></i>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            <div class="col-md-3">
-                <div class="card text-bg-success">
-                    <div class="card-body">
-                        <div class="d-flex justify-content-between">
-                            <div>
-                                <h6 class="card-title">Complete Volumes</h6>
-                                <h4 class="mb-0">{fmt.Sprintf("%d", data.CompleteVolumes)}</h4>
-                                <small>All shards present</small>
-                            </div>
-                            <div class="align-self-center">
-                                <i class="fas fa-check-circle fa-2x"></i>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            <div class="col-md-3">
-                <div class="card text-bg-warning">
-                    <div class="card-body">
-                        <div class="d-flex justify-content-between">
-                            <div>
-                                <h6 class="card-title">Incomplete Volumes</h6>
-                                <h4 class="mb-0">{fmt.Sprintf("%d", data.IncompleteVolumes)}</h4>
-                                <small>Missing shards</small>
-                            </div>
-                            <div class="align-self-center">
-                                <i class="fas fa-exclamation-triangle fa-2x"></i>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-        </div>
-
-        <!-- EC Storage Information Note -->
-        <div class="alert alert-info mb-4" role="alert">
-            <i class="fas fa-info-circle me-2"></i>
-            <strong>EC Storage Note:</strong> 
-            EC volumes use erasure coding ({ fmt.Sprintf("%d+%d", erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount) }) which stores data across { fmt.Sprintf("%d", erasure_coding.TotalShardsCount) } shards with redundancy. 
-            Physical storage is approximately { fmt.Sprintf("%.1fx", float64(erasure_coding.TotalShardsCount)/float64(erasure_coding.DataShardsCount)) } the original logical data size due to { fmt.Sprintf("%d", erasure_coding.ParityShardsCount) } parity shards.
-        </div>
-
-        <!-- Volumes Table -->
-        <div class="d-flex justify-content-between align-items-center mb-3">
-            <div class="d-flex align-items-center">
-                <span class="me-3">
-                    Showing {fmt.Sprintf("%d", (data.Page-1)*data.PageSize + 1)} to {fmt.Sprintf("%d", func() int {
-                        end := data.Page * data.PageSize
-                        if end > data.TotalVolumes {
-                            return data.TotalVolumes
-                        }
-                        return end
-                    }())} of {fmt.Sprintf("%d", data.TotalVolumes)} volumes
-                </span>
-                
-                <div class="d-flex align-items-center">
-                    <label for="pageSize" class="form-label me-2 mb-0">Show:</label>
-                    <select id="pageSize" class="form-select form-select-sm" style="width: auto;" onchange="changePageSize(this.value)">
-                        <option value="5" if data.PageSize == 5 { selected }>5</option>
-                        <option value="10" if data.PageSize == 10 { selected }>10</option>
-                        <option value="25" if data.PageSize == 25 { selected }>25</option>
-                        <option value="50" if data.PageSize == 50 { selected }>50</option>
-                        <option value="100" if data.PageSize == 100 { selected }>100</option>
-                    </select>
-                    <span class="ms-2">per page</span>
-                </div>
-            </div>
-            
+    <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
+        <div>
+            <h1 class="h2">
+                <i class="fas fa-database me-2"></i>EC Volumes
+            </h1>
            if data.Collection != "" {
-                <div>
+                <div class="d-flex align-items-center mt-2">
                    if data.Collection == "default" {
-                        <span class="badge bg-secondary text-white">Collection: default</span>
+                        <span class="badge bg-secondary text-white me-2">
+                            <i class="fas fa-filter me-1"></i>Collection: default
+                        </span>
                    } else {
-                        <span class="badge bg-info text-white">Collection: {data.Collection}</span>
+                        <span class="badge bg-info text-white me-2">
+                            <i class="fas fa-filter me-1"></i>Collection: {data.Collection}
+                        </span>
                    }
-                    <a href="/cluster/ec-shards" class="btn btn-sm btn-outline-secondary ms-2">Clear Filter</a>
+                    <a href="/cluster/ec-volumes" class="btn btn-sm btn-outline-secondary">
+                        <i class="fas fa-times me-1"></i>Clear Filter
+                    </a>
                </div>
            }
        </div>
-
-        <div class="table-responsive">
-            <table class="table table-striped table-hover" id="ecVolumesTable">
-                <thead>
-                    <tr>
-                        <th>
-                            <a href="#" onclick="sortBy('volume_id')" class="text-dark text-decoration-none">
-                                Volume ID
-                                if data.SortBy == "volume_id" {
-                                    if data.SortOrder == "asc" {
-                                        <i class="fas fa-sort-up ms-1"></i>
-                                    } else {
-                                        <i class="fas fa-sort-down ms-1"></i>
-                                    }
-                                } else {
-                                    <i class="fas fa-sort ms-1 text-muted"></i>
-                                }
-                            </a>
-                        </th>
-                        if data.ShowCollectionColumn {
-                            <th>
-                                <a href="#" onclick="sortBy('collection')" class="text-dark text-decoration-none">
-                                    Collection
-                                    if data.SortBy == "collection" {
-                                        if data.SortOrder == "asc" {
-                                            <i class="fas fa-sort-up ms-1"></i>
-                                        } else {
-                                            <i class="fas fa-sort-down ms-1"></i>
-                                        }
-                                    } else {
-                                        <i class="fas fa-sort ms-1 text-muted"></i>
-                                    }
-                                </a>
-                            </th>
-                        }
-                        <th>
-                            <a href="#" onclick="sortBy('total_shards')" class="text-dark text-decoration-none">
-                                Shard Count
-                                if data.SortBy == "total_shards" {
-                                    if data.SortOrder == "asc" {
-                                        <i class="fas fa-sort-up ms-1"></i>
-                                    } else {
-                                        <i class="fas fa-sort-down ms-1"></i>
-                                    }
-                                } else {
-                                    <i class="fas fa-sort ms-1 text-muted"></i>
-                                }
-                            </a>
-                        </th>
-                        <th class="text-dark">Shard Size</th>
-                        <th class="text-dark">Shard Locations</th>
-                        <th>
-                            <a href="#" onclick="sortBy('completeness')" class="text-dark text-decoration-none">
-                                Status
-                                if data.SortBy == "completeness" {
-                                    if data.SortOrder == "asc" {
-                                        <i class="fas fa-sort-up ms-1"></i>
-                                    } else {
-                                        <i class="fas fa-sort-down ms-1"></i>
-                                    }
-                                } else {
-                                    <i class="fas fa-sort ms-1 text-muted"></i>
-                                }
-                            </a>
-                        </th>
-                        if data.ShowDataCenterColumn {
-                            <th class="text-dark">Data Centers</th>
-                        }
-                        <th class="text-dark">Actions</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    for _, volume := range data.EcVolumes {
-                        <tr>
-                            <td>
-                                <strong>{fmt.Sprintf("%d", volume.VolumeID)}</strong>
-                            </td>
-                            if data.ShowCollectionColumn {
-                                <td>
-                                    if volume.Collection != "" {
-                                        <a href="/cluster/ec-shards?collection={volume.Collection}" class="text-decoration-none">
-                                            <span class="badge bg-info text-white">{volume.Collection}</span>
-                                        </a>
-                                    } else {
-                                        <a href="/cluster/ec-shards?collection=default" class="text-decoration-none">
-                                            <span class="badge bg-secondary text-white">default</span>
-                                        </a>
-                                    }
-                                </td>
-                            }
-                            <td>
-                                <span class="badge bg-primary">{fmt.Sprintf("%d/14", volume.TotalShards)}</span>
-                            </td>
-                            <td>
-                                @displayShardSizes(volume.ShardSizes)
-                            </td>
-                            <td>
-                                @displayVolumeDistribution(volume)
-                            </td>
-                            <td>
-                                @displayEcVolumeStatus(volume)
-                            </td>
-                            if data.ShowDataCenterColumn {
-                                <td>
-                                    for i, dc := range volume.DataCenters {
-                                        if i > 0 {
-                                            <span>, </span>
-                                        }
-                                        <span class="badge bg-primary text-white">{dc}</span>
-                                    }
-                                </td>
-                            }
-                            <td>
-                                <div class="btn-group" role="group">
-                                    <button type="button" class="btn btn-sm btn-outline-primary" 
-                                            onclick="showVolumeDetails(event)" 
-                                            data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
-                                            title="View EC volume details">
-                                        <i class="fas fa-info-circle"></i>
-                                    </button>
-                                    if !volume.IsComplete {
-                                        <button type="button" class="btn btn-sm btn-outline-warning" 
-                                                onclick="repairVolume(event)" 
-                                                data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
-                                                title="Repair missing shards">
-                                            <i class="fas fa-wrench"></i>
-                                        </button>
-                                    }
-                                </div>
-                            </td>
-                        </tr>
-                    }
-                </tbody>
-            </table>
+        <div class="btn-toolbar mb-2 mb-md-0">
+            <div class="btn-group me-2">
+                <select class="form-select form-select-sm me-2" id="pageSizeSelect" onchange="changePageSize(this.value)" style="width: auto;">
+                    <option value="5" if data.PageSize == 5 { selected="selected" }>5 per page</option>
+                    <option value="10" if data.PageSize == 10 { selected="selected" }>10 per page</option>
+                    <option value="25" if data.PageSize == 25 { selected="selected" }>25 per page</option>
+                    <option value="50" if data.PageSize == 50 { selected="selected" }>50 per page</option>
+                    <option value="100" if data.PageSize == 100 { selected="selected" }>100 per page</option>
+                </select>
+                <button type="button" class="btn btn-sm btn-outline-primary" onclick="window.location.reload()">
+                    <i class="fas fa-refresh me-1"></i>Refresh
+                </button>
+            </div>
        </div>
-
-        <!-- Pagination -->
-        if data.TotalPages > 1 {
-            <nav aria-label="EC Volumes pagination">
-                <ul class="pagination justify-content-center">
-                    if data.Page > 1 {
-                        <li class="page-item">
-                            <a class="page-link" href="#" onclick="goToPage(event)" data-page="1">First</a>
-                        </li>
-                        <li class="page-item">
-                            <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page-1) }>Previous</a>
-                        </li>
-                    }
-                    
-                    for i := 1; i <= data.TotalPages; i++ {
-                        if i == data.Page {
-                            <li class="page-item active">
-                                <span class="page-link">{fmt.Sprintf("%d", i)}</span>
-                            </li>
-                        } else if i <= 3 || i > data.TotalPages-3 || (i >= data.Page-2 && i <= data.Page+2) {
-                            <li class="page-item">
-                                <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", i) }>{fmt.Sprintf("%d", i)}</a>
-                            </li>
-                        } else if i == 4 && data.Page > 6 {
-                            <li class="page-item disabled">
-                                <span class="page-link">...</span>
-                            </li>
-                        } else if i == data.TotalPages-3 && data.Page < data.TotalPages-5 {
-                            <li class="page-item disabled">
-                                <span class="page-link">...</span>
-                            </li>
-                        }
-                    }
-                    
-                    if data.Page < data.TotalPages {
-                        <li class="page-item">
-                            <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page+1) }>Next</a>
-                        </li>
-                        <li class="page-item">
-                            <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.TotalPages) }>Last</a>
-                        </li>
-                    }
-                </ul>
-            </nav>
-        }
    </div>

-    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
+    <!-- Statistics Cards -->
+    <div class="row mb-4">
+        <div class="col-md-3">
+            <div class="card text-bg-primary">
+                <div class="card-body">
+                    <div class="d-flex justify-content-between">
+                        <div>
+                            <h6 class="card-title">Total Volumes</h6>
+                            <h4 class="mb-0">{fmt.Sprintf("%d", data.TotalVolumes)}</h4>
+                        </div>
+                        <div class="align-self-center">
+                            <i class="fas fa-cubes fa-2x"></i>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="col-md-3">
+            <div class="card text-bg-info">
+                <div class="card-body">
+                    <div class="d-flex justify-content-between">
+                        <div>
+                            <h6 class="card-title">Total Shards</h6>
+                            <h4 class="mb-0">{fmt.Sprintf("%d", data.TotalShards)}</h4>
+                        </div>
+                        <div class="align-self-center">
+                            <i class="fas fa-puzzle-piece fa-2x"></i>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="col-md-3">
+            <div class="card text-bg-success">
+                <div class="card-body">
+                    <div class="d-flex justify-content-between">
+                        <div>
+                            <h6 class="card-title">Healthy Volumes</h6>
+                            <h4 class="mb-0">{fmt.Sprintf("%d", data.CompleteVolumes)}</h4>
+                            <small>All { fmt.Sprintf("%d", erasure_coding.TotalShardsCount) } shards present</small>
+                        </div>
+                        <div class="align-self-center">
+                            <i class="fas fa-check-circle fa-2x"></i>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="col-md-3">
+            <div class="card text-bg-warning">
+                <div class="card-body">
+                    <div class="d-flex justify-content-between">
+                        <div>
+                            <h6 class="card-title">Degraded Volumes</h6>
+                            <h4 class="mb-0">{fmt.Sprintf("%d", data.IncompleteVolumes)}</h4>
+                            <small>Incomplete/Critical</small>
+                        </div>
+                        <div class="align-self-center">
+                            <i class="fas fa-exclamation-triangle fa-2x"></i>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <!-- EC Storage Information Note -->
+    <div class="alert alert-info mb-4" role="alert">
+        <i class="fas fa-info-circle me-2"></i>
+        <strong>EC Storage Note:</strong> 
+        EC volumes use erasure coding ({ fmt.Sprintf("%d+%d", erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount) }) which stores data across { fmt.Sprintf("%d", erasure_coding.TotalShardsCount) } shards with redundancy. 
+        Physical storage is approximately { fmt.Sprintf("%.1fx", float64(erasure_coding.TotalShardsCount)/float64(erasure_coding.DataShardsCount)) } the original logical data size due to { fmt.Sprintf("%d", erasure_coding.ParityShardsCount) } parity shards.
+    </div>
+
+    <!-- Volumes Table -->
+    <div class="table-responsive">
+        <table class="table table-striped table-hover" id="ecVolumesTable">
+            <thead>
+                <tr>
+                    <th>
+                        <a href="#" onclick="sortBy('volume_id')" class="text-dark text-decoration-none">
+                            Volume ID
+                            if data.SortBy == "volume_id" {
+                                if data.SortOrder == "asc" {
+                                    <i class="fas fa-sort-up ms-1"></i>
+                                } else {
+                                    <i class="fas fa-sort-down ms-1"></i>
+                                }
+                            } else {
+                                <i class="fas fa-sort ms-1 text-muted"></i>
+                            }
+                        </a>
+                    </th>
+                    if data.ShowCollectionColumn {
+                        <th>
+                            <a href="#" onclick="sortBy('collection')" class="text-dark text-decoration-none">
+                                Collection
+                                if data.SortBy == "collection" {
+                                    if data.SortOrder == "asc" {
+                                        <i class="fas fa-sort-up ms-1"></i>
+                                    } else {
+                                        <i class="fas fa-sort-down ms-1"></i>
+                                    }
+                                } else {
+                                    <i class="fas fa-sort ms-1 text-muted"></i>
+                                }
+                            </a>
+                        </th>
+                    }
+                    <th>
+                        <a href="#" onclick="sortBy('total_shards')" class="text-dark text-decoration-none">
+                            Shard Count
+                            if data.SortBy == "total_shards" {
+                                if data.SortOrder == "asc" {
+                                    <i class="fas fa-sort-up ms-1"></i>
+                                } else {
+                                    <i class="fas fa-sort-down ms-1"></i>
+                                }
+                            } else {
+                                <i class="fas fa-sort ms-1 text-muted"></i>
+                            }
+                        </a>
+                    </th>
+                    <th class="text-dark">Shard Size</th>
+                    <th class="text-dark">Shard Locations</th>
+                    <th>
+                        <a href="#" onclick="sortBy('completeness')" class="text-dark text-decoration-none">
+                            Status
+                            if data.SortBy == "completeness" {
+                                if data.SortOrder == "asc" {
+                                    <i class="fas fa-sort-up ms-1"></i>
+                                } else {
+                                    <i class="fas fa-sort-down ms-1"></i>
+                                }
+                            } else {
+                                <i class="fas fa-sort ms-1 text-muted"></i>
+                            }
+                        </a>
+                    </th>
+                    if data.ShowDataCenterColumn {
+                        <th class="text-dark">Data Centers</th>
+                    }
+                    <th class="text-dark">Actions</th>
+                </tr>
+            </thead>
+            <tbody>
+                for _, volume := range data.EcVolumes {
+                    <tr>
+                        <td>
+                            <span class="fw-bold">{fmt.Sprintf("%d", volume.VolumeID)}</span>
+                        </td>
+                        if data.ShowCollectionColumn {
+                            <td>
+                                if volume.Collection != "" {
+                                    <a href={ templ.URL(fmt.Sprintf("/cluster/ec-shards?collection=%s", volume.Collection)) } class="text-decoration-none">
+                                        <span class="badge bg-info text-white">{volume.Collection}</span>
+                                    </a>
+                                } else {
+                                    <a href={ templ.URL("/cluster/ec-shards?collection=default") } class="text-decoration-none">
+                                        <span class="badge bg-secondary text-white">default</span>
+                                    </a>
+                                }
+                            </td>
+                        }
+                        <td>
+                            <span class="badge bg-primary">{fmt.Sprintf("%d/%d", volume.TotalShards, erasure_coding.TotalShardsCount)}</span>
+                        </td>
+                        <td>
+                            @displayShardSizes(volume.ShardSizes)
+                        </td>
+                        <td>
+                            @displayVolumeDistribution(volume)
+                        </td>
+                        <td>
+                            @displayEcVolumeStatus(volume)
+                        </td>
+                        if data.ShowDataCenterColumn {
+                            <td>
+                                for i, dc := range volume.DataCenters {
+                                    if i > 0 {
+                                        <span>, </span>
+                                    }
+                                    <span class="badge bg-primary text-white">{dc}</span>
+                                }
+                            </td>
+                        }
+                        <td>
+                            <div class="btn-group" role="group">
+                                <button type="button" class="btn btn-sm btn-outline-primary" 
+                                        onclick="showVolumeDetails(event)" 
+                                        data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
+                                        title="View EC volume details">
+                                    <i class="fas fa-info-circle"></i>
+                                </button>
+                                if !volume.IsComplete {
+                                    <button type="button" class="btn btn-sm btn-outline-warning" 
+                                            onclick="repairVolume(event)" 
+                                            data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
+                                            title="Repair missing shards">
+                                        <i class="fas fa-wrench"></i>
+                                    </button>
+                                }
+                            </div>
+                        </td>
+                    </tr>
+                }
+            </tbody>
+        </table>
+    </div>
+
+    <!-- Pagination -->
+    if data.TotalPages > 1 {
+        <nav aria-label="EC Volumes pagination">
+            <ul class="pagination justify-content-center">
+                if data.Page > 1 {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page="1">
+                            <i class="fas fa-angle-double-left"></i>
+                        </a>
+                    </li>
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page-1) }>
+                            <i class="fas fa-chevron-left"></i>
+                        </a>
+                    </li>
+                }
+                
+                <!-- First page -->
+                if data.Page > 3 {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page="1">1</a>
+                    </li>
+                    if data.Page > 4 {
+                        <li class="page-item disabled">
+                            <span class="page-link">...</span>
+                        </li>
+                    }
+                }
+                
+                <!-- Current page neighbors -->
+                if data.Page > 1 {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page-1) }>{fmt.Sprintf("%d", data.Page-1)}</a>
+                    </li>
+                }
+                
+                <li class="page-item active">
+                    <span class="page-link">{fmt.Sprintf("%d", data.Page)}</span>
+                </li>
+                
+                if data.Page < data.TotalPages {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page+1) }>{fmt.Sprintf("%d", data.Page+1)}</a>
+                    </li>
+                }
+                
+                <!-- Last page -->
+                if data.Page < data.TotalPages-2 {
+                    if data.Page < data.TotalPages-3 {
+                        <li class="page-item disabled">
+                            <span class="page-link">...</span>
+                        </li>
+                    }
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.TotalPages) }>{fmt.Sprintf("%d", data.TotalPages)}</a>
+                    </li>
+                }
+                
+                if data.Page < data.TotalPages {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page+1) }>
+                            <i class="fas fa-chevron-right"></i>
+                        </a>
+                    </li>
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.TotalPages) }>
+                            <i class="fas fa-angle-double-right"></i>
+                        </a>
+                    </li>
+                }
+            </ul>
+        </nav>
+    }
+
+    <!-- JavaScript -->
    <script>
-        // Sorting functionality
        function sortBy(field) {
            const currentSort = new URLSearchParams(window.location.search).get('sort_by');
            const currentOrder = new URLSearchParams(window.location.search).get('sort_order') || 'asc';
@@ -335,65 +342,69 @@ templ ClusterEcVolumes(data dash.ClusterEcVolumesData) {
                newOrder = 'desc';
            }
            
-            const url = new URL(window.location);
-            url.searchParams.set('sort_by', field);
-            url.searchParams.set('sort_order', newOrder);
-            url.searchParams.set('page', '1'); // Reset to first page
-            window.location.href = url.toString();
+            updateUrl({
+                sort_by: field,
+                sort_order: newOrder,
+                page: 1
+            });
        }

-        // Pagination functionality
        function goToPage(event) {
            event.preventDefault();
            const page = event.target.closest('a').getAttribute('data-page');
-            const url = new URL(window.location);
-            url.searchParams.set('page', page);
-            window.location.href = url.toString();
+            updateUrl({ page: page });
        }

-        // Page size functionality
        function changePageSize(newPageSize) {
+            updateUrl({ page_size: newPageSize, page: 1 });
+        }
+
+        function updateUrl(params) {
            const url = new URL(window.location);
-            url.searchParams.set('page_size', newPageSize);
-            url.searchParams.set('page', '1'); // Reset to first page when changing page size
+            Object.keys(params).forEach(key => {
+                if (params[key] != null) {
+                    url.searchParams.set(key, params[key]);
+                } else {
+                    url.searchParams.delete(key);
+                }
+            });
            window.location.href = url.toString();
        }

-        // Volume details
        function showVolumeDetails(event) {
            const volumeId = event.target.closest('button').getAttribute('data-volume-id');
            window.location.href = `/cluster/ec-volumes/${volumeId}`;
        }

-        // Repair volume
        function repairVolume(event) {
            const volumeId = event.target.closest('button').getAttribute('data-volume-id');
            if (confirm(`Are you sure you want to repair missing shards for volume ${volumeId}?`)) {
-                // TODO: Implement repair functionality
-                alert('Repair functionality will be implemented soon.');
+                fetch(`/api/cluster/ec-volumes/${volumeId}/repair`, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                    }
+                })
+                .then(response => {
+                    if (!response.ok) {
+                        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                    }
+                    return response.json();
+                })
+                .then(data => {
+                    if (data && data.success) {
+                        alert('Repair initiated successfully');
+                        location.reload();
+                    } else {
+                        alert('Failed to initiate repair: ' + (data && data.error ? data.error : 'Unknown error'));
+                    }
+                })
+                .catch(error => {
+                    alert('Error: ' + error.message);
+                });
            }
        }
    </script>
-</body>
-</html>
-}
-
-// displayShardLocationsHTML renders shard locations as proper HTML
-templ displayShardLocationsHTML(shardLocations map[int]string) {
-	if len(shardLocations) == 0 {
-		<span class="text-muted">No shards</span>
-	} else {
-		for i, serverInfo := range groupShardsByServer(shardLocations) {
-			if i > 0 {
-				<br/>
-			}
-			<strong>
-				<a href={ templ.URL("/cluster/volume-servers/" + serverInfo.Server) } class="text-primary text-decoration-none">
-					{ serverInfo.Server }
-				</a>:
-			</strong> { serverInfo.ShardRanges }
-		}
-	}
 }

 // displayShardSizes renders shard sizes in a compact format
@@ -439,13 +450,7 @@ func groupShardsByServer(shardLocations map[int]string) []ServerShardInfo {
 	var serverInfos []ServerShardInfo
 	for server, shards := range serverShards {
 		// Sort shards for each server
-		for i := 0; i < len(shards); i++ {
-			for j := i + 1; j < len(shards); j++ {
-				if shards[i] > shards[j] {
-					shards[i], shards[j] = shards[j], shards[i]
-				}
-			}
-		}
+		sort.Ints(shards)
 		
 		// Format shard ranges compactly
 		shardRanges := formatShardRanges(shards)
@@ -456,56 +461,9 @@ func groupShardsByServer(shardLocations map[int]string) []ServerShardInfo {
 	}

 	// Sort by server name
-	for i := 0; i < len(serverInfos); i++ {
-		for j := i + 1; j < len(serverInfos); j++ {
-			if serverInfos[i].Server > serverInfos[j].Server {
-				serverInfos[i], serverInfos[j] = serverInfos[j], serverInfos[i]
-			}
-		}
-	}
-
-	return serverInfos
-}
-
-// groupShardsByServerWithSizes groups shards by server and formats ranges with sizes
-func groupShardsByServerWithSizes(shardLocations map[int]string, shardSizes map[int]int64) []ServerShardInfo {
-	if len(shardLocations) == 0 {
-		return []ServerShardInfo{}
-	}
-
-	// Group shards by server
-	serverShards := make(map[string][]int)
-	for shardId, server := range shardLocations {
-		serverShards[server] = append(serverShards[server], shardId)
-	}
-
-	var serverInfos []ServerShardInfo
-	for server, shards := range serverShards {
-		// Sort shards for each server
-		for i := 0; i < len(shards); i++ {
-			for j := i + 1; j < len(shards); j++ {
-				if shards[i] > shards[j] {
-					shards[i], shards[j] = shards[j], shards[i]
-				}
-			}
-		}
-		
-		// Format shard ranges compactly with sizes
-		shardRanges := formatShardRangesWithSizes(shards, shardSizes)
-		serverInfos = append(serverInfos, ServerShardInfo{
-			Server:      server,
-			ShardRanges: shardRanges,
-		})
-	}
-
-	// Sort by server name
-	for i := 0; i < len(serverInfos); i++ {
-		for j := i + 1; j < len(serverInfos); j++ {
-			if serverInfos[i].Server > serverInfos[j].Server {
-				serverInfos[i], serverInfos[j] = serverInfos[j], serverInfos[i]
-			}
-		}
-	}
+	sort.Slice(serverInfos, func(i, j int) bool {
+		return serverInfos[i].Server < serverInfos[j].Server
+	})

 	return serverInfos
 }
@@ -544,72 +502,6 @@ func formatShardRanges(shards []int) string {
 	return strings.Join(ranges, ",")
 }

-// Helper function to format shard ranges with sizes (e.g., "0(1.2MB),1-3(2.5MB),7(800KB)")
-func formatShardRangesWithSizes(shards []int, shardSizes map[int]int64) string {
-	if len(shards) == 0 {
-		return ""
-	}
-
-	var ranges []string
-	start := shards[0]
-	end := shards[0]
-	var totalSize int64
-
-	for i := 1; i < len(shards); i++ {
-		if shards[i] == end+1 {
-			end = shards[i]
-			totalSize += shardSizes[shards[i]]
-		} else {
-			// Add current range with size
-			if start == end {
-				size := shardSizes[start]
-				if size > 0 {
-					ranges = append(ranges, fmt.Sprintf("%d(%s)", start, bytesToHumanReadable(size)))
-				} else {
-					ranges = append(ranges, fmt.Sprintf("%d", start))
-				}
-			} else {
-				// Calculate total size for the range
-				rangeSize := shardSizes[start]
-				for j := start + 1; j <= end; j++ {
-					rangeSize += shardSizes[j]
-				}
-				if rangeSize > 0 {
-					ranges = append(ranges, fmt.Sprintf("%d-%d(%s)", start, end, bytesToHumanReadable(rangeSize)))
-				} else {
-					ranges = append(ranges, fmt.Sprintf("%d-%d", start, end))
-				}
-			}
-			start = shards[i]
-			end = shards[i]
-			totalSize = shardSizes[shards[i]]
-		}
-	}
-
-	// Add the last range
-	if start == end {
-		size := shardSizes[start]
-		if size > 0 {
-			ranges = append(ranges, fmt.Sprintf("%d(%s)", start, bytesToHumanReadable(size)))
-		} else {
-			ranges = append(ranges, fmt.Sprintf("%d", start))
-		}
-	} else {
-		// Calculate total size for the range
-		rangeSize := shardSizes[start]
-		for j := start + 1; j <= end; j++ {
-			rangeSize += shardSizes[j]
-		}
-		if rangeSize > 0 {
-			ranges = append(ranges, fmt.Sprintf("%d-%d(%s)", start, end, bytesToHumanReadable(rangeSize)))
-		} else {
-			ranges = append(ranges, fmt.Sprintf("%d-%d", start, end))
-		}
-	}
-
-	return strings.Join(ranges, ",")
-}
-
 // Helper function to convert bytes to human readable format
 func bytesToHumanReadable(bytes int64) string {
 	const unit = 1024
@@ -680,13 +572,7 @@ func formatIndividualShardSizes(shardSizes map[int]int64) string {
 		var groupStrs []string
 		for size, shardIds := range sizeGroups {
 			// Sort shard IDs
-			for i := 0; i < len(shardIds); i++ {
-				for j := i + 1; j < len(shardIds); j++ {
-					if shardIds[i] > shardIds[j] {
-						shardIds[i], shardIds[j] = shardIds[j], shardIds[i]
-					}
-				}
-			}
+			sort.Ints(shardIds)
 			
 			var idRanges []string
 			if len(shardIds) <= erasure_coding.ParityShardsCount {
@@ -715,18 +601,27 @@ templ displayVolumeDistribution(volume dash.EcVolumeWithShards) {
    </div>
 }

-// displayEcVolumeStatus shows an improved status display for EC volumes
+// displayEcVolumeStatus shows an improved status display for EC volumes.
+// Status thresholds are based on EC recovery capability:
+// - Critical: More than DataShardsCount missing (data is unrecoverable)
+// - Degraded: More than half of DataShardsCount missing (high risk)
+// - Incomplete: More than half of ParityShardsCount missing (reduced redundancy)
+// - Minor Issues: Few shards missing (still fully recoverable)
 templ displayEcVolumeStatus(volume dash.EcVolumeWithShards) {
    if volume.IsComplete {
        <span class="badge bg-success"><i class="fas fa-check me-1"></i>Complete</span>
    } else {
        if len(volume.MissingShards) > erasure_coding.DataShardsCount {
+            // Unrecoverable: more shards missing than EC can reconstruct
            <span class="badge bg-danger"><i class="fas fa-skull me-1"></i>Critical ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
        } else if len(volume.MissingShards) > (erasure_coding.DataShardsCount/2) {
+            // High risk: approaching unrecoverable state
            <span class="badge bg-warning"><i class="fas fa-exclamation-triangle me-1"></i>Degraded ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
        } else if len(volume.MissingShards) > (erasure_coding.ParityShardsCount/2) {
+            // Reduced redundancy but still recoverable
            <span class="badge bg-warning"><i class="fas fa-info-circle me-1"></i>Incomplete ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
        } else {
+            // Minor: few shards missing, fully recoverable with good margin
            <span class="badge bg-info"><i class="fas fa-info-circle me-1"></i>Minor Issues ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
        }
    }
@@ -127,7 +127,7 @@ templ S3Buckets(data dash.S3BucketsData) {
                                    for _, bucket := range data.Buckets {
                                        <tr>
                                            <td>
-                                                <a href={templ.SafeURL(fmt.Sprintf("/s3/buckets/%s", bucket.Name))} 
+                                                <a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))} 
                                                   class="text-decoration-none">
                                                    <i class="fas fa-cube me-2"></i>
                                                    {bucket.Name}
@@ -83,9 +83,9 @@ func S3Buckets(data dash.S3BucketsData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var5 templ.SafeURL
-			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/s3/buckets/%s", bucket.Name)))
+			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 130, Col: 114}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 130, Col: 123}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 			if templ_7745c5c3_Err != nil {
@@ -3,6 +3,8 @@ package cluster
 import (
 	"context"
 	"fmt"
+	"strings"
+	"sync/atomic"
 	"time"

 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
@@ -36,10 +38,11 @@ type LiveLock struct {
 	hostFiler      pb.ServerAddress
 	cancelCh       chan struct{}
 	grpcDialOption grpc.DialOption
-	isLocked       bool
+	isLocked       int32 // 0 = unlocked, 1 = locked; use atomic operations
 	self           string
 	lc             *LockClient
 	owner          string
+	lockTTL        time.Duration
 }

 // NewShortLivedLock creates a lock with a 5-second duration
@@ -58,17 +61,24 @@ func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLoc
 }

 // StartLongLivedLock starts a goroutine to lock the key and returns immediately.
-func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerChange func(newLockOwner string)) (lock *LiveLock) {
+// lockTTL specifies how long the lock should be held. The renewal interval is
+// automatically derived as lockTTL / 2 to ensure timely renewals.
+func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerChange func(newLockOwner string), lockTTL time.Duration) (lock *LiveLock) {
 	lock = &LiveLock{
 		key:            key,
 		hostFiler:      lc.seedFiler,
 		cancelCh:       make(chan struct{}),
-		expireAtNs:     time.Now().Add(lock_manager.LiveLockTTL).UnixNano(),
+		expireAtNs:     time.Now().Add(lockTTL).UnixNano(),
 		grpcDialOption: lc.grpcDialOption,
 		self:           owner,
 		lc:             lc,
+		lockTTL:        lockTTL,
+	}
+	if lock.lockTTL == 0 {
+		lock.lockTTL = lock_manager.LiveLockTTL
 	}
 	go func() {
+		renewInterval := lock.lockTTL / 2
 		isLocked := false
 		lockOwner := ""
 		for {
@@ -81,13 +91,15 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
 			}

 			if isLocked {
-				if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err != nil {
+				if err := lock.AttemptToLock(lock.lockTTL); err != nil {
 					glog.V(0).Infof("Lost lock %s: %v", key, err)
 					isLocked = false
+					atomic.StoreInt32(&lock.isLocked, 0)
 				}
 			} else {
-				if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err == nil {
+				if err := lock.AttemptToLock(lock.lockTTL); err == nil {
 					isLocked = true
+					// Note: AttemptToLock already sets lock.isLocked atomically on success
 				}
 			}
 			if lockOwner != lock.LockOwner() && lock.LockOwner() != "" {
@@ -99,7 +111,11 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
 			case <-lock.cancelCh:
 				return
 			default:
-				time.Sleep(lock_manager.RenewInterval)
+				if isLocked {
+					time.Sleep(renewInterval)
+				} else {
+					time.Sleep(5 * renewInterval)
+				}
 			}
 		}
 	}()
@@ -126,24 +142,28 @@ func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {
 		return err
 	}
 	if errorMessage != "" {
-		glog.V(1).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
+		if strings.Contains(errorMessage, "lock already owned") {
+			glog.V(3).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
+		} else {
+			glog.V(2).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
+		}
 		time.Sleep(time.Second)
 		return fmt.Errorf("%v", errorMessage)
 	}
-	if !lock.isLocked {
+	if atomic.LoadInt32(&lock.isLocked) == 0 {
 		// Only log when transitioning from unlocked to locked
 		glog.V(1).Infof("LOCK: Successfully acquired key=%s owner=%s", lock.key, lock.self)
 	}
-	lock.isLocked = true
+	atomic.StoreInt32(&lock.isLocked, 1)
 	return nil
 }

 func (lock *LiveLock) StopShortLivedLock() error {
-	if !lock.isLocked {
+	if atomic.LoadInt32(&lock.isLocked) == 0 {
 		return nil
 	}
 	defer func() {
-		lock.isLocked = false
+		atomic.StoreInt32(&lock.isLocked, 0)
 	}()
 	return pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		_, err := client.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
@@ -203,7 +223,7 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 			errorMessage = resp.Error
 			if resp.LockHostMovedTo != "" && resp.LockHostMovedTo != string(previousHostFiler) {
 				// Only log if the host actually changed
-				glog.V(1).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
+				glog.V(2).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
 				lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
 				lock.lc.seedFiler = lock.hostFiler
 			} else if resp.LockHostMovedTo != "" {
@@ -211,12 +231,12 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 			}
 			if resp.LockOwner != "" && resp.LockOwner != previousOwner {
 				// Only log if the owner actually changed
-				glog.V(1).Infof("LOCK: Owner changed from %s to %s for key=%s", previousOwner, resp.LockOwner, lock.key)
+				glog.V(2).Infof("LOCK: Owner changed from %s to %s for key=%s", previousOwner, resp.LockOwner, lock.key)
 				lock.owner = resp.LockOwner
 			} else if resp.LockOwner != "" {
 				lock.owner = resp.LockOwner
 			} else if previousOwner != "" {
-				glog.V(1).Infof("LOCK: Owner cleared for key=%s", lock.key)
+				glog.V(2).Infof("LOCK: Owner cleared for key=%s", lock.key)
 				lock.owner = ""
 			}
 		}
@@ -228,3 +248,8 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 func (lock *LiveLock) LockOwner() string {
 	return lock.owner
 }
+
+// IsLocked returns true if this instance currently holds the lock
+func (lock *LiveLock) IsLocked() bool {
+	return atomic.LoadInt32(&lock.isLocked) == 1
+}
@@ -32,6 +32,7 @@ var Commands = []*Command{
 	cmdIam,
 	cmdMaster,
 	cmdMasterFollower,
+	cmdMini,
 	cmdMount,
 	cmdMqAgent,
 	cmdMqBroker,
@@ -53,6 +53,8 @@ type SyncOptions struct {
 	bDoDeleteFiles  *bool
 	clientId        int32
 	clientEpoch     atomic.Int32
+	debug           *bool
+	debugPort       *int
 }

 const (
@@ -108,6 +110,8 @@ func init() {
 	syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
 	syncOptions.aDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("a.doDeleteFiles", true, "delete and update files when synchronizing on filer A")
 	syncOptions.bDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("b.doDeleteFiles", true, "delete and update files when synchronizing on filer B")
+	syncOptions.debug = cmdFilerSynchronize.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	syncOptions.debugPort = cmdFilerSynchronize.Flag.Int("debug.port", 6060, "http port for debugging")
 	syncOptions.clientId = util.RandomInt32()
 }

@@ -130,6 +134,9 @@ var cmdFilerSynchronize = &Command{
 }

 func runFilerSynchronize(cmd *Command, args []string) bool {
+	if *syncOptions.debug {
+		grace.StartDebugServer(*syncOptions.debugPort)
+	}

 	util.LoadSecurityConfiguration()
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
@@ -69,6 +69,8 @@ type MasterOptions struct {
 	raftBootstrap      *bool
 	telemetryUrl       *string
 	telemetryEnabled   *bool
+	debug              *bool
+	debugPort          *int
 }

 func init() {
@@ -98,6 +100,8 @@ func init() {
 	m.raftBootstrap = cmdMaster.Flag.Bool("raftBootstrap", false, "Whether to bootstrap the Raft cluster")
 	m.telemetryUrl = cmdMaster.Flag.String("telemetry.url", "https://telemetry.seaweedfs.com/api/collect", "telemetry server URL to send usage statistics")
 	m.telemetryEnabled = cmdMaster.Flag.Bool("telemetry", false, "enable telemetry reporting")
+	m.debug = cmdMaster.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	m.debugPort = cmdMaster.Flag.Int("debug.port", 6060, "http port for debugging")
 }

 var cmdMaster = &Command{
@@ -121,6 +125,9 @@ var (
 )

 func runMaster(cmd *Command, args []string) bool {
+	if *m.debug {
+		grace.StartDebugServer(*m.debugPort)
+	}

 	util.LoadSecurityConfiguration()
 	util.LoadConfiguration("master", false)
@@ -41,6 +41,9 @@ func init() {
 	mf.metricsAddress = aws.String("")
 	mf.metricsIntervalSec = aws.Int(0)
 	mf.raftResumeState = aws.Bool(false)
+	mf.maxParallelVacuumPerServer = aws.Int(1)
+	mf.telemetryUrl = aws.String("https://telemetry.seaweedfs.com/api/collect")
+	mf.telemetryEnabled = aws.Bool(false)
 }

 var cmdMasterFollower = &Command{
@@ -68,8 +68,8 @@ func init() {
 	mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
-	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers")
-	mountOptions.concurrentReaders = cmdMount.Flag.Int("concurrentReaders", 16, "limit concurrent chunk fetches for read operations")
+	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 128, "limit concurrent goroutine writers")
+	mountOptions.concurrentReaders = cmdMount.Flag.Int("concurrentReaders", 128, "limit concurrent chunk fetches for read operations")
 	mountOptions.cacheDirForRead = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
 	mountOptions.cacheSizeMBForRead = cmdMount.Flag.Int64("cacheCapacityMB", 128, "file chunk read cache capacity in MB")
 	mountOptions.cacheDirForWrite = cmdMount.Flag.String("cacheDirWrite", "", "buffer writes mostly for large files")
@@ -1,20 +1,15 @@
 package command

 import (
-	"fmt"
-	"net/http"
-	_ "net/http/pprof"
-
 	"google.golang.org/grpc/reflection"

-	"github.com/seaweedfs/seaweedfs/weed/util/grace"
-
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/broker"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/grace"
 )

 var (
@@ -27,12 +22,13 @@ type MessageQueueBrokerOptions struct {
 	filerGroup       *string
 	ip               *string
 	port             *int
-	pprofPort        *int
 	dataCenter       *string
 	rack             *string
 	cpuprofile       *string
 	memprofile       *string
 	logFlushInterval *int
+	debug            *bool
+	debugPort        *int
 }

 func init() {
@@ -41,12 +37,13 @@ func init() {
 	mqBrokerStandaloneOptions.filerGroup = cmdMqBroker.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
 	mqBrokerStandaloneOptions.ip = cmdMqBroker.Flag.String("ip", util.DetectedHostAddress(), "broker host address")
 	mqBrokerStandaloneOptions.port = cmdMqBroker.Flag.Int("port", 17777, "broker gRPC listen port")
-	mqBrokerStandaloneOptions.pprofPort = cmdMqBroker.Flag.Int("port.pprof", 0, "HTTP profiling port (0 to disable)")
 	mqBrokerStandaloneOptions.dataCenter = cmdMqBroker.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
 	mqBrokerStandaloneOptions.rack = cmdMqBroker.Flag.String("rack", "", "prefer to write to volumes in this rack")
 	mqBrokerStandaloneOptions.cpuprofile = cmdMqBroker.Flag.String("cpuprofile", "", "cpu profile output file")
 	mqBrokerStandaloneOptions.memprofile = cmdMqBroker.Flag.String("memprofile", "", "memory profile output file")
 	mqBrokerStandaloneOptions.logFlushInterval = cmdMqBroker.Flag.Int("logFlushInterval", 5, "log buffer flush interval in seconds")
+	mqBrokerStandaloneOptions.debug = cmdMqBroker.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	mqBrokerStandaloneOptions.debugPort = cmdMqBroker.Flag.Int("debug.port", 6060, "http port for debugging")
 }

 var cmdMqBroker = &Command{
@@ -61,6 +58,9 @@ var cmdMqBroker = &Command{
 }

 func runMqBroker(cmd *Command, args []string) bool {
+	if *mqBrokerStandaloneOptions.debug {
+		grace.StartDebugServer(*mqBrokerStandaloneOptions.debugPort)
+	}

 	util.LoadSecurityConfiguration()

@@ -115,18 +115,6 @@ func (mqBrokerOpt *MessageQueueBrokerOptions) startQueueServer() bool {
 		}()
 	}

-	// Start HTTP profiling server if enabled
-	if mqBrokerOpt.pprofPort != nil && *mqBrokerOpt.pprofPort > 0 {
-		go func() {
-			pprofAddr := fmt.Sprintf(":%d", *mqBrokerOpt.pprofPort)
-			glog.V(0).Infof("MQ Broker pprof server listening on %s", pprofAddr)
-			glog.V(0).Infof("Access profiling at: http://localhost:%d/debug/pprof/", *mqBrokerOpt.pprofPort)
-			if err := http.ListenAndServe(pprofAddr, nil); err != nil {
-				glog.Errorf("pprof server error: %v", err)
-			}
-		}()
-	}
-
 	glog.V(0).Infof("MQ Broker listening on %s:%d", *mqBrokerOpt.ip, *mqBrokerOpt.port)
 	grpcS.Serve(grpcL)

@@ -26,6 +26,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/grace"
 	"github.com/seaweedfs/seaweedfs/weed/util/version"
 )

@@ -59,6 +60,8 @@ type S3Options struct {
 	concurrentUploadLimitMB   *int
 	concurrentFileUploadLimit *int
 	enableIam                 *bool
+	debug                     *bool
+	debugPort                 *int
 }

 func init() {
@@ -88,6 +91,8 @@ func init() {
 	s3StandaloneOptions.concurrentUploadLimitMB = cmdS3.Flag.Int("concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited")
 	s3StandaloneOptions.concurrentFileUploadLimit = cmdS3.Flag.Int("concurrentFileUploadLimit", 0, "limit number of concurrent file uploads, 0 means unlimited")
 	s3StandaloneOptions.enableIam = cmdS3.Flag.Bool("iam", true, "enable embedded IAM API on the same port")
+	s3StandaloneOptions.debug = cmdS3.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	s3StandaloneOptions.debugPort = cmdS3.Flag.Int("debug.port", 6060, "http port for debugging")
 }

 var cmdS3 = &Command{
@@ -182,6 +187,9 @@ var cmdS3 = &Command{
 }

 func runS3(cmd *Command, args []string) bool {
+	if *s3StandaloneOptions.debug {
+		grace.StartDebugServer(*s3StandaloneOptions.debugPort)
+	}

 	util.LoadSecurityConfiguration()

@@ -238,7 +246,7 @@ func (s3opt *S3Options) startS3Server() bool {
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("wait to connect to filers %v grpc address", filerAddresses)
+			glog.V(2).Infof("wait to connect to filers %v grpc address", filerAddresses)
 			time.Sleep(time.Second)
 		} else {
 			glog.V(0).Infof("connected to filers %v", filerAddresses)
@@ -341,6 +349,11 @@ func (s3opt *S3Options) startS3Server() bool {
 	go grpcS.Serve(grpcL)

 	if *s3opt.tlsPrivateKey != "" {
+		// Check for port conflict when both HTTP and HTTPS are enabled on the same port
+		if *s3opt.portHttps > 0 && *s3opt.portHttps == *s3opt.port {
+			glog.Fatalf("S3 API Server error: -s3.port.https (%d) cannot be the same as -s3.port (%d)", *s3opt.portHttps, *s3opt.port)
+		}
+
 		pemfileOptions := pemfile.Options{
 			CertFile:        *s3opt.tlsCertificate,
 			KeyFile:         *s3opt.tlsPrivateKey,
@@ -388,8 +401,11 @@ func (s3opt *S3Options) startS3Server() bool {
 			}
 		} else {
 			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", version.Version(), *s3opt.portHttps)
-			s3ApiListenerHttps, s3ApiLocalListenerHttps, _ := util.NewIpAndLocalListeners(
+			s3ApiListenerHttps, s3ApiLocalListenerHttps, err := util.NewIpAndLocalListeners(
 				*s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second)
+			if err != nil {
+				glog.Fatalf("S3 API HTTPS listener on %s:%d error: %v", *s3opt.bindIp, *s3opt.portHttps, err)
+			}
 			if s3ApiLocalListenerHttps != nil {
 				go func() {
 					if err = newHttpServer(router, tlsConfig).ServeTLS(s3ApiLocalListenerHttps, "", ""); err != nil {
@@ -415,6 +415,16 @@ key_path=""
 # The name list used to verify the cn name
 verify_cn=""

+[foundationdb]
+# FoundationDB provides ACID transactions and horizontal scalability.
+# Requires: go build -tags foundationdb
+enabled = false
+cluster_file = "/etc/foundationdb/fdb.cluster"
+# api_version = 740
+# timeout = "5s"
+# directory_prefix = "seaweedfs"
+# For bulk ingestion, enable batching: batch_enabled = true
+
 [tarantool]
 address = "localhost:3301"
 user = "guest"
@@ -72,6 +72,8 @@ type VolumeServerOptions struct {
 	hasSlowRead                 *bool
 	readBufferSizeMB            *int
 	ldbTimeout                  *int64
+	debug                       *bool
+	debugPort                   *int
 }

 func init() {
@@ -110,6 +112,8 @@ func init() {
 	v.inflightDownloadDataTimeout = cmdVolume.Flag.Duration("inflightDownloadDataTimeout", 60*time.Second, "inflight download data wait timeout of volume servers")
 	v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
 	v.readBufferSizeMB = cmdVolume.Flag.Int("readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally.")
+	v.debug = cmdVolume.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	v.debugPort = cmdVolume.Flag.Int("debug.port", 6060, "http port for debugging")
 }

 var cmdVolume = &Command{
@@ -129,6 +133,9 @@ var (
 )

 func runVolume(cmd *Command, args []string) bool {
+	if *v.debug {
+		grace.StartDebugServer(*v.debugPort)
+	}

 	util.LoadSecurityConfiguration()

@@ -3,13 +3,14 @@ package command
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"os"
 	"os/user"
 	"strconv"
 	"time"

+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -102,7 +103,7 @@ func (wo *WebDavOption) startWebDav() bool {
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *wo.filer, filerAddress.ToGrpcAddress())
+			glog.V(2).Infof("wait to connect to filer %s grpc address %s", *wo.filer, filerAddress.ToGrpcAddress())
 			time.Sleep(time.Second)
 		} else {
 			glog.V(0).Infof("connected to filer %s grpc address %s", *wo.filer, filerAddress.ToGrpcAddress())
@@ -205,3 +205,4 @@ func (q *CleanupQueue) OldestAge() time.Duration {
 }


+
@@ -369,3 +369,4 @@ func TestCleanupQueue_Concurrent(t *testing.T) {
 }


+
@@ -27,6 +27,10 @@ export WEED_FOUNDATIONDB_API_VERSION=740
 export WEED_FOUNDATIONDB_TIMEOUT=5s
 export WEED_FOUNDATIONDB_MAX_RETRY_DELAY=1s
 export WEED_FOUNDATIONDB_DIRECTORY_PREFIX=seaweedfs
+# Write batching (disabled by default)
+export WEED_FOUNDATIONDB_BATCH_ENABLED=false
+export WEED_FOUNDATIONDB_BATCH_SIZE=100
+export WEED_FOUNDATIONDB_BATCH_INTERVAL=1ms
 ```

 ### 3. Command Line Arguments
@@ -56,6 +60,18 @@ While not directly supported, configuration can be specified via config files pa
 |--------|------|---------|-------------|
 | `directory_prefix` | string | `seaweedfs` | Directory prefix for key organization |

+### Write Batching Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `batch_enabled` | boolean | `false` | Enable write batching. Disabled by default for optimal S3 PUT latency. |
+| `batch_size` | integer | `100` | Maximum number of operations per batch (when batching is enabled) |
+| `batch_interval` | duration | `1ms` | Maximum time to wait before flushing a batch (when batching is enabled) |
+
+**Note:** Write batching is **disabled by default**. Each write commits immediately in its own 
+transaction, providing optimal latency for S3 PUT operations. Enable batching only for 
+high-throughput bulk ingestion workloads where you can tolerate slightly higher per-operation latency.
+
 ## Configuration Examples

 ### Development Environment
@@ -82,7 +98,25 @@ max_retry_delay = "5s"
 directory_prefix = "seaweedfs_prod"
 ```

-### High-Performance Setup
+### High-Performance Setup (Low Latency)
+
+For S3 workloads requiring low latency per operation:
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+api_version = 740
+timeout = "5s"
+max_retry_delay = "1s"
+directory_prefix = "sw"  # Shorter prefix for efficiency
+# Batching disabled (default) for optimal per-operation latency
+batch_enabled = false
+```
+
+### High-Throughput Bulk Ingestion
+
+For bulk data loading where throughput matters more than per-operation latency:

 ```toml
 [foundationdb]
@@ -91,7 +125,11 @@ cluster_file = "/etc/foundationdb/fdb.cluster"
 api_version = 740
 timeout = "60s"
 max_retry_delay = "10s"
-directory_prefix = "sw"  # Shorter prefix for efficiency
+directory_prefix = "sw"
+# Enable batching for higher throughput
+batch_enabled = true
+batch_size = 100
+batch_interval = "1ms"
 ```

 ### Path-Specific Configuration
@@ -51,6 +51,9 @@ directory_prefix = "seaweedfs"
 | `timeout` | Operation timeout duration | `5s` | No |
 | `max_retry_delay` | Maximum retry delay | `1s` | No |
 | `directory_prefix` | Directory prefix for organization | `seaweedfs` | No |
+| `batch_enabled` | Enable write batching (see Performance section) | `false` | No |
+| `batch_size` | Max operations per batch | `100` | No |
+| `batch_interval` | Max time before batch flush | `1ms` | No |

 ### Path-Specific Configuration

@@ -109,12 +112,38 @@ make setup

 ## Performance Considerations

+### Write Batching Configuration
+
+By default, write batching is **disabled** (`batch_enabled = false`). Each write commits 
+immediately in its own transaction. This provides optimal latency for S3 PUT operations.
+
+**When to enable batching:**
+- High-throughput bulk ingestion workloads
+- Scenarios where you can tolerate slightly higher per-operation latency
+- Workloads with many concurrent small writes
+
+**Batching configuration options:**
+
+```toml
+[foundationdb]
+# Enable write batching (disabled by default for optimal S3 latency)
+batch_enabled = true
+# Maximum operations per batch
+batch_size = 100
+# Maximum time to wait before flushing a batch
+batch_interval = "1ms"
+```
+
+**Performance comparison:**
+- **Batching disabled**: Each S3 PUT commits immediately (~1-5ms per op depending on FDB latency)
+- **Batching enabled**: Operations are grouped, reducing total commits but adding batch interval latency
+
 ### Optimal Configuration

 - **API Version**: Use the latest stable API version (720+)
 - **Directory Structure**: Use logical directory prefixes to isolate different SeaweedFS instances
 - **Transaction Size**: Keep transactions under 10MB (FDB limit)
- **Batch Operations**: Use transactions for multiple related operations
+- **Concurrency**: Use multiple client connections for parallel operations

 ### Monitoring

@@ -45,8 +45,12 @@ const (
 	MAX_DIRECTORY_LIST_LIMIT = 1000

 	// Write batching defaults
+	// Note: Batching is disabled by default because S3 semantics require waiting
+	// for durability, and the batch timer adds latency to each operation.
+	// Enable batching only for workloads that can tolerate potential latency.
 	DEFAULT_BATCH_SIZE     = 100
-	DEFAULT_BATCH_INTERVAL = 5 * time.Millisecond
+	DEFAULT_BATCH_INTERVAL = 1 * time.Millisecond
+	DEFAULT_BATCH_ENABLED  = false
 )

 func init() {
@@ -129,11 +133,33 @@ func (b *writeBatcher) run() {
 		timer.Reset(b.interval)
 	}

+	// Collect available ops without blocking
+	collectAvailable := func() {
+		for {
+			select {
+			case op := <-b.ops:
+				batch = append(batch, op)
+				batchBytes += op.size()
+				if len(batch) >= b.size || batchBytes >= FDB_BATCH_SIZE_LIMIT {
+					return
+				}
+			default:
+				return
+			}
+		}
+	}
+
 	for {
 		select {
 		case op := <-b.ops:
 			batch = append(batch, op)
 			batchBytes += op.size()
+
+			// Optimization: When an operation arrives, try to collect more
+			// available operations without blocking. This improves throughput
+			// when multiple goroutines are submitting concurrently.
+			collectAvailable()
+
 			// Flush when batch count or size limit is reached
 			if len(batch) >= b.size || batchBytes >= FDB_BATCH_SIZE_LIMIT {
 				flush()
@@ -185,8 +211,10 @@ type FoundationDBStore struct {
 	directoryPrefix string
 	timeout         time.Duration
 	maxRetryDelay   time.Duration
-	// Write batching
+	// Write batching - disabled by default for optimal S3 latency
+	// Enable for high-throughput bulk ingestion workloads
 	batcher       *writeBatcher
+	batchEnabled  bool
 	batchSize     int
 	batchInterval time.Duration
 }
@@ -225,6 +253,10 @@ func (store *FoundationDBStore) Initialize(configuration util.Configuration, pre
 	configuration.SetDefault(prefix+"timeout", "5s")
 	configuration.SetDefault(prefix+"max_retry_delay", "1s")
 	configuration.SetDefault(prefix+"directory_prefix", "seaweedfs")
+	// Batching is disabled by default - each write commits immediately.
+	// This provides optimal latency for S3 PUT operations.
+	// Enable batching for high-throughput bulk ingestion workloads.
+	configuration.SetDefault(prefix+"batch_enabled", DEFAULT_BATCH_ENABLED)
 	configuration.SetDefault(prefix+"batch_size", DEFAULT_BATCH_SIZE)
 	configuration.SetDefault(prefix+"batch_interval", DEFAULT_BATCH_INTERVAL.String())

@@ -247,6 +279,7 @@ func (store *FoundationDBStore) Initialize(configuration util.Configuration, pre
 	}

 	// Parse batch configuration
+	store.batchEnabled = configuration.GetBool(prefix + "batch_enabled")
 	store.batchSize = configuration.GetInt(prefix + "batch_size")
 	if store.batchSize <= 0 {
 		store.batchSize = DEFAULT_BATCH_SIZE
@@ -288,10 +321,16 @@ func (store *FoundationDBStore) initialize(clusterFile string, apiVersion int) e
 		return fmt.Errorf("failed to create/open kv directory: %w", err)
 	}

-	// Start write batcher for improved throughput
-	store.batcher = newWriteBatcher(store, store.batchSize, store.batchInterval)
-	glog.V(0).Infof("FoundationDB: write batching enabled (batch_size=%d, batch_interval=%v)",
-		store.batchSize, store.batchInterval)
+	// Conditionally start write batcher
+	// Batching is disabled by default for optimal S3 latency.
+	// When disabled, each write commits immediately in its own transaction.
+	if store.batchEnabled {
+		store.batcher = newWriteBatcher(store, store.batchSize, store.batchInterval)
+		glog.V(0).Infof("FoundationDB: write batching enabled (batch_size=%d, batch_interval=%v)",
+			store.batchSize, store.batchInterval)
+	} else {
+		glog.V(0).Infof("FoundationDB: write batching disabled (direct commit mode for optimal latency)")
+	}

 	glog.V(0).Infof("FoundationDB store initialized successfully with directory prefix: %s", store.directoryPrefix)
 	return nil
--- a/Show More
+++ b/Show More
				`@@ -224,3 +224,4 @@ jobs:`
				`echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"`
				`@@ -91,3 +91,4 @@ jobs:`
				`echo "\| readonly \| /public \| Read-only \|" >> $GITHUB_STEP_SUMMARY`
				`@@ -205,3 +205,4 @@ func (q *CleanupQueue) OldestAge() time.Duration {`
				`}`
				`@@ -369,3 +369,4 @@ func TestCleanupQueue_Concurrent(t *testing.T) {`
				`}`