Make lock_manager.RenewInterval configurable in LiveLock

Co-authored-by: chrislusf <1543151+chrislusf@users.noreply.github.com>

.github/workflows/binaries_dev.yml

@@ -44,7 +44,7 @@ jobs:
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -60,7 +60,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -93,7 +93,7 @@ jobs:
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -109,7 +109,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release0.yml

@@ -30,7 +30,7 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release1.yml

@@ -30,7 +30,7 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release2.yml

@@ -30,7 +30,7 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release3.yml

@@ -30,7 +30,7 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release4.yml

@@ -30,7 +30,7 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -45,7 +45,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release5.yml

@@ -30,7 +30,7 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/container_release_unified.yml

@@ -224,3 +224,4 @@ jobs:
           echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
 
 
+

.github/workflows/e2e.yml

@@ -36,7 +36,7 @@ jobs:
       uses: docker/setup-buildx-action@v3
       
     - name: Cache Docker layers
-      uses: actions/cache@v4
+      uses: actions/cache@v5
       with:
         path: /tmp/.buildx-cache
         key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
@@ -134,7 +134,7 @@ jobs:
 
     - name: Archive logs
       if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: output-logs
         path: docker/output.log

.github/workflows/ec-integration-tests.yml

@@ -35,7 +35,7 @@ jobs:
 
     - name: Archive logs
       if: failure()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: ec-integration-test-logs
         path: test/erasure_coding

.github/workflows/fuse-integration.yml

@@ -183,7 +183,7 @@ jobs:
         
     - name: Upload Test Artifacts
       if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: fuse-integration-test-results
         path: |

.github/workflows/java_integration_tests.yml

@@ -53,6 +53,7 @@ jobs:
           mkdir -p "$WEED_DATA_DIR"
           
           # Start SeaweedFS with optimized settings for CI
+          # Include S3 API for s3copier integration tests
           weed server -dir="$WEED_DATA_DIR" \
             -master.raftHashicorp \
             -master.electionTimeout=1s \
@@ -61,6 +62,9 @@ jobs:
             -volume.preStopSeconds=1 \
             -master.peers=none \
             -filer -filer.maxMB=64 \
+            -s3 -s3.port=8333 \
+            -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \
+            -s3.allowDeleteBucketNotEmpty=true \
             -master.port=9333 \
             -volume.port=8080 \
             -filer.port=8888 \
@@ -105,6 +109,16 @@ jobs:
             sleep 2
           done
           
+          # Wait for S3 API
+          for i in {1..30}; do
+            if curl -s http://localhost:8333/ > /dev/null 2>&1; then
+              echo "✓ S3 API is ready"
+              break
+            fi
+            echo "Waiting for S3 API... ($i/30)"
+            sleep 2
+          done
+          
           echo "✓ All SeaweedFS components are ready!"
           
           # Display cluster status
@@ -133,6 +147,20 @@ jobs:
         run: |
           mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
       
+      - name: Run S3 ETag Validation Tests (Issue #7768)
+        working-directory: other/java/s3copier
+        env:
+          S3_ENDPOINT: http://127.0.0.1:8333
+          S3_ACCESS_KEY: some_access_key1
+          S3_SECRET_KEY: some_secret_key1
+        run: |
+          echo "Running S3 ETag validation tests against $S3_ENDPOINT"
+          mvn test -Dtest=ETagValidationTest \
+            -DS3_ENDPOINT=$S3_ENDPOINT \
+            -DS3_ACCESS_KEY=$S3_ACCESS_KEY \
+            -DS3_SECRET_KEY=$S3_SECRET_KEY \
+            -Dmaven.javadoc.skip=true -Dgpg.skip=true
+      
       - name: Display logs on failure
         if: failure()
         run: |

.github/workflows/java_unit_tests.yml

@@ -49,7 +49,7 @@ jobs:
       
       - name: Upload Test Reports
         if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: test-reports-java-${{ matrix.java }}
           path: |

.github/workflows/metadata-subscribe-tests.yml

@@ -70,7 +70,7 @@ jobs:
         
     - name: Archive logs on failure
       if: failure()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: metadata-subscribe-test-logs
         path: |

.github/workflows/postgres-tests.yml

@@ -35,7 +35,7 @@ jobs:
       uses: docker/setup-buildx-action@v3
 
     - name: Cache Docker layers
-      uses: actions/cache@v4
+      uses: actions/cache@v5
       with:
         path: /tmp/.buildx-cache
         key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
@@ -62,7 +62,7 @@ jobs:
 
     - name: Archive logs
       if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: postgres-logs
         path: test/postgres/postgres-output.log

.github/workflows/s3-filer-group-tests.yml

@@ -77,9 +77,10 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-filer-group-test-logs
           path: test/s3/filer_group/weed-test*.log
           retention-days: 3
 
+

.github/workflows/s3-go-tests.yml

@@ -76,7 +76,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-versioning-test-logs-${{ matrix.test-type }}
           path: test/s3/versioning/weed-test*.log
@@ -124,7 +124,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-versioning-compatibility-logs
           path: test/s3/versioning/weed-test*.log
@@ -172,7 +172,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-cors-compatibility-logs
           path: test/s3/cors/weed-test*.log
@@ -239,7 +239,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-retention-test-logs-${{ matrix.test-type }}
           path: test/s3/retention/weed-test*.log
@@ -306,7 +306,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-cors-test-logs-${{ matrix.test-type }}
           path: test/s3/cors/weed-test*.log
@@ -355,7 +355,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-retention-worm-logs
           path: test/s3/retention/weed-test*.log
@@ -365,8 +365,6 @@ jobs:
     name: S3 Versioning Stress Test
     runs-on: ubuntu-22.04
     timeout-minutes: 35
-    # Only run stress tests on master branch pushes to avoid overloading PR testing
-    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     
     steps:
       - name: Check out code
@@ -391,6 +389,9 @@ jobs:
           uname -a
           free -h
           
+          # Start server for stress tests
+          make start-server
+          
           # Run stress tests (concurrent operations)
           make test-versioning-stress || {
             echo "❌ Stress test failed, checking logs..."
@@ -398,14 +399,30 @@ jobs:
               echo "=== Server logs ==="
               tail -200 weed-test.log
             fi
+            make stop-server
             make clean
             exit 1
           }
+          
+          # Run pagination stress tests (>1000 versions)
+          echo "=== Running pagination stress tests ==="
+          make test-versioning-pagination-stress || {
+            echo "❌ Pagination stress test failed, checking logs..."
+            if [ -f weed-test.log ]; then
+              echo "=== Server logs ==="
+              tail -200 weed-test.log
+            fi
+            make stop-server
+            make clean
+            exit 1
+          }
+          
+          make stop-server
           make clean
 
       - name: Upload stress test logs
         if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-versioning-stress-logs
           path: test/s3/versioning/weed-test*.log
@@ -459,10 +476,65 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-tagging-test-logs
           path: test/s3/tagging/weed-test*.log
           retention-days: 3
 
+  s3-remote-cache-tests:
+    name: S3 Remote Cache Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false
+
+      - name: Run S3 Remote Cache Tests
+        timeout-minutes: 15
+        working-directory: test/s3/remote_cache
+        run: |
+          set -x
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          
+          # Run the remote cache integration tests
+          # Tests singleflight deduplication for caching remote objects
+          make test-with-server || {
+            echo "❌ Test failed, checking logs..."
+            if [ -f primary-weed.log ]; then
+              echo "=== Primary server logs ==="
+              tail -100 primary-weed.log
+            fi
+            if [ -f remote-weed.log ]; then
+              echo "=== Remote server logs ==="
+              tail -100 remote-weed.log
+            fi
+            echo "=== Process information ==="
+            ps aux | grep -E "(weed|test)" || true
+            exit 1
+          }
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: s3-remote-cache-test-logs
+          path: |
+            test/s3/remote_cache/primary-weed.log
+            test/s3/remote_cache/remote-weed.log
+          retention-days: 3
+
   # Removed SSE-C integration tests and compatibility job

.github/workflows/s3-iam-tests.yml

@@ -65,7 +65,7 @@ jobs:
 
       - name: Upload test results on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: iam-unit-test-results
           path: |
@@ -162,7 +162,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-iam-integration-logs-${{ matrix.test-type }}
           path: test/s3/iam/weed-*.log
@@ -222,7 +222,7 @@ jobs:
 
       - name: Upload distributed test logs
         if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-iam-distributed-logs
           path: test/s3/iam/weed-*.log
@@ -274,7 +274,7 @@ jobs:
 
       - name: Upload performance test results
         if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-iam-performance-results
           path: |

.github/workflows/s3-keycloak-tests.yml

@@ -152,7 +152,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-keycloak-test-logs
           path: |

.github/workflows/s3-parquet-tests.yml

@@ -121,7 +121,7 @@ jobs:
       
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: test-logs-python-${{ matrix.python-version }}
           path: |

.github/workflows/s3-sse-tests.yml

@@ -95,7 +95,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-sse-test-logs-${{ matrix.test-type }}
           path: test/s3/sse/weed-test*.log
@@ -143,7 +143,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-sse-compatibility-logs
           path: test/s3/sse/weed-test*.log
@@ -192,7 +192,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-sse-metadata-persistence-logs
           path: test/s3/sse/weed-test*.log
@@ -241,7 +241,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-sse-copy-operations-logs
           path: test/s3/sse/weed-test*.log
@@ -290,7 +290,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-sse-multipart-logs
           path: test/s3/sse/weed-test*.log
@@ -340,7 +340,7 @@ jobs:
 
       - name: Upload performance test logs
         if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: s3-sse-performance-logs
           path: test/s3/sse/weed-test*.log

.github/workflows/sftp-tests.yml

@@ -91,3 +91,4 @@ jobs:
         echo "| readonly | /public | Read-only |" >> $GITHUB_STEP_SUMMARY
 
 
+

.github/workflows/spark-integration-tests.yml

@@ -125,7 +125,7 @@ jobs:
 
     - name: Upload test results
       if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: spark-test-results
         path: test/java/spark/target/surefire-reports/
@@ -150,7 +150,7 @@ jobs:
     - name: Cache Apache Spark
       if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
       id: cache-spark
-      uses: actions/cache@v4
+      uses: actions/cache@v5
       with:
         path: spark-3.5.0-bin-hadoop3
         key: spark-3.5.0-hadoop3

.github/workflows/test-s3-over-https-using-awscli.yml

@@ -103,7 +103,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: seaweedfs-logs
           # Note: actions don't use defaults.run.working-directory, so path is relative to workspace root

.github/workflows/tus-tests.yml

@@ -106,7 +106,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: tus-test-logs
           path: |

.gitignore

@@ -126,3 +126,8 @@ weed-iam
 test/kafka/kafka-client-loadtest/weed-linux-arm64
 /test/tus/filerldb2
 coverage.out
+/test/s3/remote_cache/test-primary-data
+/test/s3/remote_cache/test-remote-data
+test/s3/remote_cache/remote-server.pid
+test/s3/remote_cache/primary-server.pid
+/test/erasure_coding/filerldb2

go.mod

@@ -28,7 +28,7 @@ require (
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
-	github.com/go-redsync/redsync/v4 v4.14.0
+	github.com/go-redsync/redsync/v4 v4.15.0
 	github.com/go-sql-driver/mysql v1.9.3
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.7.0
@@ -71,7 +71,7 @@ require (
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
-	github.com/seaweedfs/raft v1.1.3
+	github.com/seaweedfs/raft v1.1.6
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect
@@ -98,12 +98,12 @@ require (
 	gocloud.dev/pubsub/rabbitpubsub v0.43.0
 	golang.org/x/crypto v0.45.0
 	golang.org/x/exp v0.0.0-20250811191247-51f88131bc50
-	golang.org/x/image v0.33.0
+	golang.org/x/image v0.34.0
 	golang.org/x/net v0.47.0
 	golang.org/x/oauth2 v0.32.0 // indirect
 	golang.org/x/sys v0.38.0
-	golang.org/x/text v0.31.0 // indirect
-	golang.org/x/tools v0.38.0 // indirect
+	golang.org/x/text v0.32.0 // indirect
+	golang.org/x/tools v0.39.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.247.0
 	google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 // indirect
@@ -126,9 +126,9 @@ require (
 	github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
 	github.com/arangodb/go-driver v1.6.9
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.40.1
+	github.com/aws/aws-sdk-go-v2 v1.41.0
 	github.com/aws/aws-sdk-go-v2/config v1.31.3
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.3
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.5
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
@@ -152,14 +152,14 @@ require (
 	github.com/rabbitmq/amqp091-go v1.10.0
 	github.com/rclone/rclone v1.71.2
 	github.com/rdleal/intervalst v1.5.0
-	github.com/redis/go-redis/v9 v9.14.1
+	github.com/redis/go-redis/v9 v9.17.2
 	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/shirou/gopsutil/v4 v4.25.11
 	github.com/tarantool/go-tarantool/v2 v2.4.1
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0
 	go.etcd.io/etcd/client/pkg/v3 v3.6.6
 	go.uber.org/atomic v1.11.0
 	golang.org/x/sync v0.19.0
@@ -220,7 +220,7 @@ require (
 	go.opentelemetry.io/proto/otlp v1.7.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/mod v0.29.0 // indirect
+	golang.org/x/mod v0.30.0 // indirect
 	gonum.org/v1/gonum v0.16.0 // indirect
 )
 
@@ -260,21 +260,21 @@ require (
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
 	github.com/aws/smithy-go v1.24.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect

go.sum

@@ -665,22 +665,22 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.40.1 h1:difXb4maDZkRH0x//Qkwcfpdg1XQVXEAEs2DdXldFFc=
-github.com/aws/aws-sdk-go-v2 v1.40.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
+github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
+github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
 github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
 github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.3 h1:01Ym72hK43hjwDeJUfi1l2oYLXBAOR8gNSZNmXmvuas=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.3/go.mod h1:55nWF/Sr9Zvls0bGnWkRxUdhzKqj9uRNlPvgV1vgxKc=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 h1:utxLraaifrSBkeyII9mIbVwXXWrZdlPO7FIKmyLCEcY=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15/go.mod h1:hW6zjYUDQwfz3icf4g2O41PHi77u10oAzJ84iSzR/lo=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.5 h1:xMo63RlqP3ZZydpJDMBsH9uJ10hgHYfQFIk1cHDXrR4=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.5/go.mod h1:hhbH6oRcou+LpXfA/0vPElh/e0M3aFeOblE1sssAAEk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 h1:Y5YXgygXwDI5P4RkteB5yF7v35neH7LfJKBG+hzIons=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15/go.mod h1:K+/1EpG42dFSY7CBj+Fruzm8PsCGWTXJ3jdeJ659oGQ=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 h1:AvltKnW9ewxX2hFmQS0FyJH93aSvJVUEFvXfU+HWtSE=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15/go.mod h1:3I4oCdZdmgrREhU74qS1dK9yZ62yumob+58AbFR4cQA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
@@ -689,8 +689,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEd
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 h1:3/u/4yZOffg5jdNk1sDpOQ4Y+R6Xbh+GzpDrSZjuy3U=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15/go.mod h1:4Zkjq0FKjE78NKjabuM4tRXKFzUJWXgP0ItEZK8l7JU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
@@ -699,12 +699,12 @@ github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnY
 github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 h1:8sTTiw+9yuNXcfWeqKF2x01GqCF49CpP4Z9nKrrk/ts=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.6/go.mod h1:8WYg+Y40Sn3X2hioaaWAAIngndR8n1XFdRPPX+7QBaM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 h1:E+KqWoVsSrj1tJ6I/fjDIu5xoS2Zacuu1zT+H7KtiIk=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11/go.mod h1:qyWHz+4lvkXcr3+PoGlGHEI+3DLLiU6/GdrFfMaAhB0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 h1:tzMkjh0yTChUqJDgGkcDdxvZDSrJ/WB6R6ymI5ehqJI=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.3/go.mod h1:T270C0R5sZNLbWUe8ueiAF42XSZxxPocTaGSgs5c/60=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 h1:eYnlt6QxnFINKzwxP5/Ucs1vkG7VT3Iezmvfgc2waUw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.7/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
 github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
 github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
@@ -986,8 +986,8 @@ github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOr
 github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
-github.com/go-redsync/redsync/v4 v4.14.0 h1:zyxzFJsmQHIPBl8iBT7KFKohWsjsghgGLiP8TnFMLNc=
-github.com/go-redsync/redsync/v4 v4.14.0/go.mod h1:twMlVd19upZ/juvJyJGlQOSQxor1oeHtjs62l4pRFzo=
+github.com/go-redsync/redsync/v4 v4.15.0 h1:KH/XymuxSV7vyKs6z1Cxxj+N+N18JlPxgXeP6x4JY54=
+github.com/go-redsync/redsync/v4 v4.15.0/go.mod h1:qNp+lLs3vkfZbtA/aM/OjlZHfEr5YTAYhRktFPKHC7s=
 github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
 github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
 github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
@@ -1070,8 +1070,8 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
 github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s=
-github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
+github.com/gomodule/redigo v1.9.3 h1:dNPSXeXv6HCq2jdyWfjgmhBdqnR6PRO3m/G05nvpPC8=
+github.com/gomodule/redigo v1.9.3/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
@@ -1575,12 +1575,12 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
 github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
-github.com/redis/go-redis/v9 v9.14.1 h1:nDCrEiJmfOWhD76xlaw+HXT0c9hfNWeXgl0vIRYSDvQ=
-github.com/redis/go-redis/v9 v9.14.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
-github.com/redis/rueidis v1.0.64 h1:XqgbueDuNV3qFdVdQwAHJl1uNt90zUuAJuzqjH4cw6Y=
-github.com/redis/rueidis v1.0.64/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
-github.com/redis/rueidis/rueidiscompat v1.0.64 h1:M8JbLP4LyHQhBLBRsUQIzui8/LyTtdESNIMVveqm4RY=
-github.com/redis/rueidis/rueidiscompat v1.0.64/go.mod h1:8pJVPhEjpw0izZFSxYwDziUiEYEkEklTSw/nZzga61M=
+github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
+github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
+github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
+github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
+github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
+github.com/redis/rueidis/rueidiscompat v1.0.69/go.mod h1:iC4Y8DoN0Uth0Uezg9e2trvNRC7QAgGeuP2OPLb5ccI=
 github.com/rekby/fixenv v0.3.2/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
 github.com/rekby/fixenv v0.6.1 h1:jUFiSPpajT4WY2cYuc++7Y1zWrnCxnovGCIX72PZniM=
 github.com/rekby/fixenv v0.6.1/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
@@ -1618,8 +1618,8 @@ github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138 h1:bX
 github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
 github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
 github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
-github.com/seaweedfs/raft v1.1.3 h1:5B6hgneQ7IuU4Ceom/f6QUt8pEeqjcsRo+IxlyPZCws=
-github.com/seaweedfs/raft v1.1.3/go.mod h1:9cYlEBA+djJbnf/5tWsCybtbL7ICYpi+Uxcg3MxjuNs=
+github.com/seaweedfs/raft v1.1.6 h1:e83Xn0boscPnuSiBllUPeWRVS6JKrqJPYBozgFyBiC0=
+github.com/seaweedfs/raft v1.1.6/go.mod h1:9cYlEBA+djJbnf/5tWsCybtbL7ICYpi+Uxcg3MxjuNs=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
@@ -1781,8 +1781,8 @@ github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0 h1:/NyPd9KnCJgzrEXCArqk1T
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0 h1:Ml4WMsaRY+/+Wh3DCp1A+/zjM1aMdQg3irjuYFwWVHQ=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0/go.mod h1:/LjMxb/rXmoGAAnImoqAFIlhO5ampHacbvDetQitCk4=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0 h1:uvqheUfoEJz0CLLLaYct69wpvoaosM/2joWFcbwoqfw=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0/go.mod h1:/LjMxb/rXmoGAAnImoqAFIlhO5ampHacbvDetQitCk4=
 github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
 github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
 github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
@@ -1949,8 +1949,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
 golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
-golang.org/x/image v0.33.0 h1:LXRZRnv1+zGd5XBUVRFmYEphyyKJjQjCRiOuAP3sZfQ=
-golang.org/x/image v0.33.0/go.mod h1:DD3OsTYT9chzuzTQt+zMcOlBHgfoKQb1gry8p76Y1sc=
+golang.org/x/image v0.34.0 h1:33gCkyw9hmwbZJeZkct8XyR11yH889EQt/QH4VmXMn8=
+golang.org/x/image v0.34.0/go.mod h1:2RNFBZRB+vnwwFil8GkMdRvrJOFd1AzdZI6vOY+eJVU=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1985,8 +1985,8 @@ golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
-golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
+golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
+golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2260,8 +2260,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
+golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2341,8 +2341,8 @@ golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58
 golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
-golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
+golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
 golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
 golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json

@@ -1595,6 +1595,101 @@
       "title": "S3 Bucket Traffic Sent",
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "reqps",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 41
+      },
+      "id": 86,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$NAMESPACE\"}[$__interval])) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}}",
+          "refId": "A"
+        }
+      ],
+      "title": "S3 API Calls per Bucket",
+      "type": "timeseries"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -1659,8 +1754,8 @@
       },
       "gridPos": {
         "h": 7,
-        "w": 24,
-        "x": 0,
+        "w": 12,
+        "x": 12,
         "y": 41
       },
       "id": 72,
@@ -3266,6 +3361,209 @@
       ],
       "title": "Filer Go Routines",
       "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 48
+      },
+      "id": 89,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}} (logical)",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}} (physical)",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "S3 Bucket Size",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 48
+      },
+      "id": 90,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_object_count{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "S3 Bucket Object Count",
+      "type": "timeseries"
     }
   ],
   "refresh": "",
@@ -3356,4 +3654,4 @@
   "uid": "a24009d7-cbda-4443-a132-1cc1c4677304",
   "version": 1,
   "weekStart": ""
-}
+}

k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml

@@ -203,6 +203,9 @@ spec:
             - name: client-cert
               readOnly: true
               mountPath: /usr/local/share/ca-certificates/client/
+            - name: admin-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/admin/
             {{- end }}
             {{ tpl .Values.admin.extraVolumeMounts . | nindent 12 | trim }}
           ports:
@@ -298,6 +301,9 @@ spec:
         - name: client-cert
           secret:
             secretName: {{ template "seaweedfs.name" . }}-client-cert
+        - name: admin-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-admin-cert
         {{- end }}
         {{ tpl .Values.admin.extraVolumes . | indent 8 | trim }}
       {{- if .Values.admin.nodeSelector }}

k8s/charts/seaweedfs/templates/cert/admin-cert.yaml

@@ -0,0 +1,43 @@
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
+apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
+kind: Certificate
+metadata:
+  name: {{ template "seaweedfs.name" . }}-admin-cert
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: admin
+  {{- if .Values.admin.annotations }}
+  annotations:
+    {{- toYaml .Values.admin.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  secretName: {{ template "seaweedfs.name" . }}-admin-cert
+  issuerRef:
+    name: {{ template "seaweedfs.name" . }}-ca-issuer
+    kind: Issuer
+  commonName: {{ .Values.certificates.commonName }}
+  subject:
+    organizations:
+    - "SeaweedFS CA"
+  dnsNames:
+    - '*.{{ template "seaweedfs.name" . }}-admin'
+    - '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}'
+    - '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}.svc'
+    - '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}.svc.cluster.local'
+{{- if .Values.certificates.ipAddresses }}
+  ipAddresses:
+    {{- range .Values.certificates.ipAddresses }}
+    - {{ . }}
+    {{- end }}
+{{- end }}
+  privateKey:
+    algorithm: {{ .Values.certificates.keyAlgorithm }}
+    size: {{ .Values.certificates.keySize }}
+  duration: {{ .Values.certificates.duration }}
+  renewBefore: {{ .Values.certificates.renewBefore }}
+{{- end }}
+

k8s/charts/seaweedfs/templates/cert/worker-cert.yaml

@@ -0,0 +1,43 @@
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
+apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
+kind: Certificate
+metadata:
+  name: {{ template "seaweedfs.name" . }}-worker-cert
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: worker
+  {{- if .Values.worker.annotations }}
+  annotations:
+    {{- toYaml .Values.worker.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  secretName: {{ template "seaweedfs.name" . }}-worker-cert
+  issuerRef:
+    name: {{ template "seaweedfs.name" . }}-ca-issuer
+    kind: Issuer
+  commonName: {{ .Values.certificates.commonName }}
+  subject:
+    organizations:
+    - "SeaweedFS CA"
+  dnsNames:
+    - '*.{{ template "seaweedfs.name" . }}-worker'
+    - '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}'
+    - '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}.svc'
+    - '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}.svc.cluster.local'
+{{- if .Values.certificates.ipAddresses }}
+  ipAddresses:
+    {{- range .Values.certificates.ipAddresses }}
+    - {{ . }}
+    {{- end }}
+{{- end }}
+  privateKey:
+    algorithm: {{ .Values.certificates.keyAlgorithm }}
+    size: {{ .Values.certificates.keySize }}
+  duration: {{ .Values.certificates.duration }}
+  renewBefore: {{ .Values.certificates.renewBefore }}
+{{- end }}
+

k8s/charts/seaweedfs/templates/shared/security-configmap.yaml

@@ -65,6 +65,14 @@ data:
     cert = "/usr/local/share/ca-certificates/filer/tls.crt"
     key  = "/usr/local/share/ca-certificates/filer/tls.key"
 
+    [grpc.admin]
+    cert = "/usr/local/share/ca-certificates/admin/tls.crt"
+    key  = "/usr/local/share/ca-certificates/admin/tls.key"
+
+    [grpc.worker]
+    cert = "/usr/local/share/ca-certificates/worker/tls.crt"
+    key  = "/usr/local/share/ca-certificates/worker/tls.key"
+
     # use this for any place needs a grpc client
     # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
     [grpc.client]

k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml

@@ -171,6 +171,9 @@ spec:
             - name: client-cert
               readOnly: true
               mountPath: /usr/local/share/ca-certificates/client/
+            - name: worker-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/worker/
             {{- end }}
             {{ tpl .Values.worker.extraVolumeMounts . | nindent 12 | trim }}
           ports:
@@ -262,6 +265,9 @@ spec:
         - name: client-cert
           secret:
             secretName: {{ template "seaweedfs.name" . }}-client-cert
+        - name: worker-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-worker-cert
         {{- end }}
         {{ tpl .Values.worker.extraVolumes . | indent 8 | trim }}
       {{- if .Values.worker.nodeSelector }}

other/java/s3copier/README.md

@@ -0,0 +1,110 @@
+# SeaweedFS S3 Java SDK Compatibility Tests
+
+This project contains Java-based integration tests for SeaweedFS S3 API compatibility.
+
+## Overview
+
+Tests are provided for both AWS SDK v1 and v2 to ensure compatibility with the various SDK versions commonly used in production.
+
+## SDK Versions
+
+| SDK | Version | Notes |
+|-----|---------|-------|
+| AWS SDK v1 for Java | 1.12.600 | Legacy SDK, less strict ETag validation |
+| AWS SDK v2 for Java | 2.20.127 | Modern SDK with strict checksum validation |
+
+## Running Tests
+
+### Prerequisites
+
+1. SeaweedFS running with S3 API enabled:
+   ```bash
+   weed server -s3
+   ```
+
+2. Java 18+ and Maven
+
+### Run All Tests
+
+```bash
+mvn test
+```
+
+### Run Specific Tests
+
+```bash
+# Run only ETag validation tests (AWS SDK v2)
+mvn test -Dtest=ETagValidationTest
+
+# Run with custom endpoint
+mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
+```
+
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `S3_ENDPOINT` | `http://127.0.0.1:8333` | S3 API endpoint URL |
+| `S3_ACCESS_KEY` | `some_access_key1` | Access key ID |
+| `S3_SECRET_KEY` | `some_secret_key1` | Secret access key |
+| `S3_REGION` | `us-east-1` | AWS region |
+
+## Test Coverage
+
+### ETagValidationTest (AWS SDK v2)
+
+Tests for [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768) - ETag format validation.
+
+| Test | Description |
+|------|-------------|
+| `testSmallFilePutObject` | Verify small files return pure MD5 ETag |
+| `testLargeFilePutObject_Issue7768` | **Critical**: Verify large files (>8MB) return pure MD5 ETag |
+| `testExtraLargeFilePutObject` | Verify very large files (>24MB) return pure MD5 ETag |
+| `testMultipartUploadETag` | Verify multipart uploads return composite ETag |
+| `testETagConsistency` | Verify ETag consistency across PUT/HEAD/GET |
+| `testMultipleLargeFileUploads` | Stress test multiple large uploads |
+
+### Background: Issue #7768
+
+AWS SDK v2 for Java includes checksum validation that decodes the ETag as hexadecimal. When SeaweedFS returned composite ETags (`<md5>-<count>`) for regular `PutObject` with internally auto-chunked files, the SDK failed with:
+
+```
+java.lang.IllegalArgumentException: Invalid base 16 character: '-'
+```
+
+**Per AWS S3 specification:**
+- `PutObject`: ETag is always a pure MD5 hex string (32 chars)
+- `CompleteMultipartUpload`: ETag is composite format (`<md5>-<partcount>`)
+
+The fix ensures SeaweedFS follows this specification.
+
+## Project Structure
+
+```
+src/
+├── main/java/com/seaweedfs/s3/
+│   ├── PutObject.java           # Example PutObject with SDK v1
+│   └── HighLevelMultipartUpload.java
+└── test/java/com/seaweedfs/s3/
+    ├── PutObjectTest.java       # Basic SDK v1 test
+    └── ETagValidationTest.java  # Comprehensive SDK v2 ETag tests
+```
+
+## Validated SDK Versions
+
+This Java test project validates:
+
+- ✅ AWS SDK v2 for Java 2.20.127+
+- ✅ AWS SDK v1 for Java 1.12.600+
+
+Go SDK validation is performed by separate test suites:
+- See [Go ETag Tests](/test/s3/etag/) for AWS SDK v2 for Go tests
+- See [test/s3/SDK_COMPATIBILITY.md](/test/s3/SDK_COMPATIBILITY.md) for full SDK compatibility matrix
+
+## Related
+
+- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
+- [Go ETag Tests](/test/s3/etag/)
+- [SDK Compatibility Matrix](/test/s3/SDK_COMPATIBILITY.md)
+

other/java/s3copier/pom.xml

@@ -6,18 +6,29 @@
   <packaging>jar</packaging>
   <version>1.0-SNAPSHOT</version>
   <properties>
-    <maven.compiler.source>18</maven.compiler.source>
-    <maven.compiler.target>18</maven.compiler.target>
+    <maven.compiler.source>11</maven.compiler.source>
+    <maven.compiler.target>11</maven.compiler.target>
+    <aws.sdk.v1.version>1.12.600</aws.sdk.v1.version>
+    <aws.sdk.v2.version>2.20.127</aws.sdk.v2.version>
   </properties>
   <name>copier</name>
   <url>http://maven.apache.org</url>
 
   <dependencyManagement>
     <dependencies>
+      <!-- AWS SDK v1 BOM -->
       <dependency>
         <groupId>com.amazonaws</groupId>
         <artifactId>aws-java-sdk-bom</artifactId>
-        <version>1.11.327</version>
+        <version>${aws.sdk.v1.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+      <!-- AWS SDK v2 BOM -->
+      <dependency>
+        <groupId>software.amazon.awssdk</groupId>
+        <artifactId>bom</artifactId>
+        <version>${aws.sdk.v2.version}</version>
         <type>pom</type>
         <scope>import</scope>
       </dependency>
@@ -25,15 +36,50 @@
   </dependencyManagement>
 
   <dependencies>
+    <!-- AWS SDK v1 (for backward compatibility with existing code) -->
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk-s3</artifactId>
     </dependency>
+    
+    <!-- AWS SDK v2 (for modern tests with checksum validation) -->
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3-transfer-manager</artifactId>
+    </dependency>
+    
+    <!-- Test dependencies -->
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
-      <version>4.13.1</version>
+      <version>4.13.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>5.10.0</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
+      <version>3.24.2</version>
       <scope>test</scope>
     </dependency>
   </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>3.2.2</version>
+      </plugin>
+    </plugins>
+  </build>
 </project>

other/java/s3copier/src/test/java/com/seaweedfs/s3/ETagValidationTest.java

@@ -0,0 +1,439 @@
+package com.seaweedfs.s3;
+
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.*;
+
+import java.net.URI;
+import java.security.MessageDigest;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.regex.Pattern;
+
+import static org.assertj.core.api.Assertions.*;
+
+/**
+ * AWS SDK v2 Integration Tests for S3 ETag Format Validation.
+ * 
+ * These tests verify that SeaweedFS returns correct ETag formats that are
+ * compatible with AWS SDK v2's checksum validation.
+ * 
+ * Background (GitHub Issue #7768):
+ * AWS SDK v2 for Java validates ETags as hexadecimal MD5 hashes for PutObject
+ * responses. If the ETag contains non-hex characters (like '-' in composite
+ * format), the SDK fails with "Invalid base 16 character: '-'".
+ * 
+ * Per AWS S3 specification:
+ * - Regular PutObject: ETag is always a pure MD5 hex string (32 chars)
+ * - CompleteMultipartUpload: ETag is composite format "<md5>-<partcount>"
+ * 
+ * To run these tests:
+ *   mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
+ * 
+ * Or set environment variable:
+ *   export S3_ENDPOINT=http://localhost:8333
+ *   mvn test -Dtest=ETagValidationTest
+ */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@DisplayName("S3 ETag Format Validation Tests (AWS SDK v2)")
+class ETagValidationTest {
+
+    // Configuration - can be overridden via system properties or environment variables
+    private static final String DEFAULT_ENDPOINT = "http://127.0.0.1:8333";
+    private static final String DEFAULT_ACCESS_KEY = "some_access_key1";
+    private static final String DEFAULT_SECRET_KEY = "some_secret_key1";
+    private static final String DEFAULT_REGION = "us-east-1";
+    
+    // Auto-chunking threshold in SeaweedFS (must match s3api_object_handlers_put.go)
+    private static final int AUTO_CHUNK_SIZE = 8 * 1024 * 1024; // 8MB
+    
+    // Test sizes
+    private static final int SMALL_FILE_SIZE = 1024; // 1KB
+    private static final int LARGE_FILE_SIZE = 10 * 1024 * 1024; // 10MB (triggers auto-chunking)
+    private static final int XL_FILE_SIZE = 25 * 1024 * 1024; // 25MB (multiple chunks)
+    private static final int MULTIPART_PART_SIZE = 5 * 1024 * 1024; // 5MB per part
+    
+    // ETag format patterns
+    private static final Pattern PURE_MD5_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}\"?$");
+    private static final Pattern COMPOSITE_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}-\\d+\"?$");
+    
+    private S3Client s3Client;
+    private String testBucketName;
+    private final SecureRandom random = new SecureRandom();
+
+    @BeforeAll
+    void setUp() {
+        String endpoint = getConfig("S3_ENDPOINT", DEFAULT_ENDPOINT);
+        String accessKey = getConfig("S3_ACCESS_KEY", DEFAULT_ACCESS_KEY);
+        String secretKey = getConfig("S3_SECRET_KEY", DEFAULT_SECRET_KEY);
+        String region = getConfig("S3_REGION", DEFAULT_REGION);
+        
+        System.out.println("Connecting to S3 endpoint: " + endpoint);
+        
+        s3Client = S3Client.builder()
+                .endpointOverride(URI.create(endpoint))
+                .credentialsProvider(StaticCredentialsProvider.create(
+                        AwsBasicCredentials.create(accessKey, secretKey)))
+                .region(Region.of(region))
+                .forcePathStyle(true) // Required for SeaweedFS
+                .build();
+        
+        // Create test bucket
+        testBucketName = "test-etag-" + UUID.randomUUID().toString().substring(0, 8);
+        s3Client.createBucket(CreateBucketRequest.builder()
+                .bucket(testBucketName)
+                .build());
+        
+        System.out.println("Created test bucket: " + testBucketName);
+    }
+
+    @AfterAll
+    void tearDown() {
+        if (s3Client != null && testBucketName != null) {
+            try {
+                // Delete all objects with pagination
+                String continuationToken = null;
+                do {
+                    ListObjectsV2Response listResp = s3Client.listObjectsV2(
+                            ListObjectsV2Request.builder()
+                                    .bucket(testBucketName)
+                                    .continuationToken(continuationToken)
+                                    .build());
+                    for (S3Object obj : listResp.contents()) {
+                        s3Client.deleteObject(DeleteObjectRequest.builder()
+                                .bucket(testBucketName)
+                                .key(obj.key())
+                                .build());
+                    }
+                    continuationToken = listResp.nextContinuationToken();
+                } while (continuationToken != null);
+                
+                // Abort any multipart uploads
+                ListMultipartUploadsResponse mpResp = s3Client.listMultipartUploads(
+                        ListMultipartUploadsRequest.builder().bucket(testBucketName).build());
+                for (MultipartUpload upload : mpResp.uploads()) {
+                    s3Client.abortMultipartUpload(AbortMultipartUploadRequest.builder()
+                            .bucket(testBucketName)
+                            .key(upload.key())
+                            .uploadId(upload.uploadId())
+                            .build());
+                }
+                
+                // Delete bucket
+                s3Client.deleteBucket(DeleteBucketRequest.builder()
+                        .bucket(testBucketName)
+                        .build());
+                
+                System.out.println("Cleaned up test bucket: " + testBucketName);
+            } catch (Exception e) {
+                System.err.println("Warning: Failed to cleanup test bucket: " + e.getMessage());
+            }
+            s3Client.close();
+        }
+    }
+
+    @Test
+    @DisplayName("Small file PutObject should return pure MD5 ETag")
+    void testSmallFilePutObject() throws Exception {
+        byte[] testData = generateRandomData(SMALL_FILE_SIZE);
+        String expectedMD5 = calculateMD5Hex(testData);
+        String objectKey = "small-file-" + UUID.randomUUID() + ".bin";
+        
+        PutObjectResponse response = s3Client.putObject(
+                PutObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build(),
+                RequestBody.fromBytes(testData));
+        
+        String etag = response.eTag();
+        System.out.println("Small file ETag: " + etag + " (expected MD5: " + expectedMD5 + ")");
+        
+        assertThat(etag)
+                .describedAs("Small file ETag should be pure MD5")
+                .matches(PURE_MD5_PATTERN);
+        assertThat(cleanETag(etag))
+                .describedAs("ETag should match calculated MD5")
+                .isEqualTo(expectedMD5);
+        assertThat(etag)
+                .describedAs("ETag should not contain hyphen")
+                .doesNotContain("-");
+    }
+
+    /**
+     * Critical test for GitHub Issue #7768.
+     * 
+     * This test uploads a file larger than the auto-chunking threshold (8MB),
+     * which triggers SeaweedFS to split the file into multiple internal chunks.
+     * 
+     * Previously, this caused SeaweedFS to return a composite ETag like
+     * "d41d8cd98f00b204e9800998ecf8427e-2", which AWS SDK v2 rejected because
+     * it validates the ETag as hexadecimal and '-' is not a valid hex character.
+     * 
+     * The fix ensures that regular PutObject always returns a pure MD5 ETag,
+     * regardless of internal chunking.
+     */
+    @Test
+    @DisplayName("Large file PutObject (>8MB) should return pure MD5 ETag - Issue #7768")
+    void testLargeFilePutObject_Issue7768() throws Exception {
+        byte[] testData = generateRandomData(LARGE_FILE_SIZE);
+        String expectedMD5 = calculateMD5Hex(testData);
+        String objectKey = "large-file-" + UUID.randomUUID() + ".bin";
+        
+        System.out.println("Uploading large file (" + LARGE_FILE_SIZE + " bytes, " +
+                "> " + AUTO_CHUNK_SIZE + " byte auto-chunk threshold)...");
+        
+        // This is where Issue #7768 would manifest - SDK v2 validates ETag
+        PutObjectResponse response = s3Client.putObject(
+                PutObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build(),
+                RequestBody.fromBytes(testData));
+        
+        String etag = response.eTag();
+        int expectedChunks = (LARGE_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
+        System.out.println("Large file ETag: " + etag + 
+                " (expected MD5: " + expectedMD5 + ", internal chunks: ~" + expectedChunks + ")");
+        
+        // These assertions would fail before the fix
+        assertThat(etag)
+                .describedAs("Large file PutObject ETag MUST be pure MD5 (not composite)")
+                .matches(PURE_MD5_PATTERN);
+        assertThat(etag)
+                .describedAs("Large file ETag should NOT be composite format")
+                .doesNotMatch(COMPOSITE_PATTERN);
+        assertThat(etag)
+                .describedAs("ETag should not contain hyphen for regular PutObject")
+                .doesNotContain("-");
+        assertThat(cleanETag(etag))
+                .describedAs("ETag should match calculated MD5")
+                .isEqualTo(expectedMD5);
+        
+        // Verify hex decoding works (this is what fails in Issue #7768)
+        assertThatCode(() -> hexToBytes(cleanETag(etag)))
+                .describedAs("ETag should be valid hexadecimal (AWS SDK v2 validation)")
+                .doesNotThrowAnyException();
+    }
+
+    @Test
+    @DisplayName("Extra large file PutObject (>24MB) should return pure MD5 ETag")
+    void testExtraLargeFilePutObject() throws Exception {
+        byte[] testData = generateRandomData(XL_FILE_SIZE);
+        String expectedMD5 = calculateMD5Hex(testData);
+        String objectKey = "xl-file-" + UUID.randomUUID() + ".bin";
+        
+        int expectedChunks = (XL_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
+        System.out.println("Uploading XL file (" + XL_FILE_SIZE + " bytes, ~" + 
+                expectedChunks + " internal chunks)...");
+        
+        PutObjectResponse response = s3Client.putObject(
+                PutObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build(),
+                RequestBody.fromBytes(testData));
+        
+        String etag = response.eTag();
+        System.out.println("XL file ETag: " + etag);
+        
+        assertThat(etag)
+                .describedAs("XL file PutObject ETag MUST be pure MD5")
+                .matches(PURE_MD5_PATTERN);
+        assertThat(cleanETag(etag))
+                .describedAs("ETag should match calculated MD5")
+                .isEqualTo(expectedMD5);
+    }
+
+    @Test
+    @DisplayName("Multipart upload should return composite ETag")
+    void testMultipartUploadETag() throws Exception {
+        int totalSize = 15 * 1024 * 1024; // 15MB = 3 parts
+        byte[] testData = generateRandomData(totalSize);
+        String objectKey = "multipart-file-" + UUID.randomUUID() + ".bin";
+        
+        System.out.println("Performing multipart upload (" + totalSize + " bytes)...");
+        
+        // Initiate multipart upload
+        CreateMultipartUploadResponse createResp = s3Client.createMultipartUpload(
+                CreateMultipartUploadRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build());
+        String uploadId = createResp.uploadId();
+        
+        List<CompletedPart> completedParts = new ArrayList<>();
+        int partNumber = 1;
+        
+        // Upload parts
+        for (int offset = 0; offset < totalSize; offset += MULTIPART_PART_SIZE) {
+            int end = Math.min(offset + MULTIPART_PART_SIZE, totalSize);
+            byte[] partData = new byte[end - offset];
+            System.arraycopy(testData, offset, partData, 0, partData.length);
+            
+            UploadPartResponse uploadResp = s3Client.uploadPart(
+                    UploadPartRequest.builder()
+                            .bucket(testBucketName)
+                            .key(objectKey)
+                            .uploadId(uploadId)
+                            .partNumber(partNumber)
+                            .build(),
+                    RequestBody.fromBytes(partData));
+            
+            completedParts.add(CompletedPart.builder()
+                    .partNumber(partNumber)
+                    .eTag(uploadResp.eTag())
+                    .build());
+            partNumber++;
+        }
+        
+        // Complete multipart upload
+        CompleteMultipartUploadResponse completeResp = s3Client.completeMultipartUpload(
+                CompleteMultipartUploadRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .uploadId(uploadId)
+                        .multipartUpload(CompletedMultipartUpload.builder()
+                                .parts(completedParts)
+                                .build())
+                        .build());
+        
+        String etag = completeResp.eTag();
+        System.out.println("Multipart upload ETag: " + etag + " (" + completedParts.size() + " parts)");
+        
+        // Multipart uploads SHOULD have composite ETag
+        assertThat(etag)
+                .describedAs("Multipart upload ETag SHOULD be composite format")
+                .matches(COMPOSITE_PATTERN);
+        assertThat(etag)
+                .describedAs("Multipart ETag should contain hyphen")
+                .contains("-");
+        
+        // Verify part count in ETag
+        String[] parts = cleanETag(etag).split("-");
+        assertThat(parts).hasSize(2);
+        assertThat(parts[1])
+                .describedAs("Part count in ETag should match uploaded parts")
+                .isEqualTo(String.valueOf(completedParts.size()));
+    }
+
+    @Test
+    @DisplayName("ETag should be consistent across PUT, HEAD, and GET")
+    void testETagConsistency() throws Exception {
+        byte[] testData = generateRandomData(LARGE_FILE_SIZE);
+        String objectKey = "consistency-test-" + UUID.randomUUID() + ".bin";
+        
+        // PUT
+        PutObjectResponse putResp = s3Client.putObject(
+                PutObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build(),
+                RequestBody.fromBytes(testData));
+        String putETag = putResp.eTag();
+        
+        // HEAD
+        HeadObjectResponse headResp = s3Client.headObject(
+                HeadObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build());
+        String headETag = headResp.eTag();
+        
+        // GET
+        GetObjectResponse getResp = s3Client.getObject(
+                GetObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build())
+                .response();
+        String getETag = getResp.eTag();
+        
+        System.out.println("PUT ETag: " + putETag + ", HEAD ETag: " + headETag + ", GET ETag: " + getETag);
+        
+        assertThat(putETag).isEqualTo(headETag);
+        assertThat(putETag).isEqualTo(getETag);
+    }
+
+    @Test
+    @DisplayName("Multiple large file uploads should all return pure MD5 ETags")
+    void testMultipleLargeFileUploads() throws Exception {
+        int numFiles = 3;
+        
+        for (int i = 0; i < numFiles; i++) {
+            byte[] testData = generateRandomData(LARGE_FILE_SIZE);
+            String expectedMD5 = calculateMD5Hex(testData);
+            String objectKey = "multi-large-" + i + "-" + UUID.randomUUID() + ".bin";
+            
+            PutObjectResponse response = s3Client.putObject(
+                    PutObjectRequest.builder()
+                            .bucket(testBucketName)
+                            .key(objectKey)
+                            .build(),
+                    RequestBody.fromBytes(testData));
+            
+            String etag = response.eTag();
+            System.out.println("File " + i + " ETag: " + etag);
+            
+            assertThat(etag)
+                    .describedAs("File " + i + " ETag should be pure MD5")
+                    .matches(PURE_MD5_PATTERN);
+            assertThat(cleanETag(etag))
+                    .describedAs("File " + i + " ETag should match MD5")
+                    .isEqualTo(expectedMD5);
+            
+            // Validate hex decoding
+            assertThatCode(() -> hexToBytes(cleanETag(etag)))
+                    .doesNotThrowAnyException();
+        }
+    }
+
+    // Helper methods
+    
+    private String getConfig(String key, String defaultValue) {
+        String value = System.getProperty(key);
+        if (value == null) {
+            value = System.getenv(key);
+        }
+        return value != null ? value : defaultValue;
+    }
+    
+    private byte[] generateRandomData(int size) {
+        byte[] data = new byte[size];
+        random.nextBytes(data);
+        return data;
+    }
+    
+    private String calculateMD5Hex(byte[] data) throws Exception {
+        MessageDigest md = MessageDigest.getInstance("MD5");
+        byte[] digest = md.digest(data);
+        StringBuilder sb = new StringBuilder();
+        for (byte b : digest) {
+            sb.append(String.format("%02x", b));
+        }
+        return sb.toString();
+    }
+    
+    private String cleanETag(String etag) {
+        if (etag == null) return null;
+        return etag.replace("\"", "");
+    }
+    
+    private byte[] hexToBytes(String hex) {
+        int len = hex.length();
+        byte[] data = new byte[len / 2];
+        for (int i = 0; i < len; i += 2) {
+            data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4)
+                    + Character.digit(hex.charAt(i + 1), 16));
+        }
+        return data;
+    }
+}
+

other/metrics/grafana_seaweedfs.json

@@ -1319,6 +1319,99 @@
         "alignLevel": null
       }
     },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "hiddenSeries": false,
+      "id": 90,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.1.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(SeaweedFS_s3_request_total[$__interval])) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 API Calls per Bucket",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "reqps",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
     {
       "aliasColors": {},
       "bars": false,
@@ -1332,8 +1425,8 @@
       "grid": {},
       "gridPos": {
         "h": 7,
-        "w": 24,
-        "x": 0,
+        "w": 12,
+        "x": 12,
         "y": 16
       },
       "hiddenSeries": false,
@@ -2960,6 +3053,200 @@
         "align": false,
         "alignLevel": null
       }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 37
+      },
+      "hiddenSeries": false,
+      "id": 87,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.1.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "max(SeaweedFS_s3_bucket_size_bytes) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}} (logical)",
+          "refId": "A"
+        },
+        {
+          "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}} (physical)",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 Bucket Size",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "decbytes",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 37
+      },
+      "hiddenSeries": false,
+      "id": 88,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.1.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "max(SeaweedFS_s3_bucket_object_count) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 Bucket Object Count",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
     }
   ],
   "refresh": "30s",

other/metrics/grafana_seaweedfs_heartbeat.json

@@ -875,6 +875,85 @@
             }
           ]
         },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS-DEV}",
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 16
+          },
+          "hiddenSeries": false,
+          "id": 86,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(SeaweedFS_s3_request_total[$__interval])) by (bucket)",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{bucket}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "S3 API Calls per Bucket",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "reqps",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ]
+        },
         {
           "aliasColors": {},
           "bars": false,
@@ -1087,6 +1166,202 @@
               "show": false
             }
           ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "fillGradient": 0,
+          "fieldConfig": {
+            "defaults": {
+              "unit": "decbytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 37
+          },
+          "hiddenSeries": false,
+          "id": 87,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.1.2",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(SeaweedFS_s3_bucket_size_bytes) by (bucket)",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{bucket}} (logical)",
+              "refId": "A"
+            },
+            {
+              "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes) by (bucket)",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{bucket}} (physical)",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "S3 Bucket Size",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "decbytes",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "fillGradient": 0,
+          "fieldConfig": {
+            "defaults": {
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 37
+          },
+          "hiddenSeries": false,
+          "id": 88,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.1.2",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(SeaweedFS_s3_bucket_object_count) by (bucket)",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{bucket}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "S3 Bucket Object Count",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         }
       ],
       "repeat": null,
@@ -2111,4 +2386,4 @@
   "timezone": "browser",
   "title": "SeaweedFS ks8 heartbeat",
   "version": 1
-}
+}

other/metrics/grafana_seaweedfs_k8s.json

@@ -447,6 +447,99 @@
         "alignLevel": null
       }
     },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fill": 1,
+      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 6,
+        "x": 0,
+        "y": 15
+      },
+      "hiddenSeries": false,
+      "id": 86,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.1.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$namespace\",service=~\"$service-api\"}[$__interval])) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 API Calls per Bucket",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "reqps",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
     {
       "aliasColors": {},
       "bars": false,
@@ -466,8 +559,8 @@
       "grid": {},
       "gridPos": {
         "h": 7,
-        "w": 8,
-        "x": 0,
+        "w": 6,
+        "x": 12,
         "y": 15
       },
       "hiddenSeries": false,
@@ -577,8 +670,8 @@
       "grid": {},
       "gridPos": {
         "h": 7,
-        "w": 8,
-        "x": 8,
+        "w": 6,
+        "x": 6,
         "y": 15
       },
       "hiddenSeries": false,
@@ -688,8 +781,8 @@
       "grid": {},
       "gridPos": {
         "h": 7,
-        "w": 8,
-        "x": 16,
+        "w": 6,
+        "x": 18,
         "y": 15
       },
       "hiddenSeries": false,
@@ -2354,6 +2447,209 @@
         "align": false,
         "alignLevel": null
       }
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 41
+      },
+      "id": 89,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}} (logical)",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}} (physical)",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "S3 Bucket Size",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 41
+      },
+      "id": 90,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_object_count{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "S3 Bucket Object Count",
+      "type": "timeseries"
     }
   ],
   "refresh": "30s",
@@ -2545,4 +2841,4 @@
   "timezone": "browser",
   "title": "SeaweedFS",
   "version": 2
-}
+}

test/erasure_coding/Makefile

@@ -0,0 +1,187 @@
+# Makefile for EC integration testing
+# Usage:
+#   make start      - Start the test cluster (master + 6 volume servers + filer)
+#   make stop       - Stop the test cluster
+#   make populate   - Populate test data (~300MB across 7 volumes)
+#   make shell      - Open weed shell connected to the test cluster
+#   make clean      - Stop cluster and remove all test data
+#   make setup      - Start cluster and populate data (one command)
+#
+# Requirements: curl, jq
+
+WEED_BINARY := $(shell pwd)/../../weed/weed
+TEST_DIR := /tmp/ec_manual_test
+# Use non-standard ports to avoid conflicts with existing SeaweedFS servers
+MASTER_PORT := 29333
+FILER_PORT := 28888
+VOLUME_BASE_PORT := 28080
+NUM_VOLUME_SERVERS := 6
+VOLUME_SIZE_LIMIT_MB := 30
+MAX_VOLUMES_PER_SERVER := 10
+
+# Build weed binary if it doesn't exist
+$(WEED_BINARY):
+	cd ../../weed && go build -o weed .
+
+.PHONY: build
+build: $(WEED_BINARY)
+
+.PHONY: start
+start: build
+	@echo "=== Starting SeaweedFS test cluster ==="
+	@mkdir -p $(TEST_DIR)/master $(TEST_DIR)/filer
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do mkdir -p $(TEST_DIR)/volume$$i; done
+	@# Create security.toml with JWT disabled
+	@echo "# Disable JWT for testing" > $(TEST_DIR)/security.toml
+	@echo '[jwt.signing]' >> $(TEST_DIR)/security.toml
+	@echo 'key = ""' >> $(TEST_DIR)/security.toml
+	@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
+	@echo '' >> $(TEST_DIR)/security.toml
+	@echo '[jwt.signing.read]' >> $(TEST_DIR)/security.toml
+	@echo 'key = ""' >> $(TEST_DIR)/security.toml
+	@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
+	@# Create filer.toml with leveldb2
+	@echo '[leveldb2]' > $(TEST_DIR)/filer.toml
+	@echo 'enabled = true' >> $(TEST_DIR)/filer.toml
+	@echo 'dir = "$(TEST_DIR)/filer/filerldb2"' >> $(TEST_DIR)/filer.toml
+	@# Start master
+	@echo "Starting master on port $(MASTER_PORT)..."
+	@cd $(TEST_DIR) && $(WEED_BINARY) master \
+		-port=$(MASTER_PORT) \
+		-mdir=$(TEST_DIR)/master \
+		-volumeSizeLimitMB=$(VOLUME_SIZE_LIMIT_MB) \
+		-ip=127.0.0.1 \
+		> $(TEST_DIR)/master/master.log 2>&1 & echo $$! > $(TEST_DIR)/master.pid
+	@sleep 3
+	@# Start volume servers (run from TEST_DIR to find security.toml)
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
+		port=$$(($(VOLUME_BASE_PORT) + $$i)); \
+		echo "Starting volume server $$i on port $$port (rack$$i)..."; \
+		cd $(TEST_DIR) && $(WEED_BINARY) volume \
+			-port=$$port \
+			-dir=$(TEST_DIR)/volume$$i \
+			-max=$(MAX_VOLUMES_PER_SERVER) \
+			-master=127.0.0.1:$(MASTER_PORT) \
+			-ip=127.0.0.1 \
+			-dataCenter=dc1 \
+			-rack=rack$$i \
+			> $(TEST_DIR)/volume$$i/volume.log 2>&1 & echo $$! > $(TEST_DIR)/volume$$i.pid; \
+	done
+	@sleep 3
+	@# Start filer (run from TEST_DIR to find security.toml)
+	@echo "Starting filer on port $(FILER_PORT)..."
+	@cd $(TEST_DIR) && $(WEED_BINARY) filer \
+		-port=$(FILER_PORT) \
+		-master=127.0.0.1:$(MASTER_PORT) \
+		-ip=127.0.0.1 \
+		> $(TEST_DIR)/filer/filer.log 2>&1 & echo $$! > $(TEST_DIR)/filer.pid
+	@sleep 3
+	@echo ""
+	@echo "=== Cluster started ==="
+	@echo "Master: http://127.0.0.1:$(MASTER_PORT)"
+	@echo "Filer:  http://127.0.0.1:$(FILER_PORT)"
+	@echo "Volume servers: http://127.0.0.1:$(VOLUME_BASE_PORT) - http://127.0.0.1:$$(($(VOLUME_BASE_PORT) + $(NUM_VOLUME_SERVERS) - 1))"
+	@echo ""
+	@echo "Run 'make shell' to open weed shell"
+	@echo "Run 'make populate' to add test data"
+
+.PHONY: stop
+stop:
+	@echo "=== Stopping SeaweedFS test cluster ==="
+	@# Stop filer by PID
+	@-[ -f $(TEST_DIR)/filer.pid ] && kill $$(cat $(TEST_DIR)/filer.pid) 2>/dev/null && rm -f $(TEST_DIR)/filer.pid || true
+	@# Stop volume servers by PID
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
+		[ -f $(TEST_DIR)/volume$$i.pid ] && kill $$(cat $(TEST_DIR)/volume$$i.pid) 2>/dev/null && rm -f $(TEST_DIR)/volume$$i.pid || true; \
+	done
+	@# Stop master by PID
+	@-[ -f $(TEST_DIR)/master.pid ] && kill $$(cat $(TEST_DIR)/master.pid) 2>/dev/null && rm -f $(TEST_DIR)/master.pid || true
+	@# Fallback: use pkill with specific patterns to ensure cleanup
+	@-pkill -f "weed filer.*-master=127.0.0.1:$(MASTER_PORT)" 2>/dev/null || true
+	@-pkill -f "weed volume.*-dir=$(TEST_DIR)/volume" 2>/dev/null || true
+	@-pkill -f "weed master.*-mdir=$(TEST_DIR)/master" 2>/dev/null || true
+	@echo "Cluster stopped."
+
+.PHONY: clean
+clean: stop
+	@echo "Removing test data..."
+	@rm -rf $(TEST_DIR)
+	@echo "Clean complete."
+
+.PHONY: populate
+populate:
+	@echo "=== Populating test data (~300MB) ==="
+	@# Create a 500KB test file template using mktemp for isolation
+	@tmpfile=$$(mktemp) && \
+	dd if=/dev/urandom bs=1024 count=500 of=$$tmpfile 2>/dev/null && \
+	uploaded=0; \
+	for i in $$(seq 1 600); do \
+		response=$$(curl -s "http://127.0.0.1:$(MASTER_PORT)/dir/assign?collection=ectest&replication=000"); \
+		fid=$$(echo $$response | jq -r '.fid'); \
+		url=$$(echo $$response | jq -r '.url'); \
+		if [ "$$fid" != "null" ] && [ -n "$$fid" ]; then \
+			curl -s -F "file=@$$tmpfile;filename=file_$$i.bin" "http://$$url/$$fid" > /dev/null; \
+			uploaded=$$((uploaded + 1)); \
+		fi; \
+		if [ $$((i % 100)) -eq 0 ]; then \
+			echo "Uploaded $$uploaded files..."; \
+		fi; \
+	done; \
+	rm -f $$tmpfile; \
+	echo ""; \
+	echo "=== Data population complete ==="; \
+	echo "Uploaded $$uploaded files (~$$((uploaded * 500 / 1024))MB)"
+	@echo ""
+	@echo "Volume status:"
+	@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
+		'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | "  Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack)"' 2>/dev/null || true
+
+.PHONY: shell
+shell: build
+	@echo "Opening weed shell..."
+	@echo "Commands to try:"
+	@echo "  lock"
+	@echo "  volume.list"
+	@echo "  ec.encode -collection=ectest -quietFor=1s -force"
+	@echo "  ec.balance -collection=ectest"
+	@echo "  unlock"
+	@echo ""
+	@$(WEED_BINARY) shell -master=127.0.0.1:$(MASTER_PORT) -filer=127.0.0.1:$(FILER_PORT)
+
+.PHONY: setup
+setup: clean start
+	@sleep 2
+	@$(MAKE) populate
+
+.PHONY: status
+status:
+	@echo "=== Cluster Status ==="
+	@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
+		'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | "Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack) (\($$server))"' 2>/dev/null | sort -t: -k1 -n || echo "Cluster not running"
+	@echo ""
+	@echo "=== EC Shards ==="
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
+		count=$$(ls $(TEST_DIR)/volume$$i/*.ec[0-9]* 2>/dev/null | wc -l | tr -d ' '); \
+		if [ "$$count" != "0" ]; then \
+			echo "  volume$$i (port $$(($(VOLUME_BASE_PORT) + $$i))): $$count EC shard files"; \
+		fi; \
+	done
+
+.PHONY: help
+help:
+	@echo "EC Integration Test Makefile"
+	@echo ""
+	@echo "Targets:"
+	@echo "  make start     - Start test cluster (master + 6 volume servers + filer)"
+	@echo "  make stop      - Stop test cluster"
+	@echo "  make populate  - Populate ~300MB of test data"
+	@echo "  make shell     - Open weed shell"
+	@echo "  make setup     - Clean, start, and populate (all-in-one)"
+	@echo "  make status    - Show cluster and EC shard status"
+	@echo "  make clean     - Stop cluster and remove all test data"
+	@echo "  make help      - Show this help"
+	@echo ""
+	@echo "Quick start:"
+	@echo "  make setup     # Start cluster and populate data"
+	@echo "  make shell     # Open shell to run EC commands"
+

test/erasure_coding/README.md

@@ -78,6 +78,43 @@ go test -v -run TestECEncodingMasterTimingRaceCondition
 go test -v -short
 ```
 
+## Manual Testing with Makefile
+
+A Makefile is provided for manual EC testing.
+
+**Requirements:** `curl`, `jq` (command-line JSON processor)
+
+```bash
+# Quick start: start cluster and populate data
+make setup
+
+# Open weed shell to run EC commands
+make shell
+
+# Individual targets
+make start      # Start test cluster (master + 6 volume servers + filer)
+make stop       # Stop test cluster
+make populate   # Populate ~300MB of test data
+make status     # Show cluster and EC shard status
+make clean      # Stop cluster and remove all test data
+make help       # Show all targets
+```
+
+### EC Rebalance Limited Slots (Unit Test)
+
+The "no free ec shard slots" issue is tested with a **unit test** that works directly on
+topology data structures without requiring a running cluster.
+
+**Location**: `weed/shell/ec_rebalance_slots_test.go`
+
+Tests included:
+- `TestECRebalanceWithLimitedSlots`: Tests a topology with 6 servers, 7 EC volumes (98 shards)
+- `TestECRebalanceZeroFreeSlots`: Reproduces the exact 0 free slots scenario
+
+**Known Issue**: When volume servers are at capacity (`volumeCount == maxVolumeCount`),
+the rebalance step fails with "no free ec shard slots" instead of recognizing that
+moving shards frees slots on source servers.
+
 ## Test Results
 
 **With the fix**: Shows "Collecting volume locations for N volumes before EC encoding..." message

test/erasure_coding/ec_integration_test.go

@@ -15,8 +15,10 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/seaweedfs/seaweedfs/weed/shell"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc"
@@ -386,7 +388,7 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
 	// Find weed binary
 	weedBinary := findWeedBinary()
 	if weedBinary == "" {
-		return nil, fmt.Errorf("weed binary not found")
+		return nil, fmt.Errorf("weed binary not found - build with 'go build' or 'make' first")
 	}
 
 	cluster := &TestCluster{}
@@ -395,6 +397,11 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
 	masterDir := filepath.Join(dataDir, "master")
 	os.MkdirAll(masterDir, 0755)
 
+	// Create an empty security.toml to disable JWT authentication in tests
+	// This prevents the test from picking up ~/.seaweedfs/security.toml
+	securityToml := filepath.Join(dataDir, "security.toml")
+	os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
+
 	// Start master server
 	masterCmd := exec.CommandContext(ctx, weedBinary, "master",
 		"-port", "9333",
@@ -403,6 +410,7 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
 		"-ip", "127.0.0.1",
 		"-peers", "none", // Faster startup when no multiple masters needed
 	)
+	masterCmd.Dir = dataDir // Run from test dir so it picks up our security.toml
 
 	masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
 	if err != nil {
@@ -435,6 +443,7 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
 			"-dataCenter", "dc1",
 			"-rack", rack,
 		)
+		volumeCmd.Dir = dataDir // Run from test dir so it picks up our security.toml
 
 		volumeLogFile, err := os.Create(filepath.Join(volumeDir, "volume.log"))
 		if err != nil {
@@ -469,6 +478,10 @@ func findWeedBinary() string {
 
 	for _, candidate := range candidates {
 		if _, err := os.Stat(candidate); err == nil {
+			// Convert to absolute path so it works when command's Dir is changed
+			if absPath, err := filepath.Abs(candidate); err == nil {
+				return absPath
+			}
 			return candidate
 		}
 	}
@@ -2174,3 +2187,249 @@ func countShardsPerRack(testDir string, volumeId uint32) map[string]int {
 
 	return rackDistribution
 }
+
+// TestECEncodeReplicatedVolumeSync tests that ec.encode properly syncs missing entries
+// between replicas before encoding. This addresses issue #7797.
+func TestECEncodeReplicatedVolumeSync(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping replicated volume sync integration test in short mode")
+	}
+
+	// Create temporary directory for test data
+	testDir, err := os.MkdirTemp("", "seaweedfs_ec_replica_sync_test_")
+	require.NoError(t, err)
+	defer os.RemoveAll(testDir)
+
+	// Start SeaweedFS cluster
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	cluster, err := startSeaweedFSCluster(ctx, testDir)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	// Wait for servers to be ready
+	require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
+	for i := 0; i < 6; i++ {
+		require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:%d", 8080+i), 30*time.Second))
+	}
+
+	// Create command environment
+	options := &shell.ShellOptions{
+		Masters:        stringPtr("127.0.0.1:9333"),
+		GrpcDialOption: grpc.WithInsecure(),
+		FilerGroup:     stringPtr("default"),
+	}
+	commandEnv := shell.NewCommandEnv(options)
+
+	// Connect to master with longer timeout
+	ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel2()
+	go commandEnv.MasterClient.KeepConnectedToMaster(ctx2)
+	commandEnv.MasterClient.WaitUntilConnected(ctx2)
+
+	// Wait for volume servers to register with master
+	time.Sleep(5 * time.Second)
+
+	// Test: Create replicated volume and verify sync behavior with consistent replicas
+	t.Run("sync_replicated_volume_consistent", func(t *testing.T) {
+		const testCollection = "replicated_consistent"
+		const testReplication = "010" // 2 copies on different servers on the same rack
+
+		// Retry a few times as volume servers may still be registering
+		var volumeId needle.VolumeId
+		var uploadErr error
+		for retry := 0; retry < 5; retry++ {
+			volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
+			if uploadErr == nil {
+				break
+			}
+			t.Logf("Upload attempt %d failed: %v, retrying...", retry+1, uploadErr)
+			time.Sleep(3 * time.Second)
+		}
+		if uploadErr != nil {
+			t.Skipf("Could not create replicated volume: %v", uploadErr)
+			return
+		}
+		t.Logf("Created replicated volume %d with collection %s, replication %s", volumeId, testCollection, testReplication)
+
+		// Acquire lock
+		locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
+		if !locked {
+			t.Skip("Could not acquire lock within timeout")
+		}
+		defer unlock()
+
+		// Execute EC encoding with the same collection
+		ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
+		args := []string{
+			"-volumeId", fmt.Sprintf("%d", volumeId),
+			"-collection", testCollection,
+			"-force",
+		}
+
+		outputStr, encodeErr := captureCommandOutput(t, ecEncodeCmd, args, commandEnv)
+		t.Logf("EC encode output:\n%s", outputStr)
+
+		// For consistent replicas, should see "all X replicas are consistent"
+		assert.Contains(t, outputStr, "replicas are consistent", "Should detect replicas are consistent")
+
+		if encodeErr != nil {
+			t.Logf("EC encoding result: %v", encodeErr)
+		} else {
+			t.Log("EC encoding completed successfully")
+		}
+	})
+
+	// Test: Create divergent replicas and verify sync/union code is exercised
+	t.Run("sync_replicated_volume_divergent", func(t *testing.T) {
+		const testCollection = "replicated_divergent"
+		const testReplication = "010" // 2 copies on different servers on the same rack
+
+		// Create replicated volume with initial data
+		var volumeId needle.VolumeId
+		var uploadErr error
+		for retry := 0; retry < 5; retry++ {
+			volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
+			if uploadErr == nil {
+				break
+			}
+			t.Logf("Upload attempt %d failed: %v, retrying...", retry+1, uploadErr)
+			time.Sleep(3 * time.Second)
+		}
+		if uploadErr != nil {
+			t.Skipf("Could not create replicated volume: %v", uploadErr)
+			return
+		}
+		t.Logf("Created replicated volume %d with collection %s, replication %s", volumeId, testCollection, testReplication)
+
+		// Get volume locations to identify the two replicas
+		locations, found := commandEnv.MasterClient.GetLocationsClone(uint32(volumeId))
+		if !found || len(locations) < 2 {
+			t.Skipf("Could not get 2 replica locations for volume %d (got %d)", volumeId, len(locations))
+			return
+		}
+		t.Logf("Volume %d has replicas at: %v and %v", volumeId, locations[0].Url, locations[1].Url)
+
+		// Write an extra entry to ONLY one replica to create divergence
+		// We'll use the WriteNeedleBlob gRPC call to inject data directly
+		extraNeedleId := uint64(999999) // Use a high needle ID unlikely to conflict
+		extraData := []byte("extra data written to only one replica to create divergence")
+
+		err := injectNeedleToOneReplica(grpc.WithInsecure(), volumeId, locations[0], extraNeedleId, extraData)
+		if err != nil {
+			t.Logf("Could not inject divergent needle (may not be supported): %v", err)
+			// Fall back to testing consistent path
+		} else {
+			t.Logf("Injected extra needle %d to replica %s to create divergence", extraNeedleId, locations[0].Url)
+		}
+
+		// Acquire lock
+		locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
+		if !locked {
+			t.Skip("Could not acquire lock within timeout")
+		}
+		defer unlock()
+
+		// Execute EC encoding - should detect divergence and build union
+		ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
+		args := []string{
+			"-volumeId", fmt.Sprintf("%d", volumeId),
+			"-collection", testCollection,
+			"-force",
+		}
+
+		outputStr, encodeErr := captureCommandOutput(t, ecEncodeCmd, args, commandEnv)
+		t.Logf("EC encode output:\n%s", outputStr)
+
+		// Check if divergence was detected
+		if strings.Contains(outputStr, "building union") {
+			t.Log("SUCCESS: Divergent replicas detected - sync/union code was exercised")
+			assert.Contains(t, outputStr, "selected", "Should show which replica was selected as best")
+			// The "copied" message only appears if the best replica is missing entries from other replicas
+			// In our case, we injected into what becomes the best replica (higher file count),
+			// so copying may not be needed. The key is that "building union" was triggered.
+			if strings.Contains(outputStr, "copied") {
+				t.Log("Entries were copied between replicas")
+			} else {
+				t.Log("Best replica already had all entries (injected entry was on best replica)")
+			}
+		} else if strings.Contains(outputStr, "replicas are consistent") {
+			t.Log("Replicas were consistent (injection may not have worked)")
+		} else {
+			t.Log("Single replica or sync not triggered")
+		}
+
+		if encodeErr != nil {
+			t.Logf("EC encoding result: %v", encodeErr)
+		} else {
+			t.Log("EC encoding completed successfully")
+		}
+	})
+}
+
+// injectNeedleToOneReplica writes a needle directly to one replica to create divergence
+func injectNeedleToOneReplica(grpcDialOption grpc.DialOption, vid needle.VolumeId, location wdclient.Location, needleId uint64, data []byte) error {
+	return operation.WithVolumeServerClient(false, location.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+		_, err := client.WriteNeedleBlob(context.Background(), &volume_server_pb.WriteNeedleBlobRequest{
+			VolumeId:   uint32(vid),
+			NeedleId:   needleId,
+			Size:       int32(len(data)),
+			NeedleBlob: data,
+		})
+		return err
+	})
+}
+
+// uploadTestDataWithReplication uploads test data and returns the volume ID
+// using the specified collection and replication level. All files are uploaded to the same volume.
+func uploadTestDataWithReplication(t *testing.T, masterAddr string, collection string, replication string) (needle.VolumeId, error) {
+	const numFiles = 20 // Reduced count since we're uploading to same volume
+
+	// Assign multiple file IDs from the same volume
+	assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
+		return pb.ServerAddress(masterAddr)
+	}, grpc.WithInsecure(), &operation.VolumeAssignRequest{
+		Count:       uint64(numFiles),
+		Collection:  collection,
+		Replication: replication,
+	})
+	if err != nil {
+		return 0, fmt.Errorf("failed to assign volume: %v", err)
+	}
+
+	// Parse volume ID from file ID
+	fid, err := needle.ParseFileIdFromString(assignResult.Fid)
+	if err != nil {
+		return 0, fmt.Errorf("failed to parse file ID: %v", err)
+	}
+	volumeId := fid.VolumeId
+
+	// Create uploader for upload
+	uploader, err := operation.NewUploader()
+	if err != nil {
+		return 0, fmt.Errorf("failed to create uploader: %v", err)
+	}
+
+	// Upload test data to the same volume using the assigned file IDs
+	// The first FID is assignResult.Fid, subsequent ones increment the needle key
+	baseNeedleKey := uint64(fid.Key)
+	for i := 0; i < numFiles; i++ {
+		data := []byte(fmt.Sprintf("test data for replicated volume sync test - entry %d - padding to make it larger %s",
+			i, strings.Repeat("x", 1000)))
+
+		// Construct FID for this file (same volume, incrementing needle key)
+		currentFid := fmt.Sprintf("%d,%x%08x", volumeId, baseNeedleKey+uint64(i), fid.Cookie)
+		uploadUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, currentFid)
+
+		_, _, uploadErr := uploader.Upload(context.Background(), bytes.NewReader(data), &operation.UploadOption{
+			UploadUrl: uploadUrl,
+			Filename:  fmt.Sprintf("file%d.txt", i),
+		})
+		if uploadErr != nil {
+			t.Logf("Upload %d failed: %v", i, uploadErr)
+		}
+	}
+
+	return volumeId, nil
+}

test/foundationdb/Dockerfile.build.arm64

@@ -5,13 +5,14 @@ ARG FOUNDATIONDB_VERSION=7.4.5
 ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}
 
 # Install build dependencies and download prebuilt FoundationDB clients
-RUN apt-get update && apt-get install -y \
+SHELL ["/bin/bash", "-c"]
+RUN set -euo pipefail && \
+    apt-get update && apt-get install -y \
     build-essential \
     git \
     wget \
     ca-certificates \
     && rm -rf /var/lib/apt/lists/* && \
-    set -euo pipefail && \
     case "${FOUNDATIONDB_VERSION}" in \
         "7.4.5") EXPECTED_SHA256="f2176b86b7e1b561c3632b4e6e7efb82e3b8f57c2ff0d0ac4671e742867508aa" ;; \
         *) echo "ERROR: No known ARM64 client checksum for FoundationDB ${FOUNDATIONDB_VERSION}. Please update this Dockerfile." >&2; exit 1 ;; \

test/foundationdb/Dockerfile.fdb-arm64

@@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 # Install FoundationDB server + client debs with checksum verification
+SHELL ["/bin/bash", "-c"]
 RUN set -euo pipefail && \
     apt-get update && \
     case "${FOUNDATIONDB_VERSION}" in \

test/foundationdb/Makefile

@@ -50,6 +50,24 @@ test-benchmark: ## Run performance benchmarks
 	@echo "$(YELLOW)Running FoundationDB performance benchmarks...$(NC)"
 	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb -bench=. ./test/foundationdb/...
 
+test-benchmark-filer: ## Run filer store benchmarks (batch vs no-batch comparison)
+	@echo "$(YELLOW)Running FoundationDB filer store benchmarks...$(NC)"
+	@echo "$(BLUE)Comparing batched vs non-batched write performance$(NC)"
+	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb \
+		-bench='BenchmarkFoundationDBStore_.*' \
+		-benchmem \
+		-benchtime=5s \
+		./weed/filer/foundationdb/...
+
+test-benchmark-concurrent: ## Run concurrent operation benchmarks
+	@echo "$(YELLOW)Running concurrent operation benchmarks...$(NC)"
+	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb \
+		-bench='BenchmarkFoundationDBStore_Concurrent.*' \
+		-benchmem \
+		-benchtime=10s \
+		-cpu=1,2,4,8 \
+		./weed/filer/foundationdb/...
+
 # ARM64 specific targets (Apple Silicon / M1/M2/M3 Macs)
 setup-arm64: ## Set up ARM64-native FoundationDB cluster (builds from source)
 	@echo "$(YELLOW)Setting up ARM64-native FoundationDB cluster...$(NC)"

test/foundationdb/README.md

@@ -353,12 +353,31 @@ The tests are designed to be reliable in CI environments with:
 Run performance benchmarks:
 
 ```bash
-make test-benchmark
+make test-benchmark              # Run all benchmarks
+make test-benchmark-filer        # Filer store benchmarks with batch comparison
+make test-benchmark-concurrent   # Concurrent operation benchmarks (varies CPU count)
 
 # Sample expected results:
-# BenchmarkFoundationDBStore_InsertEntry-8    1000    1.2ms per op
-# BenchmarkFoundationDBStore_FindEntry-8      5000    0.5ms per op  
-# BenchmarkFoundationDBStore_KvOperations-8   2000    0.8ms per op
+# BenchmarkFoundationDBStore_InsertEntry-8           1000    1.2ms per op
+# BenchmarkFoundationDBStore_FindEntry-8             5000    0.5ms per op  
+# BenchmarkFoundationDBStore_KvOperations-8          2000    0.8ms per op
+# BenchmarkFoundationDBStore_InsertEntry_NoBatch-8   1000    1.0ms per op  (optimal for S3)
+# BenchmarkFoundationDBStore_InsertEntry_WithBatch-8 1200    0.9ms per op  (bulk ingestion)
+# BenchmarkFoundationDBStore_ConcurrentInsert_*-8    5000    0.3ms per op  (parallel writes)
+```
+
+### Batch vs Non-Batch Performance
+
+The FoundationDB filer store supports two write modes:
+
+| Mode | Config | Best For | Latency | Throughput |
+|------|--------|----------|---------|------------|
+| **Direct Commit** (default) | `batch_enabled = false` | S3 API, low-latency workloads | ~1-5ms per op | Good |
+| **Batched** | `batch_enabled = true` | Bulk ingestion, high throughput | Variable | Higher |
+
+Run the comparison benchmark:
+```bash
+make test-benchmark-filer
 ```
 
 ## Contributing

test/foundationdb/docker-compose.arm64.yml

@@ -28,8 +28,8 @@ services:
           echo 'testing:testing@fdb1:4500,fdb2:4502,fdb3:4504' > /var/fdb/config/fdb.cluster
         fi
         # Start FDB processes
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4501 --listen_address=0.0.0.0:4501 --coordination=fdb1:4500 &
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4500 --listen_address=0.0.0.0:4500 --coordination=fdb1:4500 --class=coordination &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4501 --listen_address=0.0.0.0:4501 --coordination=fdb1:4500 &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4500 --listen_address=0.0.0.0:4500 --coordination=fdb1:4500 --class=coordination &
         wait
       "
 
@@ -59,8 +59,8 @@ services:
         # Wait for cluster file from fdb1
         while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
         # Start FDB processes
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4503 --listen_address=0.0.0.0:4503 --coordination=fdb1:4500 &
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4502 --listen_address=0.0.0.0:4502 --coordination=fdb1:4500 --class=coordination &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4503 --listen_address=0.0.0.0:4503 --coordination=fdb1:4500 &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4502 --listen_address=0.0.0.0:4502 --coordination=fdb1:4500 --class=coordination &
         wait
       "
 
@@ -90,8 +90,8 @@ services:
         # Wait for cluster file from fdb1
         while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
         # Start FDB processes
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4505 --listen_address=0.0.0.0:4505 --coordination=fdb1:4500 &
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4504 --listen_address=0.0.0.0:4504 --coordination=fdb1:4500 --class=coordination &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4505 --listen_address=0.0.0.0:4505 --coordination=fdb1:4500 &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4504 --listen_address=0.0.0.0:4504 --coordination=fdb1:4500 --class=coordination &
         wait
       "
 

test/foundationdb/filer.toml

@@ -1,19 +1,8 @@
-# FoundationDB Filer Configuration
+# FoundationDB Filer Configuration for Testing
 
 [foundationdb]
 enabled = true
 cluster_file = "/var/fdb/config/fdb.cluster"
-api_version = 740
-timeout = "5s"
-max_retry_delay = "1s"
-directory_prefix = "seaweedfs"
-
-# For testing different configurations
-[foundationdb.test]
-enabled = false
-cluster_file = "/var/fdb/config/fdb.cluster"
-api_version = 740
-timeout = "10s"
-max_retry_delay = "2s"
-directory_prefix = "seaweedfs_test"
-location = "/test"
+# api_version = 740
+# timeout = "5s"
+# directory_prefix = "seaweedfs"

test/java/spark/pom.xml

@@ -22,7 +22,7 @@
         <junit.version>4.13.2</junit.version>
         <seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
         <jackson.version>2.18.2</jackson.version> <!-- Upgraded from 2.15.3 -->
-        <netty.version>4.1.125.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
+        <netty.version>4.1.129.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
         <parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
         <parquet.format.version>2.12.0</parquet.format.version>
         <surefire.jvm.args>

test/kafka/docker-compose.yml

@@ -172,7 +172,8 @@ services:
       - -master=seaweedfs-master:9333
       - -ip=seaweedfs-mq-broker
       - -port=17777
-      - -port.pprof=18777
+      - -debug
+      - -debug.port=18777
     depends_on:
       seaweedfs-filer:
         condition: service_healthy

test/kafka/go.mod

@@ -43,25 +43,25 @@ require (
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go v1.55.8 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.40.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.40.1 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.31.3 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.18.20 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.3 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect
-	github.com/aws/smithy-go v1.23.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect
+	github.com/aws/smithy-go v1.24.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
@@ -231,13 +231,13 @@ require (
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/crypto v0.45.0 // indirect
 	golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 // indirect
-	golang.org/x/image v0.33.0 // indirect
+	golang.org/x/image v0.34.0 // indirect
 	golang.org/x/net v0.47.0 // indirect
 	golang.org/x/oauth2 v0.32.0 // indirect
 	golang.org/x/sync v0.19.0 // indirect
 	golang.org/x/sys v0.38.0 // indirect
 	golang.org/x/term v0.37.0 // indirect
-	golang.org/x/text v0.31.0 // indirect
+	golang.org/x/text v0.32.0 // indirect
 	golang.org/x/time v0.12.0 // indirect
 	google.golang.org/api v0.247.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect

test/kafka/go.sum

@@ -102,44 +102,44 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
-github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
+github.com/aws/aws-sdk-go-v2 v1.40.1 h1:difXb4maDZkRH0x//Qkwcfpdg1XQVXEAEs2DdXldFFc=
+github.com/aws/aws-sdk-go-v2 v1.40.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
 github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
 github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.3 h1:01Ym72hK43hjwDeJUfi1l2oYLXBAOR8gNSZNmXmvuas=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.3/go.mod h1:55nWF/Sr9Zvls0bGnWkRxUdhzKqj9uRNlPvgV1vgxKc=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 h1:utxLraaifrSBkeyII9mIbVwXXWrZdlPO7FIKmyLCEcY=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15/go.mod h1:hW6zjYUDQwfz3icf4g2O41PHi77u10oAzJ84iSzR/lo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 h1:Y5YXgygXwDI5P4RkteB5yF7v35neH7LfJKBG+hzIons=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15/go.mod h1:K+/1EpG42dFSY7CBj+Fruzm8PsCGWTXJ3jdeJ659oGQ=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 h1:AvltKnW9ewxX2hFmQS0FyJH93aSvJVUEFvXfU+HWtSE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15/go.mod h1:3I4oCdZdmgrREhU74qS1dK9yZ62yumob+58AbFR4cQA=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY=
-github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs=
-github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
-github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 h1:3/u/4yZOffg5jdNk1sDpOQ4Y+R6Xbh+GzpDrSZjuy3U=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15/go.mod h1:4Zkjq0FKjE78NKjabuM4tRXKFzUJWXgP0ItEZK8l7JU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 h1:8sTTiw+9yuNXcfWeqKF2x01GqCF49CpP4Z9nKrrk/ts=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.6/go.mod h1:8WYg+Y40Sn3X2hioaaWAAIngndR8n1XFdRPPX+7QBaM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 h1:E+KqWoVsSrj1tJ6I/fjDIu5xoS2Zacuu1zT+H7KtiIk=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11/go.mod h1:qyWHz+4lvkXcr3+PoGlGHEI+3DLLiU6/GdrFfMaAhB0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 h1:tzMkjh0yTChUqJDgGkcDdxvZDSrJ/WB6R6ymI5ehqJI=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.3/go.mod h1:T270C0R5sZNLbWUe8ueiAF42XSZxxPocTaGSgs5c/60=
+github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
+github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo=
@@ -536,10 +536,10 @@ github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4
 github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
-github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
-github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
-github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg=
-github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
+github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
+github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
+github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
+github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
 github.com/rclone/rclone v1.71.2 h1:3Jk5xNPFrZhVABRuN/OPvApuZQddpE2tkhYMuEn1Ud4=
 github.com/rclone/rclone v1.71.2/go.mod h1:dCK9FzPDlpkbQJ9M7MmWsmv3X5nibfWe+ogJXu6gSgM=
 github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg=
@@ -708,8 +708,6 @@ go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJr
 go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
-go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
@@ -748,8 +746,8 @@ golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 h1:3yiSh9fhy5/RhCSntf4Sy0Tnx
 golang.org/x/exp v0.0.0-20250811191247-51f88131bc50/go.mod h1:rT6SFzZ7oxADUDx58pcaKFTcZ+inxAa9fTrYx/uVYwg=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.33.0 h1:LXRZRnv1+zGd5XBUVRFmYEphyyKJjQjCRiOuAP3sZfQ=
-golang.org/x/image v0.33.0/go.mod h1:DD3OsTYT9chzuzTQt+zMcOlBHgfoKQb1gry8p76Y1sc=
+golang.org/x/image v0.34.0 h1:33gCkyw9hmwbZJeZkct8XyR11yH889EQt/QH4VmXMn8=
+golang.org/x/image v0.34.0/go.mod h1:2RNFBZRB+vnwwFil8GkMdRvrJOFd1AzdZI6vOY+eJVU=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -774,8 +772,6 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
-golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -925,8 +921,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
+golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -979,8 +975,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
-golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
+golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

test/kafka/kafka-client-loadtest/docker-compose.yml

@@ -181,7 +181,8 @@ services:
       - -ip=seaweedfs-mq-broker
       - -port=17777
       - -logFlushInterval=0
-      - -port.pprof=18777
+      - -debug
+      - -debug.port=18777
     depends_on:
       seaweedfs-filer:
         condition: service_healthy

test/metadata_subscribe/Makefile

@@ -25,3 +25,4 @@ test-resume: build-weed
 clean:
 	rm -f ../../weed/weed
 
+

test/metadata_subscribe/README.md

@@ -47,3 +47,4 @@ go test -v -timeout 5m ./test/metadata_subscribe/...
 - Tests create temporary directories that are cleaned up after completion
 - Tests use ports 9333 (master), 8080 (volume), 8888 (filer)
 
+

test/s3/SDK_COMPATIBILITY.md

@@ -0,0 +1,126 @@
+# S3 SDK Compatibility Testing
+
+This document describes the SDK versions tested against SeaweedFS S3 API and known compatibility considerations.
+
+## Validated SDK Versions
+
+### Go SDKs
+
+| SDK | Version | Test Location | Status |
+|-----|---------|---------------|--------|
+| AWS SDK v2 for Go | 1.20+ | `test/s3/etag/`, `test/s3/copying/` | ✅ Tested |
+| AWS SDK v1 for Go | 1.x | `test/s3/basic/` | ✅ Tested |
+
+### Java SDKs
+
+| SDK | Version | Test Location | Status |
+|-----|---------|---------------|--------|
+| AWS SDK v2 for Java | 2.20.127+ | `other/java/s3copier/` | ✅ Tested |
+| AWS SDK v1 for Java | 1.12.600+ | `other/java/s3copier/` | ✅ Tested |
+
+### Python SDKs
+
+| SDK | Version | Test Location | Status |
+|-----|---------|---------------|--------|
+| boto3 | 1.x | `test/s3/parquet/` | ✅ Tested |
+| PyArrow S3 | 14+ | `test/s3/parquet/` | ✅ Tested |
+
+## SDK-Specific Considerations
+
+### AWS SDK v2 for Java - ETag Validation
+
+**Issue**: [GitHub #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+
+AWS SDK v2 for Java includes strict ETag validation in `ChecksumsEnabledValidator.validatePutObjectChecksum`. It decodes the ETag as a hexadecimal MD5 hash using `Base16Codec.decode()`.
+
+**Impact**: If the ETag contains non-hexadecimal characters (like `-` in composite format), the SDK fails with:
+```text
+java.lang.IllegalArgumentException: Invalid base 16 character: '-'
+```
+
+**Resolution**: SeaweedFS now correctly returns:
+- **PutObject**: Pure MD5 hex ETag (32 characters) regardless of internal chunking
+- **CompleteMultipartUpload**: Composite ETag (`<md5>-<partcount>`)
+
+**Test Coverage**: `test/s3/etag/` and `other/java/s3copier/ETagValidationTest.java`
+
+### AWS SDK v1 vs v2 Differences
+
+| Feature | SDK v1 | SDK v2 |
+|---------|--------|--------|
+| ETag hex validation | No | Yes (strict) |
+| Checksum validation | Basic | Enhanced |
+| Async support | Limited | Full |
+| Default retry behavior | Lenient | Stricter |
+
+### Large File Handling
+
+SeaweedFS auto-chunks files larger than **8MB** for efficient storage. This is transparent to clients, but previously affected ETag format. The current implementation ensures:
+
+1. Regular `PutObject` (any size): Returns pure MD5 ETag
+2. Multipart upload: Returns composite ETag per AWS S3 specification
+
+## Test Categories by File Size
+
+| Category | Size | Chunks | ETag Format |
+|----------|------|--------|-------------|
+| Small | < 256KB | 1 (inline) | Pure MD5 |
+| Medium | 256KB - 8MB | 1 | Pure MD5 |
+| Large | 8MB - 24MB | 2-3 | Pure MD5 |
+| Extra Large | > 24MB | 4+ | Pure MD5 |
+| Multipart | N/A | Per part | Composite |
+
+## Running SDK Compatibility Tests
+
+### Go Tests
+
+```bash
+# Run all ETag tests
+cd test/s3/etag && make test
+
+# Run large file tests only
+cd test/s3/etag && make test-large
+```
+
+### Java Tests
+
+```bash
+# Run all Java SDK tests
+cd other/java/s3copier && mvn test
+
+# Run only ETag validation tests
+cd other/java/s3copier && mvn test -Dtest=ETagValidationTest
+```
+
+### Python Tests
+
+```bash
+# Run PyArrow S3 tests
+cd test/s3/parquet && make test
+```
+
+## Adding New SDK Tests
+
+When adding tests for new SDKs, ensure:
+
+1. **Large file tests (>8MB)**: Critical for verifying ETag format with auto-chunking
+2. **Multipart upload tests**: Verify composite ETag format
+3. **Checksum validation**: Test SDK-specific checksum validation if applicable
+4. **Document SDK version**: Add to this compatibility matrix
+
+## Known Issues and Workarounds
+
+### Issue: Older SDK Versions
+
+Some very old SDK versions (e.g., AWS SDK v1 for Java < 1.11.x) may have different behavior. Testing with the versions listed above is recommended.
+
+### Issue: Custom Checksum Algorithms
+
+AWS SDK v2 supports SHA-256 and CRC32 checksums in addition to MD5. SeaweedFS currently returns MD5-based ETags. For checksums other than MD5, use the `x-amz-checksum-*` headers.
+
+## References
+
+- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
+- [AWS SDK v2 Migration Guide](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/migration.html)
+- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+

test/s3/etag/Makefile

@@ -0,0 +1,50 @@
+# ETag Format Integration Tests
+#
+# These tests verify S3 ETag format compatibility, particularly for large files
+# that trigger SeaweedFS auto-chunking. This addresses GitHub Issue #7768.
+#
+# Prerequisites:
+#   - SeaweedFS running with S3 API enabled on port 8333
+#   - Go 1.21+
+#
+# Usage:
+#   make test          - Run all tests
+#   make test-large    - Run only large file tests
+#   make test-verbose  - Run with verbose output
+#   make clean         - Clean test artifacts
+
+.PHONY: all test test-large test-verbose test-quick clean help
+
+# Default S3 endpoint
+S3_ENDPOINT ?= http://127.0.0.1:8333
+
+all: test
+
+test:
+	@echo "Running ETag format tests against $(S3_ENDPOINT)..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m ./...
+
+test-large:
+	@echo "Running large file ETag tests..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m -run "LargeFile|ExtraLarge" ./...
+
+test-verbose:
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m -count=1 ./...
+
+test-quick:
+	@echo "Running quick ETag tests (small files only)..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 1m -run "SmallFile|Consistency" ./...
+
+clean:
+	go clean -testcache
+
+help:
+	@echo "ETag Format Integration Tests"
+	@echo "Targets:"
+	@echo "  test          Run all ETag format tests"
+	@echo "  test-large    Run only large file tests (>8MB)"
+	@echo "  test-quick    Run quick tests (small files only)"
+	@echo "  test-verbose  Run with verbose output"
+	@echo "  clean         Clean test cache"
+	@echo "Environment Variables:"
+	@echo "  S3_ENDPOINT   S3 endpoint URL (default: http://127.0.0.1:8333)"

test/s3/etag/README.md

@@ -0,0 +1,93 @@
+# S3 ETag Format Integration Tests
+
+This test suite verifies that SeaweedFS returns correct ETag formats for S3 operations, ensuring compatibility with AWS S3 SDKs.
+
+## Background
+
+**GitHub Issue #7768**: AWS S3 SDK for Java v2 was failing with `Invalid base 16 character: '-'` when performing `PutObject` on large files.
+
+### Root Cause
+
+SeaweedFS internally auto-chunks large files (>8MB) for efficient storage. Previously, when a regular `PutObject` request resulted in multiple internal chunks, SeaweedFS returned a composite ETag format (`<md5>-<count>`) instead of a pure MD5 hash.
+
+### AWS S3 Specification
+
+| Operation | ETag Format | Example |
+|-----------|-------------|---------|
+| PutObject (any size) | Pure MD5 hex (32 chars) | `d41d8cd98f00b204e9800998ecf8427e` |
+| CompleteMultipartUpload | Composite (`<md5>-<partcount>`) | `d41d8cd98f00b204e9800998ecf8427e-3` |
+
+AWS S3 SDK v2 for Java validates `PutObject` ETags as hexadecimal, which fails when the ETag contains a hyphen.
+
+## Test Coverage
+
+| Test | File Size | Purpose |
+|------|-----------|---------|
+| `TestPutObjectETagFormat_SmallFile` | 1KB | Verify single-chunk uploads return pure MD5 |
+| `TestPutObjectETagFormat_LargeFile` | 10MB | **Critical**: Verify auto-chunked uploads return pure MD5 |
+| `TestPutObjectETagFormat_ExtraLargeFile` | 25MB | Verify multi-chunk auto-chunked uploads return pure MD5 |
+| `TestMultipartUploadETagFormat` | 15MB | Verify multipart uploads correctly return composite ETag |
+| `TestPutObjectETagConsistency` | Various | Verify ETag consistency across PUT/HEAD/GET |
+| `TestETagHexValidation` | 10MB | Simulate AWS SDK v2 hex validation |
+| `TestMultipleLargeFileUploads` | 10MB x5 | Stress test multiple large uploads |
+
+## Prerequisites
+
+1. SeaweedFS running with S3 API enabled:
+   ```bash
+   weed server -s3
+   ```
+
+2. Go 1.21 or later
+
+3. AWS SDK v2 for Go (installed via go modules)
+
+## Running Tests
+
+```bash
+# Run all tests
+make test
+
+# Run only large file tests (the critical ones for issue #7768)
+make test-large
+
+# Run quick tests (small files only)
+make test-quick
+
+# Run with verbose output
+make test-verbose
+```
+
+## Configuration
+
+By default, tests connect to `http://127.0.0.1:8333`. To use a different endpoint:
+
+```bash
+S3_ENDPOINT=http://localhost:8333 make test
+```
+
+Or modify `defaultConfig` in `s3_etag_test.go`.
+
+## SDK Compatibility
+
+These tests use **AWS SDK v2 for Go**, which has the same ETag validation behavior as AWS SDK v2 for Java. The tests include:
+
+- ETag format validation (pure MD5 vs composite)
+- Hex decoding validation (simulates `Base16Codec.decode`)
+- Content integrity verification
+
+## Validated SDK Versions
+
+| SDK | Version | Status |
+|-----|---------|--------|
+| AWS SDK v2 for Go | 1.20+ | ✅ Tested |
+| AWS SDK v2 for Java | 2.20+ | ✅ Compatible (issue #7768 fixed) |
+| AWS SDK v1 for Go | 1.x | ✅ Compatible (less strict validation) |
+| AWS SDK v1 for Java | 1.x | ✅ Compatible (less strict validation) |
+
+## Related
+
+- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
+
+

test/s3/etag/s3_etag_test.go

@@ -0,0 +1,543 @@
+// Package etag_test provides integration tests for S3 ETag format validation.
+//
+// These tests verify that SeaweedFS returns correct ETag formats for different
+// upload scenarios, ensuring compatibility with AWS S3 SDKs that validate ETags.
+//
+// Background (GitHub Issue #7768):
+// AWS S3 SDK for Java v2 validates ETags as hexadecimal MD5 hashes for PutObject
+// responses. SeaweedFS was incorrectly returning composite ETags ("<md5>-<count>")
+// for regular PutObject when files were internally auto-chunked (>8MB), causing
+// the SDK to fail with "Invalid base 16 character: '-'".
+//
+// Per AWS S3 specification:
+// - Regular PutObject: ETag is always a pure MD5 hex string (32 chars)
+// - Multipart Upload (CompleteMultipartUpload): ETag is "<md5>-<partcount>"
+//
+// These tests ensure this behavior is maintained.
+package etag_test
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+	mathrand "math/rand"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// S3TestConfig holds configuration for S3 tests
+type S3TestConfig struct {
+	Endpoint     string
+	AccessKey    string
+	SecretKey    string
+	Region       string
+	BucketPrefix string
+}
+
+// Default test configuration
+var defaultConfig = &S3TestConfig{
+	Endpoint:     "http://127.0.0.1:8333",
+	AccessKey:    "some_access_key1",
+	SecretKey:    "some_secret_key1",
+	Region:       "us-east-1",
+	BucketPrefix: "test-etag-",
+}
+
+// Constants for auto-chunking thresholds (must match s3api_object_handlers_put.go)
+const (
+	// SeaweedFS auto-chunks files larger than 8MB
+	autoChunkSize = 8 * 1024 * 1024
+
+	// Test sizes
+	smallFileSize  = 1 * 1024              // 1KB - single chunk
+	mediumFileSize = 256 * 1024            // 256KB - single chunk (at threshold)
+	largeFileSize  = 10 * 1024 * 1024      // 10MB - triggers auto-chunking (2 chunks)
+	xlFileSize     = 25 * 1024 * 1024      // 25MB - triggers auto-chunking (4 chunks)
+	multipartSize  = 5 * 1024 * 1024       // 5MB per part for multipart uploads
+)
+
+// ETag format patterns
+var (
+	// Pure MD5 ETag: 32 hex characters (with or without quotes)
+	pureMD5Pattern = regexp.MustCompile(`^"?[a-f0-9]{32}"?$`)
+
+	// Composite ETag for multipart: 32 hex chars, hyphen, part count (with or without quotes)
+	compositePattern = regexp.MustCompile(`^"?[a-f0-9]{32}-\d+"?$`)
+)
+
+func init() {
+	mathrand.Seed(time.Now().UnixNano())
+}
+
+// getS3Client creates an AWS S3 v2 client for testing
+func getS3Client(t *testing.T) *s3.Client {
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(defaultConfig.Region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			defaultConfig.AccessKey,
+			defaultConfig.SecretKey,
+			"",
+		)),
+		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
+			func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+				return aws.Endpoint{
+					URL:               defaultConfig.Endpoint,
+					SigningRegion:     defaultConfig.Region,
+					HostnameImmutable: true,
+				}, nil
+			})),
+	)
+	require.NoError(t, err)
+
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+}
+
+// getNewBucketName generates a unique bucket name
+func getNewBucketName() string {
+	timestamp := time.Now().UnixNano()
+	randomSuffix := mathrand.Intn(100000)
+	return fmt.Sprintf("%s%d-%d", defaultConfig.BucketPrefix, timestamp, randomSuffix)
+}
+
+// generateRandomData generates random test data of specified size
+func generateRandomData(size int) []byte {
+	data := make([]byte, size)
+	if _, err := rand.Read(data); err != nil {
+		panic(fmt.Sprintf("failed to generate random test data: %v", err))
+	}
+	return data
+}
+
+// calculateMD5 calculates the MD5 hash of data and returns hex string
+func calculateMD5(data []byte) string {
+	hash := md5.Sum(data)
+	return hex.EncodeToString(hash[:])
+}
+
+// cleanETag removes quotes from ETag
+func cleanETag(etag string) string {
+	return strings.Trim(etag, `"`)
+}
+
+// isPureMD5ETag checks if ETag is a pure MD5 hex string (no composite format)
+func isPureMD5ETag(etag string) bool {
+	return pureMD5Pattern.MatchString(etag)
+}
+
+// isCompositeETag checks if ETag is in composite format (md5-count)
+func isCompositeETag(etag string) bool {
+	return compositePattern.MatchString(etag)
+}
+
+// createTestBucket creates a new bucket for testing
+func createTestBucket(ctx context.Context, client *s3.Client, bucketName string) error {
+	_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	return err
+}
+
+// cleanupTestBucket deletes all objects and the bucket
+func cleanupTestBucket(ctx context.Context, client *s3.Client, bucketName string) {
+	// Delete all objects
+	paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+	})
+
+	for paginator.HasMorePages() {
+		page, err := paginator.NextPage(ctx)
+		if err != nil {
+			break
+		}
+		for _, obj := range page.Contents {
+			client.DeleteObject(ctx, &s3.DeleteObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    obj.Key,
+			})
+		}
+	}
+
+	// Abort any in-progress multipart uploads
+	mpPaginator := s3.NewListMultipartUploadsPaginator(client, &s3.ListMultipartUploadsInput{
+		Bucket: aws.String(bucketName),
+	})
+	for mpPaginator.HasMorePages() {
+		page, err := mpPaginator.NextPage(ctx)
+		if err != nil {
+			break
+		}
+		for _, upload := range page.Uploads {
+			client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
+				Bucket:   aws.String(bucketName),
+				Key:      upload.Key,
+				UploadId: upload.UploadId,
+			})
+		}
+	}
+
+	// Delete bucket
+	client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+}
+
+// TestPutObjectETagFormat_SmallFile verifies ETag format for small files (single chunk)
+func TestPutObjectETagFormat_SmallFile(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testData := generateRandomData(smallFileSize)
+	expectedMD5 := calculateMD5(testData)
+	objectKey := "small-file.bin"
+
+	// Upload small file
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload small file")
+
+	// Verify ETag format
+	etag := aws.ToString(putResp.ETag)
+	t.Logf("Small file (%d bytes) ETag: %s", smallFileSize, etag)
+
+	assert.True(t, isPureMD5ETag(etag),
+		"Small file ETag should be pure MD5, got: %s", etag)
+	assert.False(t, isCompositeETag(etag),
+		"Small file ETag should NOT be composite format, got: %s", etag)
+	assert.Equal(t, expectedMD5, cleanETag(etag),
+		"ETag should match calculated MD5")
+}
+
+// TestPutObjectETagFormat_LargeFile verifies ETag format for large files that trigger auto-chunking
+// This is the critical test for GitHub Issue #7768
+func TestPutObjectETagFormat_LargeFile(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testData := generateRandomData(largeFileSize)
+	expectedMD5 := calculateMD5(testData)
+	objectKey := "large-file.bin"
+
+	t.Logf("Uploading large file (%d bytes, > %d byte auto-chunk threshold)...",
+		largeFileSize, autoChunkSize)
+
+	// Upload large file (triggers auto-chunking internally)
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload large file")
+
+	// Verify ETag format - MUST be pure MD5, NOT composite
+	etag := aws.ToString(putResp.ETag)
+	t.Logf("Large file (%d bytes, ~%d internal chunks) ETag: %s",
+		largeFileSize, (largeFileSize/autoChunkSize)+1, etag)
+
+	assert.True(t, isPureMD5ETag(etag),
+		"Large file PutObject ETag MUST be pure MD5 (not composite), got: %s", etag)
+	assert.False(t, isCompositeETag(etag),
+		"Large file PutObject ETag should NOT contain '-' (composite format), got: %s", etag)
+	assert.False(t, strings.Contains(cleanETag(etag), "-"),
+		"ETag should not contain hyphen for regular PutObject, got: %s", etag)
+	assert.Equal(t, expectedMD5, cleanETag(etag),
+		"ETag should match calculated MD5 of entire content")
+
+	// Verify we can read back the object correctly
+	getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to get large file")
+	defer getResp.Body.Close()
+
+	downloadedData, err := io.ReadAll(getResp.Body)
+	require.NoError(t, err, "Failed to read large file content")
+	assert.Equal(t, testData, downloadedData, "Downloaded content should match uploaded content")
+}
+
+// TestPutObjectETagFormat_ExtraLargeFile tests even larger files with multiple internal chunks
+func TestPutObjectETagFormat_ExtraLargeFile(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testData := generateRandomData(xlFileSize)
+	expectedMD5 := calculateMD5(testData)
+	objectKey := "xl-file.bin"
+
+	expectedChunks := (xlFileSize / autoChunkSize) + 1
+	t.Logf("Uploading XL file (%d bytes, expected ~%d internal chunks)...",
+		xlFileSize, expectedChunks)
+
+	// Upload extra large file
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload XL file")
+
+	// Verify ETag format
+	etag := aws.ToString(putResp.ETag)
+	t.Logf("XL file (%d bytes) ETag: %s", xlFileSize, etag)
+
+	assert.True(t, isPureMD5ETag(etag),
+		"XL file PutObject ETag MUST be pure MD5, got: %s", etag)
+	assert.False(t, isCompositeETag(etag),
+		"XL file PutObject ETag should NOT be composite, got: %s", etag)
+	assert.Equal(t, expectedMD5, cleanETag(etag),
+		"ETag should match calculated MD5")
+}
+
+// TestMultipartUploadETagFormat verifies that ONLY multipart uploads get composite ETags
+func TestMultipartUploadETagFormat(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	// Create test data for multipart upload (15MB = 3 parts of 5MB each)
+	totalSize := 15 * 1024 * 1024
+	testData := generateRandomData(totalSize)
+	objectKey := "multipart-file.bin"
+
+	expectedPartCount := (totalSize + multipartSize - 1) / multipartSize // ceiling division
+	t.Logf("Performing multipart upload (%d bytes, %d parts)...",
+		totalSize, expectedPartCount)
+
+	// Initiate multipart upload
+	createResp, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	uploadId := createResp.UploadId
+	var completedParts []types.CompletedPart
+	partNumber := int32(1)
+
+	// Upload parts
+	for offset := 0; offset < totalSize; offset += multipartSize {
+		end := offset + multipartSize
+		if end > totalSize {
+			end = totalSize
+		}
+		partData := testData[offset:end]
+
+		uploadResp, err := client.UploadPart(ctx, &s3.UploadPartInput{
+			Bucket:     aws.String(bucketName),
+			Key:        aws.String(objectKey),
+			UploadId:   uploadId,
+			PartNumber: aws.Int32(partNumber),
+			Body:       bytes.NewReader(partData),
+		})
+		require.NoError(t, err, "Failed to upload part %d", partNumber)
+
+		completedParts = append(completedParts, types.CompletedPart{
+			ETag:       uploadResp.ETag,
+			PartNumber: aws.Int32(partNumber),
+		})
+		partNumber++
+	}
+
+	// Complete multipart upload
+	completeResp, err := client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: uploadId,
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: completedParts,
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+
+	// Verify ETag format - SHOULD be composite for multipart
+	etag := aws.ToString(completeResp.ETag)
+	t.Logf("Multipart upload ETag: %s", etag)
+
+	assert.True(t, isCompositeETag(etag),
+		"Multipart upload ETag SHOULD be composite format (md5-count), got: %s", etag)
+	assert.True(t, strings.Contains(cleanETag(etag), "-"),
+		"Multipart ETag should contain hyphen, got: %s", etag)
+
+	// Verify the part count in the ETag matches
+	parts := strings.Split(cleanETag(etag), "-")
+	require.Len(t, parts, 2, "Composite ETag should have format 'hash-count'")
+	assert.Equal(t, fmt.Sprintf("%d", len(completedParts)), parts[1],
+		"Part count in ETag should match number of parts uploaded")
+}
+
+// TestPutObjectETagConsistency verifies ETag consistency between PUT and GET
+func TestPutObjectETagConsistency(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testCases := []struct {
+		name string
+		size int
+	}{
+		{"tiny", 100},
+		{"small", smallFileSize},
+		{"medium", mediumFileSize},
+		{"large", largeFileSize},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			testData := generateRandomData(tc.size)
+			objectKey := fmt.Sprintf("consistency-test-%s.bin", tc.name)
+
+			// PUT object
+			putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+				Body:   bytes.NewReader(testData),
+			})
+			require.NoError(t, err)
+			putETag := aws.ToString(putResp.ETag)
+
+			// HEAD object
+			headResp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+			})
+			require.NoError(t, err)
+			headETag := aws.ToString(headResp.ETag)
+
+			// GET object
+			getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+			})
+			require.NoError(t, err)
+			getETag := aws.ToString(getResp.ETag)
+			getResp.Body.Close()
+
+			// All ETags should match
+			t.Logf("%s (%d bytes): PUT=%s, HEAD=%s, GET=%s",
+				tc.name, tc.size, putETag, headETag, getETag)
+
+			assert.Equal(t, putETag, headETag,
+				"PUT and HEAD ETags should match")
+			assert.Equal(t, putETag, getETag,
+				"PUT and GET ETags should match")
+
+			// All should be pure MD5 (not composite) for regular PutObject
+			assert.True(t, isPureMD5ETag(putETag),
+				"PutObject ETag should be pure MD5, got: %s", putETag)
+		})
+	}
+}
+
+// TestETagHexValidation simulates the AWS SDK v2 validation that caused issue #7768
+func TestETagHexValidation(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	// Test with a file large enough to trigger auto-chunking
+	testData := generateRandomData(largeFileSize)
+	objectKey := "hex-validation-test.bin"
+
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err)
+
+	etag := cleanETag(aws.ToString(putResp.ETag))
+
+	// Simulate AWS SDK v2's hex validation (Base16Codec.decode)
+	// This is what fails in issue #7768 when ETag contains '-'
+	t.Logf("Validating ETag as hex: %s", etag)
+
+	_, err = hex.DecodeString(etag)
+	assert.NoError(t, err,
+		"ETag should be valid hexadecimal (AWS SDK v2 validation). "+
+			"Got ETag: %s. If this fails with 'invalid byte', the ETag contains non-hex chars like '-'",
+		etag)
+}
+
+// TestMultipleLargeFileUploads verifies ETag format across multiple large uploads
+func TestMultipleLargeFileUploads(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	numFiles := 5
+	for i := 0; i < numFiles; i++ {
+		testData := generateRandomData(largeFileSize)
+		expectedMD5 := calculateMD5(testData)
+		objectKey := fmt.Sprintf("large-file-%d.bin", i)
+
+		putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(objectKey),
+			Body:   bytes.NewReader(testData),
+		})
+		require.NoError(t, err, "Failed to upload file %d", i)
+
+		etag := aws.ToString(putResp.ETag)
+		t.Logf("File %d ETag: %s (expected MD5: %s)", i, etag, expectedMD5)
+
+		assert.True(t, isPureMD5ETag(etag),
+			"File %d ETag should be pure MD5, got: %s", i, etag)
+		assert.Equal(t, expectedMD5, cleanETag(etag),
+			"File %d ETag should match MD5", i)
+
+		// Validate as hex (AWS SDK v2 check)
+		_, err = hex.DecodeString(cleanETag(etag))
+		assert.NoError(t, err, "File %d ETag should be valid hex", i)
+	}
+}
+

test/s3/etag/test_config.json

@@ -0,0 +1,20 @@
+{
+  "endpoint": "http://127.0.0.1:8333",
+  "access_key": "some_access_key1",
+  "secret_key": "some_secret_key1",
+  "region": "us-east-1",
+  "bucket_prefix": "test-etag-",
+  "notes": {
+    "description": "S3 ETag format integration tests",
+    "issue": "https://github.com/seaweedfs/seaweedfs/issues/7768",
+    "auto_chunk_size": "8MB - files larger than this trigger auto-chunking",
+    "test_sizes": {
+      "small": "1KB - single chunk",
+      "medium": "256KB - at inline threshold", 
+      "large": "10MB - 2 internal chunks",
+      "xl": "25MB - 4 internal chunks"
+    }
+  }
+}
+
+

test/s3/filer_group/test_config.json

@@ -5,4 +5,4 @@
   "secret_key": "some_secret_key1",
   "region": "us-east-1",
   "filer_group": "testgroup"
-}
+}

test/s3/iam/s3_iam_integration_test.go

@@ -558,13 +558,14 @@ func TestS3IAMPresignedURLIntegration(t *testing.T) {
 	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
 	require.NoError(t, err)
 
-	// Use static bucket name but with cleanup to handle conflicts
-	err = framework.CreateBucketWithCleanup(adminClient, testBucket)
+	// Use unique bucket name to avoid conflicts with other tests
+	bucketName := framework.GenerateUniqueBucketName("test-iam-presigned")
+	err = framework.CreateBucket(adminClient, bucketName)
 	require.NoError(t, err)
 
 	// Put test object
 	_, err = adminClient.PutObject(&s3.PutObjectInput{
-		Bucket: aws.String(testBucket),
+		Bucket: aws.String(bucketName),
 		Key:    aws.String(testObjectKey),
 		Body:   strings.NewReader(testObjectData),
 	})
@@ -586,7 +587,7 @@ func TestS3IAMPresignedURLIntegration(t *testing.T) {
 
 		// Test direct object access with JWT Bearer token (recommended approach)
 		_, err := adminClient.GetObject(&s3.GetObjectInput{
-			Bucket: aws.String(testBucket),
+			Bucket: aws.String(bucketName),
 			Key:    aws.String(testObjectKey),
 		})
 		require.NoError(t, err, "Direct object access with JWT Bearer token works correctly")
@@ -597,13 +598,13 @@ func TestS3IAMPresignedURLIntegration(t *testing.T) {
 
 	// Cleanup
 	_, err = adminClient.DeleteObject(&s3.DeleteObjectInput{
-		Bucket: aws.String(testBucket),
+		Bucket: aws.String(bucketName),
 		Key:    aws.String(testObjectKey),
 	})
 	require.NoError(t, err)
 
 	_, err = adminClient.DeleteBucket(&s3.DeleteBucketInput{
-		Bucket: aws.String(testBucket),
+		Bucket: aws.String(bucketName),
 	})
 	require.NoError(t, err)
 }

test/s3/iam/test_config.json

@@ -36,7 +36,7 @@
     "sts": {
       "tokenDuration": "15m",
       "issuer": "seaweedfs-sts",
-      "signingKey": "test-sts-signing-key-for-integration-tests"
+      "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc="
     },
     "policy": {
       "defaultEffect": "Deny"

test/s3/remote_cache/Makefile

@@ -0,0 +1,210 @@
+# Remote Storage Cache Integration Tests
+# Tests the remote object caching functionality with singleflight deduplication
+# Uses two SeaweedFS instances: primary (with caching) and secondary (as remote storage)
+
+.PHONY: all help build-weed check-deps start-remote stop-remote start-primary stop-primary \
+        setup-remote test test-with-server clean logs logs-primary logs-remote health
+
+all: test-with-server
+
+# Configuration
+WEED_BINARY := ../../../weed/weed_binary
+
+# Primary SeaweedFS (the one being tested - has remote caching)
+PRIMARY_S3_PORT := 8333
+PRIMARY_FILER_PORT := 8888
+PRIMARY_MASTER_PORT := 9333
+PRIMARY_VOLUME_PORT := 8080
+PRIMARY_METRICS_PORT := 9324
+PRIMARY_DIR := ./test-primary-data
+
+# Secondary SeaweedFS (acts as "remote" S3 storage)
+REMOTE_S3_PORT := 8334
+REMOTE_FILER_PORT := 8889
+REMOTE_MASTER_PORT := 9334
+REMOTE_VOLUME_PORT := 8081
+REMOTE_METRICS_PORT := 9325
+REMOTE_DIR := ./test-remote-data
+
+# Test configuration
+TEST_TIMEOUT := 10m
+TEST_PATTERN := TestRemoteCache
+
+# Buckets
+REMOTE_BUCKET := remotesourcebucket
+
+# Default target
+help:
+	@echo "Remote Storage Cache Integration Tests"
+	@echo ""
+	@echo "Uses two SeaweedFS instances:"
+	@echo "  - Primary (port $(PRIMARY_S3_PORT)): Being tested, has remote caching"
+	@echo "  - Remote  (port $(REMOTE_S3_PORT)): Acts as remote S3 storage"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  help              - Show this help message"
+	@echo "  build-weed        - Build the SeaweedFS binary"
+	@echo "  check-deps        - Check dependencies"
+	@echo "  start-remote      - Start remote SeaweedFS (secondary)"
+	@echo "  stop-remote       - Stop remote SeaweedFS"
+	@echo "  start-primary     - Start primary SeaweedFS"
+	@echo "  stop-primary      - Stop primary SeaweedFS"
+	@echo "  setup-remote      - Configure remote storage mount"
+	@echo "  test              - Run tests (assumes servers are running)"
+	@echo "  test-with-server  - Start servers, run tests, stop servers"
+	@echo "  clean             - Clean up all resources"
+	@echo "  logs              - Show server logs"
+
+# Build the SeaweedFS binary
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd ../../../weed && go build -o weed_binary .
+	@chmod +x $(WEED_BINARY)
+	@echo "SeaweedFS binary built"
+
+check-deps: build-weed
+	@echo "Checking dependencies..."
+	@command -v go >/dev/null 2>&1 || (echo "Go is required" && exit 1)
+	@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found" && exit 1)
+	@echo "All dependencies available"
+
+# Start remote SeaweedFS (acts as the "remote" S3 storage)
+start-remote: check-deps
+	@echo "Starting remote SeaweedFS (secondary instance)..."
+	@rm -f remote-server.pid
+	@mkdir -p $(REMOTE_DIR)
+	@$(WEED_BINARY) server \
+		-s3 \
+		-s3.port=$(REMOTE_S3_PORT) \
+		-s3.allowDeleteBucketNotEmpty=true \
+		-filer \
+		-filer.port=$(REMOTE_FILER_PORT) \
+		-master.port=$(REMOTE_MASTER_PORT) \
+		-volume.port=$(REMOTE_VOLUME_PORT) \
+		-master.volumeSizeLimitMB=50 \
+		-volume.max=100 \
+		-dir=$(REMOTE_DIR) \
+		-volume.preStopSeconds=1 \
+		-metricsPort=$(REMOTE_METRICS_PORT) \
+		> remote-weed.log 2>&1 & echo $$! > remote-server.pid
+	@echo "Waiting for remote SeaweedFS to start..."
+	@for i in $$(seq 1 60); do \
+		if curl -s http://localhost:$(REMOTE_S3_PORT) >/dev/null 2>&1; then \
+			echo "Remote SeaweedFS started on port $(REMOTE_S3_PORT)"; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "ERROR: Remote SeaweedFS failed to start"; \
+	cat remote-weed.log; \
+	exit 1
+
+stop-remote:
+	@echo "Stopping remote SeaweedFS..."
+	@if [ -f remote-server.pid ]; then \
+		kill -TERM $$(cat remote-server.pid) 2>/dev/null || true; \
+		sleep 2; \
+		kill -KILL $$(cat remote-server.pid) 2>/dev/null || true; \
+		rm -f remote-server.pid; \
+	fi
+	@echo "Remote SeaweedFS stopped"
+
+# Start primary SeaweedFS (the one being tested)
+start-primary: check-deps
+	@echo "Starting primary SeaweedFS..."
+	@rm -f primary-server.pid
+	@mkdir -p $(PRIMARY_DIR)
+	@$(WEED_BINARY) server \
+		-s3 \
+		-s3.port=$(PRIMARY_S3_PORT) \
+		-s3.allowDeleteBucketNotEmpty=true \
+		-filer \
+		-filer.port=$(PRIMARY_FILER_PORT) \
+		-master.port=$(PRIMARY_MASTER_PORT) \
+		-volume.port=$(PRIMARY_VOLUME_PORT) \
+		-master.volumeSizeLimitMB=50 \
+		-volume.max=100 \
+		-dir=$(PRIMARY_DIR) \
+		-volume.preStopSeconds=1 \
+		-metricsPort=$(PRIMARY_METRICS_PORT) \
+		> primary-weed.log 2>&1 & echo $$! > primary-server.pid
+	@echo "Waiting for primary SeaweedFS to start..."
+	@for i in $$(seq 1 60); do \
+		if curl -s http://localhost:$(PRIMARY_S3_PORT) >/dev/null 2>&1; then \
+			echo "Primary SeaweedFS started on port $(PRIMARY_S3_PORT)"; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "ERROR: Primary SeaweedFS failed to start"; \
+	cat primary-weed.log; \
+	exit 1
+
+stop-primary:
+	@echo "Stopping primary SeaweedFS..."
+	@if [ -f primary-server.pid ]; then \
+		kill -TERM $$(cat primary-server.pid) 2>/dev/null || true; \
+		sleep 2; \
+		kill -KILL $$(cat primary-server.pid) 2>/dev/null || true; \
+		rm -f primary-server.pid; \
+	fi
+	@echo "Primary SeaweedFS stopped"
+
+# Create bucket on remote and configure remote storage mount on primary
+setup-remote:
+	@echo "Creating bucket on remote SeaweedFS..."
+	@curl -s -X PUT "http://localhost:$(REMOTE_S3_PORT)/$(REMOTE_BUCKET)" || echo "Bucket may already exist"
+	@sleep 1
+	@echo "Configuring remote storage on primary..."
+	@printf 'remote.configure -name=seaweedremote -type=s3 -s3.access_key=any -s3.secret_key=any -s3.endpoint=http://localhost:$(REMOTE_S3_PORT) -s3.region=us-east-1\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.configure done"
+	@sleep 2
+	@echo "Mounting remote bucket on primary..."
+	@printf 'remote.mount -dir=/buckets/remotemounted -remote=seaweedremote/$(REMOTE_BUCKET) -nonempty\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.mount done"
+	@sleep 1
+	@echo "Remote storage configured"
+
+# Run tests
+test: check-deps
+	@echo "Running remote cache tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" .
+	@echo "Tests completed"
+
+# Full test workflow
+test-with-server: start-remote start-primary
+	@sleep 3
+	@$(MAKE) setup-remote || (echo "Remote setup failed" && $(MAKE) stop-primary stop-remote && exit 1)
+	@sleep 2
+	@echo "Running remote cache tests..."
+	@$(MAKE) test || (echo "Tests failed" && tail -50 primary-weed.log && $(MAKE) stop-primary stop-remote && exit 1)
+	@$(MAKE) stop-primary stop-remote
+	@echo "All tests passed"
+
+# Show logs
+logs:
+	@echo "=== Primary SeaweedFS Logs ==="
+	@if [ -f primary-weed.log ]; then tail -50 primary-weed.log; else echo "No log file"; fi
+	@echo ""
+	@echo "=== Remote SeaweedFS Logs ==="
+	@if [ -f remote-weed.log ]; then tail -50 remote-weed.log; else echo "No log file"; fi
+
+logs-primary:
+	@if [ -f primary-weed.log ]; then tail -f primary-weed.log; else echo "No log file"; fi
+
+logs-remote:
+	@if [ -f remote-weed.log ]; then tail -f remote-weed.log; else echo "No log file"; fi
+
+# Clean up
+clean:
+	@$(MAKE) stop-primary
+	@$(MAKE) stop-remote
+	@rm -f primary-weed.log remote-weed.log primary-server.pid remote-server.pid
+	@rm -rf $(PRIMARY_DIR) $(REMOTE_DIR)
+	@rm -f remote_cache.test
+	@go clean -testcache
+	@echo "Cleanup completed"
+
+# Health check
+health:
+	@echo "Checking server status..."
+	@curl -s http://localhost:$(PRIMARY_S3_PORT) >/dev/null 2>&1 && echo "Primary S3 ($(PRIMARY_S3_PORT)): UP" || echo "Primary S3 ($(PRIMARY_S3_PORT)): DOWN"
+	@curl -s http://localhost:$(REMOTE_S3_PORT) >/dev/null 2>&1 && echo "Remote S3 ($(REMOTE_S3_PORT)): UP" || echo "Remote S3 ($(REMOTE_S3_PORT)): DOWN"

test/s3/remote_cache/README.md

@@ -0,0 +1,157 @@
+# Remote Object Cache Integration Tests
+
+This directory contains integration tests for the remote object caching feature with singleflight deduplication.
+
+## Test Flow
+
+Each test follows this pattern:
+1. **Write to local** - Upload data to primary SeaweedFS (local storage)
+2. **Uncache** - Push data to remote storage and remove local chunks
+3. **Read** - Read data (triggers caching from remote back to local)
+
+This tests the full remote caching workflow including singleflight deduplication.
+
+## Architecture
+
+```text
+┌─────────────────────────────────────────────────────────────────┐
+│                        Test Client                               │
+│                                                                  │
+│    1. PUT data to primary SeaweedFS                             │
+│    2. remote.cache.uncache (push to remote, purge local)        │
+│    3. GET data (triggers caching from remote)                   │
+│    4. Verify singleflight deduplication                         │
+└──────────────────────────────────┬──────────────────────────────┘
+                                   │
+                 ┌─────────────────┴─────────────────┐
+                 ▼                                   ▼
+┌────────────────────────────────────┐   ┌────────────────────────────────┐
+│     Primary SeaweedFS              │   │     Remote SeaweedFS           │
+│        (port 8333)                 │   │        (port 8334)             │
+│                                    │   │                                │
+│  - Being tested                    │   │  - Acts as "remote" S3         │
+│  - Has remote storage mounted      │──▶│  - Receives uncached data      │
+│  - Caches remote objects           │   │  - Serves data for caching     │
+│  - Singleflight deduplication      │   │                                │
+└────────────────────────────────────┘   └────────────────────────────────┘
+```
+
+## What's Being Tested
+
+1. **Basic Remote Caching**: Write → Uncache → Read workflow
+2. **Singleflight Deduplication**: Concurrent reads only trigger ONE caching operation
+3. **Large Object Caching**: 5MB files cache correctly
+4. **Range Requests**: Partial reads work with cached objects
+5. **Not Found Handling**: Proper error for non-existent objects
+
+## Quick Start
+
+### Run Full Test Suite (Recommended)
+
+```bash
+# Build SeaweedFS, start both servers, run tests, stop servers
+make test-with-server
+```
+
+### Manual Steps
+
+```bash
+# 1. Build SeaweedFS binary
+make build-weed
+
+# 2. Start remote SeaweedFS (acts as "remote" storage)
+make start-remote
+
+# 3. Start primary SeaweedFS (the one being tested)
+make start-primary
+
+# 4. Configure remote storage mount
+make setup-remote
+
+# 5. Run tests
+make test
+
+# 6. Clean up
+make clean
+```
+
+## Configuration
+
+### Primary SeaweedFS (Being Tested)
+
+| Service | Port |
+|---------|------|
+| S3 API | 8333 |
+| Filer | 8888 |
+| Master | 9333 |
+| Volume | 8080 |
+
+### Remote SeaweedFS (Remote Storage)
+
+| Service | Port |
+|---------|------|
+| S3 API | 8334 |
+| Filer | 8889 |
+| Master | 9334 |
+| Volume | 8081 |
+
+## Makefile Targets
+
+```bash
+make help           # Show all available targets
+make build-weed     # Build SeaweedFS binary
+make start-remote   # Start remote SeaweedFS
+make start-primary  # Start primary SeaweedFS
+make setup-remote   # Configure remote storage mount
+make test           # Run tests
+make test-with-server  # Full automated test workflow
+make logs           # Show server logs
+make health         # Check server status
+make clean          # Stop servers and clean up
+```
+
+## Test Details
+
+### TestRemoteCacheBasic
+Basic workflow test:
+1. Write object to primary (local)
+2. Uncache (push to remote, remove local chunks)
+3. Read (triggers caching from remote)
+4. Read again (from local cache - should be faster)
+
+### TestRemoteCacheConcurrent
+Singleflight deduplication test:
+1. Write 1MB object
+2. Uncache to remote
+3. Launch 10 concurrent reads
+4. All should succeed with correct data
+5. Only ONE caching operation should run (singleflight)
+
+### TestRemoteCacheLargeObject
+Large file test (5MB) to verify chunked transfer works correctly.
+
+### TestRemoteCacheRangeRequest
+Tests HTTP range requests work correctly after caching.
+
+### TestRemoteCacheNotFound
+Tests proper error handling for non-existent objects.
+
+## Troubleshooting
+
+### View logs
+```bash
+make logs           # Show recent logs from both servers
+make logs-primary   # Follow primary logs in real-time
+make logs-remote    # Follow remote logs in real-time
+```
+
+### Check server health
+```bash
+make health
+```
+
+### Clean up and retry
+```bash
+make clean
+make test-with-server
+```

test/s3/remote_cache/remote_cache_test.go

@@ -0,0 +1,375 @@
+package remote_cache
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Test configuration
+// Uses two SeaweedFS instances:
+// - Primary: The one being tested (has remote caching)
+// - Remote: Acts as the "remote" S3 storage
+const (
+	// Primary SeaweedFS
+	primaryEndpoint   = "http://localhost:8333"
+	primaryMasterPort = "9333"
+
+	// Remote SeaweedFS (acts as remote storage)
+	remoteEndpoint = "http://localhost:8334"
+
+	// Credentials (anonymous access for testing)
+	accessKey = "any"
+	secretKey = "any"
+
+	// Bucket name - mounted on primary as remote storage
+	testBucket = "remotemounted"
+
+	// Path to weed binary
+	weedBinary = "../../../weed/weed_binary"
+)
+
+var (
+	primaryClient     *s3.S3
+	primaryClientOnce sync.Once
+)
+
+func getPrimaryClient() *s3.S3 {
+	primaryClientOnce.Do(func() {
+		primaryClient = createS3Client(primaryEndpoint)
+	})
+	return primaryClient
+}
+
+func createS3Client(endpoint string) *s3.S3 {
+	sess, err := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String(endpoint),
+		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
+		DisableSSL:       aws.Bool(!strings.HasPrefix(endpoint, "https")),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	if err != nil {
+		panic(fmt.Sprintf("failed to create session: %v", err))
+	}
+	return s3.New(sess)
+}
+
+// skipIfNotRunning skips the test if the servers aren't running
+func skipIfNotRunning(t *testing.T) {
+	resp, err := http.Get(primaryEndpoint)
+	if err != nil {
+		t.Skipf("Primary SeaweedFS not running at %s: %v", primaryEndpoint, err)
+	}
+	resp.Body.Close()
+
+	resp, err = http.Get(remoteEndpoint)
+	if err != nil {
+		t.Skipf("Remote SeaweedFS not running at %s: %v", remoteEndpoint, err)
+	}
+	resp.Body.Close()
+}
+
+// runWeedShell executes a weed shell command
+func runWeedShell(t *testing.T, command string) (string, error) {
+	cmd := exec.Command(weedBinary, "shell", "-master=localhost:"+primaryMasterPort)
+	cmd.Stdin = strings.NewReader(command + "\nexit\n")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Logf("weed shell command '%s' failed: %v, output: %s", command, err, string(output))
+		return string(output), err
+	}
+	return string(output), nil
+}
+
+// uploadToPrimary uploads an object to the primary SeaweedFS (local write)
+func uploadToPrimary(t *testing.T, key string, data []byte) {
+	_, err := getPrimaryClient().PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(key),
+		Body:   bytes.NewReader(data),
+	})
+	require.NoError(t, err, "failed to upload to primary SeaweedFS")
+}
+
+// getFromPrimary gets an object from primary SeaweedFS
+func getFromPrimary(t *testing.T, key string) []byte {
+	resp, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(key),
+	})
+	require.NoError(t, err, "failed to get from primary SeaweedFS")
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
+	require.NoError(t, err, "failed to read response body")
+	return data
+}
+
+// syncToRemote syncs local data to remote storage
+func syncToRemote(t *testing.T) {
+	t.Log("Syncing to remote storage...")
+	output, err := runWeedShell(t, "remote.cache.uncache -dir=/buckets/"+testBucket+" -include=*")
+	if err != nil {
+		t.Logf("syncToRemote warning: %v", err)
+	}
+	t.Log(output)
+	time.Sleep(1 * time.Second)
+}
+
+// uncacheLocal purges the local cache, forcing data to be fetched from remote
+func uncacheLocal(t *testing.T, pattern string) {
+	t.Logf("Purging local cache for pattern: %s", pattern)
+	output, err := runWeedShell(t, fmt.Sprintf("remote.uncache -dir=/buckets/%s -include=%s", testBucket, pattern))
+	if err != nil {
+		t.Logf("uncacheLocal warning: %v", err)
+	}
+	t.Log(output)
+	time.Sleep(500 * time.Millisecond)
+}
+
+// TestRemoteCacheBasic tests the basic caching workflow:
+// 1. Write to local
+// 2. Uncache (push to remote, remove local chunks)
+// 3. Read (triggers caching from remote)
+func TestRemoteCacheBasic(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("test-basic-%d.txt", time.Now().UnixNano())
+	testData := []byte("Hello, this is test data for remote caching!")
+
+	// Step 1: Write to local
+	t.Log("Step 1: Writing object to primary SeaweedFS (local)...")
+	uploadToPrimary(t, testKey, testData)
+
+	// Verify it's readable
+	result := getFromPrimary(t, testKey)
+	assert.Equal(t, testData, result, "initial read mismatch")
+
+	// Step 2: Uncache - push to remote and remove local chunks
+	t.Log("Step 2: Uncaching (pushing to remote, removing local chunks)...")
+	uncacheLocal(t, testKey)
+
+	// Step 3: Read - this should trigger caching from remote
+	t.Log("Step 3: Reading object (should trigger caching from remote)...")
+	start := time.Now()
+	result = getFromPrimary(t, testKey)
+	firstReadDuration := time.Since(start)
+
+	assert.Equal(t, testData, result, "data mismatch after cache")
+	t.Logf("First read (from remote) took %v", firstReadDuration)
+
+	// Step 4: Read again - should be from local cache
+	t.Log("Step 4: Reading again (should be from local cache)...")
+	start = time.Now()
+	result = getFromPrimary(t, testKey)
+	secondReadDuration := time.Since(start)
+
+	assert.Equal(t, testData, result, "data mismatch on cached read")
+	t.Logf("Second read (from cache) took %v", secondReadDuration)
+
+	t.Log("Basic caching test passed")
+}
+
+// TestRemoteCacheConcurrent tests that concurrent reads of the same
+// remote object only trigger ONE caching operation (singleflight deduplication)
+func TestRemoteCacheConcurrent(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("test-concurrent-%d.txt", time.Now().UnixNano())
+	// Use larger data to make caching take measurable time
+	testData := make([]byte, 1024*1024) // 1MB
+	for i := range testData {
+		testData[i] = byte(i % 256)
+	}
+
+	// Step 1: Write to local
+	t.Log("Step 1: Writing 1MB object to primary SeaweedFS...")
+	uploadToPrimary(t, testKey, testData)
+
+	// Verify it's readable
+	result := getFromPrimary(t, testKey)
+	assert.Equal(t, len(testData), len(result), "initial size mismatch")
+
+	// Step 2: Uncache
+	t.Log("Step 2: Uncaching (pushing to remote)...")
+	uncacheLocal(t, testKey)
+
+	// Step 3: Launch many concurrent reads - singleflight should deduplicate
+	numRequests := 10
+	var wg sync.WaitGroup
+	var successCount atomic.Int32
+	var errorCount atomic.Int32
+	results := make(chan []byte, numRequests)
+
+	t.Logf("Step 3: Launching %d concurrent requests...", numRequests)
+	startTime := time.Now()
+
+	for i := 0; i < numRequests; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			resp, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
+				Bucket: aws.String(testBucket),
+				Key:    aws.String(testKey),
+			})
+			if err != nil {
+				t.Logf("Request %d failed: %v", idx, err)
+				errorCount.Add(1)
+				return
+			}
+			defer resp.Body.Close()
+
+			data, err := io.ReadAll(resp.Body)
+			if err != nil {
+				t.Logf("Request %d read failed: %v", idx, err)
+				errorCount.Add(1)
+				return
+			}
+
+			results <- data
+			successCount.Add(1)
+		}(i)
+	}
+
+	wg.Wait()
+	close(results)
+	totalDuration := time.Since(startTime)
+
+	t.Logf("All %d requests completed in %v", numRequests, totalDuration)
+	t.Logf("Successful: %d, Failed: %d", successCount.Load(), errorCount.Load())
+
+	// Verify all successful requests returned correct data
+	for data := range results {
+		assert.Equal(t, len(testData), len(data), "data length mismatch")
+	}
+
+	// All requests should succeed
+	assert.Equal(t, int32(numRequests), successCount.Load(), "some requests failed")
+	assert.Equal(t, int32(0), errorCount.Load(), "no requests should fail")
+
+	t.Log("Concurrent caching test passed")
+}
+
+// TestRemoteCacheLargeObject tests caching of larger objects
+func TestRemoteCacheLargeObject(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("test-large-%d.bin", time.Now().UnixNano())
+	// 5MB object
+	testData := make([]byte, 5*1024*1024)
+	for i := range testData {
+		testData[i] = byte(i % 256)
+	}
+
+	// Step 1: Write to local
+	t.Log("Step 1: Writing 5MB object to primary SeaweedFS...")
+	uploadToPrimary(t, testKey, testData)
+
+	// Verify it's readable
+	result := getFromPrimary(t, testKey)
+	assert.Equal(t, len(testData), len(result), "initial size mismatch")
+
+	// Step 2: Uncache
+	t.Log("Step 2: Uncaching...")
+	uncacheLocal(t, testKey)
+
+	// Step 3: Read from remote
+	t.Log("Step 3: Reading 5MB object (should cache from remote)...")
+	start := time.Now()
+	result = getFromPrimary(t, testKey)
+	duration := time.Since(start)
+
+	assert.Equal(t, len(testData), len(result), "size mismatch")
+	assert.Equal(t, testData, result, "data mismatch")
+	t.Logf("Large object cached in %v", duration)
+
+	t.Log("Large object caching test passed")
+}
+
+// TestRemoteCacheRangeRequest tests that range requests work after caching
+func TestRemoteCacheRangeRequest(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("test-range-%d.txt", time.Now().UnixNano())
+	testData := []byte("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+	// Write, uncache, then test range request
+	t.Log("Writing and uncaching object...")
+	uploadToPrimary(t, testKey, testData)
+	uncacheLocal(t, testKey)
+
+	// Range request should work and trigger caching
+	t.Log("Testing range request (bytes 10-19)...")
+	resp, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(testKey),
+		Range:  aws.String("bytes=10-19"),
+	})
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	rangeData, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	expected := testData[10:20] // "ABCDEFGHIJ"
+	assert.Equal(t, expected, rangeData, "range data mismatch")
+	t.Logf("Range request returned: %s", string(rangeData))
+
+	t.Log("Range request test passed")
+}
+
+// TestRemoteCacheNotFound tests that non-existent objects return proper errors
+func TestRemoteCacheNotFound(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("non-existent-object-%d", time.Now().UnixNano())
+
+	_, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(testKey),
+	})
+
+	assert.Error(t, err, "should get error for non-existent object")
+	t.Logf("Got expected error: %v", err)
+
+	t.Log("Not found test passed")
+}
+
+// TestMain sets up and tears down the test environment
+func TestMain(m *testing.M) {
+	if !isServerRunning(primaryEndpoint) {
+		fmt.Println("WARNING: Primary SeaweedFS not running at", primaryEndpoint)
+		fmt.Println("   Run 'make test-with-server' to start servers automatically")
+	}
+	if !isServerRunning(remoteEndpoint) {
+		fmt.Println("WARNING: Remote SeaweedFS not running at", remoteEndpoint)
+		fmt.Println("   Run 'make test-with-server' to start servers automatically")
+	}
+
+	os.Exit(m.Run())
+}
+
+func isServerRunning(url string) bool {
+	resp, err := http.Get(url)
+	if err != nil {
+		return false
+	}
+	resp.Body.Close()
+	return true
+}

test/s3/versioning/Makefile

@@ -81,7 +81,7 @@ start-server: check-deps
 	@echo "🔍 DEBUG: Creating volume directory..."
 	@mkdir -p ./test-volume-data
 	@echo "🔍 DEBUG: Launching SeaweedFS server in background..."
-	@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324"
+	@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -master.peers=none -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324"
 	@$(WEED_BINARY) server \
 		-debug \
 		-s3 \
@@ -91,6 +91,7 @@ start-server: check-deps
 		-filer \
 		-filer.maxMB=64 \
 		-master.volumeSizeLimitMB=50 \
+		-master.peers=none \
 		-volume.max=100 \
 		-dir=./test-volume-data \
 		-volume.preStopSeconds=1 \
@@ -221,13 +222,13 @@ test-with-server: start-server
 test-versioning-with-configs: check-deps
 	@echo "Testing with different S3 configurations..."
 	@echo "Testing with empty folder allowed..."
-	@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
+	@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -filer -master.volumeSizeLimitMB=100 -master.peers=none -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
 	@sleep 5
 	@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
 	@if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi
 	@sleep 2
 	@echo "Testing with delete bucket not empty disabled..."
-	@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
+	@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -master.peers=none -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
 	@sleep 5
 	@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
 	@if [ -f weed-config2.pid ]; then kill -TERM $$(cat weed-config2.pid) 2>/dev/null || true; rm -f weed-config2.pid; fi
@@ -239,6 +240,13 @@ test-versioning-stress: check-deps
 	@go test -v -timeout=20m -run "TestVersioningConcurrentOperations" . -count=5
 	@echo "✅ Stress tests completed"
 
+# Pagination stress testing (tests >1000 versions)
+test-versioning-pagination-stress: check-deps
+	@echo "Running pagination stress tests (>1000 versions)..."
+	@echo "This test creates 1500+ versions and may take several minutes..."
+	@ENABLE_STRESS_TESTS=true go test -v -timeout=30m -run "TestVersioningPagination" .
+	@echo "✅ Pagination stress tests completed"
+
 # Generate test reports
 test-report: check-deps
 	@echo "Generating test reports..."
@@ -272,6 +280,7 @@ debug-server:
 		-filer \
 		-filer.maxMB=16 \
 		-master.volumeSizeLimitMB=50 \
+		-master.peers=none \
 		-volume.max=100 \
 		-dir=./test-volume-data \
 		-volume.preStopSeconds=1 \
@@ -320,6 +329,7 @@ start-server-simple: check-deps
 		-filer \
 		-filer.maxMB=64 \
 		-master.volumeSizeLimitMB=50 \
+		-master.peers=none \
 		-volume.max=100 \
 		-volume.preStopSeconds=1 \
 		-metricsPort=9324 \

test/s3/versioning/s3_versioning_multipart_test.go

@@ -0,0 +1,521 @@
+package s3api
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestMultipartUploadVersioningListETag tests that multipart uploaded objects
+// in versioned buckets have correct ETags when listed.
+// This covers a bug where synthetic entries for versioned objects didn't include
+// proper ETag handling for multipart uploads (ETags with format "<md5>-<parts>").
+func TestMultipartUploadVersioningListETag(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err, "Failed to enable versioning")
+
+	// Create multipart upload
+	objectKey := "multipart-test-object"
+	createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	uploadId := *createResp.UploadId
+
+	// Upload 2 parts (minimum 5MB per part except last)
+	partSize := 5 * 1024 * 1024 // 5MB
+	part1Data := bytes.Repeat([]byte("a"), partSize)
+	part2Data := bytes.Repeat([]byte("b"), partSize)
+
+	// Calculate MD5 for each part
+	part1MD5 := md5.Sum(part1Data)
+	part2MD5 := md5.Sum(part2Data)
+
+	// Upload part 1
+	uploadPart1Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   aws.String(uploadId),
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(part1Data),
+	})
+	require.NoError(t, err, "Failed to upload part 1")
+
+	// Upload part 2
+	uploadPart2Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   aws.String(uploadId),
+		PartNumber: aws.Int32(2),
+		Body:       bytes.NewReader(part2Data),
+	})
+	require.NoError(t, err, "Failed to upload part 2")
+
+	// Complete multipart upload
+	completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: aws.String(uploadId),
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: []types.CompletedPart{
+				{
+					ETag:       uploadPart1Resp.ETag,
+					PartNumber: aws.Int32(1),
+				},
+				{
+					ETag:       uploadPart2Resp.ETag,
+					PartNumber: aws.Int32(2),
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+
+	// Verify the ETag from CompleteMultipartUpload has the multipart format (md5-parts)
+	completeETag := strings.Trim(*completeResp.ETag, "\"")
+	assert.Contains(t, completeETag, "-", "Multipart ETag should contain '-' (format: md5-parts)")
+	assert.True(t, strings.HasSuffix(completeETag, "-2"), "Multipart ETag should end with '-2' for 2 parts")
+
+	t.Logf("CompleteMultipartUpload ETag: %s", completeETag)
+	t.Logf("Part 1 MD5: %x", part1MD5)
+	t.Logf("Part 2 MD5: %x", part2MD5)
+
+	// HeadObject should return the same ETag
+	headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to head object")
+
+	headETag := strings.Trim(*headResp.ETag, "\"")
+	assert.Equal(t, completeETag, headETag, "HeadObject ETag should match CompleteMultipartUpload ETag")
+
+	// ListObjectsV2 should return the same ETag
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects")
+	require.Len(t, listResp.Contents, 1, "Should have exactly one object")
+
+	listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
+	assert.Equal(t, completeETag, listETag, "ListObjectsV2 ETag should match CompleteMultipartUpload ETag")
+	assert.NotEmpty(t, listETag, "ListObjectsV2 ETag should not be empty")
+
+	t.Logf("ListObjectsV2 ETag: %s", listETag)
+
+	// ListObjectVersions should also return the correct ETag
+	versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list object versions")
+	require.Len(t, versionsResp.Versions, 1, "Should have exactly one version")
+
+	versionETag := strings.Trim(*versionsResp.Versions[0].ETag, "\"")
+	assert.Equal(t, completeETag, versionETag, "ListObjectVersions ETag should match CompleteMultipartUpload ETag")
+	assert.NotEmpty(t, versionETag, "ListObjectVersions ETag should not be empty")
+
+	t.Logf("ListObjectVersions ETag: %s", versionETag)
+}
+
+// TestMultipartUploadMultipleVersionsListETag tests that multiple versions
+// of multipart uploaded objects all have correct ETags when listed.
+func TestMultipartUploadMultipleVersionsListETag(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err, "Failed to enable versioning")
+
+	objectKey := "multipart-multi-version-object"
+	partSize := 5 * 1024 * 1024 // 5MB
+	var expectedETags []string
+
+	// Create 3 versions using multipart upload
+	for version := 1; version <= 3; version++ {
+		// Create multipart upload
+		createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(objectKey),
+		})
+		require.NoError(t, err, "Failed to create multipart upload for version %d", version)
+
+		uploadId := *createResp.UploadId
+
+		// Create unique data for each version
+		partData := bytes.Repeat([]byte(fmt.Sprintf("%d", version)), partSize)
+
+		// Upload single part (still results in multipart ETag format)
+		uploadPartResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+			Bucket:     aws.String(bucketName),
+			Key:        aws.String(objectKey),
+			UploadId:   aws.String(uploadId),
+			PartNumber: aws.Int32(1),
+			Body:       bytes.NewReader(partData),
+		})
+		require.NoError(t, err, "Failed to upload part for version %d", version)
+
+		// Complete multipart upload
+		completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+			Bucket:   aws.String(bucketName),
+			Key:      aws.String(objectKey),
+			UploadId: aws.String(uploadId),
+			MultipartUpload: &types.CompletedMultipartUpload{
+				Parts: []types.CompletedPart{
+					{
+						ETag:       uploadPartResp.ETag,
+						PartNumber: aws.Int32(1),
+					},
+				},
+			},
+		})
+		require.NoError(t, err, "Failed to complete multipart upload for version %d", version)
+
+		etag := strings.Trim(*completeResp.ETag, "\"")
+		expectedETags = append(expectedETags, etag)
+		t.Logf("Version %d ETag: %s", version, etag)
+	}
+
+	// ListObjectVersions should return all versions with correct ETags
+	versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list object versions")
+	require.Len(t, versionsResp.Versions, 3, "Should have exactly 3 versions")
+
+	// Collect ETags from the listing
+	var listedETags []string
+	for _, v := range versionsResp.Versions {
+		etag := strings.Trim(*v.ETag, "\"")
+		listedETags = append(listedETags, etag)
+		assert.NotEmpty(t, etag, "Version ETag should not be empty")
+		assert.Contains(t, etag, "-", "Multipart ETag should contain '-'")
+	}
+
+	t.Logf("Expected ETags: %v", expectedETags)
+	t.Logf("Listed ETags: %v", listedETags)
+
+	// Verify all expected ETags are present (order may differ due to version ordering)
+	assert.ElementsMatch(t, expectedETags, listedETags, "Listed ETags should match all expected ETags")
+
+	// Regular ListObjectsV2 should return only the latest version with correct ETag
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects")
+	require.Len(t, listResp.Contents, 1, "Should have exactly one object in regular listing")
+
+	listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
+	// The latest version (version 3) should be the one shown
+	assert.Equal(t, expectedETags[2], listETag, "ListObjectsV2 should show latest version's ETag")
+}
+
+// TestMixedSingleAndMultipartVersionsListETag tests that a mix of
+// single-part and multipart uploaded versions all have correct ETags.
+func TestMixedSingleAndMultipartVersionsListETag(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err, "Failed to enable versioning")
+
+	objectKey := "mixed-upload-versions"
+
+	// Version 1: Regular PutObject (single-part, pure MD5 ETag)
+	content1 := []byte("This is version 1 content - single part upload")
+	putResp1, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(content1),
+	})
+	require.NoError(t, err, "Failed to put version 1")
+	etag1 := strings.Trim(*putResp1.ETag, "\"")
+	assert.NotContains(t, etag1, "-", "Single-part ETag should not contain '-'")
+	t.Logf("Version 1 (PutObject) ETag: %s", etag1)
+
+	// Version 2: Multipart upload
+	partSize := 5 * 1024 * 1024
+	partData := bytes.Repeat([]byte("x"), partSize)
+
+	createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	uploadPartResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   createResp.UploadId,
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(partData),
+	})
+	require.NoError(t, err, "Failed to upload part")
+
+	completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: createResp.UploadId,
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: []types.CompletedPart{
+				{
+					ETag:       uploadPartResp.ETag,
+					PartNumber: aws.Int32(1),
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+	etag2 := strings.Trim(*completeResp.ETag, "\"")
+	assert.Contains(t, etag2, "-", "Multipart ETag should contain '-'")
+	t.Logf("Version 2 (Multipart) ETag: %s", etag2)
+
+	// Version 3: Another regular PutObject
+	content3 := []byte("This is version 3 content - another single part upload")
+	putResp3, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(content3),
+	})
+	require.NoError(t, err, "Failed to put version 3")
+	etag3 := strings.Trim(*putResp3.ETag, "\"")
+	assert.NotContains(t, etag3, "-", "Single-part ETag should not contain '-'")
+	t.Logf("Version 3 (PutObject) ETag: %s", etag3)
+
+	// ListObjectVersions should return all 3 versions with correct ETags
+	versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list object versions")
+	require.Len(t, versionsResp.Versions, 3, "Should have exactly 3 versions")
+
+	var listedETags []string
+	for _, v := range versionsResp.Versions {
+		etag := strings.Trim(*v.ETag, "\"")
+		assert.NotEmpty(t, etag, "Version ETag should not be empty")
+		listedETags = append(listedETags, etag)
+		t.Logf("Listed version %s ETag: %s, IsLatest: %v", *v.VersionId, etag, *v.IsLatest)
+	}
+
+	// Verify all ETags were found
+	assert.ElementsMatch(t, []string{etag1, etag2, etag3}, listedETags, "Listed ETags should match all expected ETags")
+
+	// Regular ListObjectsV2 should return only the latest (version 3)
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects")
+	require.Len(t, listResp.Contents, 1, "Should have exactly one object")
+
+	listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
+	assert.Equal(t, etag3, listETag, "ListObjectsV2 should show latest version's ETag (version 3)")
+}
+
+// TestMultipartUploadDeleteMarkerListBehavior tests that delete markers work correctly
+// with multipart uploaded objects in versioned buckets.
+func TestMultipartUploadDeleteMarkerListBehavior(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err, "Failed to enable versioning")
+
+	objectKey := "multipart-delete-marker-test"
+	partSize := 5 * 1024 * 1024 // 5MB
+
+	// Create multipart upload
+	createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	// Upload 2 parts
+	part1Data := bytes.Repeat([]byte("a"), partSize)
+	part2Data := bytes.Repeat([]byte("b"), partSize)
+
+	uploadPart1Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   createResp.UploadId,
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(part1Data),
+	})
+	require.NoError(t, err, "Failed to upload part 1")
+
+	uploadPart2Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   createResp.UploadId,
+		PartNumber: aws.Int32(2),
+		Body:       bytes.NewReader(part2Data),
+	})
+	require.NoError(t, err, "Failed to upload part 2")
+
+	// Complete multipart upload
+	completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: createResp.UploadId,
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: []types.CompletedPart{
+				{ETag: uploadPart1Resp.ETag, PartNumber: aws.Int32(1)},
+				{ETag: uploadPart2Resp.ETag, PartNumber: aws.Int32(2)},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+
+	multipartETag := strings.Trim(*completeResp.ETag, "\"")
+	multipartVersionId := *completeResp.VersionId
+	t.Logf("Multipart upload completed: ETag=%s, VersionId=%s", multipartETag, multipartVersionId)
+
+	// Verify object is visible in ListObjectsV2
+	listBeforeDelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects before delete")
+	require.Len(t, listBeforeDelete.Contents, 1, "Object should be visible before delete")
+	assert.Equal(t, multipartETag, strings.Trim(*listBeforeDelete.Contents[0].ETag, "\""),
+		"Listed ETag should match multipart ETag before delete")
+
+	// Delete object (creates delete marker)
+	deleteResp, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to delete object")
+	require.NotNil(t, deleteResp.DeleteMarker, "Should create delete marker")
+	assert.True(t, *deleteResp.DeleteMarker, "DeleteMarker should be true")
+	require.NotNil(t, deleteResp.VersionId, "Delete marker should have version ID")
+
+	deleteMarkerVersionId := *deleteResp.VersionId
+	t.Logf("Delete marker created: VersionId=%s", deleteMarkerVersionId)
+
+	// ListObjectsV2 should NOT show the object anymore
+	listAfterDelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects after delete")
+	assert.Empty(t, listAfterDelete.Contents, "Object should NOT be visible after delete marker")
+
+	// ListObjectVersions should show both the original version AND the delete marker
+	versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list object versions")
+
+	// Should have 1 version (the multipart object)
+	require.Len(t, versionsResp.Versions, 1, "Should have exactly 1 version (the multipart object)")
+	version := versionsResp.Versions[0]
+	assert.Equal(t, multipartVersionId, *version.VersionId, "Version ID should match")
+	assert.Equal(t, multipartETag, strings.Trim(*version.ETag, "\""), "Version ETag should match multipart ETag")
+	assert.False(t, *version.IsLatest, "Multipart version should NOT be latest (delete marker is latest)")
+
+	// Should have 1 delete marker
+	require.Len(t, versionsResp.DeleteMarkers, 1, "Should have exactly 1 delete marker")
+	deleteMarker := versionsResp.DeleteMarkers[0]
+	assert.Equal(t, deleteMarkerVersionId, *deleteMarker.VersionId, "Delete marker version ID should match")
+	assert.True(t, *deleteMarker.IsLatest, "Delete marker should be latest")
+
+	t.Logf("ListObjectVersions: 1 version (ETag=%s), 1 delete marker", multipartETag)
+
+	// Access the specific version by version ID - should still work
+	getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(multipartVersionId),
+	})
+	require.NoError(t, err, "Should be able to get object by version ID after delete marker")
+	defer getResp.Body.Close()
+
+	assert.Equal(t, multipartETag, strings.Trim(*getResp.ETag, "\""),
+		"GetObject with version ID should return correct ETag")
+	assert.Equal(t, int64(partSize*2), *getResp.ContentLength,
+		"GetObject with version ID should return correct size")
+
+	t.Logf("Successfully retrieved version %s after delete marker", multipartVersionId)
+
+	// Delete the delete marker to "undelete" the object
+	_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(deleteMarkerVersionId),
+	})
+	require.NoError(t, err, "Failed to delete the delete marker")
+
+	// ListObjectsV2 should show the object again
+	listAfterUndelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects after undelete")
+	require.Len(t, listAfterUndelete.Contents, 1, "Object should be visible again after removing delete marker")
+	assert.Equal(t, multipartETag, strings.Trim(*listAfterUndelete.Contents[0].ETag, "\""),
+		"Undeleted object should have correct multipart ETag")
+
+	t.Logf("Object restored after delete marker removal, ETag=%s", multipartETag)
+}
+

test/s3/versioning/s3_versioning_pagination_stress_test.go

@@ -0,0 +1,322 @@
+package s3api
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestVersioningPaginationOver1000Versions tests that ListObjectVersions correctly
+// handles objects with more than 1000 versions using pagination.
+// This is a stress test that creates 1500 versions and verifies all are listed.
+//
+// Run with: ENABLE_STRESS_TESTS=true go test -v -run TestVersioningPaginationOver1000Versions -timeout 30m
+func TestVersioningPaginationOver1000Versions(t *testing.T) {
+	if os.Getenv("ENABLE_STRESS_TESTS") != "true" {
+		t.Skip("Skipping stress test. Set ENABLE_STRESS_TESTS=true to run.")
+	}
+
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	enableVersioning(t, client, bucketName)
+	checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
+
+	objectKey := "test-many-versions"
+	// Use 1100 for CI (tests pagination boundary), can override with env var for quick local testing
+	numVersions := 1100
+	if quickTest := os.Getenv("QUICK_TEST"); quickTest == "true" {
+		numVersions = 50
+	}
+	versionIds := make([]string, 0, numVersions)
+
+	t.Logf("Creating %d versions of object %s...", numVersions, objectKey)
+	startTime := time.Now()
+
+	// Create many versions of the same object
+	for i := 0; i < numVersions; i++ {
+		content := fmt.Sprintf("content-version-%d", i+1)
+		putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(objectKey),
+			Body:   strings.NewReader(content),
+		})
+		require.NoError(t, err, "Failed to create version %d", i+1)
+		require.NotNil(t, putResp.VersionId)
+		versionIds = append(versionIds, *putResp.VersionId)
+
+		if (i+1)%100 == 0 {
+			t.Logf("Created %d/%d versions...", i+1, numVersions)
+		}
+	}
+
+	createDuration := time.Since(startTime)
+	t.Logf("Created %d versions in %v", numVersions, createDuration)
+	t.Logf("Version IDs collected: %d", len(versionIds))
+
+	// Quick sanity check - list versions to see what we have
+	quickResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	t.Logf("Quick check: got %d versions, %d delete markers, truncated=%v",
+		len(quickResp.Versions), len(quickResp.DeleteMarkers),
+		quickResp.IsTruncated != nil && *quickResp.IsTruncated)
+
+	// Test 1: List all versions without pagination limit
+	t.Run("ListAllVersionsNoPagination", func(t *testing.T) {
+		allVersions := listAllVersions(t, client, bucketName, objectKey)
+		t.Logf("Got %d versions total", len(allVersions))
+		assert.Len(t, allVersions, numVersions, "Should have all %d versions", numVersions)
+
+		// Verify all version IDs are unique
+		versionIdSet := make(map[string]bool)
+		for _, v := range allVersions {
+			assert.False(t, versionIdSet[*v.VersionId], "Duplicate version ID: %s", *v.VersionId)
+			versionIdSet[*v.VersionId] = true
+		}
+
+		// Verify only one version is marked as latest
+		latestCount := 0
+		for _, v := range allVersions {
+			if *v.IsLatest {
+				latestCount++
+				assert.Equal(t, versionIds[numVersions-1], *v.VersionId, "Latest should be the last created")
+			}
+		}
+		assert.Equal(t, 1, latestCount, "Only one version should be marked as latest")
+	})
+
+	// Test 2: List with explicit small max-keys to force multiple pages
+	t.Run("ListWithSmallMaxKeys", func(t *testing.T) {
+		maxKeys := int32(100)
+		allVersions := make([]types.ObjectVersion, 0, numVersions)
+		var keyMarker, versionIdMarker *string
+		pageCount := 0
+
+		for {
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:          aws.String(bucketName),
+				Prefix:          aws.String(objectKey),
+				MaxKeys:         aws.Int32(maxKeys),
+				KeyMarker:       keyMarker,
+				VersionIdMarker: versionIdMarker,
+			})
+			require.NoError(t, err)
+			pageCount++
+
+			allVersions = append(allVersions, resp.Versions...)
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+			versionIdMarker = resp.NextVersionIdMarker
+		}
+
+		t.Logf("Listed %d versions in %d pages (max-keys=%d)", len(allVersions), pageCount, maxKeys)
+		assert.Len(t, allVersions, numVersions, "Should have all %d versions", numVersions)
+		if numVersions > int(maxKeys) {
+			assert.Greater(t, pageCount, 1, "Should require multiple pages when versions > max-keys")
+		}
+	})
+
+	// Test 3: List with max-keys=1000 (S3 default) to test the boundary
+	t.Run("ListWithDefaultMaxKeys", func(t *testing.T) {
+		maxKeys := int32(1000)
+		allVersions := make([]types.ObjectVersion, 0, numVersions)
+		var keyMarker, versionIdMarker *string
+		pageCount := 0
+
+		for {
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:          aws.String(bucketName),
+				Prefix:          aws.String(objectKey),
+				MaxKeys:         aws.Int32(maxKeys),
+				KeyMarker:       keyMarker,
+				VersionIdMarker: versionIdMarker,
+			})
+			require.NoError(t, err)
+			pageCount++
+
+			allVersions = append(allVersions, resp.Versions...)
+			t.Logf("Page %d: got %d versions, truncated=%v", pageCount, len(resp.Versions), resp.IsTruncated != nil && *resp.IsTruncated)
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+			versionIdMarker = resp.NextVersionIdMarker
+		}
+
+		t.Logf("Listed %d versions in %d pages (max-keys=%d)", len(allVersions), pageCount, maxKeys)
+		assert.Len(t, allVersions, numVersions, "Should have all %d versions", numVersions)
+		if numVersions > 1000 {
+			assert.GreaterOrEqual(t, pageCount, 2, "Should require at least 2 pages for >1000 versions with max-keys=1000")
+		}
+	})
+
+	// Test 4: Verify we can retrieve specific versions
+	t.Run("RetrieveSpecificVersions", func(t *testing.T) {
+		// Test first, middle, and last versions
+		testIndices := []int{0, numVersions / 2, numVersions - 1}
+		for _, idx := range testIndices {
+			expectedContent := fmt.Sprintf("content-version-%d", idx+1)
+			getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+				Bucket:    aws.String(bucketName),
+				Key:       aws.String(objectKey),
+				VersionId: aws.String(versionIds[idx]),
+			})
+			require.NoError(t, err, "Failed to get version at index %d", idx)
+
+			buf := make([]byte, len(expectedContent))
+			_, err = getResp.Body.Read(buf)
+			getResp.Body.Close()
+
+			assert.Equal(t, expectedContent, string(buf), "Content mismatch for version %d", idx+1)
+		}
+	})
+}
+
+// TestVersioningPaginationMultipleObjectsManyVersions tests pagination with multiple objects
+// each having many versions, ensuring correct handling of key-marker and version-id-marker.
+//
+// Run with: ENABLE_STRESS_TESTS=true go test -v -run TestVersioningPaginationMultipleObjectsManyVersions -timeout 30m
+func TestVersioningPaginationMultipleObjectsManyVersions(t *testing.T) {
+	if os.Getenv("ENABLE_STRESS_TESTS") != "true" {
+		t.Skip("Skipping stress test. Set ENABLE_STRESS_TESTS=true to run.")
+	}
+
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	enableVersioning(t, client, bucketName)
+	checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
+
+	numObjects := 5
+	versionsPerObject := 300
+	totalExpectedVersions := numObjects * versionsPerObject
+
+	t.Logf("Creating %d objects with %d versions each (%d total)...", numObjects, versionsPerObject, totalExpectedVersions)
+	startTime := time.Now()
+
+	// Create multiple objects with many versions each
+	for i := 0; i < numObjects; i++ {
+		objectKey := fmt.Sprintf("object-%03d", i)
+		for j := 0; j < versionsPerObject; j++ {
+			content := fmt.Sprintf("obj%d-ver%d", i, j+1)
+			_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+				Body:   strings.NewReader(content),
+			})
+			require.NoError(t, err)
+		}
+		t.Logf("Created object %s with %d versions", objectKey, versionsPerObject)
+	}
+
+	createDuration := time.Since(startTime)
+	t.Logf("Created %d total versions in %v", totalExpectedVersions, createDuration)
+
+	// List all versions using pagination
+	t.Run("ListAllWithPagination", func(t *testing.T) {
+		maxKeys := int32(500)
+		allVersions := make([]types.ObjectVersion, 0, totalExpectedVersions)
+		var keyMarker, versionIdMarker *string
+		pageCount := 0
+
+		for {
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:          aws.String(bucketName),
+				MaxKeys:         aws.Int32(maxKeys),
+				KeyMarker:       keyMarker,
+				VersionIdMarker: versionIdMarker,
+			})
+			require.NoError(t, err)
+			pageCount++
+
+			allVersions = append(allVersions, resp.Versions...)
+			t.Logf("Page %d: got %d versions, truncated=%v", pageCount, len(resp.Versions), resp.IsTruncated != nil && *resp.IsTruncated)
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+			versionIdMarker = resp.NextVersionIdMarker
+		}
+
+		t.Logf("Listed %d versions in %d pages", len(allVersions), pageCount)
+		assert.Len(t, allVersions, totalExpectedVersions, "Should have all versions")
+
+		// Verify each object has correct number of versions
+		versionsByKey := make(map[string]int)
+		for _, v := range allVersions {
+			versionsByKey[*v.Key]++
+		}
+		assert.Len(t, versionsByKey, numObjects, "Should have %d unique objects", numObjects)
+		for key, count := range versionsByKey {
+			assert.Equal(t, versionsPerObject, count, "Object %s should have %d versions", key, versionsPerObject)
+		}
+
+		// Verify each object has exactly one latest version
+		latestByKey := make(map[string]int)
+		for _, v := range allVersions {
+			if *v.IsLatest {
+				latestByKey[*v.Key]++
+			}
+		}
+		for key, count := range latestByKey {
+			assert.Equal(t, 1, count, "Object %s should have exactly one latest version", key)
+		}
+	})
+}
+
+// listAllVersions is a helper to list all versions of a specific object using pagination
+func listAllVersions(t *testing.T, client *s3.Client, bucketName, objectKey string) []types.ObjectVersion {
+	var allVersions []types.ObjectVersion
+	var keyMarker, versionIdMarker *string
+	pageCount := 0
+
+	for {
+		resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+			Bucket:          aws.String(bucketName),
+			Prefix:          aws.String(objectKey),
+			KeyMarker:       keyMarker,
+			VersionIdMarker: versionIdMarker,
+		})
+		require.NoError(t, err)
+		pageCount++
+
+		t.Logf("Page %d: got %d versions, truncated=%v", pageCount, len(resp.Versions), resp.IsTruncated != nil && *resp.IsTruncated)
+		allVersions = append(allVersions, resp.Versions...)
+
+		// Check if truncated (handle nil pointer)
+		if resp.IsTruncated == nil || !*resp.IsTruncated {
+			break
+		}
+		keyMarker = resp.NextKeyMarker
+		versionIdMarker = resp.NextVersionIdMarker
+	}
+
+	t.Logf("Total: %d versions in %d pages", len(allVersions), pageCount)
+	return allVersions
+}
+

test/sftp/README.md

@@ -90,3 +90,4 @@ To debug test failures:
    ```
 
 
+

test/sftp/testdata/userstore.json

@@ -35,3 +35,4 @@
 ]
 
 
+

weed/admin/dash/admin_data.go

@@ -109,13 +109,6 @@ func (s *AdminServer) GetAdminData(username string) (AdminData, error) {
 		glog.Errorf("Failed to get cluster volume servers: %v", err)
 		return AdminData{}, err
 	}
-	// Sort the servers so they show up in consistent order after each reload
-	sort.Slice(volumeServersData.VolumeServers, func(i, j int) bool {
-		s1Name := volumeServersData.VolumeServers[i].GetDisplayAddress()
-		s2Name := volumeServersData.VolumeServers[j].GetDisplayAddress()
-
-		return s1Name < s2Name
-	})
 
 	// Get master nodes status
 	masterNodes := s.getMasterNodesStatus()
@@ -265,6 +258,11 @@ func (s *AdminServer) getFilerNodesStatus() []FilerNode {
 		return []FilerNode{}
 	}
 
+	// Sort filer nodes by address for consistent ordering on page refresh
+	sort.Slice(filerNodes, func(i, j int) bool {
+		return filerNodes[i].Address < filerNodes[j].Address
+	})
+
 	return filerNodes
 }
 
@@ -301,5 +299,10 @@ func (s *AdminServer) getMessageBrokerNodesStatus() []MessageBrokerNode {
 		return []MessageBrokerNode{}
 	}
 
+	// Sort message broker nodes by address for consistent ordering on page refresh
+	sort.Slice(messageBrokers, func(i, j int) bool {
+		return messageBrokers[i].Address < messageBrokers[j].Address
+	})
+
 	return messageBrokers
 }

weed/admin/dash/admin_server.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"sort"
 	"strconv"
 	"time"
 
@@ -720,6 +721,11 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 		masters = append(masters, *masterInfo)
 	}
 
+	// Sort masters by address for consistent ordering on page refresh
+	sort.Slice(masters, func(i, j int) bool {
+		return masters[i].Address < masters[j].Address
+	})
+
 	// If no masters found at all, add the current master as fallback
 	if len(masters) == 0 {
 		currentMaster := s.masterClient.GetMaster(context.Background())
@@ -776,6 +782,11 @@ func (s *AdminServer) GetClusterFilers() (*ClusterFilersData, error) {
 		return nil, fmt.Errorf("failed to get filer nodes from master: %w", err)
 	}
 
+	// Sort filers by address for consistent ordering on page refresh
+	sort.Slice(filers, func(i, j int) bool {
+		return filers[i].Address < filers[j].Address
+	})
+
 	return &ClusterFilersData{
 		Filers:      filers,
 		TotalFilers: len(filers),
@@ -818,6 +829,11 @@ func (s *AdminServer) GetClusterBrokers() (*ClusterBrokersData, error) {
 		return nil, fmt.Errorf("failed to get broker nodes from master: %w", err)
 	}
 
+	// Sort brokers by address for consistent ordering on page refresh
+	sort.Slice(brokers, func(i, j int) bool {
+		return brokers[i].Address < brokers[j].Address
+	})
+
 	return &ClusterBrokersData{
 		Brokers:      brokers,
 		TotalBrokers: len(brokers),

weed/admin/dash/volume_management.go

@@ -544,6 +544,11 @@ func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, erro
 		volumeServers = append(volumeServers, *vs)
 	}
 
+	// Sort volume servers by address for consistent ordering on page refresh
+	sort.Slice(volumeServers, func(i, j int) bool {
+		return volumeServers[i].GetDisplayAddress() < volumeServers[j].GetDisplayAddress()
+	})
+
 	var totalCapacity int64
 	var totalVolumes int
 	for _, vs := range volumeServers {

weed/admin/dash/worker_grpc_server.go

@@ -117,7 +117,7 @@ func (s *WorkerGrpcServer) Stop() error {
 	s.connMutex.Lock()
 	for _, conn := range s.connections {
 		conn.cancel()
-		close(conn.outgoing)
+		s.safeCloseOutgoingChannel(conn, "Stop")
 	}
 	s.connections = make(map[string]*WorkerConnection)
 	s.connMutex.Unlock()
@@ -182,15 +182,27 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
 	}
 	conn.capabilities = capabilities
 
-	// Register connection
+	// Register connection - clean up old connection if worker is reconnecting
 	s.connMutex.Lock()
+	if oldConn, exists := s.connections[workerID]; exists {
+		glog.Infof("Worker %s reconnected, cleaning up old connection", workerID)
+		// Cancel old connection to stop its goroutines
+		oldConn.cancel()
+		// Don't close oldConn.outgoing here as it may cause panic in handleOutgoingMessages
+		// Let the goroutine exit naturally when it detects context cancellation
+	}
 	s.connections[workerID] = conn
 	s.connMutex.Unlock()
 
 	// Register worker with maintenance manager
 	s.registerWorkerWithManager(conn)
 
-	// Send registration response
+	// IMPORTANT: Start outgoing message handler BEFORE sending registration response
+	// This ensures the handler is ready to process messages and prevents race conditions
+	// where the worker might send requests before we're ready to respond
+	go s.handleOutgoingMessages(conn)
+
+	// Send registration response (after handler is started)
 	regResponse := &worker_pb.AdminMessage{
 		Timestamp: time.Now().Unix(),
 		Message: &worker_pb.AdminMessage_RegistrationResponse{
@@ -203,13 +215,11 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
 
 	select {
 	case conn.outgoing <- regResponse:
+		glog.V(1).Infof("Registration response sent to worker %s", workerID)
 	case <-time.After(5 * time.Second):
 		glog.Errorf("Failed to send registration response to worker %s", workerID)
 	}
 
-	// Start outgoing message handler
-	go s.handleOutgoingMessages(conn)
-
 	// Handle incoming messages
 	for {
 		select {
@@ -235,7 +245,9 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
 			return err
 		}
 
+		s.connMutex.Lock()
 		conn.lastSeen = time.Now()
+		s.connMutex.Unlock()
 		s.handleWorkerMessage(conn, msg)
 	}
 }
@@ -440,16 +452,36 @@ func (s *WorkerGrpcServer) handleTaskLogResponse(conn *WorkerConnection, respons
 	s.logRequestsMutex.Unlock()
 }
 
+// safeCloseOutgoingChannel safely closes the outgoing channel for a worker connection.
+func (s *WorkerGrpcServer) safeCloseOutgoingChannel(conn *WorkerConnection, source string) {
+	defer func() {
+		if r := recover(); r != nil {
+			glog.V(1).Infof("%s: recovered from panic closing outgoing channel for worker %s: %v", source, conn.workerID, r)
+		}
+	}()
+	close(conn.outgoing)
+}
+
 // unregisterWorker removes a worker connection
 func (s *WorkerGrpcServer) unregisterWorker(workerID string) {
 	s.connMutex.Lock()
-	if conn, exists := s.connections[workerID]; exists {
-		conn.cancel()
-		close(conn.outgoing)
-		delete(s.connections, workerID)
+	conn, exists := s.connections[workerID]
+	if !exists {
+		s.connMutex.Unlock()
+		glog.V(2).Infof("unregisterWorker: worker %s not found in connections map (already unregistered)", workerID)
+		return
 	}
+
+	// Remove from map first to prevent duplicate cleanup attempts
+	delete(s.connections, workerID)
 	s.connMutex.Unlock()
 
+	// Cancel context to signal goroutines to stop
+	conn.cancel()
+
+	// Safely close the outgoing channel with recover to handle potential double-close
+	s.safeCloseOutgoingChannel(conn, "unregisterWorker")
+
 	glog.V(1).Infof("Unregistered worker %s", workerID)
 }
 
@@ -479,7 +511,7 @@ func (s *WorkerGrpcServer) cleanupStaleConnections() {
 		if conn.lastSeen.Before(cutoff) {
 			glog.Warningf("Cleaning up stale worker connection: %s", workerID)
 			conn.cancel()
-			close(conn.outgoing)
+			s.safeCloseOutgoingChannel(conn, "cleanupStaleConnections")
 			delete(s.connections, workerID)
 		}
 	}

weed/admin/maintenance/maintenance_integration.go

@@ -266,7 +266,26 @@ func (s *MaintenanceIntegration) ScanWithTaskDetectors(volumeMetrics []*types.Vo
 
 // UpdateTopologyInfo updates the volume shard tracker with topology information for empty servers
 func (s *MaintenanceIntegration) UpdateTopologyInfo(topologyInfo *master_pb.TopologyInfo) error {
-	return s.activeTopology.UpdateTopology(topologyInfo)
+	// Log topology details before update for diagnostics
+	if topologyInfo != nil {
+		dcCount, nodeCount, diskCount := topology.CountTopologyResources(topologyInfo)
+		glog.V(2).Infof("UpdateTopologyInfo: received topology with %d datacenters, %d nodes, %d disks",
+			dcCount, nodeCount, diskCount)
+	} else {
+		glog.Warningf("UpdateTopologyInfo: received nil topologyInfo")
+	}
+
+	err := s.activeTopology.UpdateTopology(topologyInfo)
+
+	if err != nil {
+		glog.Errorf("UpdateTopologyInfo: topology update failed: %v", err)
+	} else {
+		// Log success with current disk count
+		currentDiskCount := s.activeTopology.GetDiskCount()
+		glog.V(1).Infof("UpdateTopologyInfo: topology update successful, active topology now has %d disks", currentDiskCount)
+	}
+
+	return err
 }
 
 // convertToExistingFormat converts task results to existing system format using dynamic mapping

weed/admin/maintenance/maintenance_manager.go

@@ -137,6 +137,7 @@ func (mm *MaintenanceManager) Start() error {
 	// Start background processes
 	go mm.scanLoop()
 	go mm.cleanupLoop()
+	go mm.topologyStatusLoop() // Periodic diagnostic logging
 
 	glog.Infof("Maintenance manager started with scan interval %ds", mm.config.ScanIntervalSeconds)
 	return nil
@@ -255,6 +256,54 @@ func (mm *MaintenanceManager) cleanupLoop() {
 	}
 }
 
+// topologyStatusLoop periodically logs topology status for diagnostics
+func (mm *MaintenanceManager) topologyStatusLoop() {
+	// Log topology status every 5 minutes for diagnostic purposes
+	statusInterval := 5 * time.Minute
+	ticker := time.NewTicker(statusInterval)
+	defer ticker.Stop()
+
+	for mm.running {
+		select {
+		case <-mm.stopChan:
+			return
+		case <-ticker.C:
+			mm.logTopologyStatus()
+		}
+	}
+}
+
+// logTopologyStatus logs current topology and worker status for diagnostics
+func (mm *MaintenanceManager) logTopologyStatus() {
+	if mm.scanner == nil || mm.scanner.integration == nil {
+		glog.V(2).Infof("Topology status: scanner/integration not available")
+		return
+	}
+
+	activeTopology := mm.scanner.integration.GetActiveTopology()
+	if activeTopology == nil {
+		glog.V(1).Infof("Topology status: ActiveTopology is nil")
+		return
+	}
+
+	diskCount := activeTopology.GetDiskCount()
+	nodeCount := len(activeTopology.GetAllNodes())
+
+	// Get queue stats
+	stats := mm.queue.GetStats()
+	workerCount := len(mm.queue.GetWorkers())
+
+	mm.mutex.RLock()
+	errorCount := mm.errorCount
+	mm.mutex.RUnlock()
+
+	glog.V(0).Infof("Topology status: %d nodes, %d disks, %d workers, %d pending tasks, %d running tasks, errors: %d",
+		nodeCount, diskCount, workerCount,
+		stats.TasksByStatus[TaskStatusPending],
+		stats.TasksByStatus[TaskStatusInProgress]+stats.TasksByStatus[TaskStatusAssigned],
+		errorCount)
+}
+
 // performScan executes a maintenance scan with error handling and backoff
 func (mm *MaintenanceManager) performScan() {
 	defer func() {

weed/admin/topology/topology_management.go

@@ -8,11 +8,53 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 )
 
+// CountTopologyResources counts datacenters, nodes, and disks in topology info
+func CountTopologyResources(topologyInfo *master_pb.TopologyInfo) (dcCount, nodeCount, diskCount int) {
+	if topologyInfo == nil {
+		return 0, 0, 0
+	}
+	dcCount = len(topologyInfo.DataCenterInfos)
+	for _, dc := range topologyInfo.DataCenterInfos {
+		for _, rack := range dc.RackInfos {
+			nodeCount += len(rack.DataNodeInfos)
+			for _, node := range rack.DataNodeInfos {
+				diskCount += len(node.DiskInfos)
+			}
+		}
+	}
+	return
+}
+
 // UpdateTopology updates the topology information from master
 func (at *ActiveTopology) UpdateTopology(topologyInfo *master_pb.TopologyInfo) error {
 	at.mutex.Lock()
 	defer at.mutex.Unlock()
 
+	// Validate topology updates to prevent clearing disk maps with invalid data
+	if topologyInfo == nil {
+		glog.Warningf("UpdateTopology received nil topologyInfo, preserving last-known-good topology")
+		return fmt.Errorf("rejected invalid topology update: nil topologyInfo")
+	}
+
+	if len(topologyInfo.DataCenterInfos) == 0 {
+		glog.Warningf("UpdateTopology received empty DataCenterInfos, preserving last-known-good topology (had %d nodes, %d disks)",
+			len(at.nodes), len(at.disks))
+		return fmt.Errorf("rejected invalid topology update: empty DataCenterInfos (had %d nodes, %d disks)", len(at.nodes), len(at.disks))
+	}
+
+	// Count incoming topology for validation logging
+	dcCount, incomingNodes, incomingDisks := CountTopologyResources(topologyInfo)
+
+	// Reject updates that would wipe out a valid topology with an empty one (e.g. during master restart)
+	if incomingNodes == 0 && len(at.nodes) > 0 {
+		glog.Warningf("UpdateTopology received topology with 0 nodes, preserving last-known-good topology (had %d nodes, %d disks)",
+			len(at.nodes), len(at.disks))
+		return fmt.Errorf("rejected invalid topology update: 0 nodes (had %d nodes, %d disks)", len(at.nodes), len(at.disks))
+	}
+
+	glog.V(2).Infof("UpdateTopology: validating update with %d datacenters, %d nodes, %d disks (current: %d nodes, %d disks)",
+		dcCount, incomingNodes, incomingDisks, len(at.nodes), len(at.disks))
+
 	at.topologyInfo = topologyInfo
 	at.lastUpdated = time.Now()
 
@@ -142,8 +184,21 @@ func (at *ActiveTopology) GetNodeDisks(nodeID string) []*DiskInfo {
 	return disks
 }
 
+// GetDiskCount returns the total number of disks in the active topology
+func (at *ActiveTopology) GetDiskCount() int {
+	at.mutex.RLock()
+	defer at.mutex.RUnlock()
+	return len(at.disks)
+}
+
 // rebuildIndexes rebuilds the volume and EC shard indexes for O(1) lookups
 func (at *ActiveTopology) rebuildIndexes() {
+	// Nil-safety guard: return early if topology is not valid
+	if at.topologyInfo == nil || at.topologyInfo.DataCenterInfos == nil {
+		glog.V(1).Infof("rebuildIndexes: skipping rebuild due to nil topology or DataCenterInfos")
+		return
+	}
+
 	// Clear existing indexes
 	at.volumeIndex = make(map[uint32][]string)
 	at.ecShardIndex = make(map[uint32][]string)

weed/admin/view/app/cluster_ec_volumes.templ

@@ -2,330 +2,337 @@ package app
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 )
 
 templ ClusterEcVolumes(data dash.ClusterEcVolumesData) {
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <title>EC Volumes - SeaweedFS</title>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
-    <link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css" rel="stylesheet">
-</head>
-<body>
-    <div class="container-fluid">
-        <div class="row">
-            <div class="col-12">
-                <h2 class="mb-4">
-                    <i class="fas fa-database me-2"></i>EC Volumes
-                    <small class="text-muted">({fmt.Sprintf("%d", data.TotalVolumes)} volumes)</small>
-                </h2>
-            </div>
-        </div>
-
-        <!-- Statistics Cards -->
-        <div class="row mb-4">
-            <div class="col-md-3">
-                <div class="card text-bg-primary">
-                    <div class="card-body">
-                        <div class="d-flex justify-content-between">
-                            <div>
-                                <h6 class="card-title">Total Volumes</h6>
-                                <h4 class="mb-0">{fmt.Sprintf("%d", data.TotalVolumes)}</h4>
-                                <small>EC encoded volumes</small>
-                            </div>
-                            <div class="align-self-center">
-                                <i class="fas fa-cubes fa-2x"></i>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            <div class="col-md-3">
-                <div class="card text-bg-info">
-                    <div class="card-body">
-                        <div class="d-flex justify-content-between">
-                            <div>
-                                <h6 class="card-title">Total Shards</h6>
-                                <h4 class="mb-0">{fmt.Sprintf("%d", data.TotalShards)}</h4>
-                                <small>Distributed shards</small>
-                            </div>
-                            <div class="align-self-center">
-                                <i class="fas fa-puzzle-piece fa-2x"></i>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            <div class="col-md-3">
-                <div class="card text-bg-success">
-                    <div class="card-body">
-                        <div class="d-flex justify-content-between">
-                            <div>
-                                <h6 class="card-title">Complete Volumes</h6>
-                                <h4 class="mb-0">{fmt.Sprintf("%d", data.CompleteVolumes)}</h4>
-                                <small>All shards present</small>
-                            </div>
-                            <div class="align-self-center">
-                                <i class="fas fa-check-circle fa-2x"></i>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            <div class="col-md-3">
-                <div class="card text-bg-warning">
-                    <div class="card-body">
-                        <div class="d-flex justify-content-between">
-                            <div>
-                                <h6 class="card-title">Incomplete Volumes</h6>
-                                <h4 class="mb-0">{fmt.Sprintf("%d", data.IncompleteVolumes)}</h4>
-                                <small>Missing shards</small>
-                            </div>
-                            <div class="align-self-center">
-                                <i class="fas fa-exclamation-triangle fa-2x"></i>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-        </div>
-
-        <!-- EC Storage Information Note -->
-        <div class="alert alert-info mb-4" role="alert">
-            <i class="fas fa-info-circle me-2"></i>
-            <strong>EC Storage Note:</strong> 
-            EC volumes use erasure coding ({ fmt.Sprintf("%d+%d", erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount) }) which stores data across { fmt.Sprintf("%d", erasure_coding.TotalShardsCount) } shards with redundancy. 
-            Physical storage is approximately { fmt.Sprintf("%.1fx", float64(erasure_coding.TotalShardsCount)/float64(erasure_coding.DataShardsCount)) } the original logical data size due to { fmt.Sprintf("%d", erasure_coding.ParityShardsCount) } parity shards.
-        </div>
-
-        <!-- Volumes Table -->
-        <div class="d-flex justify-content-between align-items-center mb-3">
-            <div class="d-flex align-items-center">
-                <span class="me-3">
-                    Showing {fmt.Sprintf("%d", (data.Page-1)*data.PageSize + 1)} to {fmt.Sprintf("%d", func() int {
-                        end := data.Page * data.PageSize
-                        if end > data.TotalVolumes {
-                            return data.TotalVolumes
-                        }
-                        return end
-                    }())} of {fmt.Sprintf("%d", data.TotalVolumes)} volumes
-                </span>
-                
-                <div class="d-flex align-items-center">
-                    <label for="pageSize" class="form-label me-2 mb-0">Show:</label>
-                    <select id="pageSize" class="form-select form-select-sm" style="width: auto;" onchange="changePageSize(this.value)">
-                        <option value="5" if data.PageSize == 5 { selected }>5</option>
-                        <option value="10" if data.PageSize == 10 { selected }>10</option>
-                        <option value="25" if data.PageSize == 25 { selected }>25</option>
-                        <option value="50" if data.PageSize == 50 { selected }>50</option>
-                        <option value="100" if data.PageSize == 100 { selected }>100</option>
-                    </select>
-                    <span class="ms-2">per page</span>
-                </div>
-            </div>
-            
+    <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
+        <div>
+            <h1 class="h2">
+                <i class="fas fa-database me-2"></i>EC Volumes
+            </h1>
             if data.Collection != "" {
-                <div>
+                <div class="d-flex align-items-center mt-2">
                     if data.Collection == "default" {
-                        <span class="badge bg-secondary text-white">Collection: default</span>
+                        <span class="badge bg-secondary text-white me-2">
+                            <i class="fas fa-filter me-1"></i>Collection: default
+                        </span>
                     } else {
-                        <span class="badge bg-info text-white">Collection: {data.Collection}</span>
+                        <span class="badge bg-info text-white me-2">
+                            <i class="fas fa-filter me-1"></i>Collection: {data.Collection}
+                        </span>
                     }
-                    <a href="/cluster/ec-shards" class="btn btn-sm btn-outline-secondary ms-2">Clear Filter</a>
+                    <a href="/cluster/ec-volumes" class="btn btn-sm btn-outline-secondary">
+                        <i class="fas fa-times me-1"></i>Clear Filter
+                    </a>
                 </div>
             }
         </div>
-
-        <div class="table-responsive">
-            <table class="table table-striped table-hover" id="ecVolumesTable">
-                <thead>
-                    <tr>
-                        <th>
-                            <a href="#" onclick="sortBy('volume_id')" class="text-dark text-decoration-none">
-                                Volume ID
-                                if data.SortBy == "volume_id" {
-                                    if data.SortOrder == "asc" {
-                                        <i class="fas fa-sort-up ms-1"></i>
-                                    } else {
-                                        <i class="fas fa-sort-down ms-1"></i>
-                                    }
-                                } else {
-                                    <i class="fas fa-sort ms-1 text-muted"></i>
-                                }
-                            </a>
-                        </th>
-                        if data.ShowCollectionColumn {
-                            <th>
-                                <a href="#" onclick="sortBy('collection')" class="text-dark text-decoration-none">
-                                    Collection
-                                    if data.SortBy == "collection" {
-                                        if data.SortOrder == "asc" {
-                                            <i class="fas fa-sort-up ms-1"></i>
-                                        } else {
-                                            <i class="fas fa-sort-down ms-1"></i>
-                                        }
-                                    } else {
-                                        <i class="fas fa-sort ms-1 text-muted"></i>
-                                    }
-                                </a>
-                            </th>
-                        }
-                        <th>
-                            <a href="#" onclick="sortBy('total_shards')" class="text-dark text-decoration-none">
-                                Shard Count
-                                if data.SortBy == "total_shards" {
-                                    if data.SortOrder == "asc" {
-                                        <i class="fas fa-sort-up ms-1"></i>
-                                    } else {
-                                        <i class="fas fa-sort-down ms-1"></i>
-                                    }
-                                } else {
-                                    <i class="fas fa-sort ms-1 text-muted"></i>
-                                }
-                            </a>
-                        </th>
-                        <th class="text-dark">Shard Size</th>
-                        <th class="text-dark">Shard Locations</th>
-                        <th>
-                            <a href="#" onclick="sortBy('completeness')" class="text-dark text-decoration-none">
-                                Status
-                                if data.SortBy == "completeness" {
-                                    if data.SortOrder == "asc" {
-                                        <i class="fas fa-sort-up ms-1"></i>
-                                    } else {
-                                        <i class="fas fa-sort-down ms-1"></i>
-                                    }
-                                } else {
-                                    <i class="fas fa-sort ms-1 text-muted"></i>
-                                }
-                            </a>
-                        </th>
-                        if data.ShowDataCenterColumn {
-                            <th class="text-dark">Data Centers</th>
-                        }
-                        <th class="text-dark">Actions</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    for _, volume := range data.EcVolumes {
-                        <tr>
-                            <td>
-                                <strong>{fmt.Sprintf("%d", volume.VolumeID)}</strong>
-                            </td>
-                            if data.ShowCollectionColumn {
-                                <td>
-                                    if volume.Collection != "" {
-                                        <a href="/cluster/ec-shards?collection={volume.Collection}" class="text-decoration-none">
-                                            <span class="badge bg-info text-white">{volume.Collection}</span>
-                                        </a>
-                                    } else {
-                                        <a href="/cluster/ec-shards?collection=default" class="text-decoration-none">
-                                            <span class="badge bg-secondary text-white">default</span>
-                                        </a>
-                                    }
-                                </td>
-                            }
-                            <td>
-                                <span class="badge bg-primary">{fmt.Sprintf("%d/14", volume.TotalShards)}</span>
-                            </td>
-                            <td>
-                                @displayShardSizes(volume.ShardSizes)
-                            </td>
-                            <td>
-                                @displayVolumeDistribution(volume)
-                            </td>
-                            <td>
-                                @displayEcVolumeStatus(volume)
-                            </td>
-                            if data.ShowDataCenterColumn {
-                                <td>
-                                    for i, dc := range volume.DataCenters {
-                                        if i > 0 {
-                                            <span>, </span>
-                                        }
-                                        <span class="badge bg-primary text-white">{dc}</span>
-                                    }
-                                </td>
-                            }
-                            <td>
-                                <div class="btn-group" role="group">
-                                    <button type="button" class="btn btn-sm btn-outline-primary" 
-                                            onclick="showVolumeDetails(event)" 
-                                            data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
-                                            title="View EC volume details">
-                                        <i class="fas fa-info-circle"></i>
-                                    </button>
-                                    if !volume.IsComplete {
-                                        <button type="button" class="btn btn-sm btn-outline-warning" 
-                                                onclick="repairVolume(event)" 
-                                                data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
-                                                title="Repair missing shards">
-                                            <i class="fas fa-wrench"></i>
-                                        </button>
-                                    }
-                                </div>
-                            </td>
-                        </tr>
-                    }
-                </tbody>
-            </table>
+        <div class="btn-toolbar mb-2 mb-md-0">
+            <div class="btn-group me-2">
+                <select class="form-select form-select-sm me-2" id="pageSizeSelect" onchange="changePageSize(this.value)" style="width: auto;">
+                    <option value="5" if data.PageSize == 5 { selected="selected" }>5 per page</option>
+                    <option value="10" if data.PageSize == 10 { selected="selected" }>10 per page</option>
+                    <option value="25" if data.PageSize == 25 { selected="selected" }>25 per page</option>
+                    <option value="50" if data.PageSize == 50 { selected="selected" }>50 per page</option>
+                    <option value="100" if data.PageSize == 100 { selected="selected" }>100 per page</option>
+                </select>
+                <button type="button" class="btn btn-sm btn-outline-primary" onclick="window.location.reload()">
+                    <i class="fas fa-refresh me-1"></i>Refresh
+                </button>
+            </div>
         </div>
-
-        <!-- Pagination -->
-        if data.TotalPages > 1 {
-            <nav aria-label="EC Volumes pagination">
-                <ul class="pagination justify-content-center">
-                    if data.Page > 1 {
-                        <li class="page-item">
-                            <a class="page-link" href="#" onclick="goToPage(event)" data-page="1">First</a>
-                        </li>
-                        <li class="page-item">
-                            <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page-1) }>Previous</a>
-                        </li>
-                    }
-                    
-                    for i := 1; i <= data.TotalPages; i++ {
-                        if i == data.Page {
-                            <li class="page-item active">
-                                <span class="page-link">{fmt.Sprintf("%d", i)}</span>
-                            </li>
-                        } else if i <= 3 || i > data.TotalPages-3 || (i >= data.Page-2 && i <= data.Page+2) {
-                            <li class="page-item">
-                                <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", i) }>{fmt.Sprintf("%d", i)}</a>
-                            </li>
-                        } else if i == 4 && data.Page > 6 {
-                            <li class="page-item disabled">
-                                <span class="page-link">...</span>
-                            </li>
-                        } else if i == data.TotalPages-3 && data.Page < data.TotalPages-5 {
-                            <li class="page-item disabled">
-                                <span class="page-link">...</span>
-                            </li>
-                        }
-                    }
-                    
-                    if data.Page < data.TotalPages {
-                        <li class="page-item">
-                            <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page+1) }>Next</a>
-                        </li>
-                        <li class="page-item">
-                            <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.TotalPages) }>Last</a>
-                        </li>
-                    }
-                </ul>
-            </nav>
-        }
     </div>
 
-    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
+    <!-- Statistics Cards -->
+    <div class="row mb-4">
+        <div class="col-md-3">
+            <div class="card text-bg-primary">
+                <div class="card-body">
+                    <div class="d-flex justify-content-between">
+                        <div>
+                            <h6 class="card-title">Total Volumes</h6>
+                            <h4 class="mb-0">{fmt.Sprintf("%d", data.TotalVolumes)}</h4>
+                        </div>
+                        <div class="align-self-center">
+                            <i class="fas fa-cubes fa-2x"></i>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="col-md-3">
+            <div class="card text-bg-info">
+                <div class="card-body">
+                    <div class="d-flex justify-content-between">
+                        <div>
+                            <h6 class="card-title">Total Shards</h6>
+                            <h4 class="mb-0">{fmt.Sprintf("%d", data.TotalShards)}</h4>
+                        </div>
+                        <div class="align-self-center">
+                            <i class="fas fa-puzzle-piece fa-2x"></i>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="col-md-3">
+            <div class="card text-bg-success">
+                <div class="card-body">
+                    <div class="d-flex justify-content-between">
+                        <div>
+                            <h6 class="card-title">Healthy Volumes</h6>
+                            <h4 class="mb-0">{fmt.Sprintf("%d", data.CompleteVolumes)}</h4>
+                            <small>All { fmt.Sprintf("%d", erasure_coding.TotalShardsCount) } shards present</small>
+                        </div>
+                        <div class="align-self-center">
+                            <i class="fas fa-check-circle fa-2x"></i>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="col-md-3">
+            <div class="card text-bg-warning">
+                <div class="card-body">
+                    <div class="d-flex justify-content-between">
+                        <div>
+                            <h6 class="card-title">Degraded Volumes</h6>
+                            <h4 class="mb-0">{fmt.Sprintf("%d", data.IncompleteVolumes)}</h4>
+                            <small>Incomplete/Critical</small>
+                        </div>
+                        <div class="align-self-center">
+                            <i class="fas fa-exclamation-triangle fa-2x"></i>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <!-- EC Storage Information Note -->
+    <div class="alert alert-info mb-4" role="alert">
+        <i class="fas fa-info-circle me-2"></i>
+        <strong>EC Storage Note:</strong> 
+        EC volumes use erasure coding ({ fmt.Sprintf("%d+%d", erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount) }) which stores data across { fmt.Sprintf("%d", erasure_coding.TotalShardsCount) } shards with redundancy. 
+        Physical storage is approximately { fmt.Sprintf("%.1fx", float64(erasure_coding.TotalShardsCount)/float64(erasure_coding.DataShardsCount)) } the original logical data size due to { fmt.Sprintf("%d", erasure_coding.ParityShardsCount) } parity shards.
+    </div>
+
+    <!-- Volumes Table -->
+    <div class="table-responsive">
+        <table class="table table-striped table-hover" id="ecVolumesTable">
+            <thead>
+                <tr>
+                    <th>
+                        <a href="#" onclick="sortBy('volume_id')" class="text-dark text-decoration-none">
+                            Volume ID
+                            if data.SortBy == "volume_id" {
+                                if data.SortOrder == "asc" {
+                                    <i class="fas fa-sort-up ms-1"></i>
+                                } else {
+                                    <i class="fas fa-sort-down ms-1"></i>
+                                }
+                            } else {
+                                <i class="fas fa-sort ms-1 text-muted"></i>
+                            }
+                        </a>
+                    </th>
+                    if data.ShowCollectionColumn {
+                        <th>
+                            <a href="#" onclick="sortBy('collection')" class="text-dark text-decoration-none">
+                                Collection
+                                if data.SortBy == "collection" {
+                                    if data.SortOrder == "asc" {
+                                        <i class="fas fa-sort-up ms-1"></i>
+                                    } else {
+                                        <i class="fas fa-sort-down ms-1"></i>
+                                    }
+                                } else {
+                                    <i class="fas fa-sort ms-1 text-muted"></i>
+                                }
+                            </a>
+                        </th>
+                    }
+                    <th>
+                        <a href="#" onclick="sortBy('total_shards')" class="text-dark text-decoration-none">
+                            Shard Count
+                            if data.SortBy == "total_shards" {
+                                if data.SortOrder == "asc" {
+                                    <i class="fas fa-sort-up ms-1"></i>
+                                } else {
+                                    <i class="fas fa-sort-down ms-1"></i>
+                                }
+                            } else {
+                                <i class="fas fa-sort ms-1 text-muted"></i>
+                            }
+                        </a>
+                    </th>
+                    <th class="text-dark">Shard Size</th>
+                    <th class="text-dark">Shard Locations</th>
+                    <th>
+                        <a href="#" onclick="sortBy('completeness')" class="text-dark text-decoration-none">
+                            Status
+                            if data.SortBy == "completeness" {
+                                if data.SortOrder == "asc" {
+                                    <i class="fas fa-sort-up ms-1"></i>
+                                } else {
+                                    <i class="fas fa-sort-down ms-1"></i>
+                                }
+                            } else {
+                                <i class="fas fa-sort ms-1 text-muted"></i>
+                            }
+                        </a>
+                    </th>
+                    if data.ShowDataCenterColumn {
+                        <th class="text-dark">Data Centers</th>
+                    }
+                    <th class="text-dark">Actions</th>
+                </tr>
+            </thead>
+            <tbody>
+                for _, volume := range data.EcVolumes {
+                    <tr>
+                        <td>
+                            <span class="fw-bold">{fmt.Sprintf("%d", volume.VolumeID)}</span>
+                        </td>
+                        if data.ShowCollectionColumn {
+                            <td>
+                                if volume.Collection != "" {
+                                    <a href={ templ.URL(fmt.Sprintf("/cluster/ec-shards?collection=%s", volume.Collection)) } class="text-decoration-none">
+                                        <span class="badge bg-info text-white">{volume.Collection}</span>
+                                    </a>
+                                } else {
+                                    <a href={ templ.URL("/cluster/ec-shards?collection=default") } class="text-decoration-none">
+                                        <span class="badge bg-secondary text-white">default</span>
+                                    </a>
+                                }
+                            </td>
+                        }
+                        <td>
+                            <span class="badge bg-primary">{fmt.Sprintf("%d/%d", volume.TotalShards, erasure_coding.TotalShardsCount)}</span>
+                        </td>
+                        <td>
+                            @displayShardSizes(volume.ShardSizes)
+                        </td>
+                        <td>
+                            @displayVolumeDistribution(volume)
+                        </td>
+                        <td>
+                            @displayEcVolumeStatus(volume)
+                        </td>
+                        if data.ShowDataCenterColumn {
+                            <td>
+                                for i, dc := range volume.DataCenters {
+                                    if i > 0 {
+                                        <span>, </span>
+                                    }
+                                    <span class="badge bg-primary text-white">{dc}</span>
+                                }
+                            </td>
+                        }
+                        <td>
+                            <div class="btn-group" role="group">
+                                <button type="button" class="btn btn-sm btn-outline-primary" 
+                                        onclick="showVolumeDetails(event)" 
+                                        data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
+                                        title="View EC volume details">
+                                    <i class="fas fa-info-circle"></i>
+                                </button>
+                                if !volume.IsComplete {
+                                    <button type="button" class="btn btn-sm btn-outline-warning" 
+                                            onclick="repairVolume(event)" 
+                                            data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
+                                            title="Repair missing shards">
+                                        <i class="fas fa-wrench"></i>
+                                    </button>
+                                }
+                            </div>
+                        </td>
+                    </tr>
+                }
+            </tbody>
+        </table>
+    </div>
+
+    <!-- Pagination -->
+    if data.TotalPages > 1 {
+        <nav aria-label="EC Volumes pagination">
+            <ul class="pagination justify-content-center">
+                if data.Page > 1 {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page="1">
+                            <i class="fas fa-angle-double-left"></i>
+                        </a>
+                    </li>
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page-1) }>
+                            <i class="fas fa-chevron-left"></i>
+                        </a>
+                    </li>
+                }
+                
+                <!-- First page -->
+                if data.Page > 3 {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page="1">1</a>
+                    </li>
+                    if data.Page > 4 {
+                        <li class="page-item disabled">
+                            <span class="page-link">...</span>
+                        </li>
+                    }
+                }
+                
+                <!-- Current page neighbors -->
+                if data.Page > 1 {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page-1) }>{fmt.Sprintf("%d", data.Page-1)}</a>
+                    </li>
+                }
+                
+                <li class="page-item active">
+                    <span class="page-link">{fmt.Sprintf("%d", data.Page)}</span>
+                </li>
+                
+                if data.Page < data.TotalPages {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page+1) }>{fmt.Sprintf("%d", data.Page+1)}</a>
+                    </li>
+                }
+                
+                <!-- Last page -->
+                if data.Page < data.TotalPages-2 {
+                    if data.Page < data.TotalPages-3 {
+                        <li class="page-item disabled">
+                            <span class="page-link">...</span>
+                        </li>
+                    }
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.TotalPages) }>{fmt.Sprintf("%d", data.TotalPages)}</a>
+                    </li>
+                }
+                
+                if data.Page < data.TotalPages {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page+1) }>
+                            <i class="fas fa-chevron-right"></i>
+                        </a>
+                    </li>
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.TotalPages) }>
+                            <i class="fas fa-angle-double-right"></i>
+                        </a>
+                    </li>
+                }
+            </ul>
+        </nav>
+    }
+
+    <!-- JavaScript -->
     <script>
-        // Sorting functionality
         function sortBy(field) {
             const currentSort = new URLSearchParams(window.location.search).get('sort_by');
             const currentOrder = new URLSearchParams(window.location.search).get('sort_order') || 'asc';
@@ -335,65 +342,69 @@ templ ClusterEcVolumes(data dash.ClusterEcVolumesData) {
                 newOrder = 'desc';
             }
             
-            const url = new URL(window.location);
-            url.searchParams.set('sort_by', field);
-            url.searchParams.set('sort_order', newOrder);
-            url.searchParams.set('page', '1'); // Reset to first page
-            window.location.href = url.toString();
+            updateUrl({
+                sort_by: field,
+                sort_order: newOrder,
+                page: 1
+            });
         }
 
-        // Pagination functionality
         function goToPage(event) {
             event.preventDefault();
             const page = event.target.closest('a').getAttribute('data-page');
-            const url = new URL(window.location);
-            url.searchParams.set('page', page);
-            window.location.href = url.toString();
+            updateUrl({ page: page });
         }
 
-        // Page size functionality
         function changePageSize(newPageSize) {
+            updateUrl({ page_size: newPageSize, page: 1 });
+        }
+
+        function updateUrl(params) {
             const url = new URL(window.location);
-            url.searchParams.set('page_size', newPageSize);
-            url.searchParams.set('page', '1'); // Reset to first page when changing page size
+            Object.keys(params).forEach(key => {
+                if (params[key] != null) {
+                    url.searchParams.set(key, params[key]);
+                } else {
+                    url.searchParams.delete(key);
+                }
+            });
             window.location.href = url.toString();
         }
 
-        // Volume details
         function showVolumeDetails(event) {
             const volumeId = event.target.closest('button').getAttribute('data-volume-id');
             window.location.href = `/cluster/ec-volumes/${volumeId}`;
         }
 
-        // Repair volume
         function repairVolume(event) {
             const volumeId = event.target.closest('button').getAttribute('data-volume-id');
             if (confirm(`Are you sure you want to repair missing shards for volume ${volumeId}?`)) {
-                // TODO: Implement repair functionality
-                alert('Repair functionality will be implemented soon.');
+                fetch(`/api/cluster/ec-volumes/${volumeId}/repair`, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                    }
+                })
+                .then(response => {
+                    if (!response.ok) {
+                        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                    }
+                    return response.json();
+                })
+                .then(data => {
+                    if (data && data.success) {
+                        alert('Repair initiated successfully');
+                        location.reload();
+                    } else {
+                        alert('Failed to initiate repair: ' + (data && data.error ? data.error : 'Unknown error'));
+                    }
+                })
+                .catch(error => {
+                    alert('Error: ' + error.message);
+                });
             }
         }
     </script>
-</body>
-</html>
-}
-
-// displayShardLocationsHTML renders shard locations as proper HTML
-templ displayShardLocationsHTML(shardLocations map[int]string) {
-	if len(shardLocations) == 0 {
-		<span class="text-muted">No shards</span>
-	} else {
-		for i, serverInfo := range groupShardsByServer(shardLocations) {
-			if i > 0 {
-				<br/>
-			}
-			<strong>
-				<a href={ templ.URL("/cluster/volume-servers/" + serverInfo.Server) } class="text-primary text-decoration-none">
-					{ serverInfo.Server }
-				</a>:
-			</strong> { serverInfo.ShardRanges }
-		}
-	}
 }
 
 // displayShardSizes renders shard sizes in a compact format
@@ -439,13 +450,7 @@ func groupShardsByServer(shardLocations map[int]string) []ServerShardInfo {
 	var serverInfos []ServerShardInfo
 	for server, shards := range serverShards {
 		// Sort shards for each server
-		for i := 0; i < len(shards); i++ {
-			for j := i + 1; j < len(shards); j++ {
-				if shards[i] > shards[j] {
-					shards[i], shards[j] = shards[j], shards[i]
-				}
-			}
-		}
+		sort.Ints(shards)
 		
 		// Format shard ranges compactly
 		shardRanges := formatShardRanges(shards)
@@ -456,56 +461,9 @@ func groupShardsByServer(shardLocations map[int]string) []ServerShardInfo {
 	}
 
 	// Sort by server name
-	for i := 0; i < len(serverInfos); i++ {
-		for j := i + 1; j < len(serverInfos); j++ {
-			if serverInfos[i].Server > serverInfos[j].Server {
-				serverInfos[i], serverInfos[j] = serverInfos[j], serverInfos[i]
-			}
-		}
-	}
-
-	return serverInfos
-}
-
-// groupShardsByServerWithSizes groups shards by server and formats ranges with sizes
-func groupShardsByServerWithSizes(shardLocations map[int]string, shardSizes map[int]int64) []ServerShardInfo {
-	if len(shardLocations) == 0 {
-		return []ServerShardInfo{}
-	}
-
-	// Group shards by server
-	serverShards := make(map[string][]int)
-	for shardId, server := range shardLocations {
-		serverShards[server] = append(serverShards[server], shardId)
-	}
-
-	var serverInfos []ServerShardInfo
-	for server, shards := range serverShards {
-		// Sort shards for each server
-		for i := 0; i < len(shards); i++ {
-			for j := i + 1; j < len(shards); j++ {
-				if shards[i] > shards[j] {
-					shards[i], shards[j] = shards[j], shards[i]
-				}
-			}
-		}
-		
-		// Format shard ranges compactly with sizes
-		shardRanges := formatShardRangesWithSizes(shards, shardSizes)
-		serverInfos = append(serverInfos, ServerShardInfo{
-			Server:      server,
-			ShardRanges: shardRanges,
-		})
-	}
-
-	// Sort by server name
-	for i := 0; i < len(serverInfos); i++ {
-		for j := i + 1; j < len(serverInfos); j++ {
-			if serverInfos[i].Server > serverInfos[j].Server {
-				serverInfos[i], serverInfos[j] = serverInfos[j], serverInfos[i]
-			}
-		}
-	}
+	sort.Slice(serverInfos, func(i, j int) bool {
+		return serverInfos[i].Server < serverInfos[j].Server
+	})
 
 	return serverInfos
 }
@@ -544,72 +502,6 @@ func formatShardRanges(shards []int) string {
 	return strings.Join(ranges, ",")
 }
 
-// Helper function to format shard ranges with sizes (e.g., "0(1.2MB),1-3(2.5MB),7(800KB)")
-func formatShardRangesWithSizes(shards []int, shardSizes map[int]int64) string {
-	if len(shards) == 0 {
-		return ""
-	}
-
-	var ranges []string
-	start := shards[0]
-	end := shards[0]
-	var totalSize int64
-
-	for i := 1; i < len(shards); i++ {
-		if shards[i] == end+1 {
-			end = shards[i]
-			totalSize += shardSizes[shards[i]]
-		} else {
-			// Add current range with size
-			if start == end {
-				size := shardSizes[start]
-				if size > 0 {
-					ranges = append(ranges, fmt.Sprintf("%d(%s)", start, bytesToHumanReadable(size)))
-				} else {
-					ranges = append(ranges, fmt.Sprintf("%d", start))
-				}
-			} else {
-				// Calculate total size for the range
-				rangeSize := shardSizes[start]
-				for j := start + 1; j <= end; j++ {
-					rangeSize += shardSizes[j]
-				}
-				if rangeSize > 0 {
-					ranges = append(ranges, fmt.Sprintf("%d-%d(%s)", start, end, bytesToHumanReadable(rangeSize)))
-				} else {
-					ranges = append(ranges, fmt.Sprintf("%d-%d", start, end))
-				}
-			}
-			start = shards[i]
-			end = shards[i]
-			totalSize = shardSizes[shards[i]]
-		}
-	}
-
-	// Add the last range
-	if start == end {
-		size := shardSizes[start]
-		if size > 0 {
-			ranges = append(ranges, fmt.Sprintf("%d(%s)", start, bytesToHumanReadable(size)))
-		} else {
-			ranges = append(ranges, fmt.Sprintf("%d", start))
-		}
-	} else {
-		// Calculate total size for the range
-		rangeSize := shardSizes[start]
-		for j := start + 1; j <= end; j++ {
-			rangeSize += shardSizes[j]
-		}
-		if rangeSize > 0 {
-			ranges = append(ranges, fmt.Sprintf("%d-%d(%s)", start, end, bytesToHumanReadable(rangeSize)))
-		} else {
-			ranges = append(ranges, fmt.Sprintf("%d-%d", start, end))
-		}
-	}
-
-	return strings.Join(ranges, ",")
-}
-
 // Helper function to convert bytes to human readable format
 func bytesToHumanReadable(bytes int64) string {
 	const unit = 1024
@@ -680,13 +572,7 @@ func formatIndividualShardSizes(shardSizes map[int]int64) string {
 		var groupStrs []string
 		for size, shardIds := range sizeGroups {
 			// Sort shard IDs
-			for i := 0; i < len(shardIds); i++ {
-				for j := i + 1; j < len(shardIds); j++ {
-					if shardIds[i] > shardIds[j] {
-						shardIds[i], shardIds[j] = shardIds[j], shardIds[i]
-					}
-				}
-			}
+			sort.Ints(shardIds)
 			
 			var idRanges []string
 			if len(shardIds) <= erasure_coding.ParityShardsCount {
@@ -715,18 +601,27 @@ templ displayVolumeDistribution(volume dash.EcVolumeWithShards) {
     </div>
 }
 
-// displayEcVolumeStatus shows an improved status display for EC volumes
+// displayEcVolumeStatus shows an improved status display for EC volumes.
+// Status thresholds are based on EC recovery capability:
+// - Critical: More than DataShardsCount missing (data is unrecoverable)
+// - Degraded: More than half of DataShardsCount missing (high risk)
+// - Incomplete: More than half of ParityShardsCount missing (reduced redundancy)
+// - Minor Issues: Few shards missing (still fully recoverable)
 templ displayEcVolumeStatus(volume dash.EcVolumeWithShards) {
     if volume.IsComplete {
         <span class="badge bg-success"><i class="fas fa-check me-1"></i>Complete</span>
     } else {
         if len(volume.MissingShards) > erasure_coding.DataShardsCount {
+            // Unrecoverable: more shards missing than EC can reconstruct
             <span class="badge bg-danger"><i class="fas fa-skull me-1"></i>Critical ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
         } else if len(volume.MissingShards) > (erasure_coding.DataShardsCount/2) {
+            // High risk: approaching unrecoverable state
             <span class="badge bg-warning"><i class="fas fa-exclamation-triangle me-1"></i>Degraded ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
         } else if len(volume.MissingShards) > (erasure_coding.ParityShardsCount/2) {
+            // Reduced redundancy but still recoverable
             <span class="badge bg-warning"><i class="fas fa-info-circle me-1"></i>Incomplete ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
         } else {
+            // Minor: few shards missing, fully recoverable with good margin
             <span class="badge bg-info"><i class="fas fa-info-circle me-1"></i>Minor Issues ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
         }
     }

weed/admin/view/app/s3_buckets.templ

@@ -127,7 +127,7 @@ templ S3Buckets(data dash.S3BucketsData) {
                                     for _, bucket := range data.Buckets {
                                         <tr>
                                             <td>
-                                                <a href={templ.SafeURL(fmt.Sprintf("/s3/buckets/%s", bucket.Name))} 
+                                                <a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))} 
                                                    class="text-decoration-none">
                                                     <i class="fas fa-cube me-2"></i>
                                                     {bucket.Name}

weed/admin/view/app/s3_buckets_templ.go

@@ -83,9 +83,9 @@ func S3Buckets(data dash.S3BucketsData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var5 templ.SafeURL
-			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/s3/buckets/%s", bucket.Name)))
+			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 130, Col: 114}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 130, Col: 123}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 			if templ_7745c5c3_Err != nil {

weed/cluster/lock_client.go

@@ -3,6 +3,8 @@ package cluster
 import (
 	"context"
 	"fmt"
+	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
@@ -36,10 +38,11 @@ type LiveLock struct {
 	hostFiler      pb.ServerAddress
 	cancelCh       chan struct{}
 	grpcDialOption grpc.DialOption
-	isLocked       bool
+	isLocked       int32 // 0 = unlocked, 1 = locked; use atomic operations
 	self           string
 	lc             *LockClient
 	owner          string
+	renewInterval  time.Duration
 }
 
 // NewShortLivedLock creates a lock with a 5-second duration
@@ -58,7 +61,11 @@ func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLoc
 }
 
 // StartLongLivedLock starts a goroutine to lock the key and returns immediately.
-func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerChange func(newLockOwner string)) (lock *LiveLock) {
+func (lc *LockClient) StartLongLivedLock(key string, owner string, renewInterval time.Duration, onLockOwnerChange func(newLockOwner string)) (lock *LiveLock) {
+	// Default to lock_manager.RenewInterval if zero duration is passed
+	if renewInterval == 0 {
+		renewInterval = lock_manager.RenewInterval
+	}
 	lock = &LiveLock{
 		key:            key,
 		hostFiler:      lc.seedFiler,
@@ -67,6 +74,7 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
 		grpcDialOption: lc.grpcDialOption,
 		self:           owner,
 		lc:             lc,
+		renewInterval:  renewInterval,
 	}
 	go func() {
 		isLocked := false
@@ -84,10 +92,12 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
 				if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err != nil {
 					glog.V(0).Infof("Lost lock %s: %v", key, err)
 					isLocked = false
+					atomic.StoreInt32(&lock.isLocked, 0)
 				}
 			} else {
 				if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err == nil {
 					isLocked = true
+					// Note: AttemptToLock already sets lock.isLocked atomically on success
 				}
 			}
 			if lockOwner != lock.LockOwner() && lock.LockOwner() != "" {
@@ -99,7 +109,11 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
 			case <-lock.cancelCh:
 				return
 			default:
-				time.Sleep(lock_manager.RenewInterval)
+				if isLocked {
+					time.Sleep(5 * lock.renewInterval)
+				} else {
+					time.Sleep(lock.renewInterval)
+				}
 			}
 		}
 	}()
@@ -126,24 +140,28 @@ func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {
 		return err
 	}
 	if errorMessage != "" {
-		glog.V(1).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
+		if strings.Contains(errorMessage, "lock already owned") {
+			glog.V(3).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
+		} else {
+			glog.V(2).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
+		}
 		time.Sleep(time.Second)
 		return fmt.Errorf("%v", errorMessage)
 	}
-	if !lock.isLocked {
+	if atomic.LoadInt32(&lock.isLocked) == 0 {
 		// Only log when transitioning from unlocked to locked
 		glog.V(1).Infof("LOCK: Successfully acquired key=%s owner=%s", lock.key, lock.self)
 	}
-	lock.isLocked = true
+	atomic.StoreInt32(&lock.isLocked, 1)
 	return nil
 }
 
 func (lock *LiveLock) StopShortLivedLock() error {
-	if !lock.isLocked {
+	if atomic.LoadInt32(&lock.isLocked) == 0 {
 		return nil
 	}
 	defer func() {
-		lock.isLocked = false
+		atomic.StoreInt32(&lock.isLocked, 0)
 	}()
 	return pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		_, err := client.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
@@ -203,7 +221,7 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 			errorMessage = resp.Error
 			if resp.LockHostMovedTo != "" && resp.LockHostMovedTo != string(previousHostFiler) {
 				// Only log if the host actually changed
-				glog.V(1).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
+				glog.V(2).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
 				lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
 				lock.lc.seedFiler = lock.hostFiler
 			} else if resp.LockHostMovedTo != "" {
@@ -211,12 +229,12 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 			}
 			if resp.LockOwner != "" && resp.LockOwner != previousOwner {
 				// Only log if the owner actually changed
-				glog.V(1).Infof("LOCK: Owner changed from %s to %s for key=%s", previousOwner, resp.LockOwner, lock.key)
+				glog.V(2).Infof("LOCK: Owner changed from %s to %s for key=%s", previousOwner, resp.LockOwner, lock.key)
 				lock.owner = resp.LockOwner
 			} else if resp.LockOwner != "" {
 				lock.owner = resp.LockOwner
 			} else if previousOwner != "" {
-				glog.V(1).Infof("LOCK: Owner cleared for key=%s", lock.key)
+				glog.V(2).Infof("LOCK: Owner cleared for key=%s", lock.key)
 				lock.owner = ""
 			}
 		}
@@ -228,3 +246,8 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 func (lock *LiveLock) LockOwner() string {
 	return lock.owner
 }
+
+// IsLocked returns true if this instance currently holds the lock
+func (lock *LiveLock) IsLocked() bool {
+	return atomic.LoadInt32(&lock.isLocked) == 1
+}

weed/command/filer_sync.go

@@ -53,6 +53,8 @@ type SyncOptions struct {
 	bDoDeleteFiles  *bool
 	clientId        int32
 	clientEpoch     atomic.Int32
+	debug           *bool
+	debugPort       *int
 }
 
 const (
@@ -108,6 +110,8 @@ func init() {
 	syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
 	syncOptions.aDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("a.doDeleteFiles", true, "delete and update files when synchronizing on filer A")
 	syncOptions.bDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("b.doDeleteFiles", true, "delete and update files when synchronizing on filer B")
+	syncOptions.debug = cmdFilerSynchronize.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	syncOptions.debugPort = cmdFilerSynchronize.Flag.Int("debug.port", 6060, "http port for debugging")
 	syncOptions.clientId = util.RandomInt32()
 }
 
@@ -130,6 +134,9 @@ var cmdFilerSynchronize = &Command{
 }
 
 func runFilerSynchronize(cmd *Command, args []string) bool {
+	if *syncOptions.debug {
+		grace.StartDebugServer(*syncOptions.debugPort)
+	}
 
 	util.LoadSecurityConfiguration()
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")

weed/command/master.go

@@ -69,6 +69,8 @@ type MasterOptions struct {
 	raftBootstrap      *bool
 	telemetryUrl       *string
 	telemetryEnabled   *bool
+	debug              *bool
+	debugPort          *int
 }
 
 func init() {
@@ -98,6 +100,8 @@ func init() {
 	m.raftBootstrap = cmdMaster.Flag.Bool("raftBootstrap", false, "Whether to bootstrap the Raft cluster")
 	m.telemetryUrl = cmdMaster.Flag.String("telemetry.url", "https://telemetry.seaweedfs.com/api/collect", "telemetry server URL to send usage statistics")
 	m.telemetryEnabled = cmdMaster.Flag.Bool("telemetry", false, "enable telemetry reporting")
+	m.debug = cmdMaster.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	m.debugPort = cmdMaster.Flag.Int("debug.port", 6060, "http port for debugging")
 }
 
 var cmdMaster = &Command{
@@ -121,6 +125,9 @@ var (
 )
 
 func runMaster(cmd *Command, args []string) bool {
+	if *m.debug {
+		grace.StartDebugServer(*m.debugPort)
+	}
 
 	util.LoadSecurityConfiguration()
 	util.LoadConfiguration("master", false)

weed/command/master_follower.go

@@ -41,6 +41,9 @@ func init() {
 	mf.metricsAddress = aws.String("")
 	mf.metricsIntervalSec = aws.Int(0)
 	mf.raftResumeState = aws.Bool(false)
+	mf.maxParallelVacuumPerServer = aws.Int(1)
+	mf.telemetryUrl = aws.String("https://telemetry.seaweedfs.com/api/collect")
+	mf.telemetryEnabled = aws.Bool(false)
 }
 
 var cmdMasterFollower = &Command{

weed/command/mount.go

@@ -68,8 +68,8 @@ func init() {
 	mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
-	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers")
-	mountOptions.concurrentReaders = cmdMount.Flag.Int("concurrentReaders", 16, "limit concurrent chunk fetches for read operations")
+	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 128, "limit concurrent goroutine writers")
+	mountOptions.concurrentReaders = cmdMount.Flag.Int("concurrentReaders", 128, "limit concurrent chunk fetches for read operations")
 	mountOptions.cacheDirForRead = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
 	mountOptions.cacheSizeMBForRead = cmdMount.Flag.Int64("cacheCapacityMB", 128, "file chunk read cache capacity in MB")
 	mountOptions.cacheDirForWrite = cmdMount.Flag.String("cacheDirWrite", "", "buffer writes mostly for large files")

weed/command/mq_broker.go

@@ -1,20 +1,15 @@
 package command
 
 import (
-	"fmt"
-	"net/http"
-	_ "net/http/pprof"
-
 	"google.golang.org/grpc/reflection"
 
-	"github.com/seaweedfs/seaweedfs/weed/util/grace"
-
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/broker"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/grace"
 )
 
 var (
@@ -27,12 +22,13 @@ type MessageQueueBrokerOptions struct {
 	filerGroup       *string
 	ip               *string
 	port             *int
-	pprofPort        *int
 	dataCenter       *string
 	rack             *string
 	cpuprofile       *string
 	memprofile       *string
 	logFlushInterval *int
+	debug            *bool
+	debugPort        *int
 }
 
 func init() {
@@ -41,12 +37,13 @@ func init() {
 	mqBrokerStandaloneOptions.filerGroup = cmdMqBroker.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
 	mqBrokerStandaloneOptions.ip = cmdMqBroker.Flag.String("ip", util.DetectedHostAddress(), "broker host address")
 	mqBrokerStandaloneOptions.port = cmdMqBroker.Flag.Int("port", 17777, "broker gRPC listen port")
-	mqBrokerStandaloneOptions.pprofPort = cmdMqBroker.Flag.Int("port.pprof", 0, "HTTP profiling port (0 to disable)")
 	mqBrokerStandaloneOptions.dataCenter = cmdMqBroker.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
 	mqBrokerStandaloneOptions.rack = cmdMqBroker.Flag.String("rack", "", "prefer to write to volumes in this rack")
 	mqBrokerStandaloneOptions.cpuprofile = cmdMqBroker.Flag.String("cpuprofile", "", "cpu profile output file")
 	mqBrokerStandaloneOptions.memprofile = cmdMqBroker.Flag.String("memprofile", "", "memory profile output file")
 	mqBrokerStandaloneOptions.logFlushInterval = cmdMqBroker.Flag.Int("logFlushInterval", 5, "log buffer flush interval in seconds")
+	mqBrokerStandaloneOptions.debug = cmdMqBroker.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	mqBrokerStandaloneOptions.debugPort = cmdMqBroker.Flag.Int("debug.port", 6060, "http port for debugging")
 }
 
 var cmdMqBroker = &Command{
@@ -61,6 +58,9 @@ var cmdMqBroker = &Command{
 }
 
 func runMqBroker(cmd *Command, args []string) bool {
+	if *mqBrokerStandaloneOptions.debug {
+		grace.StartDebugServer(*mqBrokerStandaloneOptions.debugPort)
+	}
 
 	util.LoadSecurityConfiguration()
 
@@ -115,18 +115,6 @@ func (mqBrokerOpt *MessageQueueBrokerOptions) startQueueServer() bool {
 		}()
 	}
 
-	// Start HTTP profiling server if enabled
-	if mqBrokerOpt.pprofPort != nil && *mqBrokerOpt.pprofPort > 0 {
-		go func() {
-			pprofAddr := fmt.Sprintf(":%d", *mqBrokerOpt.pprofPort)
-			glog.V(0).Infof("MQ Broker pprof server listening on %s", pprofAddr)
-			glog.V(0).Infof("Access profiling at: http://localhost:%d/debug/pprof/", *mqBrokerOpt.pprofPort)
-			if err := http.ListenAndServe(pprofAddr, nil); err != nil {
-				glog.Errorf("pprof server error: %v", err)
-			}
-		}()
-	}
-
 	glog.V(0).Infof("MQ Broker listening on %s:%d", *mqBrokerOpt.ip, *mqBrokerOpt.port)
 	grpcS.Serve(grpcL)
 

weed/command/s3.go

@@ -26,6 +26,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/grace"
 	"github.com/seaweedfs/seaweedfs/weed/util/version"
 )
 
@@ -59,6 +60,8 @@ type S3Options struct {
 	concurrentUploadLimitMB   *int
 	concurrentFileUploadLimit *int
 	enableIam                 *bool
+	debug                     *bool
+	debugPort                 *int
 }
 
 func init() {
@@ -88,6 +91,8 @@ func init() {
 	s3StandaloneOptions.concurrentUploadLimitMB = cmdS3.Flag.Int("concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited")
 	s3StandaloneOptions.concurrentFileUploadLimit = cmdS3.Flag.Int("concurrentFileUploadLimit", 0, "limit number of concurrent file uploads, 0 means unlimited")
 	s3StandaloneOptions.enableIam = cmdS3.Flag.Bool("iam", true, "enable embedded IAM API on the same port")
+	s3StandaloneOptions.debug = cmdS3.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	s3StandaloneOptions.debugPort = cmdS3.Flag.Int("debug.port", 6060, "http port for debugging")
 }
 
 var cmdS3 = &Command{
@@ -182,6 +187,9 @@ var cmdS3 = &Command{
 }
 
 func runS3(cmd *Command, args []string) bool {
+	if *s3StandaloneOptions.debug {
+		grace.StartDebugServer(*s3StandaloneOptions.debugPort)
+	}
 
 	util.LoadSecurityConfiguration()
 
@@ -341,6 +349,11 @@ func (s3opt *S3Options) startS3Server() bool {
 	go grpcS.Serve(grpcL)
 
 	if *s3opt.tlsPrivateKey != "" {
+		// Check for port conflict when both HTTP and HTTPS are enabled on the same port
+		if *s3opt.portHttps > 0 && *s3opt.portHttps == *s3opt.port {
+			glog.Fatalf("S3 API Server error: -s3.port.https (%d) cannot be the same as -s3.port (%d)", *s3opt.portHttps, *s3opt.port)
+		}
+
 		pemfileOptions := pemfile.Options{
 			CertFile:        *s3opt.tlsCertificate,
 			KeyFile:         *s3opt.tlsPrivateKey,
@@ -388,8 +401,11 @@ func (s3opt *S3Options) startS3Server() bool {
 			}
 		} else {
 			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", version.Version(), *s3opt.portHttps)
-			s3ApiListenerHttps, s3ApiLocalListenerHttps, _ := util.NewIpAndLocalListeners(
+			s3ApiListenerHttps, s3ApiLocalListenerHttps, err := util.NewIpAndLocalListeners(
 				*s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second)
+			if err != nil {
+				glog.Fatalf("S3 API HTTPS listener on %s:%d error: %v", *s3opt.bindIp, *s3opt.portHttps, err)
+			}
 			if s3ApiLocalListenerHttps != nil {
 				go func() {
 					if err = newHttpServer(router, tlsConfig).ServeTLS(s3ApiLocalListenerHttps, "", ""); err != nil {

weed/command/scaffold/filer.toml

@@ -415,6 +415,16 @@ key_path=""
 # The name list used to verify the cn name
 verify_cn=""
 
+[foundationdb]
+# FoundationDB provides ACID transactions and horizontal scalability.
+# Requires: go build -tags foundationdb
+enabled = false
+cluster_file = "/etc/foundationdb/fdb.cluster"
+# api_version = 740
+# timeout = "5s"
+# directory_prefix = "seaweedfs"
+# For bulk ingestion, enable batching: batch_enabled = true
+
 [tarantool]
 address = "localhost:3301"
 user = "guest"

weed/command/volume.go

@@ -72,6 +72,8 @@ type VolumeServerOptions struct {
 	hasSlowRead                 *bool
 	readBufferSizeMB            *int
 	ldbTimeout                  *int64
+	debug                       *bool
+	debugPort                   *int
 }
 
 func init() {
@@ -110,6 +112,8 @@ func init() {
 	v.inflightDownloadDataTimeout = cmdVolume.Flag.Duration("inflightDownloadDataTimeout", 60*time.Second, "inflight download data wait timeout of volume servers")
 	v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
 	v.readBufferSizeMB = cmdVolume.Flag.Int("readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally.")
+	v.debug = cmdVolume.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	v.debugPort = cmdVolume.Flag.Int("debug.port", 6060, "http port for debugging")
 }
 
 var cmdVolume = &Command{
@@ -129,6 +133,9 @@ var (
 )
 
 func runVolume(cmd *Command, args []string) bool {
+	if *v.debug {
+		grace.StartDebugServer(*v.debugPort)
+	}
 
 	util.LoadSecurityConfiguration()
 

weed/filer/empty_folder_cleanup/cleanup_queue.go

@@ -205,3 +205,4 @@ func (q *CleanupQueue) OldestAge() time.Duration {
 }
 
 
+

weed/filer/empty_folder_cleanup/cleanup_queue_test.go

@@ -369,3 +369,4 @@ func TestCleanupQueue_Concurrent(t *testing.T) {
 }
 
 
+

weed/filer/foundationdb/CONFIGURATION.md

@@ -27,6 +27,10 @@ export WEED_FOUNDATIONDB_API_VERSION=740
 export WEED_FOUNDATIONDB_TIMEOUT=5s
 export WEED_FOUNDATIONDB_MAX_RETRY_DELAY=1s
 export WEED_FOUNDATIONDB_DIRECTORY_PREFIX=seaweedfs
+# Write batching (disabled by default)
+export WEED_FOUNDATIONDB_BATCH_ENABLED=false
+export WEED_FOUNDATIONDB_BATCH_SIZE=100
+export WEED_FOUNDATIONDB_BATCH_INTERVAL=1ms
 ```
 
 ### 3. Command Line Arguments
@@ -56,6 +60,18 @@ While not directly supported, configuration can be specified via config files pa
 |--------|------|---------|-------------|
 | `directory_prefix` | string | `seaweedfs` | Directory prefix for key organization |
 
+### Write Batching Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `batch_enabled` | boolean | `false` | Enable write batching. Disabled by default for optimal S3 PUT latency. |
+| `batch_size` | integer | `100` | Maximum number of operations per batch (when batching is enabled) |
+| `batch_interval` | duration | `1ms` | Maximum time to wait before flushing a batch (when batching is enabled) |
+
+**Note:** Write batching is **disabled by default**. Each write commits immediately in its own 
+transaction, providing optimal latency for S3 PUT operations. Enable batching only for 
+high-throughput bulk ingestion workloads where you can tolerate slightly higher per-operation latency.
+
 ## Configuration Examples
 
 ### Development Environment
@@ -82,7 +98,25 @@ max_retry_delay = "5s"
 directory_prefix = "seaweedfs_prod"
 ```
 
-### High-Performance Setup
+### High-Performance Setup (Low Latency)
+
+For S3 workloads requiring low latency per operation:
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+api_version = 740
+timeout = "5s"
+max_retry_delay = "1s"
+directory_prefix = "sw"  # Shorter prefix for efficiency
+# Batching disabled (default) for optimal per-operation latency
+batch_enabled = false
+```
+
+### High-Throughput Bulk Ingestion
+
+For bulk data loading where throughput matters more than per-operation latency:
 
 ```toml
 [foundationdb]
@@ -91,7 +125,11 @@ cluster_file = "/etc/foundationdb/fdb.cluster"
 api_version = 740
 timeout = "60s"
 max_retry_delay = "10s"
-directory_prefix = "sw"  # Shorter prefix for efficiency
+directory_prefix = "sw"
+# Enable batching for higher throughput
+batch_enabled = true
+batch_size = 100
+batch_interval = "1ms"
 ```
 
 ### Path-Specific Configuration

weed/filer/foundationdb/README.md

@@ -51,6 +51,9 @@ directory_prefix = "seaweedfs"
 | `timeout` | Operation timeout duration | `5s` | No |
 | `max_retry_delay` | Maximum retry delay | `1s` | No |
 | `directory_prefix` | Directory prefix for organization | `seaweedfs` | No |
+| `batch_enabled` | Enable write batching (see Performance section) | `false` | No |
+| `batch_size` | Max operations per batch | `100` | No |
+| `batch_interval` | Max time before batch flush | `1ms` | No |
 
 ### Path-Specific Configuration
 
@@ -109,12 +112,38 @@ make setup
 
 ## Performance Considerations
 
+### Write Batching Configuration
+
+By default, write batching is **disabled** (`batch_enabled = false`). Each write commits 
+immediately in its own transaction. This provides optimal latency for S3 PUT operations.
+
+**When to enable batching:**
+- High-throughput bulk ingestion workloads
+- Scenarios where you can tolerate slightly higher per-operation latency
+- Workloads with many concurrent small writes
+
+**Batching configuration options:**
+
+```toml
+[foundationdb]
+# Enable write batching (disabled by default for optimal S3 latency)
+batch_enabled = true
+# Maximum operations per batch
+batch_size = 100
+# Maximum time to wait before flushing a batch
+batch_interval = "1ms"
+```
+
+**Performance comparison:**
+- **Batching disabled**: Each S3 PUT commits immediately (~1-5ms per op depending on FDB latency)
+- **Batching enabled**: Operations are grouped, reducing total commits but adding batch interval latency
+
 ### Optimal Configuration
 
 - **API Version**: Use the latest stable API version (720+)
 - **Directory Structure**: Use logical directory prefixes to isolate different SeaweedFS instances
 - **Transaction Size**: Keep transactions under 10MB (FDB limit)
-- **Batch Operations**: Use transactions for multiple related operations
+- **Concurrency**: Use multiple client connections for parallel operations
 
 ### Monitoring
 

weed/filer/foundationdb/foundationdb_store.go

@@ -45,8 +45,12 @@ const (
 	MAX_DIRECTORY_LIST_LIMIT = 1000
 
 	// Write batching defaults
+	// Note: Batching is disabled by default because S3 semantics require waiting
+	// for durability, and the batch timer adds latency to each operation.
+	// Enable batching only for workloads that can tolerate potential latency.
 	DEFAULT_BATCH_SIZE     = 100
-	DEFAULT_BATCH_INTERVAL = 5 * time.Millisecond
+	DEFAULT_BATCH_INTERVAL = 1 * time.Millisecond
+	DEFAULT_BATCH_ENABLED  = false
 )
 
 func init() {
@@ -129,11 +133,33 @@ func (b *writeBatcher) run() {
 		timer.Reset(b.interval)
 	}
 
+	// Collect available ops without blocking
+	collectAvailable := func() {
+		for {
+			select {
+			case op := <-b.ops:
+				batch = append(batch, op)
+				batchBytes += op.size()
+				if len(batch) >= b.size || batchBytes >= FDB_BATCH_SIZE_LIMIT {
+					return
+				}
+			default:
+				return
+			}
+		}
+	}
+
 	for {
 		select {
 		case op := <-b.ops:
 			batch = append(batch, op)
 			batchBytes += op.size()
+
+			// Optimization: When an operation arrives, try to collect more
+			// available operations without blocking. This improves throughput
+			// when multiple goroutines are submitting concurrently.
+			collectAvailable()
+
 			// Flush when batch count or size limit is reached
 			if len(batch) >= b.size || batchBytes >= FDB_BATCH_SIZE_LIMIT {
 				flush()
@@ -185,8 +211,10 @@ type FoundationDBStore struct {
 	directoryPrefix string
 	timeout         time.Duration
 	maxRetryDelay   time.Duration
-	// Write batching
+	// Write batching - disabled by default for optimal S3 latency
+	// Enable for high-throughput bulk ingestion workloads
 	batcher       *writeBatcher
+	batchEnabled  bool
 	batchSize     int
 	batchInterval time.Duration
 }
@@ -225,6 +253,10 @@ func (store *FoundationDBStore) Initialize(configuration util.Configuration, pre
 	configuration.SetDefault(prefix+"timeout", "5s")
 	configuration.SetDefault(prefix+"max_retry_delay", "1s")
 	configuration.SetDefault(prefix+"directory_prefix", "seaweedfs")
+	// Batching is disabled by default - each write commits immediately.
+	// This provides optimal latency for S3 PUT operations.
+	// Enable batching for high-throughput bulk ingestion workloads.
+	configuration.SetDefault(prefix+"batch_enabled", DEFAULT_BATCH_ENABLED)
 	configuration.SetDefault(prefix+"batch_size", DEFAULT_BATCH_SIZE)
 	configuration.SetDefault(prefix+"batch_interval", DEFAULT_BATCH_INTERVAL.String())
 
@@ -247,6 +279,7 @@ func (store *FoundationDBStore) Initialize(configuration util.Configuration, pre
 	}
 
 	// Parse batch configuration
+	store.batchEnabled = configuration.GetBool(prefix + "batch_enabled")
 	store.batchSize = configuration.GetInt(prefix + "batch_size")
 	if store.batchSize <= 0 {
 		store.batchSize = DEFAULT_BATCH_SIZE
@@ -288,10 +321,16 @@ func (store *FoundationDBStore) initialize(clusterFile string, apiVersion int) e
 		return fmt.Errorf("failed to create/open kv directory: %w", err)
 	}
 
-	// Start write batcher for improved throughput
-	store.batcher = newWriteBatcher(store, store.batchSize, store.batchInterval)
-	glog.V(0).Infof("FoundationDB: write batching enabled (batch_size=%d, batch_interval=%v)",
-		store.batchSize, store.batchInterval)
+	// Conditionally start write batcher
+	// Batching is disabled by default for optimal S3 latency.
+	// When disabled, each write commits immediately in its own transaction.
+	if store.batchEnabled {
+		store.batcher = newWriteBatcher(store, store.batchSize, store.batchInterval)
+		glog.V(0).Infof("FoundationDB: write batching enabled (batch_size=%d, batch_interval=%v)",
+			store.batchSize, store.batchInterval)
+	} else {
+		glog.V(0).Infof("FoundationDB: write batching disabled (direct commit mode for optimal latency)")
+	}
 
 	glog.V(0).Infof("FoundationDB store initialized successfully with directory prefix: %s", store.directoryPrefix)
 	return nil

weed/filer/foundationdb/foundationdb_store_test.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -325,6 +326,120 @@ func BenchmarkFoundationDBStore_KvOperations(b *testing.B) {
 	}
 }
 
+// BenchmarkFoundationDBStore_InsertEntry_NoBatch benchmarks insert performance
+// with batching disabled (direct commit mode - optimal for S3 PUT latency)
+func BenchmarkFoundationDBStore_InsertEntry_NoBatch(b *testing.B) {
+	store := createBenchmarkStoreWithBatching(b, false, 100, 1*time.Millisecond)
+	defer store.Shutdown()
+
+	ctx := context.Background()
+	entry := &filer.Entry{
+		FullPath: "/benchmark_nobatch/file.txt",
+		Attr: filer.Attr{
+			Mode:  0644,
+			Uid:   1000,
+			Gid:   1000,
+			Mtime: time.Now(),
+		},
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		entry.FullPath = util.NewFullPath("/benchmark_nobatch", fmt.Sprintf("%x", uint64(i))+".txt")
+		err := store.InsertEntry(ctx, entry)
+		if err != nil {
+			b.Fatalf("InsertEntry failed: %v", err)
+		}
+	}
+}
+
+// BenchmarkFoundationDBStore_InsertEntry_WithBatch benchmarks insert performance
+// with batching enabled (higher throughput for bulk ingestion)
+func BenchmarkFoundationDBStore_InsertEntry_WithBatch(b *testing.B) {
+	store := createBenchmarkStoreWithBatching(b, true, 100, 1*time.Millisecond)
+	defer store.Shutdown()
+
+	ctx := context.Background()
+	entry := &filer.Entry{
+		FullPath: "/benchmark_batch/file.txt",
+		Attr: filer.Attr{
+			Mode:  0644,
+			Uid:   1000,
+			Gid:   1000,
+			Mtime: time.Now(),
+		},
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		entry.FullPath = util.NewFullPath("/benchmark_batch", fmt.Sprintf("%x", uint64(i))+".txt")
+		err := store.InsertEntry(ctx, entry)
+		if err != nil {
+			b.Fatalf("InsertEntry failed: %v", err)
+		}
+	}
+}
+
+// BenchmarkFoundationDBStore_ConcurrentInsert_NoBatch benchmarks concurrent inserts
+// with batching disabled (simulates S3 PUT concurrency)
+func BenchmarkFoundationDBStore_ConcurrentInsert_NoBatch(b *testing.B) {
+	store := createBenchmarkStoreWithBatching(b, false, 100, 1*time.Millisecond)
+	defer store.Shutdown()
+
+	var counter atomic.Uint64
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		ctx := context.Background()
+		for pb.Next() {
+			n := counter.Add(1)
+			entry := &filer.Entry{
+				FullPath: util.NewFullPath("/benchmark_concurrent_nobatch", fmt.Sprintf("%d.txt", n)),
+				Attr: filer.Attr{
+					Mode:  0644,
+					Uid:   1000,
+					Gid:   1000,
+					Mtime: time.Now(),
+				},
+			}
+			err := store.InsertEntry(ctx, entry)
+			if err != nil {
+				b.Fatalf("InsertEntry failed: %v", err)
+			}
+		}
+	})
+}
+
+// BenchmarkFoundationDBStore_ConcurrentInsert_WithBatch benchmarks concurrent inserts
+// with batching enabled (tests batch efficiency under concurrent load)
+func BenchmarkFoundationDBStore_ConcurrentInsert_WithBatch(b *testing.B) {
+	store := createBenchmarkStoreWithBatching(b, true, 100, 1*time.Millisecond)
+	defer store.Shutdown()
+
+	var counter atomic.Uint64
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		ctx := context.Background()
+		for pb.Next() {
+			n := counter.Add(1)
+			entry := &filer.Entry{
+				FullPath: util.NewFullPath("/benchmark_concurrent_batch", fmt.Sprintf("%d.txt", n)),
+				Attr: filer.Attr{
+					Mode:  0644,
+					Uid:   1000,
+					Gid:   1000,
+					Mtime: time.Now(),
+				},
+			}
+			err := store.InsertEntry(ctx, entry)
+			if err != nil {
+				b.Fatalf("InsertEntry failed: %v", err)
+			}
+		}
+	})
+}
+
 // Helper functions
 func getTestClusterFile() string {
 	clusterFile := os.Getenv("FDB_CLUSTER_FILE")
@@ -349,6 +464,32 @@ func createBenchmarkStore(b *testing.B) *FoundationDBStore {
 	return store
 }
 
+// createBenchmarkStoreWithBatching creates a store with specific batching configuration
+// for comparing performance between batched and non-batched modes
+func createBenchmarkStoreWithBatching(b *testing.B, batchEnabled bool, batchSize int, batchInterval time.Duration) *FoundationDBStore {
+	clusterFile := getTestClusterFile()
+	if _, err := os.Stat(clusterFile); os.IsNotExist(err) {
+		b.Skip("FoundationDB cluster file not found, skipping benchmark")
+	}
+
+	store := &FoundationDBStore{
+		batchEnabled:    batchEnabled,
+		batchSize:       batchSize,
+		batchInterval:   batchInterval,
+		directoryPrefix: "benchmark",
+		timeout:         5 * time.Second,
+		maxRetryDelay:   1 * time.Second,
+	}
+	err := store.initialize(clusterFile, 740)
+	if err != nil {
+		b.Skipf("Failed to initialize FoundationDB store: %v", err)
+	}
+
+	// Note: initialize() already creates the batcher if batchEnabled is true
+
+	return store
+}
+
 func getTestStore(t *testing.T) *FoundationDBStore {
 	t.Helper()
 

weed/filer/read_remote.go

@@ -25,12 +25,20 @@ func MapRemoteStorageLocationPathToFullPath(localMountedDir util.FullPath, remot
 	return localMountedDir.Child(remoteLocationPath[len(remoteMountedLocation.Path):])
 }
 
-func CacheRemoteObjectToLocalCluster(filerClient filer_pb.FilerClient, remoteConf *remote_pb.RemoteConf, remoteLocation *remote_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) error {
-	return filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		_, err := client.CacheRemoteObjectToLocalCluster(context.Background(), &filer_pb.CacheRemoteObjectToLocalClusterRequest{
+// CacheRemoteObjectToLocalCluster caches a remote object to the local cluster.
+// It returns the updated entry with local chunk locations.
+// Parameters remoteConf and remoteLocation are kept for backward compatibility but are not used.
+func CacheRemoteObjectToLocalCluster(filerClient filer_pb.FilerClient, remoteConf *remote_pb.RemoteConf, remoteLocation *remote_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) (*filer_pb.Entry, error) {
+	var cachedEntry *filer_pb.Entry
+	err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		resp, cacheErr := client.CacheRemoteObjectToLocalCluster(context.Background(), &filer_pb.CacheRemoteObjectToLocalClusterRequest{
 			Directory: string(parent),
 			Name:      entry.Name,
 		})
-		return err
+		if cacheErr == nil && resp != nil {
+			cachedEntry = resp.Entry
+		}
+		return cacheErr
 	})
+	return cachedEntry, err
 }

weed/iam/helpers.go

@@ -124,3 +124,4 @@ func MaskAccessKey(accessKeyId string) string {
 	return accessKeyId
 }
 
+