refactor: remove unused gRPC connection age parameters

The GrpcMaxConnectionAge and GrpcMaxConnectionAgeGrace constants have a troubled history - they were removed in 2022 due to gRPC issues, reverted later, and recently re-added. However, they are not essential to the core worker reconnection fix which was solved through proper goroutine ordering. The Docker Swarm DNS handling mentioned in the comments is not critical, and these parameters have caused problems in the past. Removing them simplifies the configuration without losing functionality.
refactor: extract drain and close channel logic into helper function
2026-10-12 01:25:51 +00:00 · 2025-12-22 18:19:52 -08:00 · 2025-12-22 17:56:42 -08:00 · 2025-12-22 17:55:43 -08:00 · 2025-12-22 17:55:36 -08:00 · 2025-12-22 17:55:25 -08:00
256 changed files with 28171 additions and 3267 deletions
@@ -44,7 +44,7 @@ jobs:
        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}

      - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -60,7 +60,7 @@ jobs:
          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"

      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -93,7 +93,7 @@ jobs:
        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}

      - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -109,7 +109,7 @@ jobs:
          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"

      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -30,7 +30,7 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -30,7 +30,7 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -30,7 +30,7 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -30,7 +30,7 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -30,7 +30,7 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -45,7 +45,7 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -30,7 +30,7 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
+        uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
@@ -224,3 +224,4 @@ jobs:
          echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"


+
@@ -36,7 +36,7 @@ jobs:
      uses: docker/setup-buildx-action@v3
      
    - name: Cache Docker layers
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: /tmp/.buildx-cache
        key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
@@ -134,7 +134,7 @@ jobs:

    - name: Archive logs
      if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
      with:
        name: output-logs
        path: docker/output.log
@@ -35,7 +35,7 @@ jobs:

    - name: Archive logs
      if: failure()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
      with:
        name: ec-integration-test-logs
        path: test/erasure_coding
@@ -183,7 +183,7 @@ jobs:
        
    - name: Upload Test Artifacts
      if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
      with:
        name: fuse-integration-test-results
        path: |
@@ -53,6 +53,7 @@ jobs:
          mkdir -p "$WEED_DATA_DIR"
          
          # Start SeaweedFS with optimized settings for CI
+          # Include S3 API for s3copier integration tests
          weed server -dir="$WEED_DATA_DIR" \
            -master.raftHashicorp \
            -master.electionTimeout=1s \
@@ -61,6 +62,9 @@ jobs:
            -volume.preStopSeconds=1 \
            -master.peers=none \
            -filer -filer.maxMB=64 \
+            -s3 -s3.port=8333 \
+            -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \
+            -s3.allowDeleteBucketNotEmpty=true \
            -master.port=9333 \
            -volume.port=8080 \
            -filer.port=8888 \
@@ -105,6 +109,16 @@ jobs:
            sleep 2
          done
          
+          # Wait for S3 API
+          for i in {1..30}; do
+            if curl -s http://localhost:8333/ > /dev/null 2>&1; then
+              echo "✓ S3 API is ready"
+              break
+            fi
+            echo "Waiting for S3 API... ($i/30)"
+            sleep 2
+          done
+          
          echo "✓ All SeaweedFS components are ready!"
          
          # Display cluster status
@@ -133,6 +147,20 @@ jobs:
        run: |
          mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
      
+      - name: Run S3 ETag Validation Tests (Issue #7768)
+        working-directory: other/java/s3copier
+        env:
+          S3_ENDPOINT: http://127.0.0.1:8333
+          S3_ACCESS_KEY: some_access_key1
+          S3_SECRET_KEY: some_secret_key1
+        run: |
+          echo "Running S3 ETag validation tests against $S3_ENDPOINT"
+          mvn test -Dtest=ETagValidationTest \
+            -DS3_ENDPOINT=$S3_ENDPOINT \
+            -DS3_ACCESS_KEY=$S3_ACCESS_KEY \
+            -DS3_SECRET_KEY=$S3_SECRET_KEY \
+            -Dmaven.javadoc.skip=true -Dgpg.skip=true
+      
      - name: Display logs on failure
        if: failure()
        run: |
@@ -49,7 +49,7 @@ jobs:
      
      - name: Upload Test Reports
        if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: test-reports-java-${{ matrix.java }}
          path: |
@@ -70,7 +70,7 @@ jobs:
        
    - name: Archive logs on failure
      if: failure()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
      with:
        name: metadata-subscribe-test-logs
        path: |
@@ -35,7 +35,7 @@ jobs:
      uses: docker/setup-buildx-action@v3

    - name: Cache Docker layers
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: /tmp/.buildx-cache
        key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
@@ -62,7 +62,7 @@ jobs:

    - name: Archive logs
      if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
      with:
        name: postgres-logs
        path: test/postgres/postgres-output.log
@@ -0,0 +1,86 @@
+name: "S3 Filer Group Tests"
+
+on:
+  pull_request:
+  
+concurrency:
+  group: ${{ github.head_ref }}/s3-filer-group-tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    working-directory: weed
+
+jobs:
+  s3-filer-group-tests:
+    name: S3 Filer Group Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false
+
+      - name: Run S3 Filer Group Tests
+        timeout-minutes: 15
+        working-directory: test/s3/filer_group
+        run: |
+          set -x
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          df -h
+          echo "=== Starting Filer Group Tests ==="
+          
+          # Run tests with automatic server management
+          # The test-with-server target handles server startup/shutdown automatically
+          # Server is started with -filer.group=testgroup
+          make test-with-server || {
+            echo "❌ Filer group tests failed, checking logs..."
+            if [ -f weed-test.log ]; then
+              echo "=== Server logs ==="
+              tail -100 weed-test.log
+            fi
+            echo "=== Process information ==="
+            ps aux | grep -E "(weed|test)" || true
+            exit 1
+          }
+
+      - name: Show server logs on failure
+        if: failure()
+        working-directory: test/s3/filer_group
+        run: |
+          echo "=== Server Logs ==="
+          if [ -f weed-test.log ]; then
+            echo "Last 100 lines of server logs:"
+            tail -100 weed-test.log
+          else
+            echo "No server log file found"
+          fi
+          
+          echo "=== Test Environment ==="
+          ps aux | grep -E "(weed|test)" || true
+          netstat -tlnp | grep -E "(8333|9333|8080)" || true
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: s3-filer-group-test-logs
+          path: test/s3/filer_group/weed-test*.log
+          retention-days: 3
+
+
@@ -76,7 +76,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-versioning-test-logs-${{ matrix.test-type }}
          path: test/s3/versioning/weed-test*.log
@@ -124,7 +124,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-versioning-compatibility-logs
          path: test/s3/versioning/weed-test*.log
@@ -172,7 +172,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-cors-compatibility-logs
          path: test/s3/cors/weed-test*.log
@@ -239,7 +239,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-retention-test-logs-${{ matrix.test-type }}
          path: test/s3/retention/weed-test*.log
@@ -306,7 +306,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-cors-test-logs-${{ matrix.test-type }}
          path: test/s3/cors/weed-test*.log
@@ -355,7 +355,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-retention-worm-logs
          path: test/s3/retention/weed-test*.log
@@ -365,8 +365,6 @@ jobs:
    name: S3 Versioning Stress Test
    runs-on: ubuntu-22.04
    timeout-minutes: 35
-    # Only run stress tests on master branch pushes to avoid overloading PR testing
-    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
    
    steps:
      - name: Check out code
@@ -391,6 +389,9 @@ jobs:
          uname -a
          free -h
          
+          # Start server for stress tests
+          make start-server
+          
          # Run stress tests (concurrent operations)
          make test-versioning-stress || {
            echo "❌ Stress test failed, checking logs..."
@@ -398,14 +399,30 @@ jobs:
              echo "=== Server logs ==="
              tail -200 weed-test.log
            fi
+            make stop-server
            make clean
            exit 1
          }
+          
+          # Run pagination stress tests (>1000 versions)
+          echo "=== Running pagination stress tests ==="
+          make test-versioning-pagination-stress || {
+            echo "❌ Pagination stress test failed, checking logs..."
+            if [ -f weed-test.log ]; then
+              echo "=== Server logs ==="
+              tail -200 weed-test.log
+            fi
+            make stop-server
+            make clean
+            exit 1
+          }
+          
+          make stop-server
          make clean

      - name: Upload stress test logs
        if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-versioning-stress-logs
          path: test/s3/versioning/weed-test*.log
@@ -459,10 +476,65 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-tagging-test-logs
          path: test/s3/tagging/weed-test*.log
          retention-days: 3

+  s3-remote-cache-tests:
+    name: S3 Remote Cache Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false
+
+      - name: Run S3 Remote Cache Tests
+        timeout-minutes: 15
+        working-directory: test/s3/remote_cache
+        run: |
+          set -x
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          
+          # Run the remote cache integration tests
+          # Tests singleflight deduplication for caching remote objects
+          make test-with-server || {
+            echo "❌ Test failed, checking logs..."
+            if [ -f primary-weed.log ]; then
+              echo "=== Primary server logs ==="
+              tail -100 primary-weed.log
+            fi
+            if [ -f remote-weed.log ]; then
+              echo "=== Remote server logs ==="
+              tail -100 remote-weed.log
+            fi
+            echo "=== Process information ==="
+            ps aux | grep -E "(weed|test)" || true
+            exit 1
+          }
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: s3-remote-cache-test-logs
+          path: |
+            test/s3/remote_cache/primary-weed.log
+            test/s3/remote_cache/remote-weed.log
+          retention-days: 3
+
  # Removed SSE-C integration tests and compatibility job
@@ -65,7 +65,7 @@ jobs:

      - name: Upload test results on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: iam-unit-test-results
          path: |
@@ -162,7 +162,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-iam-integration-logs-${{ matrix.test-type }}
          path: test/s3/iam/weed-*.log
@@ -222,7 +222,7 @@ jobs:

      - name: Upload distributed test logs
        if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-iam-distributed-logs
          path: test/s3/iam/weed-*.log
@@ -274,7 +274,7 @@ jobs:

      - name: Upload performance test results
        if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-iam-performance-results
          path: |
@@ -152,7 +152,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-keycloak-test-logs
          path: |
@@ -121,7 +121,7 @@ jobs:
      
      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: test-logs-python-${{ matrix.python-version }}
          path: |
@@ -95,7 +95,7 @@ jobs:

      - name: Upload test logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-sse-test-logs-${{ matrix.test-type }}
          path: test/s3/sse/weed-test*.log
@@ -143,7 +143,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-sse-compatibility-logs
          path: test/s3/sse/weed-test*.log
@@ -192,7 +192,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-sse-metadata-persistence-logs
          path: test/s3/sse/weed-test*.log
@@ -241,7 +241,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-sse-copy-operations-logs
          path: test/s3/sse/weed-test*.log
@@ -290,7 +290,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-sse-multipart-logs
          path: test/s3/sse/weed-test*.log
@@ -340,7 +340,7 @@ jobs:

      - name: Upload performance test logs
        if: always()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: s3-sse-performance-logs
          path: test/s3/sse/weed-test*.log
@@ -91,3 +91,4 @@ jobs:
        echo "| readonly | /public | Read-only |" >> $GITHUB_STEP_SUMMARY


+
@@ -125,7 +125,7 @@ jobs:

    - name: Upload test results
      if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
      with:
        name: spark-test-results
        path: test/java/spark/target/surefire-reports/
@@ -150,7 +150,7 @@ jobs:
    - name: Cache Apache Spark
      if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
      id: cache-spark
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: spark-3.5.0-bin-hadoop3
        key: spark-3.5.0-hadoop3
@@ -18,7 +18,7 @@ defaults:
 jobs:
  awscli-tests:
    runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

@@ -103,7 +103,7 @@ jobs:

      - name: Upload server logs on failure
        if: failure()
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
        with:
          name: seaweedfs-logs
          # Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
@@ -0,0 +1,114 @@
+name: "TUS Protocol Tests"
+
+on:
+  pull_request:
+    paths:
+      - 'weed/server/filer_server_tus*.go'
+      - 'weed/server/filer_server.go'
+      - 'test/tus/**'
+      - '.github/workflows/tus-tests.yml'
+  push:
+    branches: [ master, main ]
+    paths:
+      - 'weed/server/filer_server_tus*.go'
+      - 'weed/server/filer_server.go'
+      - 'test/tus/**'
+
+concurrency:
+  group: ${{ github.head_ref || github.ref }}/tus-tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    working-directory: weed
+
+jobs:
+  tus-integration-tests:
+    name: TUS Protocol Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false
+
+      - name: Run TUS Integration Tests
+        timeout-minutes: 15
+        working-directory: test/tus
+        run: |
+          set -x
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          df -h
+          echo "=== Starting TUS Tests ==="
+          
+          # Run tests with automatic server management
+          make test-with-server || {
+            echo "TUS integration tests failed, checking logs..."
+            if [ -f /tmp/seaweedfs-tus-filer.log ]; then
+              echo "=== Filer logs ==="
+              tail -100 /tmp/seaweedfs-tus-filer.log
+            fi
+            if [ -f /tmp/seaweedfs-tus-master.log ]; then
+              echo "=== Master logs ==="
+              tail -50 /tmp/seaweedfs-tus-master.log
+            fi
+            if [ -f /tmp/seaweedfs-tus-volume.log ]; then
+              echo "=== Volume logs ==="
+              tail -50 /tmp/seaweedfs-tus-volume.log
+            fi
+            exit 1
+          }
+
+      - name: Show server logs on failure
+        if: failure()
+        working-directory: test/tus
+        run: |
+          echo "=== Filer Server Logs ==="
+          if [ -f /tmp/seaweedfs-tus-filer.log ]; then
+            echo "Last 100 lines of filer logs:"
+            tail -100 /tmp/seaweedfs-tus-filer.log
+          else
+            echo "No filer log file found"
+          fi
+          
+          echo "=== Master Server Logs ==="
+          if [ -f /tmp/seaweedfs-tus-master.log ]; then
+            tail -50 /tmp/seaweedfs-tus-master.log
+          else
+            echo "No master log file found"
+          fi
+          
+          echo "=== Volume Server Logs ==="
+          if [ -f /tmp/seaweedfs-tus-volume.log ]; then
+            tail -50 /tmp/seaweedfs-tus-volume.log
+          else
+            echo "No volume log file found"
+          fi
+          
+          echo "=== Test Environment ==="
+          ps aux | grep -E "(weed|test)" || true
+          netstat -tlnp 2>/dev/null | grep -E "(18888|19333|18080)" || true
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: tus-test-logs
+          path: |
+            /tmp/seaweedfs-tus-*.log
+          retention-days: 3
@@ -124,3 +124,10 @@ ADVANCED_IAM_DEVELOPMENT_PLAN.md
 *.log
 weed-iam
 test/kafka/kafka-client-loadtest/weed-linux-arm64
+/test/tus/filerldb2
+coverage.out
+/test/s3/remote_cache/test-primary-data
+/test/s3/remote_cache/test-remote-data
+test/s3/remote_cache/remote-server.pid
+test/s3/remote_cache/primary-server.pid
+/test/erasure_coding/filerldb2
@@ -28,7 +28,7 @@ require (
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
-	github.com/go-redsync/redsync/v4 v4.14.0
+	github.com/go-redsync/redsync/v4 v4.15.0
 	github.com/go-sql-driver/mysql v1.9.3
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.7.0
@@ -71,7 +71,7 @@ require (
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
-	github.com/seaweedfs/raft v1.1.3
+	github.com/seaweedfs/raft v1.1.6
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect
@@ -96,14 +96,14 @@ require (
 	gocloud.dev v0.43.0
 	gocloud.dev/pubsub/natspubsub v0.43.0
 	gocloud.dev/pubsub/rabbitpubsub v0.43.0
-	golang.org/x/crypto v0.45.0
+	golang.org/x/crypto v0.46.0
 	golang.org/x/exp v0.0.0-20250811191247-51f88131bc50
-	golang.org/x/image v0.33.0
-	golang.org/x/net v0.47.0
+	golang.org/x/image v0.34.0
+	golang.org/x/net v0.48.0
 	golang.org/x/oauth2 v0.32.0 // indirect
-	golang.org/x/sys v0.38.0
-	golang.org/x/text v0.31.0 // indirect
-	golang.org/x/tools v0.38.0 // indirect
+	golang.org/x/sys v0.39.0
+	golang.org/x/text v0.32.0 // indirect
+	golang.org/x/tools v0.39.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.247.0
 	google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 // indirect
@@ -126,9 +126,9 @@ require (
 	github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
 	github.com/arangodb/go-driver v1.6.9
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.40.1
-	github.com/aws/aws-sdk-go-v2/config v1.31.3
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.3
+	github.com/aws/aws-sdk-go-v2 v1.41.0
+	github.com/aws/aws-sdk-go-v2/config v1.32.6
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.6
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
@@ -152,14 +152,14 @@ require (
 	github.com/rabbitmq/amqp091-go v1.10.0
 	github.com/rclone/rclone v1.71.2
 	github.com/rdleal/intervalst v1.5.0
-	github.com/redis/go-redis/v9 v9.14.1
+	github.com/redis/go-redis/v9 v9.17.2
 	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/shirou/gopsutil/v4 v4.25.11
 	github.com/tarantool/go-tarantool/v2 v2.4.1
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/xeipuuv/gojsonschema v1.2.0
-	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0
+	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0
 	go.etcd.io/etcd/client/pkg/v3 v3.6.6
 	go.uber.org/atomic v1.11.0
 	golang.org/x/sync v0.19.0
@@ -173,6 +173,7 @@ require (
 	cloud.google.com/go/longrunning v0.6.7 // indirect
 	cloud.google.com/go/pubsub/v2 v2.0.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
 	github.com/bazelbuild/rules_go v0.46.0 // indirect
 	github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
 	github.com/blevesearch/snowballstem v0.9.0 // indirect
@@ -203,8 +204,8 @@ require (
 	github.com/openzipkin/zipkin-go v0.4.3 // indirect
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/pierrre/geohash v1.0.0 // indirect
-	github.com/quic-go/qpack v0.5.1 // indirect
-	github.com/quic-go/quic-go v0.54.1 // indirect
+	github.com/quic-go/qpack v0.6.0 // indirect
+	github.com/quic-go/quic-go v0.57.0 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sasha-s/go-deadlock v0.3.1 // indirect
@@ -218,10 +219,9 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
 	go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.7.0 // indirect
-	go.uber.org/mock v0.5.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/mod v0.29.0 // indirect
+	golang.org/x/mod v0.30.0 // indirect
 	gonum.org/v1/gonum v0.16.0 // indirect
 )

@@ -261,21 +261,21 @@ require (
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
 	github.com/aws/smithy-go v1.24.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
@@ -446,7 +446,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/arch v0.20.0 // indirect
-	golang.org/x/term v0.37.0 // indirect
+	golang.org/x/term v0.38.0 // indirect
 	golang.org/x/time v0.12.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
@@ -651,8 +651,6 @@ github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmg
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
 github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
-github.com/apple/foundationdb/bindings/go v0.0.0-20240515141816-262c6fe778ad h1:fQBkhYv86zyW95PWhzBlkgz3NoY1ue0L+8oYBaoCMbg=
-github.com/apple/foundationdb/bindings/go v0.0.0-20240515141816-262c6fe778ad/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
 github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3 h1:WZaTKNHCfcw7fWSR6/RKnCldVzvYZC+Y20Su4lffEIg=
 github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
 github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc h1:LoL75er+LKDHDUfU5tRvFwxH0LjPpZN8OoG8Ll+liGU=
@@ -667,46 +665,48 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.40.1 h1:difXb4maDZkRH0x//Qkwcfpdg1XQVXEAEs2DdXldFFc=
-github.com/aws/aws-sdk-go-v2 v1.40.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
+github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
+github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
-github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.3 h1:01Ym72hK43hjwDeJUfi1l2oYLXBAOR8gNSZNmXmvuas=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.3/go.mod h1:55nWF/Sr9Zvls0bGnWkRxUdhzKqj9uRNlPvgV1vgxKc=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 h1:utxLraaifrSBkeyII9mIbVwXXWrZdlPO7FIKmyLCEcY=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15/go.mod h1:hW6zjYUDQwfz3icf4g2O41PHi77u10oAzJ84iSzR/lo=
+github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8=
+github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 h1:Y5YXgygXwDI5P4RkteB5yF7v35neH7LfJKBG+hzIons=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15/go.mod h1:K+/1EpG42dFSY7CBj+Fruzm8PsCGWTXJ3jdeJ659oGQ=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 h1:AvltKnW9ewxX2hFmQS0FyJH93aSvJVUEFvXfU+HWtSE=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15/go.mod h1:3I4oCdZdmgrREhU74qS1dK9yZ62yumob+58AbFR4cQA=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 h1:3/u/4yZOffg5jdNk1sDpOQ4Y+R6Xbh+GzpDrSZjuy3U=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15/go.mod h1:4Zkjq0FKjE78NKjabuM4tRXKFzUJWXgP0ItEZK8l7JU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
 github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE=
 github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 h1:8sTTiw+9yuNXcfWeqKF2x01GqCF49CpP4Z9nKrrk/ts=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.6/go.mod h1:8WYg+Y40Sn3X2hioaaWAAIngndR8n1XFdRPPX+7QBaM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 h1:E+KqWoVsSrj1tJ6I/fjDIu5xoS2Zacuu1zT+H7KtiIk=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11/go.mod h1:qyWHz+4lvkXcr3+PoGlGHEI+3DLLiU6/GdrFfMaAhB0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 h1:tzMkjh0yTChUqJDgGkcDdxvZDSrJ/WB6R6ymI5ehqJI=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.3/go.mod h1:T270C0R5sZNLbWUe8ueiAF42XSZxxPocTaGSgs5c/60=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
 github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
 github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
@@ -988,8 +988,8 @@ github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOr
 github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
-github.com/go-redsync/redsync/v4 v4.14.0 h1:zyxzFJsmQHIPBl8iBT7KFKohWsjsghgGLiP8TnFMLNc=
-github.com/go-redsync/redsync/v4 v4.14.0/go.mod h1:twMlVd19upZ/juvJyJGlQOSQxor1oeHtjs62l4pRFzo=
+github.com/go-redsync/redsync/v4 v4.15.0 h1:KH/XymuxSV7vyKs6z1Cxxj+N+N18JlPxgXeP6x4JY54=
+github.com/go-redsync/redsync/v4 v4.15.0/go.mod h1:qNp+lLs3vkfZbtA/aM/OjlZHfEr5YTAYhRktFPKHC7s=
 github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
 github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
 github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
@@ -1072,8 +1072,8 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
 github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s=
-github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
+github.com/gomodule/redigo v1.9.3 h1:dNPSXeXv6HCq2jdyWfjgmhBdqnR6PRO3m/G05nvpPC8=
+github.com/gomodule/redigo v1.9.3/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
@@ -1565,10 +1565,10 @@ github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4
 github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
-github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
-github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
-github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg=
-github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
+github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
+github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
+github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
+github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
 github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
 github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
 github.com/rclone/rclone v1.71.2 h1:3Jk5xNPFrZhVABRuN/OPvApuZQddpE2tkhYMuEn1Ud4=
@@ -1577,12 +1577,12 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
 github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
-github.com/redis/go-redis/v9 v9.14.1 h1:nDCrEiJmfOWhD76xlaw+HXT0c9hfNWeXgl0vIRYSDvQ=
-github.com/redis/go-redis/v9 v9.14.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
-github.com/redis/rueidis v1.0.64 h1:XqgbueDuNV3qFdVdQwAHJl1uNt90zUuAJuzqjH4cw6Y=
-github.com/redis/rueidis v1.0.64/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
-github.com/redis/rueidis/rueidiscompat v1.0.64 h1:M8JbLP4LyHQhBLBRsUQIzui8/LyTtdESNIMVveqm4RY=
-github.com/redis/rueidis/rueidiscompat v1.0.64/go.mod h1:8pJVPhEjpw0izZFSxYwDziUiEYEkEklTSw/nZzga61M=
+github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
+github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
+github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
+github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
+github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
+github.com/redis/rueidis/rueidiscompat v1.0.69/go.mod h1:iC4Y8DoN0Uth0Uezg9e2trvNRC7QAgGeuP2OPLb5ccI=
 github.com/rekby/fixenv v0.3.2/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
 github.com/rekby/fixenv v0.6.1 h1:jUFiSPpajT4WY2cYuc++7Y1zWrnCxnovGCIX72PZniM=
 github.com/rekby/fixenv v0.6.1/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
@@ -1620,8 +1620,8 @@ github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138 h1:bX
 github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
 github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
 github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
-github.com/seaweedfs/raft v1.1.3 h1:5B6hgneQ7IuU4Ceom/f6QUt8pEeqjcsRo+IxlyPZCws=
-github.com/seaweedfs/raft v1.1.3/go.mod h1:9cYlEBA+djJbnf/5tWsCybtbL7ICYpi+Uxcg3MxjuNs=
+github.com/seaweedfs/raft v1.1.6 h1:e83Xn0boscPnuSiBllUPeWRVS6JKrqJPYBozgFyBiC0=
+github.com/seaweedfs/raft v1.1.6/go.mod h1:9cYlEBA+djJbnf/5tWsCybtbL7ICYpi+Uxcg3MxjuNs=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
@@ -1779,12 +1779,12 @@ github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mo
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
-github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0 h1:/NyPd9KnCJgzrEXCArqk1ThqCH2Dh31uUwl88o/VkuM=
-github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
+github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaFdZ6zevvYe+TRxOR8ifa2s=
+github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0 h1:Ml4WMsaRY+/+Wh3DCp1A+/zjM1aMdQg3irjuYFwWVHQ=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0/go.mod h1:/LjMxb/rXmoGAAnImoqAFIlhO5ampHacbvDetQitCk4=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0 h1:uvqheUfoEJz0CLLLaYct69wpvoaosM/2joWFcbwoqfw=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0/go.mod h1:/LjMxb/rXmoGAAnImoqAFIlhO5ampHacbvDetQitCk4=
 github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
 github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
 github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
@@ -1874,8 +1874,8 @@ go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A
 go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
-go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
+go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
+go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
@@ -1919,8 +1919,8 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
+golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1951,8 +1951,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
 golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
-golang.org/x/image v0.33.0 h1:LXRZRnv1+zGd5XBUVRFmYEphyyKJjQjCRiOuAP3sZfQ=
-golang.org/x/image v0.33.0/go.mod h1:DD3OsTYT9chzuzTQt+zMcOlBHgfoKQb1gry8p76Y1sc=
+golang.org/x/image v0.34.0 h1:33gCkyw9hmwbZJeZkct8XyR11yH889EQt/QH4VmXMn8=
+golang.org/x/image v0.34.0/go.mod h1:2RNFBZRB+vnwwFil8GkMdRvrJOFd1AzdZI6vOY+eJVU=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1987,8 +1987,8 @@ golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
-golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
+golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
+golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2058,8 +2058,8 @@ golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
+golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2220,8 +2220,8 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2239,8 +2239,8 @@ golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
-golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
-golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
+golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2262,8 +2262,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
+golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2343,8 +2343,8 @@ golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58
 golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
-golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
+golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
 golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
 golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.02"
+appVersion: "4.03"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.402
+version: 4.0.403
@@ -1595,6 +1595,101 @@
      "title": "S3 Bucket Traffic Sent",
      "type": "timeseries"
    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "reqps",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 41
+      },
+      "id": 86,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$NAMESPACE\"}[$__interval])) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}}",
+          "refId": "A"
+        }
+      ],
+      "title": "S3 API Calls per Bucket",
+      "type": "timeseries"
+    },
    {
      "datasource": {
        "type": "prometheus",
@@ -1659,8 +1754,8 @@
      },
      "gridPos": {
        "h": 7,
-        "w": 24,
-        "x": 0,
+        "w": 12,
+        "x": 12,
        "y": 41
      },
      "id": 72,
@@ -3266,6 +3361,209 @@
      ],
      "title": "Filer Go Routines",
      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 48
+      },
+      "id": 89,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}} (logical)",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}} (physical)",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "S3 Bucket Size",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 48
+      },
+      "id": 90,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_object_count{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "S3 Bucket Object Count",
+      "type": "timeseries"
    }
  ],
  "refresh": "",
@@ -3356,4 +3654,4 @@
  "uid": "a24009d7-cbda-4443-a132-1cc1c4677304",
  "version": 1,
  "weekStart": ""
-}
+}
@@ -203,6 +203,9 @@ spec:
            - name: client-cert
              readOnly: true
              mountPath: /usr/local/share/ca-certificates/client/
+            - name: admin-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/admin/
            {{- end }}
            {{ tpl .Values.admin.extraVolumeMounts . | nindent 12 | trim }}
          ports:
@@ -298,6 +301,9 @@ spec:
        - name: client-cert
          secret:
            secretName: {{ template "seaweedfs.name" . }}-client-cert
+        - name: admin-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-admin-cert
        {{- end }}
        {{ tpl .Values.admin.extraVolumes . | indent 8 | trim }}
      {{- if .Values.admin.nodeSelector }}
@@ -0,0 +1,43 @@
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
+apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
+kind: Certificate
+metadata:
+  name: {{ template "seaweedfs.name" . }}-admin-cert
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: admin
+  {{- if .Values.admin.annotations }}
+  annotations:
+    {{- toYaml .Values.admin.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  secretName: {{ template "seaweedfs.name" . }}-admin-cert
+  issuerRef:
+    name: {{ template "seaweedfs.name" . }}-ca-issuer
+    kind: Issuer
+  commonName: {{ .Values.certificates.commonName }}
+  subject:
+    organizations:
+    - "SeaweedFS CA"
+  dnsNames:
+    - '*.{{ template "seaweedfs.name" . }}-admin'
+    - '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}'
+    - '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}.svc'
+    - '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}.svc.cluster.local'
+{{- if .Values.certificates.ipAddresses }}
+  ipAddresses:
+    {{- range .Values.certificates.ipAddresses }}
+    - {{ . }}
+    {{- end }}
+{{- end }}
+  privateKey:
+    algorithm: {{ .Values.certificates.keyAlgorithm }}
+    size: {{ .Values.certificates.keySize }}
+  duration: {{ .Values.certificates.duration }}
+  renewBefore: {{ .Values.certificates.renewBefore }}
+{{- end }}
+
@@ -0,0 +1,43 @@
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
+apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
+kind: Certificate
+metadata:
+  name: {{ template "seaweedfs.name" . }}-worker-cert
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: worker
+  {{- if .Values.worker.annotations }}
+  annotations:
+    {{- toYaml .Values.worker.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  secretName: {{ template "seaweedfs.name" . }}-worker-cert
+  issuerRef:
+    name: {{ template "seaweedfs.name" . }}-ca-issuer
+    kind: Issuer
+  commonName: {{ .Values.certificates.commonName }}
+  subject:
+    organizations:
+    - "SeaweedFS CA"
+  dnsNames:
+    - '*.{{ template "seaweedfs.name" . }}-worker'
+    - '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}'
+    - '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}.svc'
+    - '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}.svc.cluster.local'
+{{- if .Values.certificates.ipAddresses }}
+  ipAddresses:
+    {{- range .Values.certificates.ipAddresses }}
+    - {{ . }}
+    {{- end }}
+{{- end }}
+  privateKey:
+    algorithm: {{ .Values.certificates.keyAlgorithm }}
+    size: {{ .Values.certificates.keySize }}
+  duration: {{ .Values.certificates.duration }}
+  renewBefore: {{ .Values.certificates.renewBefore }}
+{{- end }}
+
@@ -65,6 +65,14 @@ data:
    cert = "/usr/local/share/ca-certificates/filer/tls.crt"
    key  = "/usr/local/share/ca-certificates/filer/tls.key"

+    [grpc.admin]
+    cert = "/usr/local/share/ca-certificates/admin/tls.crt"
+    key  = "/usr/local/share/ca-certificates/admin/tls.key"
+
+    [grpc.worker]
+    cert = "/usr/local/share/ca-certificates/worker/tls.crt"
+    key  = "/usr/local/share/ca-certificates/worker/tls.key"
+
    # use this for any place needs a grpc client
    # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
    [grpc.client]
@@ -171,6 +171,9 @@ spec:
            - name: client-cert
              readOnly: true
              mountPath: /usr/local/share/ca-certificates/client/
+            - name: worker-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/worker/
            {{- end }}
            {{ tpl .Values.worker.extraVolumeMounts . | nindent 12 | trim }}
          ports:
@@ -262,6 +265,9 @@ spec:
        - name: client-cert
          secret:
            secretName: {{ template "seaweedfs.name" . }}-client-cert
+        - name: worker-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-worker-cert
        {{- end }}
        {{ tpl .Values.worker.extraVolumes . | indent 8 | trim }}
      {{- if .Values.worker.nodeSelector }}
@@ -0,0 +1,110 @@
+# SeaweedFS S3 Java SDK Compatibility Tests
+
+This project contains Java-based integration tests for SeaweedFS S3 API compatibility.
+
+## Overview
+
+Tests are provided for both AWS SDK v1 and v2 to ensure compatibility with the various SDK versions commonly used in production.
+
+## SDK Versions
+
+| SDK | Version | Notes |
+|-----|---------|-------|
+| AWS SDK v1 for Java | 1.12.600 | Legacy SDK, less strict ETag validation |
+| AWS SDK v2 for Java | 2.20.127 | Modern SDK with strict checksum validation |
+
+## Running Tests
+
+### Prerequisites
+
+1. SeaweedFS running with S3 API enabled:
+   ```bash
+   weed server -s3
+   ```
+
+2. Java 18+ and Maven
+
+### Run All Tests
+
+```bash
+mvn test
+```
+
+### Run Specific Tests
+
+```bash
+# Run only ETag validation tests (AWS SDK v2)
+mvn test -Dtest=ETagValidationTest
+
+# Run with custom endpoint
+mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
+```
+
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `S3_ENDPOINT` | `http://127.0.0.1:8333` | S3 API endpoint URL |
+| `S3_ACCESS_KEY` | `some_access_key1` | Access key ID |
+| `S3_SECRET_KEY` | `some_secret_key1` | Secret access key |
+| `S3_REGION` | `us-east-1` | AWS region |
+
+## Test Coverage
+
+### ETagValidationTest (AWS SDK v2)
+
+Tests for [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768) - ETag format validation.
+
+| Test | Description |
+|------|-------------|
+| `testSmallFilePutObject` | Verify small files return pure MD5 ETag |
+| `testLargeFilePutObject_Issue7768` | **Critical**: Verify large files (>8MB) return pure MD5 ETag |
+| `testExtraLargeFilePutObject` | Verify very large files (>24MB) return pure MD5 ETag |
+| `testMultipartUploadETag` | Verify multipart uploads return composite ETag |
+| `testETagConsistency` | Verify ETag consistency across PUT/HEAD/GET |
+| `testMultipleLargeFileUploads` | Stress test multiple large uploads |
+
+### Background: Issue #7768
+
+AWS SDK v2 for Java includes checksum validation that decodes the ETag as hexadecimal. When SeaweedFS returned composite ETags (`<md5>-<count>`) for regular `PutObject` with internally auto-chunked files, the SDK failed with:
+
+```
+java.lang.IllegalArgumentException: Invalid base 16 character: '-'
+```
+
+**Per AWS S3 specification:**
+- `PutObject`: ETag is always a pure MD5 hex string (32 chars)
+- `CompleteMultipartUpload`: ETag is composite format (`<md5>-<partcount>`)
+
+The fix ensures SeaweedFS follows this specification.
+
+## Project Structure
+
+```
+src/
+├── main/java/com/seaweedfs/s3/
+│   ├── PutObject.java           # Example PutObject with SDK v1
+│   └── HighLevelMultipartUpload.java
+└── test/java/com/seaweedfs/s3/
+    ├── PutObjectTest.java       # Basic SDK v1 test
+    └── ETagValidationTest.java  # Comprehensive SDK v2 ETag tests
+```
+
+## Validated SDK Versions
+
+This Java test project validates:
+
+- ✅ AWS SDK v2 for Java 2.20.127+
+- ✅ AWS SDK v1 for Java 1.12.600+
+
+Go SDK validation is performed by separate test suites:
+- See [Go ETag Tests](/test/s3/etag/) for AWS SDK v2 for Go tests
+- See [test/s3/SDK_COMPATIBILITY.md](/test/s3/SDK_COMPATIBILITY.md) for full SDK compatibility matrix
+
+## Related
+
+- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
+- [Go ETag Tests](/test/s3/etag/)
+- [SDK Compatibility Matrix](/test/s3/SDK_COMPATIBILITY.md)
+
@@ -6,18 +6,29 @@
  <packaging>jar</packaging>
  <version>1.0-SNAPSHOT</version>
  <properties>
-    <maven.compiler.source>18</maven.compiler.source>
-    <maven.compiler.target>18</maven.compiler.target>
+    <maven.compiler.source>11</maven.compiler.source>
+    <maven.compiler.target>11</maven.compiler.target>
+    <aws.sdk.v1.version>1.12.600</aws.sdk.v1.version>
+    <aws.sdk.v2.version>2.20.127</aws.sdk.v2.version>
  </properties>
  <name>copier</name>
  <url>http://maven.apache.org</url>

  <dependencyManagement>
    <dependencies>
+      <!-- AWS SDK v1 BOM -->
      <dependency>
        <groupId>com.amazonaws</groupId>
        <artifactId>aws-java-sdk-bom</artifactId>
-        <version>1.11.327</version>
+        <version>${aws.sdk.v1.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+      <!-- AWS SDK v2 BOM -->
+      <dependency>
+        <groupId>software.amazon.awssdk</groupId>
+        <artifactId>bom</artifactId>
+        <version>${aws.sdk.v2.version}</version>
        <type>pom</type>
        <scope>import</scope>
      </dependency>
@@ -25,15 +36,50 @@
  </dependencyManagement>

  <dependencies>
+    <!-- AWS SDK v1 (for backward compatibility with existing code) -->
    <dependency>
      <groupId>com.amazonaws</groupId>
      <artifactId>aws-java-sdk-s3</artifactId>
    </dependency>
+    
+    <!-- AWS SDK v2 (for modern tests with checksum validation) -->
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3-transfer-manager</artifactId>
+    </dependency>
+    
+    <!-- Test dependencies -->
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
-      <version>4.13.1</version>
+      <version>4.13.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>5.10.0</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
+      <version>3.24.2</version>
      <scope>test</scope>
    </dependency>
  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>3.2.2</version>
+      </plugin>
+    </plugins>
+  </build>
 </project>
@@ -0,0 +1,439 @@
+package com.seaweedfs.s3;
+
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.*;
+
+import java.net.URI;
+import java.security.MessageDigest;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.regex.Pattern;
+
+import static org.assertj.core.api.Assertions.*;
+
+/**
+ * AWS SDK v2 Integration Tests for S3 ETag Format Validation.
+ * 
+ * These tests verify that SeaweedFS returns correct ETag formats that are
+ * compatible with AWS SDK v2's checksum validation.
+ * 
+ * Background (GitHub Issue #7768):
+ * AWS SDK v2 for Java validates ETags as hexadecimal MD5 hashes for PutObject
+ * responses. If the ETag contains non-hex characters (like '-' in composite
+ * format), the SDK fails with "Invalid base 16 character: '-'".
+ * 
+ * Per AWS S3 specification:
+ * - Regular PutObject: ETag is always a pure MD5 hex string (32 chars)
+ * - CompleteMultipartUpload: ETag is composite format "<md5>-<partcount>"
+ * 
+ * To run these tests:
+ *   mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
+ * 
+ * Or set environment variable:
+ *   export S3_ENDPOINT=http://localhost:8333
+ *   mvn test -Dtest=ETagValidationTest
+ */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@DisplayName("S3 ETag Format Validation Tests (AWS SDK v2)")
+class ETagValidationTest {
+
+    // Configuration - can be overridden via system properties or environment variables
+    private static final String DEFAULT_ENDPOINT = "http://127.0.0.1:8333";
+    private static final String DEFAULT_ACCESS_KEY = "some_access_key1";
+    private static final String DEFAULT_SECRET_KEY = "some_secret_key1";
+    private static final String DEFAULT_REGION = "us-east-1";
+    
+    // Auto-chunking threshold in SeaweedFS (must match s3api_object_handlers_put.go)
+    private static final int AUTO_CHUNK_SIZE = 8 * 1024 * 1024; // 8MB
+    
+    // Test sizes
+    private static final int SMALL_FILE_SIZE = 1024; // 1KB
+    private static final int LARGE_FILE_SIZE = 10 * 1024 * 1024; // 10MB (triggers auto-chunking)
+    private static final int XL_FILE_SIZE = 25 * 1024 * 1024; // 25MB (multiple chunks)
+    private static final int MULTIPART_PART_SIZE = 5 * 1024 * 1024; // 5MB per part
+    
+    // ETag format patterns
+    private static final Pattern PURE_MD5_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}\"?$");
+    private static final Pattern COMPOSITE_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}-\\d+\"?$");
+    
+    private S3Client s3Client;
+    private String testBucketName;
+    private final SecureRandom random = new SecureRandom();
+
+    @BeforeAll
+    void setUp() {
+        String endpoint = getConfig("S3_ENDPOINT", DEFAULT_ENDPOINT);
+        String accessKey = getConfig("S3_ACCESS_KEY", DEFAULT_ACCESS_KEY);
+        String secretKey = getConfig("S3_SECRET_KEY", DEFAULT_SECRET_KEY);
+        String region = getConfig("S3_REGION", DEFAULT_REGION);
+        
+        System.out.println("Connecting to S3 endpoint: " + endpoint);
+        
+        s3Client = S3Client.builder()
+                .endpointOverride(URI.create(endpoint))
+                .credentialsProvider(StaticCredentialsProvider.create(
+                        AwsBasicCredentials.create(accessKey, secretKey)))
+                .region(Region.of(region))
+                .forcePathStyle(true) // Required for SeaweedFS
+                .build();
+        
+        // Create test bucket
+        testBucketName = "test-etag-" + UUID.randomUUID().toString().substring(0, 8);
+        s3Client.createBucket(CreateBucketRequest.builder()
+                .bucket(testBucketName)
+                .build());
+        
+        System.out.println("Created test bucket: " + testBucketName);
+    }
+
+    @AfterAll
+    void tearDown() {
+        if (s3Client != null && testBucketName != null) {
+            try {
+                // Delete all objects with pagination
+                String continuationToken = null;
+                do {
+                    ListObjectsV2Response listResp = s3Client.listObjectsV2(
+                            ListObjectsV2Request.builder()
+                                    .bucket(testBucketName)
+                                    .continuationToken(continuationToken)
+                                    .build());
+                    for (S3Object obj : listResp.contents()) {
+                        s3Client.deleteObject(DeleteObjectRequest.builder()
+                                .bucket(testBucketName)
+                                .key(obj.key())
+                                .build());
+                    }
+                    continuationToken = listResp.nextContinuationToken();
+                } while (continuationToken != null);
+                
+                // Abort any multipart uploads
+                ListMultipartUploadsResponse mpResp = s3Client.listMultipartUploads(
+                        ListMultipartUploadsRequest.builder().bucket(testBucketName).build());
+                for (MultipartUpload upload : mpResp.uploads()) {
+                    s3Client.abortMultipartUpload(AbortMultipartUploadRequest.builder()
+                            .bucket(testBucketName)
+                            .key(upload.key())
+                            .uploadId(upload.uploadId())
+                            .build());
+                }
+                
+                // Delete bucket
+                s3Client.deleteBucket(DeleteBucketRequest.builder()
+                        .bucket(testBucketName)
+                        .build());
+                
+                System.out.println("Cleaned up test bucket: " + testBucketName);
+            } catch (Exception e) {
+                System.err.println("Warning: Failed to cleanup test bucket: " + e.getMessage());
+            }
+            s3Client.close();
+        }
+    }
+
+    @Test
+    @DisplayName("Small file PutObject should return pure MD5 ETag")
+    void testSmallFilePutObject() throws Exception {
+        byte[] testData = generateRandomData(SMALL_FILE_SIZE);
+        String expectedMD5 = calculateMD5Hex(testData);
+        String objectKey = "small-file-" + UUID.randomUUID() + ".bin";
+        
+        PutObjectResponse response = s3Client.putObject(
+                PutObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build(),
+                RequestBody.fromBytes(testData));
+        
+        String etag = response.eTag();
+        System.out.println("Small file ETag: " + etag + " (expected MD5: " + expectedMD5 + ")");
+        
+        assertThat(etag)
+                .describedAs("Small file ETag should be pure MD5")
+                .matches(PURE_MD5_PATTERN);
+        assertThat(cleanETag(etag))
+                .describedAs("ETag should match calculated MD5")
+                .isEqualTo(expectedMD5);
+        assertThat(etag)
+                .describedAs("ETag should not contain hyphen")
+                .doesNotContain("-");
+    }
+
+    /**
+     * Critical test for GitHub Issue #7768.
+     * 
+     * This test uploads a file larger than the auto-chunking threshold (8MB),
+     * which triggers SeaweedFS to split the file into multiple internal chunks.
+     * 
+     * Previously, this caused SeaweedFS to return a composite ETag like
+     * "d41d8cd98f00b204e9800998ecf8427e-2", which AWS SDK v2 rejected because
+     * it validates the ETag as hexadecimal and '-' is not a valid hex character.
+     * 
+     * The fix ensures that regular PutObject always returns a pure MD5 ETag,
+     * regardless of internal chunking.
+     */
+    @Test
+    @DisplayName("Large file PutObject (>8MB) should return pure MD5 ETag - Issue #7768")
+    void testLargeFilePutObject_Issue7768() throws Exception {
+        byte[] testData = generateRandomData(LARGE_FILE_SIZE);
+        String expectedMD5 = calculateMD5Hex(testData);
+        String objectKey = "large-file-" + UUID.randomUUID() + ".bin";
+        
+        System.out.println("Uploading large file (" + LARGE_FILE_SIZE + " bytes, " +
+                "> " + AUTO_CHUNK_SIZE + " byte auto-chunk threshold)...");
+        
+        // This is where Issue #7768 would manifest - SDK v2 validates ETag
+        PutObjectResponse response = s3Client.putObject(
+                PutObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build(),
+                RequestBody.fromBytes(testData));
+        
+        String etag = response.eTag();
+        int expectedChunks = (LARGE_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
+        System.out.println("Large file ETag: " + etag + 
+                " (expected MD5: " + expectedMD5 + ", internal chunks: ~" + expectedChunks + ")");
+        
+        // These assertions would fail before the fix
+        assertThat(etag)
+                .describedAs("Large file PutObject ETag MUST be pure MD5 (not composite)")
+                .matches(PURE_MD5_PATTERN);
+        assertThat(etag)
+                .describedAs("Large file ETag should NOT be composite format")
+                .doesNotMatch(COMPOSITE_PATTERN);
+        assertThat(etag)
+                .describedAs("ETag should not contain hyphen for regular PutObject")
+                .doesNotContain("-");
+        assertThat(cleanETag(etag))
+                .describedAs("ETag should match calculated MD5")
+                .isEqualTo(expectedMD5);
+        
+        // Verify hex decoding works (this is what fails in Issue #7768)
+        assertThatCode(() -> hexToBytes(cleanETag(etag)))
+                .describedAs("ETag should be valid hexadecimal (AWS SDK v2 validation)")
+                .doesNotThrowAnyException();
+    }
+
+    @Test
+    @DisplayName("Extra large file PutObject (>24MB) should return pure MD5 ETag")
+    void testExtraLargeFilePutObject() throws Exception {
+        byte[] testData = generateRandomData(XL_FILE_SIZE);
+        String expectedMD5 = calculateMD5Hex(testData);
+        String objectKey = "xl-file-" + UUID.randomUUID() + ".bin";
+        
+        int expectedChunks = (XL_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
+        System.out.println("Uploading XL file (" + XL_FILE_SIZE + " bytes, ~" + 
+                expectedChunks + " internal chunks)...");
+        
+        PutObjectResponse response = s3Client.putObject(
+                PutObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build(),
+                RequestBody.fromBytes(testData));
+        
+        String etag = response.eTag();
+        System.out.println("XL file ETag: " + etag);
+        
+        assertThat(etag)
+                .describedAs("XL file PutObject ETag MUST be pure MD5")
+                .matches(PURE_MD5_PATTERN);
+        assertThat(cleanETag(etag))
+                .describedAs("ETag should match calculated MD5")
+                .isEqualTo(expectedMD5);
+    }
+
+    @Test
+    @DisplayName("Multipart upload should return composite ETag")
+    void testMultipartUploadETag() throws Exception {
+        int totalSize = 15 * 1024 * 1024; // 15MB = 3 parts
+        byte[] testData = generateRandomData(totalSize);
+        String objectKey = "multipart-file-" + UUID.randomUUID() + ".bin";
+        
+        System.out.println("Performing multipart upload (" + totalSize + " bytes)...");
+        
+        // Initiate multipart upload
+        CreateMultipartUploadResponse createResp = s3Client.createMultipartUpload(
+                CreateMultipartUploadRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build());
+        String uploadId = createResp.uploadId();
+        
+        List<CompletedPart> completedParts = new ArrayList<>();
+        int partNumber = 1;
+        
+        // Upload parts
+        for (int offset = 0; offset < totalSize; offset += MULTIPART_PART_SIZE) {
+            int end = Math.min(offset + MULTIPART_PART_SIZE, totalSize);
+            byte[] partData = new byte[end - offset];
+            System.arraycopy(testData, offset, partData, 0, partData.length);
+            
+            UploadPartResponse uploadResp = s3Client.uploadPart(
+                    UploadPartRequest.builder()
+                            .bucket(testBucketName)
+                            .key(objectKey)
+                            .uploadId(uploadId)
+                            .partNumber(partNumber)
+                            .build(),
+                    RequestBody.fromBytes(partData));
+            
+            completedParts.add(CompletedPart.builder()
+                    .partNumber(partNumber)
+                    .eTag(uploadResp.eTag())
+                    .build());
+            partNumber++;
+        }
+        
+        // Complete multipart upload
+        CompleteMultipartUploadResponse completeResp = s3Client.completeMultipartUpload(
+                CompleteMultipartUploadRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .uploadId(uploadId)
+                        .multipartUpload(CompletedMultipartUpload.builder()
+                                .parts(completedParts)
+                                .build())
+                        .build());
+        
+        String etag = completeResp.eTag();
+        System.out.println("Multipart upload ETag: " + etag + " (" + completedParts.size() + " parts)");
+        
+        // Multipart uploads SHOULD have composite ETag
+        assertThat(etag)
+                .describedAs("Multipart upload ETag SHOULD be composite format")
+                .matches(COMPOSITE_PATTERN);
+        assertThat(etag)
+                .describedAs("Multipart ETag should contain hyphen")
+                .contains("-");
+        
+        // Verify part count in ETag
+        String[] parts = cleanETag(etag).split("-");
+        assertThat(parts).hasSize(2);
+        assertThat(parts[1])
+                .describedAs("Part count in ETag should match uploaded parts")
+                .isEqualTo(String.valueOf(completedParts.size()));
+    }
+
+    @Test
+    @DisplayName("ETag should be consistent across PUT, HEAD, and GET")
+    void testETagConsistency() throws Exception {
+        byte[] testData = generateRandomData(LARGE_FILE_SIZE);
+        String objectKey = "consistency-test-" + UUID.randomUUID() + ".bin";
+        
+        // PUT
+        PutObjectResponse putResp = s3Client.putObject(
+                PutObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build(),
+                RequestBody.fromBytes(testData));
+        String putETag = putResp.eTag();
+        
+        // HEAD
+        HeadObjectResponse headResp = s3Client.headObject(
+                HeadObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build());
+        String headETag = headResp.eTag();
+        
+        // GET
+        GetObjectResponse getResp = s3Client.getObject(
+                GetObjectRequest.builder()
+                        .bucket(testBucketName)
+                        .key(objectKey)
+                        .build())
+                .response();
+        String getETag = getResp.eTag();
+        
+        System.out.println("PUT ETag: " + putETag + ", HEAD ETag: " + headETag + ", GET ETag: " + getETag);
+        
+        assertThat(putETag).isEqualTo(headETag);
+        assertThat(putETag).isEqualTo(getETag);
+    }
+
+    @Test
+    @DisplayName("Multiple large file uploads should all return pure MD5 ETags")
+    void testMultipleLargeFileUploads() throws Exception {
+        int numFiles = 3;
+        
+        for (int i = 0; i < numFiles; i++) {
+            byte[] testData = generateRandomData(LARGE_FILE_SIZE);
+            String expectedMD5 = calculateMD5Hex(testData);
+            String objectKey = "multi-large-" + i + "-" + UUID.randomUUID() + ".bin";
+            
+            PutObjectResponse response = s3Client.putObject(
+                    PutObjectRequest.builder()
+                            .bucket(testBucketName)
+                            .key(objectKey)
+                            .build(),
+                    RequestBody.fromBytes(testData));
+            
+            String etag = response.eTag();
+            System.out.println("File " + i + " ETag: " + etag);
+            
+            assertThat(etag)
+                    .describedAs("File " + i + " ETag should be pure MD5")
+                    .matches(PURE_MD5_PATTERN);
+            assertThat(cleanETag(etag))
+                    .describedAs("File " + i + " ETag should match MD5")
+                    .isEqualTo(expectedMD5);
+            
+            // Validate hex decoding
+            assertThatCode(() -> hexToBytes(cleanETag(etag)))
+                    .doesNotThrowAnyException();
+        }
+    }
+
+    // Helper methods
+    
+    private String getConfig(String key, String defaultValue) {
+        String value = System.getProperty(key);
+        if (value == null) {
+            value = System.getenv(key);
+        }
+        return value != null ? value : defaultValue;
+    }
+    
+    private byte[] generateRandomData(int size) {
+        byte[] data = new byte[size];
+        random.nextBytes(data);
+        return data;
+    }
+    
+    private String calculateMD5Hex(byte[] data) throws Exception {
+        MessageDigest md = MessageDigest.getInstance("MD5");
+        byte[] digest = md.digest(data);
+        StringBuilder sb = new StringBuilder();
+        for (byte b : digest) {
+            sb.append(String.format("%02x", b));
+        }
+        return sb.toString();
+    }
+    
+    private String cleanETag(String etag) {
+        if (etag == null) return null;
+        return etag.replace("\"", "");
+    }
+    
+    private byte[] hexToBytes(String hex) {
+        int len = hex.length();
+        byte[] data = new byte[len / 2];
+        for (int i = 0; i < len; i += 2) {
+            data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4)
+                    + Character.digit(hex.charAt(i + 1), 16));
+        }
+        return data;
+    }
+}
+
@@ -1319,6 +1319,99 @@
        "alignLevel": null
      }
    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "hiddenSeries": false,
+      "id": 90,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.1.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(SeaweedFS_s3_request_total[$__interval])) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 API Calls per Bucket",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "reqps",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
    {
      "aliasColors": {},
      "bars": false,
@@ -1332,8 +1425,8 @@
      "grid": {},
      "gridPos": {
        "h": 7,
-        "w": 24,
-        "x": 0,
+        "w": 12,
+        "x": 12,
        "y": 16
      },
      "hiddenSeries": false,
@@ -2960,6 +3053,200 @@
        "align": false,
        "alignLevel": null
      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 37
+      },
+      "hiddenSeries": false,
+      "id": 87,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.1.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "max(SeaweedFS_s3_bucket_size_bytes) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}} (logical)",
+          "refId": "A"
+        },
+        {
+          "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}} (physical)",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 Bucket Size",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "decbytes",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 37
+      },
+      "hiddenSeries": false,
+      "id": 88,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.1.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "max(SeaweedFS_s3_bucket_object_count) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 Bucket Object Count",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
    }
  ],
  "refresh": "30s",
@@ -875,6 +875,85 @@
            }
          ]
        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS-DEV}",
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 16
+          },
+          "hiddenSeries": false,
+          "id": 86,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(SeaweedFS_s3_request_total[$__interval])) by (bucket)",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{bucket}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "S3 API Calls per Bucket",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "reqps",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ]
+        },
        {
          "aliasColors": {},
          "bars": false,
@@ -1087,6 +1166,202 @@
              "show": false
            }
          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "fillGradient": 0,
+          "fieldConfig": {
+            "defaults": {
+              "unit": "decbytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 37
+          },
+          "hiddenSeries": false,
+          "id": 87,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.1.2",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(SeaweedFS_s3_bucket_size_bytes) by (bucket)",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{bucket}} (logical)",
+              "refId": "A"
+            },
+            {
+              "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes) by (bucket)",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{bucket}} (physical)",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "S3 Bucket Size",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "decbytes",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "fillGradient": 0,
+          "fieldConfig": {
+            "defaults": {
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 37
+          },
+          "hiddenSeries": false,
+          "id": 88,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.1.2",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(SeaweedFS_s3_bucket_object_count) by (bucket)",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{bucket}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "S3 Bucket Object Count",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
        }
      ],
      "repeat": null,
@@ -2111,4 +2386,4 @@
  "timezone": "browser",
  "title": "SeaweedFS ks8 heartbeat",
  "version": 1
-}
+}
@@ -447,6 +447,99 @@
        "alignLevel": null
      }
    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fill": 1,
+      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 6,
+        "x": 0,
+        "y": 15
+      },
+      "hiddenSeries": false,
+      "id": 86,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.1.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$namespace\",service=~\"$service-api\"}[$__interval])) by (bucket)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 API Calls per Bucket",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "reqps",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
    {
      "aliasColors": {},
      "bars": false,
@@ -466,8 +559,8 @@
      "grid": {},
      "gridPos": {
        "h": 7,
-        "w": 8,
-        "x": 0,
+        "w": 6,
+        "x": 12,
        "y": 15
      },
      "hiddenSeries": false,
@@ -577,8 +670,8 @@
      "grid": {},
      "gridPos": {
        "h": 7,
-        "w": 8,
-        "x": 8,
+        "w": 6,
+        "x": 6,
        "y": 15
      },
      "hiddenSeries": false,
@@ -688,8 +781,8 @@
      "grid": {},
      "gridPos": {
        "h": 7,
-        "w": 8,
-        "x": 16,
+        "w": 6,
+        "x": 18,
        "y": 15
      },
      "hiddenSeries": false,
@@ -2354,6 +2447,209 @@
        "align": false,
        "alignLevel": null
      }
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 41
+      },
+      "id": 89,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}} (logical)",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}} (physical)",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "S3 Bucket Size",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 41
+      },
+      "id": 90,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max(SeaweedFS_s3_bucket_object_count{namespace=\"$NAMESPACE\"}) by (bucket)",
+          "legendFormat": "{{bucket}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "S3 Bucket Object Count",
+      "type": "timeseries"
    }
  ],
  "refresh": "30s",
@@ -2545,4 +2841,4 @@
  "timezone": "browser",
  "title": "SeaweedFS",
  "version": 2
-}
+}
@@ -0,0 +1,187 @@
+# Makefile for EC integration testing
+# Usage:
+#   make start      - Start the test cluster (master + 6 volume servers + filer)
+#   make stop       - Stop the test cluster
+#   make populate   - Populate test data (~300MB across 7 volumes)
+#   make shell      - Open weed shell connected to the test cluster
+#   make clean      - Stop cluster and remove all test data
+#   make setup      - Start cluster and populate data (one command)
+#
+# Requirements: curl, jq
+
+WEED_BINARY := $(shell pwd)/../../weed/weed
+TEST_DIR := /tmp/ec_manual_test
+# Use non-standard ports to avoid conflicts with existing SeaweedFS servers
+MASTER_PORT := 29333
+FILER_PORT := 28888
+VOLUME_BASE_PORT := 28080
+NUM_VOLUME_SERVERS := 6
+VOLUME_SIZE_LIMIT_MB := 30
+MAX_VOLUMES_PER_SERVER := 10
+
+# Build weed binary if it doesn't exist
+$(WEED_BINARY):
+	cd ../../weed && go build -o weed .
+
+.PHONY: build
+build: $(WEED_BINARY)
+
+.PHONY: start
+start: build
+	@echo "=== Starting SeaweedFS test cluster ==="
+	@mkdir -p $(TEST_DIR)/master $(TEST_DIR)/filer
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do mkdir -p $(TEST_DIR)/volume$$i; done
+	@# Create security.toml with JWT disabled
+	@echo "# Disable JWT for testing" > $(TEST_DIR)/security.toml
+	@echo '[jwt.signing]' >> $(TEST_DIR)/security.toml
+	@echo 'key = ""' >> $(TEST_DIR)/security.toml
+	@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
+	@echo '' >> $(TEST_DIR)/security.toml
+	@echo '[jwt.signing.read]' >> $(TEST_DIR)/security.toml
+	@echo 'key = ""' >> $(TEST_DIR)/security.toml
+	@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
+	@# Create filer.toml with leveldb2
+	@echo '[leveldb2]' > $(TEST_DIR)/filer.toml
+	@echo 'enabled = true' >> $(TEST_DIR)/filer.toml
+	@echo 'dir = "$(TEST_DIR)/filer/filerldb2"' >> $(TEST_DIR)/filer.toml
+	@# Start master
+	@echo "Starting master on port $(MASTER_PORT)..."
+	@cd $(TEST_DIR) && $(WEED_BINARY) master \
+		-port=$(MASTER_PORT) \
+		-mdir=$(TEST_DIR)/master \
+		-volumeSizeLimitMB=$(VOLUME_SIZE_LIMIT_MB) \
+		-ip=127.0.0.1 \
+		> $(TEST_DIR)/master/master.log 2>&1 & echo $$! > $(TEST_DIR)/master.pid
+	@sleep 3
+	@# Start volume servers (run from TEST_DIR to find security.toml)
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
+		port=$$(($(VOLUME_BASE_PORT) + $$i)); \
+		echo "Starting volume server $$i on port $$port (rack$$i)..."; \
+		cd $(TEST_DIR) && $(WEED_BINARY) volume \
+			-port=$$port \
+			-dir=$(TEST_DIR)/volume$$i \
+			-max=$(MAX_VOLUMES_PER_SERVER) \
+			-master=127.0.0.1:$(MASTER_PORT) \
+			-ip=127.0.0.1 \
+			-dataCenter=dc1 \
+			-rack=rack$$i \
+			> $(TEST_DIR)/volume$$i/volume.log 2>&1 & echo $$! > $(TEST_DIR)/volume$$i.pid; \
+	done
+	@sleep 3
+	@# Start filer (run from TEST_DIR to find security.toml)
+	@echo "Starting filer on port $(FILER_PORT)..."
+	@cd $(TEST_DIR) && $(WEED_BINARY) filer \
+		-port=$(FILER_PORT) \
+		-master=127.0.0.1:$(MASTER_PORT) \
+		-ip=127.0.0.1 \
+		> $(TEST_DIR)/filer/filer.log 2>&1 & echo $$! > $(TEST_DIR)/filer.pid
+	@sleep 3
+	@echo ""
+	@echo "=== Cluster started ==="
+	@echo "Master: http://127.0.0.1:$(MASTER_PORT)"
+	@echo "Filer:  http://127.0.0.1:$(FILER_PORT)"
+	@echo "Volume servers: http://127.0.0.1:$(VOLUME_BASE_PORT) - http://127.0.0.1:$$(($(VOLUME_BASE_PORT) + $(NUM_VOLUME_SERVERS) - 1))"
+	@echo ""
+	@echo "Run 'make shell' to open weed shell"
+	@echo "Run 'make populate' to add test data"
+
+.PHONY: stop
+stop:
+	@echo "=== Stopping SeaweedFS test cluster ==="
+	@# Stop filer by PID
+	@-[ -f $(TEST_DIR)/filer.pid ] && kill $$(cat $(TEST_DIR)/filer.pid) 2>/dev/null && rm -f $(TEST_DIR)/filer.pid || true
+	@# Stop volume servers by PID
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
+		[ -f $(TEST_DIR)/volume$$i.pid ] && kill $$(cat $(TEST_DIR)/volume$$i.pid) 2>/dev/null && rm -f $(TEST_DIR)/volume$$i.pid || true; \
+	done
+	@# Stop master by PID
+	@-[ -f $(TEST_DIR)/master.pid ] && kill $$(cat $(TEST_DIR)/master.pid) 2>/dev/null && rm -f $(TEST_DIR)/master.pid || true
+	@# Fallback: use pkill with specific patterns to ensure cleanup
+	@-pkill -f "weed filer.*-master=127.0.0.1:$(MASTER_PORT)" 2>/dev/null || true
+	@-pkill -f "weed volume.*-dir=$(TEST_DIR)/volume" 2>/dev/null || true
+	@-pkill -f "weed master.*-mdir=$(TEST_DIR)/master" 2>/dev/null || true
+	@echo "Cluster stopped."
+
+.PHONY: clean
+clean: stop
+	@echo "Removing test data..."
+	@rm -rf $(TEST_DIR)
+	@echo "Clean complete."
+
+.PHONY: populate
+populate:
+	@echo "=== Populating test data (~300MB) ==="
+	@# Create a 500KB test file template using mktemp for isolation
+	@tmpfile=$$(mktemp) && \
+	dd if=/dev/urandom bs=1024 count=500 of=$$tmpfile 2>/dev/null && \
+	uploaded=0; \
+	for i in $$(seq 1 600); do \
+		response=$$(curl -s "http://127.0.0.1:$(MASTER_PORT)/dir/assign?collection=ectest&replication=000"); \
+		fid=$$(echo $$response | jq -r '.fid'); \
+		url=$$(echo $$response | jq -r '.url'); \
+		if [ "$$fid" != "null" ] && [ -n "$$fid" ]; then \
+			curl -s -F "file=@$$tmpfile;filename=file_$$i.bin" "http://$$url/$$fid" > /dev/null; \
+			uploaded=$$((uploaded + 1)); \
+		fi; \
+		if [ $$((i % 100)) -eq 0 ]; then \
+			echo "Uploaded $$uploaded files..."; \
+		fi; \
+	done; \
+	rm -f $$tmpfile; \
+	echo ""; \
+	echo "=== Data population complete ==="; \
+	echo "Uploaded $$uploaded files (~$$((uploaded * 500 / 1024))MB)"
+	@echo ""
+	@echo "Volume status:"
+	@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
+		'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | "  Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack)"' 2>/dev/null || true
+
+.PHONY: shell
+shell: build
+	@echo "Opening weed shell..."
+	@echo "Commands to try:"
+	@echo "  lock"
+	@echo "  volume.list"
+	@echo "  ec.encode -collection=ectest -quietFor=1s -force"
+	@echo "  ec.balance -collection=ectest"
+	@echo "  unlock"
+	@echo ""
+	@$(WEED_BINARY) shell -master=127.0.0.1:$(MASTER_PORT) -filer=127.0.0.1:$(FILER_PORT)
+
+.PHONY: setup
+setup: clean start
+	@sleep 2
+	@$(MAKE) populate
+
+.PHONY: status
+status:
+	@echo "=== Cluster Status ==="
+	@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
+		'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | "Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack) (\($$server))"' 2>/dev/null | sort -t: -k1 -n || echo "Cluster not running"
+	@echo ""
+	@echo "=== EC Shards ==="
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
+		count=$$(ls $(TEST_DIR)/volume$$i/*.ec[0-9]* 2>/dev/null | wc -l | tr -d ' '); \
+		if [ "$$count" != "0" ]; then \
+			echo "  volume$$i (port $$(($(VOLUME_BASE_PORT) + $$i))): $$count EC shard files"; \
+		fi; \
+	done
+
+.PHONY: help
+help:
+	@echo "EC Integration Test Makefile"
+	@echo ""
+	@echo "Targets:"
+	@echo "  make start     - Start test cluster (master + 6 volume servers + filer)"
+	@echo "  make stop      - Stop test cluster"
+	@echo "  make populate  - Populate ~300MB of test data"
+	@echo "  make shell     - Open weed shell"
+	@echo "  make setup     - Clean, start, and populate (all-in-one)"
+	@echo "  make status    - Show cluster and EC shard status"
+	@echo "  make clean     - Stop cluster and remove all test data"
+	@echo "  make help      - Show this help"
+	@echo ""
+	@echo "Quick start:"
+	@echo "  make setup     # Start cluster and populate data"
+	@echo "  make shell     # Open shell to run EC commands"
+
@@ -78,6 +78,43 @@ go test -v -run TestECEncodingMasterTimingRaceCondition
 go test -v -short
 ```

+## Manual Testing with Makefile
+
+A Makefile is provided for manual EC testing.
+
+**Requirements:** `curl`, `jq` (command-line JSON processor)
+
+```bash
+# Quick start: start cluster and populate data
+make setup
+
+# Open weed shell to run EC commands
+make shell
+
+# Individual targets
+make start      # Start test cluster (master + 6 volume servers + filer)
+make stop       # Stop test cluster
+make populate   # Populate ~300MB of test data
+make status     # Show cluster and EC shard status
+make clean      # Stop cluster and remove all test data
+make help       # Show all targets
+```
+
+### EC Rebalance Limited Slots (Unit Test)
+
+The "no free ec shard slots" issue is tested with a **unit test** that works directly on
+topology data structures without requiring a running cluster.
+
+**Location**: `weed/shell/ec_rebalance_slots_test.go`
+
+Tests included:
+- `TestECRebalanceWithLimitedSlots`: Tests a topology with 6 servers, 7 EC volumes (98 shards)
+- `TestECRebalanceZeroFreeSlots`: Reproduces the exact 0 free slots scenario
+
+**Known Issue**: When volume servers are at capacity (`volumeCount == maxVolumeCount`),
+the rebalance step fails with "no free ec shard slots" instead of recognizing that
+moving shards frees slots on source servers.
+
 ## Test Results

 **With the fix**: Shows "Collecting volume locations for N volumes before EC encoding..." message
@@ -5,13 +5,14 @@ ARG FOUNDATIONDB_VERSION=7.4.5
 ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}

 # Install build dependencies and download prebuilt FoundationDB clients
-RUN apt-get update && apt-get install -y \
+SHELL ["/bin/bash", "-c"]
+RUN set -euo pipefail && \
+    apt-get update && apt-get install -y \
    build-essential \
    git \
    wget \
    ca-certificates \
    && rm -rf /var/lib/apt/lists/* && \
-    set -euo pipefail && \
    case "${FOUNDATIONDB_VERSION}" in \
        "7.4.5") EXPECTED_SHA256="f2176b86b7e1b561c3632b4e6e7efb82e3b8f57c2ff0d0ac4671e742867508aa" ;; \
        *) echo "ERROR: No known ARM64 client checksum for FoundationDB ${FOUNDATIONDB_VERSION}. Please update this Dockerfile." >&2; exit 1 ;; \
@@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y \
    && rm -rf /var/lib/apt/lists/*

 # Install FoundationDB server + client debs with checksum verification
+SHELL ["/bin/bash", "-c"]
 RUN set -euo pipefail && \
    apt-get update && \
    case "${FOUNDATIONDB_VERSION}" in \
@@ -50,6 +50,24 @@ test-benchmark: ## Run performance benchmarks
 	@echo "$(YELLOW)Running FoundationDB performance benchmarks...$(NC)"
 	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb -bench=. ./test/foundationdb/...

+test-benchmark-filer: ## Run filer store benchmarks (batch vs no-batch comparison)
+	@echo "$(YELLOW)Running FoundationDB filer store benchmarks...$(NC)"
+	@echo "$(BLUE)Comparing batched vs non-batched write performance$(NC)"
+	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb \
+		-bench='BenchmarkFoundationDBStore_.*' \
+		-benchmem \
+		-benchtime=5s \
+		./weed/filer/foundationdb/...
+
+test-benchmark-concurrent: ## Run concurrent operation benchmarks
+	@echo "$(YELLOW)Running concurrent operation benchmarks...$(NC)"
+	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb \
+		-bench='BenchmarkFoundationDBStore_Concurrent.*' \
+		-benchmem \
+		-benchtime=10s \
+		-cpu=1,2,4,8 \
+		./weed/filer/foundationdb/...
+
 # ARM64 specific targets (Apple Silicon / M1/M2/M3 Macs)
 setup-arm64: ## Set up ARM64-native FoundationDB cluster (builds from source)
 	@echo "$(YELLOW)Setting up ARM64-native FoundationDB cluster...$(NC)"
@@ -353,12 +353,31 @@ The tests are designed to be reliable in CI environments with:
 Run performance benchmarks:

 ```bash
-make test-benchmark
+make test-benchmark              # Run all benchmarks
+make test-benchmark-filer        # Filer store benchmarks with batch comparison
+make test-benchmark-concurrent   # Concurrent operation benchmarks (varies CPU count)

 # Sample expected results:
-# BenchmarkFoundationDBStore_InsertEntry-8    1000    1.2ms per op
-# BenchmarkFoundationDBStore_FindEntry-8      5000    0.5ms per op  
-# BenchmarkFoundationDBStore_KvOperations-8   2000    0.8ms per op
+# BenchmarkFoundationDBStore_InsertEntry-8           1000    1.2ms per op
+# BenchmarkFoundationDBStore_FindEntry-8             5000    0.5ms per op  
+# BenchmarkFoundationDBStore_KvOperations-8          2000    0.8ms per op
+# BenchmarkFoundationDBStore_InsertEntry_NoBatch-8   1000    1.0ms per op  (optimal for S3)
+# BenchmarkFoundationDBStore_InsertEntry_WithBatch-8 1200    0.9ms per op  (bulk ingestion)
+# BenchmarkFoundationDBStore_ConcurrentInsert_*-8    5000    0.3ms per op  (parallel writes)
+```
+
+### Batch vs Non-Batch Performance
+
+The FoundationDB filer store supports two write modes:
+
+| Mode | Config | Best For | Latency | Throughput |
+|------|--------|----------|---------|------------|
+| **Direct Commit** (default) | `batch_enabled = false` | S3 API, low-latency workloads | ~1-5ms per op | Good |
+| **Batched** | `batch_enabled = true` | Bulk ingestion, high throughput | Variable | Higher |
+
+Run the comparison benchmark:
+```bash
+make test-benchmark-filer
 ```

 ## Contributing
@@ -28,8 +28,8 @@ services:
          echo 'testing:testing@fdb1:4500,fdb2:4502,fdb3:4504' > /var/fdb/config/fdb.cluster
        fi
        # Start FDB processes
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4501 --listen_address=0.0.0.0:4501 --coordination=fdb1:4500 &
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4500 --listen_address=0.0.0.0:4500 --coordination=fdb1:4500 --class=coordination &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4501 --listen_address=0.0.0.0:4501 --coordination=fdb1:4500 &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4500 --listen_address=0.0.0.0:4500 --coordination=fdb1:4500 --class=coordination &
        wait
      "

@@ -59,8 +59,8 @@ services:
        # Wait for cluster file from fdb1
        while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
        # Start FDB processes
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4503 --listen_address=0.0.0.0:4503 --coordination=fdb1:4500 &
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4502 --listen_address=0.0.0.0:4502 --coordination=fdb1:4500 --class=coordination &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4503 --listen_address=0.0.0.0:4503 --coordination=fdb1:4500 &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4502 --listen_address=0.0.0.0:4502 --coordination=fdb1:4500 --class=coordination &
        wait
      "

@@ -90,8 +90,8 @@ services:
        # Wait for cluster file from fdb1
        while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
        # Start FDB processes
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4505 --listen_address=0.0.0.0:4505 --coordination=fdb1:4500 &
-        /usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4504 --listen_address=0.0.0.0:4504 --coordination=fdb1:4500 --class=coordination &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4505 --listen_address=0.0.0.0:4505 --coordination=fdb1:4500 &
+        /usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4504 --listen_address=0.0.0.0:4504 --coordination=fdb1:4500 --class=coordination &
        wait
      "

@@ -1,19 +1,8 @@
-# FoundationDB Filer Configuration
+# FoundationDB Filer Configuration for Testing

 [foundationdb]
 enabled = true
 cluster_file = "/var/fdb/config/fdb.cluster"
-api_version = 740
-timeout = "5s"
-max_retry_delay = "1s"
-directory_prefix = "seaweedfs"
-
-# For testing different configurations
-[foundationdb.test]
-enabled = false
-cluster_file = "/var/fdb/config/fdb.cluster"
-api_version = 740
-timeout = "10s"
-max_retry_delay = "2s"
-directory_prefix = "seaweedfs_test"
-location = "/test"
+# api_version = 740
+# timeout = "5s"
+# directory_prefix = "seaweedfs"
@@ -22,7 +22,7 @@
        <junit.version>4.13.2</junit.version>
        <seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
        <jackson.version>2.18.2</jackson.version> <!-- Upgraded from 2.15.3 -->
-        <netty.version>4.1.125.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
+        <netty.version>4.1.129.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
        <parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
        <parquet.format.version>2.12.0</parquet.format.version>
        <surefire.jvm.args>
@@ -172,7 +172,8 @@ services:
      - -master=seaweedfs-master:9333
      - -ip=seaweedfs-mq-broker
      - -port=17777
-      - -port.pprof=18777
+      - -debug
+      - -debug.port=18777
    depends_on:
      seaweedfs-filer:
        condition: service_healthy
@@ -43,25 +43,25 @@ require (
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go v1.55.8 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.40.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.41.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.31.3 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.18.20 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.5 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect
-	github.com/aws/smithy-go v1.23.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
+	github.com/aws/smithy-go v1.24.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
@@ -229,15 +229,15 @@ require (
 	go.opentelemetry.io/otel/trace v1.38.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/crypto v0.46.0 // indirect
 	golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 // indirect
-	golang.org/x/image v0.33.0 // indirect
-	golang.org/x/net v0.47.0 // indirect
+	golang.org/x/image v0.34.0 // indirect
+	golang.org/x/net v0.48.0 // indirect
 	golang.org/x/oauth2 v0.32.0 // indirect
 	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.38.0 // indirect
-	golang.org/x/term v0.37.0 // indirect
-	golang.org/x/text v0.31.0 // indirect
+	golang.org/x/sys v0.39.0 // indirect
+	golang.org/x/term v0.38.0 // indirect
+	golang.org/x/text v0.32.0 // indirect
 	golang.org/x/time v0.12.0 // indirect
 	google.golang.org/api v0.247.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
@@ -102,44 +102,44 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
-github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
+github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
+github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
 github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
 github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.5 h1:xMo63RlqP3ZZydpJDMBsH9uJ10hgHYfQFIk1cHDXrR4=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.5/go.mod h1:hhbH6oRcou+LpXfA/0vPElh/e0M3aFeOblE1sssAAEk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY=
-github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs=
-github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
-github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 h1:eYnlt6QxnFINKzwxP5/Ucs1vkG7VT3Iezmvfgc2waUw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.7/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
+github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
+github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo=
@@ -536,10 +536,10 @@ github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4
 github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
-github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
-github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
-github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg=
-github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
+github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
+github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
+github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
+github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
 github.com/rclone/rclone v1.71.2 h1:3Jk5xNPFrZhVABRuN/OPvApuZQddpE2tkhYMuEn1Ud4=
 github.com/rclone/rclone v1.71.2/go.mod h1:dCK9FzPDlpkbQJ9M7MmWsmv3X5nibfWe+ogJXu6gSgM=
 github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg=
@@ -708,8 +708,6 @@ go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJr
 go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
-go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
@@ -732,8 +730,8 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
+golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -748,8 +746,8 @@ golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 h1:3yiSh9fhy5/RhCSntf4Sy0Tnx
 golang.org/x/exp v0.0.0-20250811191247-51f88131bc50/go.mod h1:rT6SFzZ7oxADUDx58pcaKFTcZ+inxAa9fTrYx/uVYwg=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.33.0 h1:LXRZRnv1+zGd5XBUVRFmYEphyyKJjQjCRiOuAP3sZfQ=
-golang.org/x/image v0.33.0/go.mod h1:DD3OsTYT9chzuzTQt+zMcOlBHgfoKQb1gry8p76Y1sc=
+golang.org/x/image v0.34.0 h1:33gCkyw9hmwbZJeZkct8XyR11yH889EQt/QH4VmXMn8=
+golang.org/x/image v0.34.0/go.mod h1:2RNFBZRB+vnwwFil8GkMdRvrJOFd1AzdZI6vOY+eJVU=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -774,8 +772,6 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
-golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -815,8 +811,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
+golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -894,8 +890,8 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -908,8 +904,8 @@ golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
-golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
-golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
+golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -925,8 +921,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
+golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -979,8 +975,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
-golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
+golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -181,7 +181,8 @@ services:
      - -ip=seaweedfs-mq-broker
      - -port=17777
      - -logFlushInterval=0
-      - -port.pprof=18777
+      - -debug
+      - -debug.port=18777
    depends_on:
      seaweedfs-filer:
        condition: service_healthy
@@ -25,3 +25,4 @@ test-resume: build-weed
 clean:
 	rm -f ../../weed/weed

+
@@ -47,3 +47,4 @@ go test -v -timeout 5m ./test/metadata_subscribe/...
 - Tests create temporary directories that are cleaned up after completion
 - Tests use ports 9333 (master), 8080 (volume), 8888 (filer)

+
@@ -0,0 +1,126 @@
+# S3 SDK Compatibility Testing
+
+This document describes the SDK versions tested against SeaweedFS S3 API and known compatibility considerations.
+
+## Validated SDK Versions
+
+### Go SDKs
+
+| SDK | Version | Test Location | Status |
+|-----|---------|---------------|--------|
+| AWS SDK v2 for Go | 1.20+ | `test/s3/etag/`, `test/s3/copying/` | ✅ Tested |
+| AWS SDK v1 for Go | 1.x | `test/s3/basic/` | ✅ Tested |
+
+### Java SDKs
+
+| SDK | Version | Test Location | Status |
+|-----|---------|---------------|--------|
+| AWS SDK v2 for Java | 2.20.127+ | `other/java/s3copier/` | ✅ Tested |
+| AWS SDK v1 for Java | 1.12.600+ | `other/java/s3copier/` | ✅ Tested |
+
+### Python SDKs
+
+| SDK | Version | Test Location | Status |
+|-----|---------|---------------|--------|
+| boto3 | 1.x | `test/s3/parquet/` | ✅ Tested |
+| PyArrow S3 | 14+ | `test/s3/parquet/` | ✅ Tested |
+
+## SDK-Specific Considerations
+
+### AWS SDK v2 for Java - ETag Validation
+
+**Issue**: [GitHub #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+
+AWS SDK v2 for Java includes strict ETag validation in `ChecksumsEnabledValidator.validatePutObjectChecksum`. It decodes the ETag as a hexadecimal MD5 hash using `Base16Codec.decode()`.
+
+**Impact**: If the ETag contains non-hexadecimal characters (like `-` in composite format), the SDK fails with:
+```text
+java.lang.IllegalArgumentException: Invalid base 16 character: '-'
+```
+
+**Resolution**: SeaweedFS now correctly returns:
+- **PutObject**: Pure MD5 hex ETag (32 characters) regardless of internal chunking
+- **CompleteMultipartUpload**: Composite ETag (`<md5>-<partcount>`)
+
+**Test Coverage**: `test/s3/etag/` and `other/java/s3copier/ETagValidationTest.java`
+
+### AWS SDK v1 vs v2 Differences
+
+| Feature | SDK v1 | SDK v2 |
+|---------|--------|--------|
+| ETag hex validation | No | Yes (strict) |
+| Checksum validation | Basic | Enhanced |
+| Async support | Limited | Full |
+| Default retry behavior | Lenient | Stricter |
+
+### Large File Handling
+
+SeaweedFS auto-chunks files larger than **8MB** for efficient storage. This is transparent to clients, but previously affected ETag format. The current implementation ensures:
+
+1. Regular `PutObject` (any size): Returns pure MD5 ETag
+2. Multipart upload: Returns composite ETag per AWS S3 specification
+
+## Test Categories by File Size
+
+| Category | Size | Chunks | ETag Format |
+|----------|------|--------|-------------|
+| Small | < 256KB | 1 (inline) | Pure MD5 |
+| Medium | 256KB - 8MB | 1 | Pure MD5 |
+| Large | 8MB - 24MB | 2-3 | Pure MD5 |
+| Extra Large | > 24MB | 4+ | Pure MD5 |
+| Multipart | N/A | Per part | Composite |
+
+## Running SDK Compatibility Tests
+
+### Go Tests
+
+```bash
+# Run all ETag tests
+cd test/s3/etag && make test
+
+# Run large file tests only
+cd test/s3/etag && make test-large
+```
+
+### Java Tests
+
+```bash
+# Run all Java SDK tests
+cd other/java/s3copier && mvn test
+
+# Run only ETag validation tests
+cd other/java/s3copier && mvn test -Dtest=ETagValidationTest
+```
+
+### Python Tests
+
+```bash
+# Run PyArrow S3 tests
+cd test/s3/parquet && make test
+```
+
+## Adding New SDK Tests
+
+When adding tests for new SDKs, ensure:
+
+1. **Large file tests (>8MB)**: Critical for verifying ETag format with auto-chunking
+2. **Multipart upload tests**: Verify composite ETag format
+3. **Checksum validation**: Test SDK-specific checksum validation if applicable
+4. **Document SDK version**: Add to this compatibility matrix
+
+## Known Issues and Workarounds
+
+### Issue: Older SDK Versions
+
+Some very old SDK versions (e.g., AWS SDK v1 for Java < 1.11.x) may have different behavior. Testing with the versions listed above is recommended.
+
+### Issue: Custom Checksum Algorithms
+
+AWS SDK v2 supports SHA-256 and CRC32 checksums in addition to MD5. SeaweedFS currently returns MD5-based ETags. For checksums other than MD5, use the `x-amz-checksum-*` headers.
+
+## References
+
+- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
+- [AWS SDK v2 Migration Guide](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/migration.html)
+- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+
@@ -0,0 +1,50 @@
+# ETag Format Integration Tests
+#
+# These tests verify S3 ETag format compatibility, particularly for large files
+# that trigger SeaweedFS auto-chunking. This addresses GitHub Issue #7768.
+#
+# Prerequisites:
+#   - SeaweedFS running with S3 API enabled on port 8333
+#   - Go 1.21+
+#
+# Usage:
+#   make test          - Run all tests
+#   make test-large    - Run only large file tests
+#   make test-verbose  - Run with verbose output
+#   make clean         - Clean test artifacts
+
+.PHONY: all test test-large test-verbose test-quick clean help
+
+# Default S3 endpoint
+S3_ENDPOINT ?= http://127.0.0.1:8333
+
+all: test
+
+test:
+	@echo "Running ETag format tests against $(S3_ENDPOINT)..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m ./...
+
+test-large:
+	@echo "Running large file ETag tests..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m -run "LargeFile|ExtraLarge" ./...
+
+test-verbose:
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m -count=1 ./...
+
+test-quick:
+	@echo "Running quick ETag tests (small files only)..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 1m -run "SmallFile|Consistency" ./...
+
+clean:
+	go clean -testcache
+
+help:
+	@echo "ETag Format Integration Tests"
+	@echo "Targets:"
+	@echo "  test          Run all ETag format tests"
+	@echo "  test-large    Run only large file tests (>8MB)"
+	@echo "  test-quick    Run quick tests (small files only)"
+	@echo "  test-verbose  Run with verbose output"
+	@echo "  clean         Clean test cache"
+	@echo "Environment Variables:"
+	@echo "  S3_ENDPOINT   S3 endpoint URL (default: http://127.0.0.1:8333)"
@@ -0,0 +1,93 @@
+# S3 ETag Format Integration Tests
+
+This test suite verifies that SeaweedFS returns correct ETag formats for S3 operations, ensuring compatibility with AWS S3 SDKs.
+
+## Background
+
+**GitHub Issue #7768**: AWS S3 SDK for Java v2 was failing with `Invalid base 16 character: '-'` when performing `PutObject` on large files.
+
+### Root Cause
+
+SeaweedFS internally auto-chunks large files (>8MB) for efficient storage. Previously, when a regular `PutObject` request resulted in multiple internal chunks, SeaweedFS returned a composite ETag format (`<md5>-<count>`) instead of a pure MD5 hash.
+
+### AWS S3 Specification
+
+| Operation | ETag Format | Example |
+|-----------|-------------|---------|
+| PutObject (any size) | Pure MD5 hex (32 chars) | `d41d8cd98f00b204e9800998ecf8427e` |
+| CompleteMultipartUpload | Composite (`<md5>-<partcount>`) | `d41d8cd98f00b204e9800998ecf8427e-3` |
+
+AWS S3 SDK v2 for Java validates `PutObject` ETags as hexadecimal, which fails when the ETag contains a hyphen.
+
+## Test Coverage
+
+| Test | File Size | Purpose |
+|------|-----------|---------|
+| `TestPutObjectETagFormat_SmallFile` | 1KB | Verify single-chunk uploads return pure MD5 |
+| `TestPutObjectETagFormat_LargeFile` | 10MB | **Critical**: Verify auto-chunked uploads return pure MD5 |
+| `TestPutObjectETagFormat_ExtraLargeFile` | 25MB | Verify multi-chunk auto-chunked uploads return pure MD5 |
+| `TestMultipartUploadETagFormat` | 15MB | Verify multipart uploads correctly return composite ETag |
+| `TestPutObjectETagConsistency` | Various | Verify ETag consistency across PUT/HEAD/GET |
+| `TestETagHexValidation` | 10MB | Simulate AWS SDK v2 hex validation |
+| `TestMultipleLargeFileUploads` | 10MB x5 | Stress test multiple large uploads |
+
+## Prerequisites
+
+1. SeaweedFS running with S3 API enabled:
+   ```bash
+   weed server -s3
+   ```
+
+2. Go 1.21 or later
+
+3. AWS SDK v2 for Go (installed via go modules)
+
+## Running Tests
+
+```bash
+# Run all tests
+make test
+
+# Run only large file tests (the critical ones for issue #7768)
+make test-large
+
+# Run quick tests (small files only)
+make test-quick
+
+# Run with verbose output
+make test-verbose
+```
+
+## Configuration
+
+By default, tests connect to `http://127.0.0.1:8333`. To use a different endpoint:
+
+```bash
+S3_ENDPOINT=http://localhost:8333 make test
+```
+
+Or modify `defaultConfig` in `s3_etag_test.go`.
+
+## SDK Compatibility
+
+These tests use **AWS SDK v2 for Go**, which has the same ETag validation behavior as AWS SDK v2 for Java. The tests include:
+
+- ETag format validation (pure MD5 vs composite)
+- Hex decoding validation (simulates `Base16Codec.decode`)
+- Content integrity verification
+
+## Validated SDK Versions
+
+| SDK | Version | Status |
+|-----|---------|--------|
+| AWS SDK v2 for Go | 1.20+ | ✅ Tested |
+| AWS SDK v2 for Java | 2.20+ | ✅ Compatible (issue #7768 fixed) |
+| AWS SDK v1 for Go | 1.x | ✅ Compatible (less strict validation) |
+| AWS SDK v1 for Java | 1.x | ✅ Compatible (less strict validation) |
+
+## Related
+
+- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
+
+
@@ -0,0 +1,543 @@
+// Package etag_test provides integration tests for S3 ETag format validation.
+//
+// These tests verify that SeaweedFS returns correct ETag formats for different
+// upload scenarios, ensuring compatibility with AWS S3 SDKs that validate ETags.
+//
+// Background (GitHub Issue #7768):
+// AWS S3 SDK for Java v2 validates ETags as hexadecimal MD5 hashes for PutObject
+// responses. SeaweedFS was incorrectly returning composite ETags ("<md5>-<count>")
+// for regular PutObject when files were internally auto-chunked (>8MB), causing
+// the SDK to fail with "Invalid base 16 character: '-'".
+//
+// Per AWS S3 specification:
+// - Regular PutObject: ETag is always a pure MD5 hex string (32 chars)
+// - Multipart Upload (CompleteMultipartUpload): ETag is "<md5>-<partcount>"
+//
+// These tests ensure this behavior is maintained.
+package etag_test
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+	mathrand "math/rand"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// S3TestConfig holds configuration for S3 tests
+type S3TestConfig struct {
+	Endpoint     string
+	AccessKey    string
+	SecretKey    string
+	Region       string
+	BucketPrefix string
+}
+
+// Default test configuration
+var defaultConfig = &S3TestConfig{
+	Endpoint:     "http://127.0.0.1:8333",
+	AccessKey:    "some_access_key1",
+	SecretKey:    "some_secret_key1",
+	Region:       "us-east-1",
+	BucketPrefix: "test-etag-",
+}
+
+// Constants for auto-chunking thresholds (must match s3api_object_handlers_put.go)
+const (
+	// SeaweedFS auto-chunks files larger than 8MB
+	autoChunkSize = 8 * 1024 * 1024
+
+	// Test sizes
+	smallFileSize  = 1 * 1024              // 1KB - single chunk
+	mediumFileSize = 256 * 1024            // 256KB - single chunk (at threshold)
+	largeFileSize  = 10 * 1024 * 1024      // 10MB - triggers auto-chunking (2 chunks)
+	xlFileSize     = 25 * 1024 * 1024      // 25MB - triggers auto-chunking (4 chunks)
+	multipartSize  = 5 * 1024 * 1024       // 5MB per part for multipart uploads
+)
+
+// ETag format patterns
+var (
+	// Pure MD5 ETag: 32 hex characters (with or without quotes)
+	pureMD5Pattern = regexp.MustCompile(`^"?[a-f0-9]{32}"?$`)
+
+	// Composite ETag for multipart: 32 hex chars, hyphen, part count (with or without quotes)
+	compositePattern = regexp.MustCompile(`^"?[a-f0-9]{32}-\d+"?$`)
+)
+
+func init() {
+	mathrand.Seed(time.Now().UnixNano())
+}
+
+// getS3Client creates an AWS S3 v2 client for testing
+func getS3Client(t *testing.T) *s3.Client {
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(defaultConfig.Region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			defaultConfig.AccessKey,
+			defaultConfig.SecretKey,
+			"",
+		)),
+		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
+			func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+				return aws.Endpoint{
+					URL:               defaultConfig.Endpoint,
+					SigningRegion:     defaultConfig.Region,
+					HostnameImmutable: true,
+				}, nil
+			})),
+	)
+	require.NoError(t, err)
+
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+}
+
+// getNewBucketName generates a unique bucket name
+func getNewBucketName() string {
+	timestamp := time.Now().UnixNano()
+	randomSuffix := mathrand.Intn(100000)
+	return fmt.Sprintf("%s%d-%d", defaultConfig.BucketPrefix, timestamp, randomSuffix)
+}
+
+// generateRandomData generates random test data of specified size
+func generateRandomData(size int) []byte {
+	data := make([]byte, size)
+	if _, err := rand.Read(data); err != nil {
+		panic(fmt.Sprintf("failed to generate random test data: %v", err))
+	}
+	return data
+}
+
+// calculateMD5 calculates the MD5 hash of data and returns hex string
+func calculateMD5(data []byte) string {
+	hash := md5.Sum(data)
+	return hex.EncodeToString(hash[:])
+}
+
+// cleanETag removes quotes from ETag
+func cleanETag(etag string) string {
+	return strings.Trim(etag, `"`)
+}
+
+// isPureMD5ETag checks if ETag is a pure MD5 hex string (no composite format)
+func isPureMD5ETag(etag string) bool {
+	return pureMD5Pattern.MatchString(etag)
+}
+
+// isCompositeETag checks if ETag is in composite format (md5-count)
+func isCompositeETag(etag string) bool {
+	return compositePattern.MatchString(etag)
+}
+
+// createTestBucket creates a new bucket for testing
+func createTestBucket(ctx context.Context, client *s3.Client, bucketName string) error {
+	_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	return err
+}
+
+// cleanupTestBucket deletes all objects and the bucket
+func cleanupTestBucket(ctx context.Context, client *s3.Client, bucketName string) {
+	// Delete all objects
+	paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+	})
+
+	for paginator.HasMorePages() {
+		page, err := paginator.NextPage(ctx)
+		if err != nil {
+			break
+		}
+		for _, obj := range page.Contents {
+			client.DeleteObject(ctx, &s3.DeleteObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    obj.Key,
+			})
+		}
+	}
+
+	// Abort any in-progress multipart uploads
+	mpPaginator := s3.NewListMultipartUploadsPaginator(client, &s3.ListMultipartUploadsInput{
+		Bucket: aws.String(bucketName),
+	})
+	for mpPaginator.HasMorePages() {
+		page, err := mpPaginator.NextPage(ctx)
+		if err != nil {
+			break
+		}
+		for _, upload := range page.Uploads {
+			client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
+				Bucket:   aws.String(bucketName),
+				Key:      upload.Key,
+				UploadId: upload.UploadId,
+			})
+		}
+	}
+
+	// Delete bucket
+	client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+}
+
+// TestPutObjectETagFormat_SmallFile verifies ETag format for small files (single chunk)
+func TestPutObjectETagFormat_SmallFile(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testData := generateRandomData(smallFileSize)
+	expectedMD5 := calculateMD5(testData)
+	objectKey := "small-file.bin"
+
+	// Upload small file
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload small file")
+
+	// Verify ETag format
+	etag := aws.ToString(putResp.ETag)
+	t.Logf("Small file (%d bytes) ETag: %s", smallFileSize, etag)
+
+	assert.True(t, isPureMD5ETag(etag),
+		"Small file ETag should be pure MD5, got: %s", etag)
+	assert.False(t, isCompositeETag(etag),
+		"Small file ETag should NOT be composite format, got: %s", etag)
+	assert.Equal(t, expectedMD5, cleanETag(etag),
+		"ETag should match calculated MD5")
+}
+
+// TestPutObjectETagFormat_LargeFile verifies ETag format for large files that trigger auto-chunking
+// This is the critical test for GitHub Issue #7768
+func TestPutObjectETagFormat_LargeFile(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testData := generateRandomData(largeFileSize)
+	expectedMD5 := calculateMD5(testData)
+	objectKey := "large-file.bin"
+
+	t.Logf("Uploading large file (%d bytes, > %d byte auto-chunk threshold)...",
+		largeFileSize, autoChunkSize)
+
+	// Upload large file (triggers auto-chunking internally)
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload large file")
+
+	// Verify ETag format - MUST be pure MD5, NOT composite
+	etag := aws.ToString(putResp.ETag)
+	t.Logf("Large file (%d bytes, ~%d internal chunks) ETag: %s",
+		largeFileSize, (largeFileSize/autoChunkSize)+1, etag)
+
+	assert.True(t, isPureMD5ETag(etag),
+		"Large file PutObject ETag MUST be pure MD5 (not composite), got: %s", etag)
+	assert.False(t, isCompositeETag(etag),
+		"Large file PutObject ETag should NOT contain '-' (composite format), got: %s", etag)
+	assert.False(t, strings.Contains(cleanETag(etag), "-"),
+		"ETag should not contain hyphen for regular PutObject, got: %s", etag)
+	assert.Equal(t, expectedMD5, cleanETag(etag),
+		"ETag should match calculated MD5 of entire content")
+
+	// Verify we can read back the object correctly
+	getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to get large file")
+	defer getResp.Body.Close()
+
+	downloadedData, err := io.ReadAll(getResp.Body)
+	require.NoError(t, err, "Failed to read large file content")
+	assert.Equal(t, testData, downloadedData, "Downloaded content should match uploaded content")
+}
+
+// TestPutObjectETagFormat_ExtraLargeFile tests even larger files with multiple internal chunks
+func TestPutObjectETagFormat_ExtraLargeFile(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testData := generateRandomData(xlFileSize)
+	expectedMD5 := calculateMD5(testData)
+	objectKey := "xl-file.bin"
+
+	expectedChunks := (xlFileSize / autoChunkSize) + 1
+	t.Logf("Uploading XL file (%d bytes, expected ~%d internal chunks)...",
+		xlFileSize, expectedChunks)
+
+	// Upload extra large file
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload XL file")
+
+	// Verify ETag format
+	etag := aws.ToString(putResp.ETag)
+	t.Logf("XL file (%d bytes) ETag: %s", xlFileSize, etag)
+
+	assert.True(t, isPureMD5ETag(etag),
+		"XL file PutObject ETag MUST be pure MD5, got: %s", etag)
+	assert.False(t, isCompositeETag(etag),
+		"XL file PutObject ETag should NOT be composite, got: %s", etag)
+	assert.Equal(t, expectedMD5, cleanETag(etag),
+		"ETag should match calculated MD5")
+}
+
+// TestMultipartUploadETagFormat verifies that ONLY multipart uploads get composite ETags
+func TestMultipartUploadETagFormat(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	// Create test data for multipart upload (15MB = 3 parts of 5MB each)
+	totalSize := 15 * 1024 * 1024
+	testData := generateRandomData(totalSize)
+	objectKey := "multipart-file.bin"
+
+	expectedPartCount := (totalSize + multipartSize - 1) / multipartSize // ceiling division
+	t.Logf("Performing multipart upload (%d bytes, %d parts)...",
+		totalSize, expectedPartCount)
+
+	// Initiate multipart upload
+	createResp, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	uploadId := createResp.UploadId
+	var completedParts []types.CompletedPart
+	partNumber := int32(1)
+
+	// Upload parts
+	for offset := 0; offset < totalSize; offset += multipartSize {
+		end := offset + multipartSize
+		if end > totalSize {
+			end = totalSize
+		}
+		partData := testData[offset:end]
+
+		uploadResp, err := client.UploadPart(ctx, &s3.UploadPartInput{
+			Bucket:     aws.String(bucketName),
+			Key:        aws.String(objectKey),
+			UploadId:   uploadId,
+			PartNumber: aws.Int32(partNumber),
+			Body:       bytes.NewReader(partData),
+		})
+		require.NoError(t, err, "Failed to upload part %d", partNumber)
+
+		completedParts = append(completedParts, types.CompletedPart{
+			ETag:       uploadResp.ETag,
+			PartNumber: aws.Int32(partNumber),
+		})
+		partNumber++
+	}
+
+	// Complete multipart upload
+	completeResp, err := client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: uploadId,
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: completedParts,
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+
+	// Verify ETag format - SHOULD be composite for multipart
+	etag := aws.ToString(completeResp.ETag)
+	t.Logf("Multipart upload ETag: %s", etag)
+
+	assert.True(t, isCompositeETag(etag),
+		"Multipart upload ETag SHOULD be composite format (md5-count), got: %s", etag)
+	assert.True(t, strings.Contains(cleanETag(etag), "-"),
+		"Multipart ETag should contain hyphen, got: %s", etag)
+
+	// Verify the part count in the ETag matches
+	parts := strings.Split(cleanETag(etag), "-")
+	require.Len(t, parts, 2, "Composite ETag should have format 'hash-count'")
+	assert.Equal(t, fmt.Sprintf("%d", len(completedParts)), parts[1],
+		"Part count in ETag should match number of parts uploaded")
+}
+
+// TestPutObjectETagConsistency verifies ETag consistency between PUT and GET
+func TestPutObjectETagConsistency(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testCases := []struct {
+		name string
+		size int
+	}{
+		{"tiny", 100},
+		{"small", smallFileSize},
+		{"medium", mediumFileSize},
+		{"large", largeFileSize},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			testData := generateRandomData(tc.size)
+			objectKey := fmt.Sprintf("consistency-test-%s.bin", tc.name)
+
+			// PUT object
+			putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+				Body:   bytes.NewReader(testData),
+			})
+			require.NoError(t, err)
+			putETag := aws.ToString(putResp.ETag)
+
+			// HEAD object
+			headResp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+			})
+			require.NoError(t, err)
+			headETag := aws.ToString(headResp.ETag)
+
+			// GET object
+			getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+			})
+			require.NoError(t, err)
+			getETag := aws.ToString(getResp.ETag)
+			getResp.Body.Close()
+
+			// All ETags should match
+			t.Logf("%s (%d bytes): PUT=%s, HEAD=%s, GET=%s",
+				tc.name, tc.size, putETag, headETag, getETag)
+
+			assert.Equal(t, putETag, headETag,
+				"PUT and HEAD ETags should match")
+			assert.Equal(t, putETag, getETag,
+				"PUT and GET ETags should match")
+
+			// All should be pure MD5 (not composite) for regular PutObject
+			assert.True(t, isPureMD5ETag(putETag),
+				"PutObject ETag should be pure MD5, got: %s", putETag)
+		})
+	}
+}
+
+// TestETagHexValidation simulates the AWS SDK v2 validation that caused issue #7768
+func TestETagHexValidation(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	// Test with a file large enough to trigger auto-chunking
+	testData := generateRandomData(largeFileSize)
+	objectKey := "hex-validation-test.bin"
+
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err)
+
+	etag := cleanETag(aws.ToString(putResp.ETag))
+
+	// Simulate AWS SDK v2's hex validation (Base16Codec.decode)
+	// This is what fails in issue #7768 when ETag contains '-'
+	t.Logf("Validating ETag as hex: %s", etag)
+
+	_, err = hex.DecodeString(etag)
+	assert.NoError(t, err,
+		"ETag should be valid hexadecimal (AWS SDK v2 validation). "+
+			"Got ETag: %s. If this fails with 'invalid byte', the ETag contains non-hex chars like '-'",
+		etag)
+}
+
+// TestMultipleLargeFileUploads verifies ETag format across multiple large uploads
+func TestMultipleLargeFileUploads(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	numFiles := 5
+	for i := 0; i < numFiles; i++ {
+		testData := generateRandomData(largeFileSize)
+		expectedMD5 := calculateMD5(testData)
+		objectKey := fmt.Sprintf("large-file-%d.bin", i)
+
+		putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(objectKey),
+			Body:   bytes.NewReader(testData),
+		})
+		require.NoError(t, err, "Failed to upload file %d", i)
+
+		etag := aws.ToString(putResp.ETag)
+		t.Logf("File %d ETag: %s (expected MD5: %s)", i, etag, expectedMD5)
+
+		assert.True(t, isPureMD5ETag(etag),
+			"File %d ETag should be pure MD5, got: %s", i, etag)
+		assert.Equal(t, expectedMD5, cleanETag(etag),
+			"File %d ETag should match MD5", i)
+
+		// Validate as hex (AWS SDK v2 check)
+		_, err = hex.DecodeString(cleanETag(etag))
+		assert.NoError(t, err, "File %d ETag should be valid hex", i)
+	}
+}
+
@@ -0,0 +1,20 @@
+{
+  "endpoint": "http://127.0.0.1:8333",
+  "access_key": "some_access_key1",
+  "secret_key": "some_secret_key1",
+  "region": "us-east-1",
+  "bucket_prefix": "test-etag-",
+  "notes": {
+    "description": "S3 ETag format integration tests",
+    "issue": "https://github.com/seaweedfs/seaweedfs/issues/7768",
+    "auto_chunk_size": "8MB - files larger than this trigger auto-chunking",
+    "test_sizes": {
+      "small": "1KB - single chunk",
+      "medium": "256KB - at inline threshold", 
+      "large": "10MB - 2 internal chunks",
+      "xl": "25MB - 4 internal chunks"
+    }
+  }
+}
+
+
@@ -0,0 +1,166 @@
+# S3 Filer Group Test Makefile
+# This Makefile provides targets for running S3 filer group integration tests
+# These tests verify that S3 bucket operations work correctly when a filer group is configured.
+
+.PHONY: help build-weed check-deps start-server stop-server test test-with-server clean logs health-check
+
+# Configuration
+WEED_BINARY := ../../../weed/weed_binary
+S3_PORT := 8333
+MASTER_PORT := 9333
+MASTER_GRPC_PORT := 19333
+VOLUME_PORT := 8080
+FILER_PORT := 8888
+TEST_TIMEOUT := 10m
+TEST_PATTERN := Test
+
+# Filer group configuration
+FILER_GROUP := testgroup
+
+# Default target
+help:
+	@echo "S3 Filer Group Test Makefile"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  help                    - Show this help message"
+	@echo "  build-weed             - Build the SeaweedFS binary"
+	@echo "  check-deps             - Check dependencies and build binary if needed"
+	@echo "  start-server           - Start SeaweedFS server with filer group"
+	@echo "  stop-server            - Stop SeaweedFS server"
+	@echo "  test                   - Run filer group tests (server must be running)"
+	@echo "  test-with-server       - Start server, run tests, stop server"
+	@echo "  logs                   - Show server logs"
+	@echo "  clean                  - Clean up test artifacts and stop server"
+	@echo "  health-check           - Check if server is accessible"
+	@echo ""
+	@echo "Configuration:"
+	@echo "  S3_PORT=${S3_PORT}"
+	@echo "  MASTER_GRPC_PORT=${MASTER_GRPC_PORT}"
+	@echo "  FILER_GROUP=${FILER_GROUP}"
+	@echo "  TEST_TIMEOUT=${TEST_TIMEOUT}"
+
+# Build the SeaweedFS binary
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd ../../../weed && go build -o weed_binary .
+	@chmod +x $(WEED_BINARY)
+	@echo "✅ SeaweedFS binary built at $(WEED_BINARY)"
+
+check-deps: build-weed
+	@echo "Checking dependencies..."
+	@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
+	@echo "Go version: $$(go version)"
+	@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
+	@go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1)
+	@go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1)
+	@echo "✅ All dependencies are available"
+
+# Start SeaweedFS server with filer group configured
+start-server: check-deps
+	@echo "Starting SeaweedFS server with filer group: $(FILER_GROUP)..."
+	@rm -f weed-server.pid
+	@mkdir -p ./test-volume-data
+	@if netstat -tlnp 2>/dev/null | grep $(S3_PORT) >/dev/null; then \
+		echo "⚠️  Port $(S3_PORT) is already in use"; \
+		exit 1; \
+	fi
+	@echo "Launching SeaweedFS server with filer group $(FILER_GROUP)..."
+	@$(WEED_BINARY) server \
+		-debug \
+		-s3 \
+		-s3.port=$(S3_PORT) \
+		-s3.allowDeleteBucketNotEmpty=true \
+		-s3.config=../../../docker/compose/s3.json \
+		-filer \
+		-filer.maxMB=64 \
+		-filer.filerGroup=$(FILER_GROUP) \
+		-master.volumeSizeLimitMB=50 \
+		-volume.max=100 \
+		-dir=./test-volume-data \
+		-volume.preStopSeconds=1 \
+		-metricsPort=9325 \
+		> weed-test.log 2>&1 & echo $$! > weed-server.pid
+	@echo "Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
+	@echo "Waiting for server to start (up to 90 seconds)..."
+	@for i in $$(seq 1 90); do \
+		if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
+			echo "✅ SeaweedFS server started successfully on port $(S3_PORT) with filer group $(FILER_GROUP)"; \
+			exit 0; \
+		fi; \
+		if [ $$i -eq 30 ]; then \
+			echo "⚠️  Server taking longer than expected (30s), checking logs..."; \
+			if [ -f weed-test.log ]; then \
+				tail -20 weed-test.log; \
+			fi; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "❌ Server failed to start within 90 seconds"; \
+	if [ -f weed-test.log ]; then \
+		cat weed-test.log; \
+	fi; \
+	exit 1
+
+# Stop SeaweedFS server
+stop-server:
+	@echo "Stopping SeaweedFS server..."
+	@if [ -f weed-server.pid ]; then \
+		SERVER_PID=$$(cat weed-server.pid); \
+		echo "Killing server PID $$SERVER_PID"; \
+		if ps -p $$SERVER_PID >/dev/null 2>&1; then \
+			kill -TERM $$SERVER_PID 2>/dev/null || true; \
+			sleep 2; \
+			if ps -p $$SERVER_PID >/dev/null 2>&1; then \
+				echo "Process still running, sending KILL signal..."; \
+				kill -KILL $$SERVER_PID 2>/dev/null || true; \
+				sleep 1; \
+			fi; \
+		else \
+			echo "Process $$SERVER_PID not found (already stopped)"; \
+		fi; \
+		rm -f weed-server.pid; \
+	else \
+		echo "No PID file found"; \
+	fi
+	@echo "✅ SeaweedFS server stopped"
+
+# Show server logs
+logs:
+	@if test -f weed-test.log; then \
+		echo "=== SeaweedFS Server Logs ==="; \
+		tail -f weed-test.log; \
+	else \
+		echo "No log file found. Server may not be running."; \
+	fi
+
+# Run filer group tests (assumes server is already running)
+test: check-deps
+	@echo "Running filer group S3 tests..."
+	@FILER_GROUP=$(FILER_GROUP) S3_ENDPOINT=http://localhost:$(S3_PORT) MASTER_ADDRESS=localhost:$(MASTER_GRPC_PORT) \
+		go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" .
+	@echo "✅ Filer group tests completed"
+
+# Run tests with automatic server management
+test-with-server: start-server
+	@echo "Server started successfully, running filer group tests..."
+	@echo "Test pattern: $(TEST_PATTERN)"
+	@echo "Test timeout: $(TEST_TIMEOUT)"
+	@trap "$(MAKE) stop-server" EXIT; \
+	$(MAKE) test || (echo "❌ Tests failed, showing server logs:" && echo "=== Last 50 lines of server logs ===" && tail -50 weed-test.log && echo "=== End of server logs ===" && exit 1)
+	@$(MAKE) stop-server
+	@echo "✅ Tests completed and server stopped"
+
+# Clean up test artifacts
+clean:
+	@echo "Cleaning up test artifacts..."
+	@$(MAKE) stop-server
+	@rm -f weed-test*.log weed-server.pid
+	@rm -rf test-volume-data/
+	@go clean -testcache
+	@echo "✅ Cleanup completed"
+
+# Quick health check
+health-check:
+	@echo "Running health check..."
+	@curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1 && echo "✅ S3 API is accessible" || echo "❌ S3 API is not accessible"
+	@curl -s http://localhost:9325/metrics >/dev/null 2>&1 && echo "✅ Metrics endpoint is accessible" || echo "❌ Metrics endpoint is not accessible"
@@ -0,0 +1,99 @@
+# Filer Group S3 Tests
+
+These tests verify that S3 bucket operations work correctly when a filer group is configured.
+
+## Background
+
+When SeaweedFS is configured with a filer group (via `-filer.group` option), collections are named with the filer group prefix:
+
+```text
+Collection name = {filerGroup}_{bucketName}
+```
+
+For example, with filer group `mygroup` and bucket `mybucket`, the collection will be named `mygroup_mybucket`.
+
+## Issue Being Tested
+
+This test suite was created to verify the fix for a bug where:
+- The admin UI was using just the bucket name when deleting collections
+- This caused collection deletion to fail when a filer group was configured
+- After bucket deletion via admin UI, the collection data would be orphaned
+
+## Running the Tests
+
+### Prerequisites
+
+1. SeaweedFS servers must be running with a filer group configured
+2. The S3 gateway must be accessible
+3. Master server must be accessible for collection verification
+
+### Quick Start
+
+```bash
+# Set environment variables
+export FILER_GROUP=testgroup
+export S3_ENDPOINT=http://localhost:8333
+export MASTER_ADDRESS=localhost:9333
+
+# Run tests
+go test -v ./...
+```
+
+### Using the Makefile
+
+```bash
+# Start test servers with filer group configured
+make start-servers FILER_GROUP=testgroup
+
+# Run tests
+make test
+
+# Stop servers
+make stop-servers
+
+# Or run full test cycle
+make full-test
+```
+
+### Configuration
+
+Tests can be configured via:
+
+1. Environment variables:
+   - `FILER_GROUP`: The filer group name (required for tests to run)
+   - `S3_ENDPOINT`: S3 API endpoint (default: `http://localhost:8333`)
+   - `MASTER_ADDRESS`: Master server address (default: localhost:9333)
+
+2. `test_config.json` file
+
+## Test Cases
+
+### TestFilerGroupCollectionNaming
+Verifies that when a bucket is created and objects are uploaded:
+1. The collection is created with the correct filer group prefix
+2. Bucket deletion removes the correctly-named collection
+
+### TestBucketDeletionWithFilerGroup
+Specifically tests that bucket deletion via S3 API correctly deletes
+the collection when filer group is configured.
+
+### TestMultipleBucketsWithFilerGroup
+Tests creating and deleting multiple buckets to ensure the filer group
+prefix is correctly applied and removed for all buckets.
+
+## Expected Behavior
+
+With filer group `testgroup`:
+
+| Bucket Name | Expected Collection Name |
+|-------------|-------------------------|
+| mybucket    | testgroup_mybucket      |
+| test-123    | testgroup_test-123      |
+
+Without filer group:
+
+| Bucket Name | Expected Collection Name |
+|-------------|-------------------------|
+| mybucket    | mybucket                |
+| test-123    | test-123                |
+
@@ -0,0 +1,332 @@
+package filer_group
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// TestConfig holds configuration for filer group S3 tests
+type TestConfig struct {
+	S3Endpoint    string `json:"s3_endpoint"`
+	MasterAddress string `json:"master_address"`
+	AccessKey     string `json:"access_key"`
+	SecretKey     string `json:"secret_key"`
+	Region        string `json:"region"`
+	FilerGroup    string `json:"filer_group"`
+}
+
+var testConfig = &TestConfig{
+	S3Endpoint:    "http://localhost:8333",
+	MasterAddress: "localhost:19333", // gRPC port = 10000 + master HTTP port (9333)
+	AccessKey:     "some_access_key1",
+	SecretKey:     "some_secret_key1",
+	Region:        "us-east-1",
+	FilerGroup:    "testgroup", // Expected filer group for these tests
+}
+
+func init() {
+	// Load config from file if exists
+	if data, err := os.ReadFile("test_config.json"); err == nil {
+		if err := json.Unmarshal(data, testConfig); err != nil {
+			// Log but don't fail - env vars can still override
+			fmt.Fprintf(os.Stderr, "Warning: failed to parse test_config.json: %v\n", err)
+		}
+	}
+
+	// Override from environment variables
+	if v := os.Getenv("S3_ENDPOINT"); v != "" {
+		testConfig.S3Endpoint = v
+	}
+	if v := os.Getenv("MASTER_ADDRESS"); v != "" {
+		testConfig.MasterAddress = v
+	}
+	if v := os.Getenv("FILER_GROUP"); v != "" {
+		testConfig.FilerGroup = v
+	}
+}
+
+func getS3Client(t *testing.T) *s3.Client {
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(testConfig.Region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			testConfig.AccessKey,
+			testConfig.SecretKey,
+			"",
+		)),
+	)
+	require.NoError(t, err)
+
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.BaseEndpoint = aws.String(testConfig.S3Endpoint)
+		o.UsePathStyle = true
+	})
+}
+
+func getMasterClient(t *testing.T) master_pb.SeaweedClient {
+	conn, err := grpc.NewClient(testConfig.MasterAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.NoError(t, err)
+	t.Cleanup(func() { conn.Close() })
+	return master_pb.NewSeaweedClient(conn)
+}
+
+func getNewBucketName() string {
+	return fmt.Sprintf("filergroup-test-%d", time.Now().UnixNano())
+}
+
+// getExpectedCollectionName returns the expected collection name for a bucket
+// When filer group is configured, collections are named: {filerGroup}_{bucketName}
+func getExpectedCollectionName(bucketName string) string {
+	if testConfig.FilerGroup != "" {
+		return fmt.Sprintf("%s_%s", testConfig.FilerGroup, bucketName)
+	}
+	return bucketName
+}
+
+// listAllCollections returns a list of all collection names from the master
+func listAllCollections(t *testing.T, masterClient master_pb.SeaweedClient) []string {
+	collectionResp, err := masterClient.CollectionList(context.Background(), &master_pb.CollectionListRequest{
+		IncludeNormalVolumes: true,
+		IncludeEcVolumes:     true,
+	})
+	if err != nil {
+		t.Logf("Warning: failed to list collections: %v", err)
+		return nil
+	}
+	var names []string
+	for _, c := range collectionResp.Collections {
+		names = append(names, c.Name)
+	}
+	return names
+}
+
+// collectionExists checks if a collection exists in the master
+func collectionExists(t *testing.T, masterClient master_pb.SeaweedClient, collectionName string) bool {
+	for _, name := range listAllCollections(t, masterClient) {
+		if name == collectionName {
+			return true
+		}
+	}
+	return false
+}
+
+// waitForCollectionExists waits for a collection to exist using polling
+func waitForCollectionExists(t *testing.T, masterClient master_pb.SeaweedClient, collectionName string) {
+	var lastCollections []string
+	success := assert.Eventually(t, func() bool {
+		lastCollections = listAllCollections(t, masterClient)
+		for _, name := range lastCollections {
+			if name == collectionName {
+				return true
+			}
+		}
+		return false
+	}, 10*time.Second, 200*time.Millisecond)
+	if !success {
+		t.Fatalf("collection %s should be created; existing collections: %v", collectionName, lastCollections)
+	}
+}
+
+// waitForCollectionDeleted waits for a collection to be deleted using polling
+func waitForCollectionDeleted(t *testing.T, masterClient master_pb.SeaweedClient, collectionName string) {
+	require.Eventually(t, func() bool {
+		return !collectionExists(t, masterClient, collectionName)
+	}, 10*time.Second, 200*time.Millisecond, "collection %s should be deleted", collectionName)
+}
+
+// TestFilerGroupCollectionNaming verifies that when a filer group is configured,
+// collections are created with the correct prefix ({filerGroup}_{bucketName})
+func TestFilerGroupCollectionNaming(t *testing.T) {
+	if testConfig.FilerGroup == "" {
+		t.Skip("Skipping test: FILER_GROUP not configured. Set FILER_GROUP environment variable or configure in test_config.json")
+	}
+
+	s3Client := getS3Client(t)
+	masterClient := getMasterClient(t)
+	bucketName := getNewBucketName()
+	expectedCollection := getExpectedCollectionName(bucketName)
+
+	t.Logf("Testing with filer group: %s", testConfig.FilerGroup)
+	t.Logf("Bucket name: %s", bucketName)
+	t.Logf("Expected collection name: %s", expectedCollection)
+
+	// Create bucket
+	_, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err, "CreateBucket should succeed")
+
+	// Upload an object to trigger collection creation
+	_, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("test-object"),
+		Body:   strings.NewReader("test content"),
+	})
+	require.NoError(t, err, "PutObject should succeed")
+
+	// Wait for collection to be visible using polling
+	waitForCollectionExists(t, masterClient, expectedCollection)
+
+	// Verify collection exists with correct name
+	require.True(t, collectionExists(t, masterClient, expectedCollection),
+		"Collection %s should exist (filer group prefix applied)", expectedCollection)
+
+	// Cleanup: delete object and bucket
+	_, err = s3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("test-object"),
+	})
+	require.NoError(t, err, "DeleteObject should succeed")
+
+	_, err = s3Client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err, "DeleteBucket should succeed")
+
+	// Wait for collection to be deleted using polling
+	waitForCollectionDeleted(t, masterClient, expectedCollection)
+
+	t.Log("SUCCESS: Collection naming with filer group works correctly")
+}
+
+// TestBucketDeletionWithFilerGroup verifies that bucket deletion correctly
+// deletes the collection when filer group is configured
+func TestBucketDeletionWithFilerGroup(t *testing.T) {
+	if testConfig.FilerGroup == "" {
+		t.Skip("Skipping test: FILER_GROUP not configured")
+	}
+
+	s3Client := getS3Client(t)
+	masterClient := getMasterClient(t)
+	bucketName := getNewBucketName()
+	expectedCollection := getExpectedCollectionName(bucketName)
+
+	// Create bucket and add an object
+	_, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+
+	_, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("test-object"),
+		Body:   strings.NewReader("test content"),
+	})
+	require.NoError(t, err)
+
+	// Wait for collection to be created using polling
+	waitForCollectionExists(t, masterClient, expectedCollection)
+
+	// Verify collection exists before deletion
+	require.True(t, collectionExists(t, masterClient, expectedCollection),
+		"Collection should exist before bucket deletion")
+
+	// Delete object first
+	_, err = s3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("test-object"),
+	})
+	require.NoError(t, err)
+
+	// Delete bucket
+	_, err = s3Client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err, "DeleteBucket should succeed")
+
+	// Wait for collection to be deleted using polling
+	waitForCollectionDeleted(t, masterClient, expectedCollection)
+
+	// Verify collection was deleted
+	require.False(t, collectionExists(t, masterClient, expectedCollection),
+		"Collection %s should be deleted after bucket deletion", expectedCollection)
+
+	t.Log("SUCCESS: Bucket deletion with filer group correctly deletes collection")
+}
+
+// TestMultipleBucketsWithFilerGroup tests creating and deleting multiple buckets
+func TestMultipleBucketsWithFilerGroup(t *testing.T) {
+	if testConfig.FilerGroup == "" {
+		t.Skip("Skipping test: FILER_GROUP not configured")
+	}
+
+	s3Client := getS3Client(t)
+	masterClient := getMasterClient(t)
+
+	buckets := make([]string, 3)
+	for i := 0; i < 3; i++ {
+		buckets[i] = getNewBucketName()
+	}
+
+	// Create all buckets and add objects
+	for _, bucket := range buckets {
+		_, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{
+			Bucket: aws.String(bucket),
+		})
+		require.NoError(t, err)
+
+		_, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String("test-object"),
+			Body:   strings.NewReader("test content"),
+		})
+		require.NoError(t, err)
+	}
+
+	// Wait for all collections to be created using polling
+	for _, bucket := range buckets {
+		expectedCollection := getExpectedCollectionName(bucket)
+		waitForCollectionExists(t, masterClient, expectedCollection)
+	}
+
+	// Verify all collections exist with correct naming
+	for _, bucket := range buckets {
+		expectedCollection := getExpectedCollectionName(bucket)
+		require.True(t, collectionExists(t, masterClient, expectedCollection),
+			"Collection %s should exist for bucket %s", expectedCollection, bucket)
+	}
+
+	// Delete all buckets
+	for _, bucket := range buckets {
+		_, err := s3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String("test-object"),
+		})
+		require.NoError(t, err)
+
+		_, err = s3Client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
+			Bucket: aws.String(bucket),
+		})
+		require.NoError(t, err)
+	}
+
+	// Wait for all collections to be deleted using polling
+	for _, bucket := range buckets {
+		expectedCollection := getExpectedCollectionName(bucket)
+		waitForCollectionDeleted(t, masterClient, expectedCollection)
+	}
+
+	// Verify all collections are deleted
+	for _, bucket := range buckets {
+		expectedCollection := getExpectedCollectionName(bucket)
+		require.False(t, collectionExists(t, masterClient, expectedCollection),
+			"Collection %s should be deleted for bucket %s", expectedCollection, bucket)
+	}
+
+	t.Log("SUCCESS: Multiple bucket operations with filer group work correctly")
+}
@@ -0,0 +1,8 @@
+{
+  "s3_endpoint": "http://localhost:8333",
+  "master_address": "localhost:9333",
+  "access_key": "some_access_key1",
+  "secret_key": "some_secret_key1",
+  "region": "us-east-1",
+  "filer_group": "testgroup"
+}
@@ -558,13 +558,14 @@ func TestS3IAMPresignedURLIntegration(t *testing.T) {
 	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
 	require.NoError(t, err)

-	// Use static bucket name but with cleanup to handle conflicts
-	err = framework.CreateBucketWithCleanup(adminClient, testBucket)
+	// Use unique bucket name to avoid conflicts with other tests
+	bucketName := framework.GenerateUniqueBucketName("test-iam-presigned")
+	err = framework.CreateBucket(adminClient, bucketName)
 	require.NoError(t, err)

 	// Put test object
 	_, err = adminClient.PutObject(&s3.PutObjectInput{
-		Bucket: aws.String(testBucket),
+		Bucket: aws.String(bucketName),
 		Key:    aws.String(testObjectKey),
 		Body:   strings.NewReader(testObjectData),
 	})
@@ -586,7 +587,7 @@ func TestS3IAMPresignedURLIntegration(t *testing.T) {

 		// Test direct object access with JWT Bearer token (recommended approach)
 		_, err := adminClient.GetObject(&s3.GetObjectInput{
-			Bucket: aws.String(testBucket),
+			Bucket: aws.String(bucketName),
 			Key:    aws.String(testObjectKey),
 		})
 		require.NoError(t, err, "Direct object access with JWT Bearer token works correctly")
@@ -597,13 +598,13 @@ func TestS3IAMPresignedURLIntegration(t *testing.T) {

 	// Cleanup
 	_, err = adminClient.DeleteObject(&s3.DeleteObjectInput{
-		Bucket: aws.String(testBucket),
+		Bucket: aws.String(bucketName),
 		Key:    aws.String(testObjectKey),
 	})
 	require.NoError(t, err)

 	_, err = adminClient.DeleteBucket(&s3.DeleteBucketInput{
-		Bucket: aws.String(testBucket),
+		Bucket: aws.String(bucketName),
 	})
 	require.NoError(t, err)
 }
@@ -36,7 +36,7 @@
    "sts": {
      "tokenDuration": "15m",
      "issuer": "seaweedfs-sts",
-      "signingKey": "test-sts-signing-key-for-integration-tests"
+      "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc="
    },
    "policy": {
      "defaultEffect": "Deny"
@@ -0,0 +1,210 @@
+# Remote Storage Cache Integration Tests
+# Tests the remote object caching functionality with singleflight deduplication
+# Uses two SeaweedFS instances: primary (with caching) and secondary (as remote storage)
+
+.PHONY: all help build-weed check-deps start-remote stop-remote start-primary stop-primary \
+        setup-remote test test-with-server clean logs logs-primary logs-remote health
+
+all: test-with-server
+
+# Configuration
+WEED_BINARY := ../../../weed/weed_binary
+
+# Primary SeaweedFS (the one being tested - has remote caching)
+PRIMARY_S3_PORT := 8333
+PRIMARY_FILER_PORT := 8888
+PRIMARY_MASTER_PORT := 9333
+PRIMARY_VOLUME_PORT := 8080
+PRIMARY_METRICS_PORT := 9324
+PRIMARY_DIR := ./test-primary-data
+
+# Secondary SeaweedFS (acts as "remote" S3 storage)
+REMOTE_S3_PORT := 8334
+REMOTE_FILER_PORT := 8889
+REMOTE_MASTER_PORT := 9334
+REMOTE_VOLUME_PORT := 8081
+REMOTE_METRICS_PORT := 9325
+REMOTE_DIR := ./test-remote-data
+
+# Test configuration
+TEST_TIMEOUT := 10m
+TEST_PATTERN := TestRemoteCache
+
+# Buckets
+REMOTE_BUCKET := remotesourcebucket
+
+# Default target
+help:
+	@echo "Remote Storage Cache Integration Tests"
+	@echo ""
+	@echo "Uses two SeaweedFS instances:"
+	@echo "  - Primary (port $(PRIMARY_S3_PORT)): Being tested, has remote caching"
+	@echo "  - Remote  (port $(REMOTE_S3_PORT)): Acts as remote S3 storage"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  help              - Show this help message"
+	@echo "  build-weed        - Build the SeaweedFS binary"
+	@echo "  check-deps        - Check dependencies"
+	@echo "  start-remote      - Start remote SeaweedFS (secondary)"
+	@echo "  stop-remote       - Stop remote SeaweedFS"
+	@echo "  start-primary     - Start primary SeaweedFS"
+	@echo "  stop-primary      - Stop primary SeaweedFS"
+	@echo "  setup-remote      - Configure remote storage mount"
+	@echo "  test              - Run tests (assumes servers are running)"
+	@echo "  test-with-server  - Start servers, run tests, stop servers"
+	@echo "  clean             - Clean up all resources"
+	@echo "  logs              - Show server logs"
+
+# Build the SeaweedFS binary
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd ../../../weed && go build -o weed_binary .
+	@chmod +x $(WEED_BINARY)
+	@echo "SeaweedFS binary built"
+
+check-deps: build-weed
+	@echo "Checking dependencies..."
+	@command -v go >/dev/null 2>&1 || (echo "Go is required" && exit 1)
+	@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found" && exit 1)
+	@echo "All dependencies available"
+
+# Start remote SeaweedFS (acts as the "remote" S3 storage)
+start-remote: check-deps
+	@echo "Starting remote SeaweedFS (secondary instance)..."
+	@rm -f remote-server.pid
+	@mkdir -p $(REMOTE_DIR)
+	@$(WEED_BINARY) server \
+		-s3 \
+		-s3.port=$(REMOTE_S3_PORT) \
+		-s3.allowDeleteBucketNotEmpty=true \
+		-filer \
+		-filer.port=$(REMOTE_FILER_PORT) \
+		-master.port=$(REMOTE_MASTER_PORT) \
+		-volume.port=$(REMOTE_VOLUME_PORT) \
+		-master.volumeSizeLimitMB=50 \
+		-volume.max=100 \
+		-dir=$(REMOTE_DIR) \
+		-volume.preStopSeconds=1 \
+		-metricsPort=$(REMOTE_METRICS_PORT) \
+		> remote-weed.log 2>&1 & echo $$! > remote-server.pid
+	@echo "Waiting for remote SeaweedFS to start..."
+	@for i in $$(seq 1 60); do \
+		if curl -s http://localhost:$(REMOTE_S3_PORT) >/dev/null 2>&1; then \
+			echo "Remote SeaweedFS started on port $(REMOTE_S3_PORT)"; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "ERROR: Remote SeaweedFS failed to start"; \
+	cat remote-weed.log; \
+	exit 1
+
+stop-remote:
+	@echo "Stopping remote SeaweedFS..."
+	@if [ -f remote-server.pid ]; then \
+		kill -TERM $$(cat remote-server.pid) 2>/dev/null || true; \
+		sleep 2; \
+		kill -KILL $$(cat remote-server.pid) 2>/dev/null || true; \
+		rm -f remote-server.pid; \
+	fi
+	@echo "Remote SeaweedFS stopped"
+
+# Start primary SeaweedFS (the one being tested)
+start-primary: check-deps
+	@echo "Starting primary SeaweedFS..."
+	@rm -f primary-server.pid
+	@mkdir -p $(PRIMARY_DIR)
+	@$(WEED_BINARY) server \
+		-s3 \
+		-s3.port=$(PRIMARY_S3_PORT) \
+		-s3.allowDeleteBucketNotEmpty=true \
+		-filer \
+		-filer.port=$(PRIMARY_FILER_PORT) \
+		-master.port=$(PRIMARY_MASTER_PORT) \
+		-volume.port=$(PRIMARY_VOLUME_PORT) \
+		-master.volumeSizeLimitMB=50 \
+		-volume.max=100 \
+		-dir=$(PRIMARY_DIR) \
+		-volume.preStopSeconds=1 \
+		-metricsPort=$(PRIMARY_METRICS_PORT) \
+		> primary-weed.log 2>&1 & echo $$! > primary-server.pid
+	@echo "Waiting for primary SeaweedFS to start..."
+	@for i in $$(seq 1 60); do \
+		if curl -s http://localhost:$(PRIMARY_S3_PORT) >/dev/null 2>&1; then \
+			echo "Primary SeaweedFS started on port $(PRIMARY_S3_PORT)"; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "ERROR: Primary SeaweedFS failed to start"; \
+	cat primary-weed.log; \
+	exit 1
+
+stop-primary:
+	@echo "Stopping primary SeaweedFS..."
+	@if [ -f primary-server.pid ]; then \
+		kill -TERM $$(cat primary-server.pid) 2>/dev/null || true; \
+		sleep 2; \
+		kill -KILL $$(cat primary-server.pid) 2>/dev/null || true; \
+		rm -f primary-server.pid; \
+	fi
+	@echo "Primary SeaweedFS stopped"
+
+# Create bucket on remote and configure remote storage mount on primary
+setup-remote:
+	@echo "Creating bucket on remote SeaweedFS..."
+	@curl -s -X PUT "http://localhost:$(REMOTE_S3_PORT)/$(REMOTE_BUCKET)" || echo "Bucket may already exist"
+	@sleep 1
+	@echo "Configuring remote storage on primary..."
+	@printf 'remote.configure -name=seaweedremote -type=s3 -s3.access_key=any -s3.secret_key=any -s3.endpoint=http://localhost:$(REMOTE_S3_PORT) -s3.region=us-east-1\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.configure done"
+	@sleep 2
+	@echo "Mounting remote bucket on primary..."
+	@printf 'remote.mount -dir=/buckets/remotemounted -remote=seaweedremote/$(REMOTE_BUCKET) -nonempty\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.mount done"
+	@sleep 1
+	@echo "Remote storage configured"
+
+# Run tests
+test: check-deps
+	@echo "Running remote cache tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" .
+	@echo "Tests completed"
+
+# Full test workflow
+test-with-server: start-remote start-primary
+	@sleep 3
+	@$(MAKE) setup-remote || (echo "Remote setup failed" && $(MAKE) stop-primary stop-remote && exit 1)
+	@sleep 2
+	@echo "Running remote cache tests..."
+	@$(MAKE) test || (echo "Tests failed" && tail -50 primary-weed.log && $(MAKE) stop-primary stop-remote && exit 1)
+	@$(MAKE) stop-primary stop-remote
+	@echo "All tests passed"
+
+# Show logs
+logs:
+	@echo "=== Primary SeaweedFS Logs ==="
+	@if [ -f primary-weed.log ]; then tail -50 primary-weed.log; else echo "No log file"; fi
+	@echo ""
+	@echo "=== Remote SeaweedFS Logs ==="
+	@if [ -f remote-weed.log ]; then tail -50 remote-weed.log; else echo "No log file"; fi
+
+logs-primary:
+	@if [ -f primary-weed.log ]; then tail -f primary-weed.log; else echo "No log file"; fi
+
+logs-remote:
+	@if [ -f remote-weed.log ]; then tail -f remote-weed.log; else echo "No log file"; fi
+
+# Clean up
+clean:
+	@$(MAKE) stop-primary
+	@$(MAKE) stop-remote
+	@rm -f primary-weed.log remote-weed.log primary-server.pid remote-server.pid
+	@rm -rf $(PRIMARY_DIR) $(REMOTE_DIR)
+	@rm -f remote_cache.test
+	@go clean -testcache
+	@echo "Cleanup completed"
+
+# Health check
+health:
+	@echo "Checking server status..."
+	@curl -s http://localhost:$(PRIMARY_S3_PORT) >/dev/null 2>&1 && echo "Primary S3 ($(PRIMARY_S3_PORT)): UP" || echo "Primary S3 ($(PRIMARY_S3_PORT)): DOWN"
+	@curl -s http://localhost:$(REMOTE_S3_PORT) >/dev/null 2>&1 && echo "Remote S3 ($(REMOTE_S3_PORT)): UP" || echo "Remote S3 ($(REMOTE_S3_PORT)): DOWN"
@@ -0,0 +1,157 @@
+# Remote Object Cache Integration Tests
+
+This directory contains integration tests for the remote object caching feature with singleflight deduplication.
+
+## Test Flow
+
+Each test follows this pattern:
+1. **Write to local** - Upload data to primary SeaweedFS (local storage)
+2. **Uncache** - Push data to remote storage and remove local chunks
+3. **Read** - Read data (triggers caching from remote back to local)
+
+This tests the full remote caching workflow including singleflight deduplication.
+
+## Architecture
+
+```text
+┌─────────────────────────────────────────────────────────────────┐
+│                        Test Client                               │
+│                                                                  │
+│    1. PUT data to primary SeaweedFS                             │
+│    2. remote.cache.uncache (push to remote, purge local)        │
+│    3. GET data (triggers caching from remote)                   │
+│    4. Verify singleflight deduplication                         │
+└──────────────────────────────────┬──────────────────────────────┘
+                                   │
+                 ┌─────────────────┴─────────────────┐
+                 ▼                                   ▼
+┌────────────────────────────────────┐   ┌────────────────────────────────┐
+│     Primary SeaweedFS              │   │     Remote SeaweedFS           │
+│        (port 8333)                 │   │        (port 8334)             │
+│                                    │   │                                │
+│  - Being tested                    │   │  - Acts as "remote" S3         │
+│  - Has remote storage mounted      │──▶│  - Receives uncached data      │
+│  - Caches remote objects           │   │  - Serves data for caching     │
+│  - Singleflight deduplication      │   │                                │
+└────────────────────────────────────┘   └────────────────────────────────┘
+```
+
+## What's Being Tested
+
+1. **Basic Remote Caching**: Write → Uncache → Read workflow
+2. **Singleflight Deduplication**: Concurrent reads only trigger ONE caching operation
+3. **Large Object Caching**: 5MB files cache correctly
+4. **Range Requests**: Partial reads work with cached objects
+5. **Not Found Handling**: Proper error for non-existent objects
+
+## Quick Start
+
+### Run Full Test Suite (Recommended)
+
+```bash
+# Build SeaweedFS, start both servers, run tests, stop servers
+make test-with-server
+```
+
+### Manual Steps
+
+```bash
+# 1. Build SeaweedFS binary
+make build-weed
+
+# 2. Start remote SeaweedFS (acts as "remote" storage)
+make start-remote
+
+# 3. Start primary SeaweedFS (the one being tested)
+make start-primary
+
+# 4. Configure remote storage mount
+make setup-remote
+
+# 5. Run tests
+make test
+
+# 6. Clean up
+make clean
+```
+
+## Configuration
+
+### Primary SeaweedFS (Being Tested)
+
+| Service | Port |
+|---------|------|
+| S3 API | 8333 |
+| Filer | 8888 |
+| Master | 9333 |
+| Volume | 8080 |
+
+### Remote SeaweedFS (Remote Storage)
+
+| Service | Port |
+|---------|------|
+| S3 API | 8334 |
+| Filer | 8889 |
+| Master | 9334 |
+| Volume | 8081 |
+
+## Makefile Targets
+
+```bash
+make help           # Show all available targets
+make build-weed     # Build SeaweedFS binary
+make start-remote   # Start remote SeaweedFS
+make start-primary  # Start primary SeaweedFS
+make setup-remote   # Configure remote storage mount
+make test           # Run tests
+make test-with-server  # Full automated test workflow
+make logs           # Show server logs
+make health         # Check server status
+make clean          # Stop servers and clean up
+```
+
+## Test Details
+
+### TestRemoteCacheBasic
+Basic workflow test:
+1. Write object to primary (local)
+2. Uncache (push to remote, remove local chunks)
+3. Read (triggers caching from remote)
+4. Read again (from local cache - should be faster)
+
+### TestRemoteCacheConcurrent
+Singleflight deduplication test:
+1. Write 1MB object
+2. Uncache to remote
+3. Launch 10 concurrent reads
+4. All should succeed with correct data
+5. Only ONE caching operation should run (singleflight)
+
+### TestRemoteCacheLargeObject
+Large file test (5MB) to verify chunked transfer works correctly.
+
+### TestRemoteCacheRangeRequest
+Tests HTTP range requests work correctly after caching.
+
+### TestRemoteCacheNotFound
+Tests proper error handling for non-existent objects.
+
+## Troubleshooting
+
+### View logs
+```bash
+make logs           # Show recent logs from both servers
+make logs-primary   # Follow primary logs in real-time
+make logs-remote    # Follow remote logs in real-time
+```
+
+### Check server health
+```bash
+make health
+```
+
+### Clean up and retry
+```bash
+make clean
+make test-with-server
+```
@@ -0,0 +1,375 @@
+package remote_cache
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Test configuration
+// Uses two SeaweedFS instances:
+// - Primary: The one being tested (has remote caching)
+// - Remote: Acts as the "remote" S3 storage
+const (
+	// Primary SeaweedFS
+	primaryEndpoint   = "http://localhost:8333"
+	primaryMasterPort = "9333"
+
+	// Remote SeaweedFS (acts as remote storage)
+	remoteEndpoint = "http://localhost:8334"
+
+	// Credentials (anonymous access for testing)
+	accessKey = "any"
+	secretKey = "any"
+
+	// Bucket name - mounted on primary as remote storage
+	testBucket = "remotemounted"
+
+	// Path to weed binary
+	weedBinary = "../../../weed/weed_binary"
+)
+
+var (
+	primaryClient     *s3.S3
+	primaryClientOnce sync.Once
+)
+
+func getPrimaryClient() *s3.S3 {
+	primaryClientOnce.Do(func() {
+		primaryClient = createS3Client(primaryEndpoint)
+	})
+	return primaryClient
+}
+
+func createS3Client(endpoint string) *s3.S3 {
+	sess, err := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String(endpoint),
+		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
+		DisableSSL:       aws.Bool(!strings.HasPrefix(endpoint, "https")),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	if err != nil {
+		panic(fmt.Sprintf("failed to create session: %v", err))
+	}
+	return s3.New(sess)
+}
+
+// skipIfNotRunning skips the test if the servers aren't running
+func skipIfNotRunning(t *testing.T) {
+	resp, err := http.Get(primaryEndpoint)
+	if err != nil {
+		t.Skipf("Primary SeaweedFS not running at %s: %v", primaryEndpoint, err)
+	}
+	resp.Body.Close()
+
+	resp, err = http.Get(remoteEndpoint)
+	if err != nil {
+		t.Skipf("Remote SeaweedFS not running at %s: %v", remoteEndpoint, err)
+	}
+	resp.Body.Close()
+}
+
+// runWeedShell executes a weed shell command
+func runWeedShell(t *testing.T, command string) (string, error) {
+	cmd := exec.Command(weedBinary, "shell", "-master=localhost:"+primaryMasterPort)
+	cmd.Stdin = strings.NewReader(command + "\nexit\n")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Logf("weed shell command '%s' failed: %v, output: %s", command, err, string(output))
+		return string(output), err
+	}
+	return string(output), nil
+}
+
+// uploadToPrimary uploads an object to the primary SeaweedFS (local write)
+func uploadToPrimary(t *testing.T, key string, data []byte) {
+	_, err := getPrimaryClient().PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(key),
+		Body:   bytes.NewReader(data),
+	})
+	require.NoError(t, err, "failed to upload to primary SeaweedFS")
+}
+
+// getFromPrimary gets an object from primary SeaweedFS
+func getFromPrimary(t *testing.T, key string) []byte {
+	resp, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(key),
+	})
+	require.NoError(t, err, "failed to get from primary SeaweedFS")
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
+	require.NoError(t, err, "failed to read response body")
+	return data
+}
+
+// syncToRemote syncs local data to remote storage
+func syncToRemote(t *testing.T) {
+	t.Log("Syncing to remote storage...")
+	output, err := runWeedShell(t, "remote.cache.uncache -dir=/buckets/"+testBucket+" -include=*")
+	if err != nil {
+		t.Logf("syncToRemote warning: %v", err)
+	}
+	t.Log(output)
+	time.Sleep(1 * time.Second)
+}
+
+// uncacheLocal purges the local cache, forcing data to be fetched from remote
+func uncacheLocal(t *testing.T, pattern string) {
+	t.Logf("Purging local cache for pattern: %s", pattern)
+	output, err := runWeedShell(t, fmt.Sprintf("remote.uncache -dir=/buckets/%s -include=%s", testBucket, pattern))
+	if err != nil {
+		t.Logf("uncacheLocal warning: %v", err)
+	}
+	t.Log(output)
+	time.Sleep(500 * time.Millisecond)
+}
+
+// TestRemoteCacheBasic tests the basic caching workflow:
+// 1. Write to local
+// 2. Uncache (push to remote, remove local chunks)
+// 3. Read (triggers caching from remote)
+func TestRemoteCacheBasic(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("test-basic-%d.txt", time.Now().UnixNano())
+	testData := []byte("Hello, this is test data for remote caching!")
+
+	// Step 1: Write to local
+	t.Log("Step 1: Writing object to primary SeaweedFS (local)...")
+	uploadToPrimary(t, testKey, testData)
+
+	// Verify it's readable
+	result := getFromPrimary(t, testKey)
+	assert.Equal(t, testData, result, "initial read mismatch")
+
+	// Step 2: Uncache - push to remote and remove local chunks
+	t.Log("Step 2: Uncaching (pushing to remote, removing local chunks)...")
+	uncacheLocal(t, testKey)
+
+	// Step 3: Read - this should trigger caching from remote
+	t.Log("Step 3: Reading object (should trigger caching from remote)...")
+	start := time.Now()
+	result = getFromPrimary(t, testKey)
+	firstReadDuration := time.Since(start)
+
+	assert.Equal(t, testData, result, "data mismatch after cache")
+	t.Logf("First read (from remote) took %v", firstReadDuration)
+
+	// Step 4: Read again - should be from local cache
+	t.Log("Step 4: Reading again (should be from local cache)...")
+	start = time.Now()
+	result = getFromPrimary(t, testKey)
+	secondReadDuration := time.Since(start)
+
+	assert.Equal(t, testData, result, "data mismatch on cached read")
+	t.Logf("Second read (from cache) took %v", secondReadDuration)
+
+	t.Log("Basic caching test passed")
+}
+
+// TestRemoteCacheConcurrent tests that concurrent reads of the same
+// remote object only trigger ONE caching operation (singleflight deduplication)
+func TestRemoteCacheConcurrent(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("test-concurrent-%d.txt", time.Now().UnixNano())
+	// Use larger data to make caching take measurable time
+	testData := make([]byte, 1024*1024) // 1MB
+	for i := range testData {
+		testData[i] = byte(i % 256)
+	}
+
+	// Step 1: Write to local
+	t.Log("Step 1: Writing 1MB object to primary SeaweedFS...")
+	uploadToPrimary(t, testKey, testData)
+
+	// Verify it's readable
+	result := getFromPrimary(t, testKey)
+	assert.Equal(t, len(testData), len(result), "initial size mismatch")
+
+	// Step 2: Uncache
+	t.Log("Step 2: Uncaching (pushing to remote)...")
+	uncacheLocal(t, testKey)
+
+	// Step 3: Launch many concurrent reads - singleflight should deduplicate
+	numRequests := 10
+	var wg sync.WaitGroup
+	var successCount atomic.Int32
+	var errorCount atomic.Int32
+	results := make(chan []byte, numRequests)
+
+	t.Logf("Step 3: Launching %d concurrent requests...", numRequests)
+	startTime := time.Now()
+
+	for i := 0; i < numRequests; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			resp, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
+				Bucket: aws.String(testBucket),
+				Key:    aws.String(testKey),
+			})
+			if err != nil {
+				t.Logf("Request %d failed: %v", idx, err)
+				errorCount.Add(1)
+				return
+			}
+			defer resp.Body.Close()
+
+			data, err := io.ReadAll(resp.Body)
+			if err != nil {
+				t.Logf("Request %d read failed: %v", idx, err)
+				errorCount.Add(1)
+				return
+			}
+
+			results <- data
+			successCount.Add(1)
+		}(i)
+	}
+
+	wg.Wait()
+	close(results)
+	totalDuration := time.Since(startTime)
+
+	t.Logf("All %d requests completed in %v", numRequests, totalDuration)
+	t.Logf("Successful: %d, Failed: %d", successCount.Load(), errorCount.Load())
+
+	// Verify all successful requests returned correct data
+	for data := range results {
+		assert.Equal(t, len(testData), len(data), "data length mismatch")
+	}
+
+	// All requests should succeed
+	assert.Equal(t, int32(numRequests), successCount.Load(), "some requests failed")
+	assert.Equal(t, int32(0), errorCount.Load(), "no requests should fail")
+
+	t.Log("Concurrent caching test passed")
+}
+
+// TestRemoteCacheLargeObject tests caching of larger objects
+func TestRemoteCacheLargeObject(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("test-large-%d.bin", time.Now().UnixNano())
+	// 5MB object
+	testData := make([]byte, 5*1024*1024)
+	for i := range testData {
+		testData[i] = byte(i % 256)
+	}
+
+	// Step 1: Write to local
+	t.Log("Step 1: Writing 5MB object to primary SeaweedFS...")
+	uploadToPrimary(t, testKey, testData)
+
+	// Verify it's readable
+	result := getFromPrimary(t, testKey)
+	assert.Equal(t, len(testData), len(result), "initial size mismatch")
+
+	// Step 2: Uncache
+	t.Log("Step 2: Uncaching...")
+	uncacheLocal(t, testKey)
+
+	// Step 3: Read from remote
+	t.Log("Step 3: Reading 5MB object (should cache from remote)...")
+	start := time.Now()
+	result = getFromPrimary(t, testKey)
+	duration := time.Since(start)
+
+	assert.Equal(t, len(testData), len(result), "size mismatch")
+	assert.Equal(t, testData, result, "data mismatch")
+	t.Logf("Large object cached in %v", duration)
+
+	t.Log("Large object caching test passed")
+}
+
+// TestRemoteCacheRangeRequest tests that range requests work after caching
+func TestRemoteCacheRangeRequest(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("test-range-%d.txt", time.Now().UnixNano())
+	testData := []byte("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+	// Write, uncache, then test range request
+	t.Log("Writing and uncaching object...")
+	uploadToPrimary(t, testKey, testData)
+	uncacheLocal(t, testKey)
+
+	// Range request should work and trigger caching
+	t.Log("Testing range request (bytes 10-19)...")
+	resp, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(testKey),
+		Range:  aws.String("bytes=10-19"),
+	})
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	rangeData, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	expected := testData[10:20] // "ABCDEFGHIJ"
+	assert.Equal(t, expected, rangeData, "range data mismatch")
+	t.Logf("Range request returned: %s", string(rangeData))
+
+	t.Log("Range request test passed")
+}
+
+// TestRemoteCacheNotFound tests that non-existent objects return proper errors
+func TestRemoteCacheNotFound(t *testing.T) {
+	skipIfNotRunning(t)
+
+	testKey := fmt.Sprintf("non-existent-object-%d", time.Now().UnixNano())
+
+	_, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(testKey),
+	})
+
+	assert.Error(t, err, "should get error for non-existent object")
+	t.Logf("Got expected error: %v", err)
+
+	t.Log("Not found test passed")
+}
+
+// TestMain sets up and tears down the test environment
+func TestMain(m *testing.M) {
+	if !isServerRunning(primaryEndpoint) {
+		fmt.Println("WARNING: Primary SeaweedFS not running at", primaryEndpoint)
+		fmt.Println("   Run 'make test-with-server' to start servers automatically")
+	}
+	if !isServerRunning(remoteEndpoint) {
+		fmt.Println("WARNING: Remote SeaweedFS not running at", remoteEndpoint)
+		fmt.Println("   Run 'make test-with-server' to start servers automatically")
+	}
+
+	os.Exit(m.Run())
+}
+
+func isServerRunning(url string) bool {
+	resp, err := http.Get(url)
+	if err != nil {
+		return false
+	}
+	resp.Body.Close()
+	return true
+}
@@ -0,0 +1,505 @@
+package sse_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestGitHub7562CopyFromEncryptedToTempToEncrypted reproduces the exact scenario from
+// GitHub issue #7562: copying from an encrypted bucket to a temp bucket, then to another
+// encrypted bucket fails with InternalError.
+//
+// Reproduction steps:
+// 1. Create source bucket with SSE-S3 encryption enabled
+// 2. Upload object (automatically encrypted)
+// 3. Create temp bucket (no encryption)
+// 4. Copy object from source to temp (decrypts)
+// 5. Delete source bucket
+// 6. Create destination bucket with SSE-S3 encryption
+// 7. Copy object from temp to dest (should re-encrypt) - THIS FAILS
+func TestGitHub7562CopyFromEncryptedToTempToEncrypted(t *testing.T) {
+	ctx := context.Background()
+	client, err := createS3Client(ctx, defaultConfig)
+	require.NoError(t, err, "Failed to create S3 client")
+
+	// Create three buckets
+	srcBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-src-")
+	require.NoError(t, err, "Failed to create source bucket")
+
+	tempBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-temp-")
+	require.NoError(t, err, "Failed to create temp bucket")
+
+	destBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-dest-")
+	require.NoError(t, err, "Failed to create destination bucket")
+
+	// Cleanup at the end
+	defer func() {
+		// Clean up in reverse order of creation
+		cleanupTestBucket(ctx, client, destBucket)
+		cleanupTestBucket(ctx, client, tempBucket)
+		// Note: srcBucket is deleted during the test
+	}()
+
+	testData := []byte("Test data for GitHub issue #7562 - copy from encrypted to temp to encrypted bucket")
+	objectKey := "demo-file.txt"
+
+	t.Logf("[1] Creating source bucket with SSE-S3 default encryption: %s", srcBucket)
+
+	// Step 1: Enable SSE-S3 default encryption on source bucket
+	_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
+		Bucket: aws.String(srcBucket),
+		ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
+			Rules: []types.ServerSideEncryptionRule{
+				{
+					ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
+						SSEAlgorithm: types.ServerSideEncryptionAes256,
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to set source bucket default encryption")
+
+	t.Log("[2] Uploading demo object to source bucket")
+
+	// Step 2: Upload object to source bucket (will be automatically encrypted)
+	_, err = client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(srcBucket),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+		// No encryption header - bucket default applies
+	})
+	require.NoError(t, err, "Failed to upload to source bucket")
+
+	// Verify source object is encrypted
+	srcHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(srcBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to HEAD source object")
+	assert.Equal(t, types.ServerSideEncryptionAes256, srcHead.ServerSideEncryption,
+		"Source object should be SSE-S3 encrypted")
+	t.Logf("Source object encryption: %v", srcHead.ServerSideEncryption)
+
+	t.Logf("[3] Creating temp bucket (no encryption): %s", tempBucket)
+	// Temp bucket already created without encryption
+
+	t.Log("[4] Copying object from source to temp (should decrypt)")
+
+	// Step 4: Copy to temp bucket (no encryption = decrypts)
+	_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
+		Bucket:     aws.String(tempBucket),
+		Key:        aws.String(objectKey),
+		CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)),
+		// No encryption header - data stored unencrypted
+	})
+	require.NoError(t, err, "Failed to copy to temp bucket")
+
+	// Verify temp object is NOT encrypted
+	tempHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(tempBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to HEAD temp object")
+	assert.Empty(t, tempHead.ServerSideEncryption, "Temp object should NOT be encrypted")
+	t.Logf("Temp object encryption: %v (should be empty)", tempHead.ServerSideEncryption)
+
+	// Verify temp object content
+	tempGet, err := client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(tempBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to GET temp object")
+	tempData, err := io.ReadAll(tempGet.Body)
+	tempGet.Body.Close()
+	require.NoError(t, err, "Failed to read temp object")
+	assertDataEqual(t, testData, tempData, "Temp object data should match original")
+
+	t.Log("[5] Deleting original source bucket")
+
+	// Step 5: Delete source bucket
+	err = cleanupTestBucket(ctx, client, srcBucket)
+	require.NoError(t, err, "Failed to delete source bucket")
+
+	t.Logf("[6] Creating destination bucket with SSE-S3 encryption: %s", destBucket)
+
+	// Step 6: Enable SSE-S3 default encryption on destination bucket
+	_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
+		Bucket: aws.String(destBucket),
+		ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
+			Rules: []types.ServerSideEncryptionRule{
+				{
+					ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
+						SSEAlgorithm: types.ServerSideEncryptionAes256,
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to set destination bucket default encryption")
+
+	t.Log("[7] Copying object from temp to dest (should re-encrypt) - THIS IS WHERE #7562 FAILS")
+
+	// Step 7: Copy from temp to dest bucket (should re-encrypt with SSE-S3)
+	// THIS IS THE STEP THAT FAILS IN GITHUB ISSUE #7562
+	_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
+		Bucket:     aws.String(destBucket),
+		Key:        aws.String(objectKey),
+		CopySource: aws.String(fmt.Sprintf("%s/%s", tempBucket, objectKey)),
+		// No encryption header - bucket default should apply
+	})
+	require.NoError(t, err, "GitHub #7562: Failed to copy from temp to encrypted dest bucket")
+
+	// Verify destination object is encrypted
+	destHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(destBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to HEAD destination object")
+	assert.Equal(t, types.ServerSideEncryptionAes256, destHead.ServerSideEncryption,
+		"Destination object should be SSE-S3 encrypted via bucket default")
+	t.Logf("Destination object encryption: %v", destHead.ServerSideEncryption)
+
+	// Verify destination object content is correct
+	destGet, err := client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(destBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to GET destination object")
+	destData, err := io.ReadAll(destGet.Body)
+	destGet.Body.Close()
+	require.NoError(t, err, "Failed to read destination object")
+	assertDataEqual(t, testData, destData, "GitHub #7562: Destination object data mismatch after re-encryption")
+
+	t.Log("[done] GitHub #7562 reproduction test completed successfully!")
+}
+
+// TestGitHub7562SimpleScenario tests the simpler variant: just copy unencrypted to encrypted bucket
+func TestGitHub7562SimpleScenario(t *testing.T) {
+	ctx := context.Background()
+	client, err := createS3Client(ctx, defaultConfig)
+	require.NoError(t, err, "Failed to create S3 client")
+
+	// Create two buckets
+	srcBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-simple-src-")
+	require.NoError(t, err, "Failed to create source bucket")
+	defer cleanupTestBucket(ctx, client, srcBucket)
+
+	destBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-simple-dest-")
+	require.NoError(t, err, "Failed to create destination bucket")
+	defer cleanupTestBucket(ctx, client, destBucket)
+
+	testData := []byte("Simple test for unencrypted to encrypted copy")
+	objectKey := "test-object.txt"
+
+	t.Logf("Source bucket (no encryption): %s", srcBucket)
+	t.Logf("Dest bucket (SSE-S3 default): %s", destBucket)
+
+	// Upload to unencrypted source bucket
+	_, err = client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(srcBucket),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload to source bucket")
+
+	// Enable SSE-S3 on destination bucket
+	_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
+		Bucket: aws.String(destBucket),
+		ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
+			Rules: []types.ServerSideEncryptionRule{
+				{
+					ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
+						SSEAlgorithm: types.ServerSideEncryptionAes256,
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to set dest bucket encryption")
+
+	// Copy to encrypted bucket (should use bucket default encryption)
+	_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
+		Bucket:     aws.String(destBucket),
+		Key:        aws.String(objectKey),
+		CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)),
+	})
+	require.NoError(t, err, "Failed to copy to encrypted bucket")
+
+	// Verify destination is encrypted
+	destHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(destBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to HEAD dest object")
+	assert.Equal(t, types.ServerSideEncryptionAes256, destHead.ServerSideEncryption,
+		"Object should be encrypted via bucket default")
+
+	// Verify content
+	destGet, err := client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(destBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to GET dest object")
+	destData, err := io.ReadAll(destGet.Body)
+	destGet.Body.Close()
+	require.NoError(t, err, "Failed to read dest object")
+	assertDataEqual(t, testData, destData, "Data mismatch")
+}
+
+// TestGitHub7562DebugMetadata helps debug what metadata is present on objects at each step
+func TestGitHub7562DebugMetadata(t *testing.T) {
+	ctx := context.Background()
+	client, err := createS3Client(ctx, defaultConfig)
+	require.NoError(t, err, "Failed to create S3 client")
+
+	// Create three buckets
+	srcBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-debug-src-")
+	require.NoError(t, err, "Failed to create source bucket")
+
+	tempBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-debug-temp-")
+	require.NoError(t, err, "Failed to create temp bucket")
+
+	destBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-debug-dest-")
+	require.NoError(t, err, "Failed to create destination bucket")
+
+	defer func() {
+		cleanupTestBucket(ctx, client, destBucket)
+		cleanupTestBucket(ctx, client, tempBucket)
+	}()
+
+	testData := []byte("Debug metadata test for GitHub #7562")
+	objectKey := "debug-file.txt"
+
+	// Enable SSE-S3 on source
+	_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
+		Bucket: aws.String(srcBucket),
+		ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
+			Rules: []types.ServerSideEncryptionRule{
+				{
+					ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
+						SSEAlgorithm: types.ServerSideEncryptionAes256,
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to set source bucket encryption")
+
+	// Upload
+	_, err = client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(srcBucket),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload")
+
+	// Log source object headers
+	srcHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(srcBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to HEAD source")
+	t.Logf("=== SOURCE OBJECT (encrypted) ===")
+	t.Logf("ServerSideEncryption: %v", srcHead.ServerSideEncryption)
+	t.Logf("Metadata: %v", srcHead.Metadata)
+	t.Logf("ContentLength: %d", aws.ToInt64(srcHead.ContentLength))
+
+	// Copy to temp
+	_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
+		Bucket:     aws.String(tempBucket),
+		Key:        aws.String(objectKey),
+		CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)),
+	})
+	require.NoError(t, err, "Failed to copy to temp")
+
+	// Log temp object headers
+	tempHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(tempBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to HEAD temp")
+	t.Logf("=== TEMP OBJECT (should be unencrypted) ===")
+	t.Logf("ServerSideEncryption: %v (should be empty)", tempHead.ServerSideEncryption)
+	t.Logf("Metadata: %v", tempHead.Metadata)
+	t.Logf("ContentLength: %d", aws.ToInt64(tempHead.ContentLength))
+
+	// Verify temp is NOT encrypted
+	if tempHead.ServerSideEncryption != "" {
+		t.Logf("WARNING: Temp object unexpectedly has encryption: %v", tempHead.ServerSideEncryption)
+	}
+
+	// Delete source bucket
+	cleanupTestBucket(ctx, client, srcBucket)
+
+	// Enable SSE-S3 on dest
+	_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
+		Bucket: aws.String(destBucket),
+		ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
+			Rules: []types.ServerSideEncryptionRule{
+				{
+					ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
+						SSEAlgorithm: types.ServerSideEncryptionAes256,
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to set dest bucket encryption")
+
+	// Copy to dest - THIS IS WHERE #7562 FAILS
+	t.Log("=== COPYING TO ENCRYPTED DEST ===")
+	_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
+		Bucket:     aws.String(destBucket),
+		Key:        aws.String(objectKey),
+		CopySource: aws.String(fmt.Sprintf("%s/%s", tempBucket, objectKey)),
+	})
+	if err != nil {
+		t.Logf("!!! COPY FAILED (GitHub #7562): %v", err)
+		t.FailNow()
+	}
+
+	// Log dest object headers
+	destHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(destBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to HEAD dest")
+	t.Logf("=== DEST OBJECT (should be encrypted) ===")
+	t.Logf("ServerSideEncryption: %v", destHead.ServerSideEncryption)
+	t.Logf("Metadata: %v", destHead.Metadata)
+	t.Logf("ContentLength: %d", aws.ToInt64(destHead.ContentLength))
+
+	// Verify dest IS encrypted
+	assert.Equal(t, types.ServerSideEncryptionAes256, destHead.ServerSideEncryption,
+		"Dest object should be encrypted")
+
+	// Verify content is readable
+	destGet, err := client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(destBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to GET dest")
+	destData, err := io.ReadAll(destGet.Body)
+	destGet.Body.Close()
+	require.NoError(t, err, "Failed to read dest")
+	assertDataEqual(t, testData, destData, "Data mismatch")
+
+	t.Log("=== DEBUG TEST PASSED ===")
+}
+
+// TestGitHub7562LargeFile tests the issue with larger files that might trigger multipart handling
+func TestGitHub7562LargeFile(t *testing.T) {
+	ctx := context.Background()
+	client, err := createS3Client(ctx, defaultConfig)
+	require.NoError(t, err, "Failed to create S3 client")
+
+	srcBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-large-src-")
+	require.NoError(t, err, "Failed to create source bucket")
+
+	tempBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-large-temp-")
+	require.NoError(t, err, "Failed to create temp bucket")
+
+	destBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-large-dest-")
+	require.NoError(t, err, "Failed to create destination bucket")
+
+	defer func() {
+		cleanupTestBucket(ctx, client, destBucket)
+		cleanupTestBucket(ctx, client, tempBucket)
+	}()
+
+	// Use larger file to potentially trigger different code paths
+	testData := generateTestData(5 * 1024 * 1024) // 5MB
+	objectKey := "large-file.bin"
+
+	t.Logf("Testing with %d byte file", len(testData))
+
+	// Enable SSE-S3 on source
+	_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
+		Bucket: aws.String(srcBucket),
+		ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
+			Rules: []types.ServerSideEncryptionRule{
+				{
+					ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
+						SSEAlgorithm: types.ServerSideEncryptionAes256,
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to set source bucket encryption")
+
+	// Upload
+	_, err = client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(srcBucket),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload")
+
+	// Copy to temp (decrypt)
+	_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
+		Bucket:     aws.String(tempBucket),
+		Key:        aws.String(objectKey),
+		CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)),
+	})
+	require.NoError(t, err, "Failed to copy to temp")
+
+	// Delete source
+	cleanupTestBucket(ctx, client, srcBucket)
+
+	// Enable SSE-S3 on dest
+	_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
+		Bucket: aws.String(destBucket),
+		ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
+			Rules: []types.ServerSideEncryptionRule{
+				{
+					ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
+						SSEAlgorithm: types.ServerSideEncryptionAes256,
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to set dest bucket encryption")
+
+	// Copy to dest (re-encrypt) - GitHub #7562
+	_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
+		Bucket:     aws.String(destBucket),
+		Key:        aws.String(objectKey),
+		CopySource: aws.String(fmt.Sprintf("%s/%s", tempBucket, objectKey)),
+	})
+	require.NoError(t, err, "GitHub #7562: Large file copy to encrypted bucket failed")
+
+	// Verify
+	destHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(destBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to HEAD dest")
+	assert.Equal(t, types.ServerSideEncryptionAes256, destHead.ServerSideEncryption)
+	assert.Equal(t, int64(len(testData)), aws.ToInt64(destHead.ContentLength))
+
+	// Verify content
+	destGet, err := client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(destBucket),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to GET dest")
+	destData, err := io.ReadAll(destGet.Body)
+	destGet.Body.Close()
+	require.NoError(t, err, "Failed to read dest")
+	assertDataEqual(t, testData, destData, "Large file data mismatch")
+
+	t.Log("Large file test passed!")
+}
+
@@ -81,7 +81,7 @@ start-server: check-deps
 	@echo "🔍 DEBUG: Creating volume directory..."
 	@mkdir -p ./test-volume-data
 	@echo "🔍 DEBUG: Launching SeaweedFS server in background..."
-	@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324"
+	@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -master.peers=none -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324"
 	@$(WEED_BINARY) server \
 		-debug \
 		-s3 \
@@ -91,6 +91,7 @@ start-server: check-deps
 		-filer \
 		-filer.maxMB=64 \
 		-master.volumeSizeLimitMB=50 \
+		-master.peers=none \
 		-volume.max=100 \
 		-dir=./test-volume-data \
 		-volume.preStopSeconds=1 \
@@ -221,13 +222,13 @@ test-with-server: start-server
 test-versioning-with-configs: check-deps
 	@echo "Testing with different S3 configurations..."
 	@echo "Testing with empty folder allowed..."
-	@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
+	@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -filer -master.volumeSizeLimitMB=100 -master.peers=none -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
 	@sleep 5
 	@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
 	@if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi
 	@sleep 2
 	@echo "Testing with delete bucket not empty disabled..."
-	@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
+	@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -master.peers=none -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
 	@sleep 5
 	@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
 	@if [ -f weed-config2.pid ]; then kill -TERM $$(cat weed-config2.pid) 2>/dev/null || true; rm -f weed-config2.pid; fi
@@ -239,6 +240,13 @@ test-versioning-stress: check-deps
 	@go test -v -timeout=20m -run "TestVersioningConcurrentOperations" . -count=5
 	@echo "✅ Stress tests completed"

+# Pagination stress testing (tests >1000 versions)
+test-versioning-pagination-stress: check-deps
+	@echo "Running pagination stress tests (>1000 versions)..."
+	@echo "This test creates 1500+ versions and may take several minutes..."
+	@ENABLE_STRESS_TESTS=true go test -v -timeout=30m -run "TestVersioningPagination" .
+	@echo "✅ Pagination stress tests completed"
+
 # Generate test reports
 test-report: check-deps
 	@echo "Generating test reports..."
@@ -272,6 +280,7 @@ debug-server:
 		-filer \
 		-filer.maxMB=16 \
 		-master.volumeSizeLimitMB=50 \
+		-master.peers=none \
 		-volume.max=100 \
 		-dir=./test-volume-data \
 		-volume.preStopSeconds=1 \
@@ -320,6 +329,7 @@ start-server-simple: check-deps
 		-filer \
 		-filer.maxMB=64 \
 		-master.volumeSizeLimitMB=50 \
+		-master.peers=none \
 		-volume.max=100 \
 		-volume.preStopSeconds=1 \
 		-metricsPort=9324 \
@@ -0,0 +1,327 @@
+package s3api
+
+import (
+	"context"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestVeleroKopiaVersionedObjectListing tests the exact access patterns used by Velero/Kopia
+// when backing up to a versioned S3 bucket. This test validates the fix for:
+// https://github.com/seaweedfs/seaweedfs/discussions/7573
+//
+// The bug was that when listing versioned objects with nested paths like
+// "kopia/logpaste/kopia.blobcfg", the returned Key would be doubled:
+// "kopia/logpaste/kopia/logpaste/kopia.blobcfg" instead of the correct path.
+//
+// This was caused by getLatestVersionEntryForListOperation setting the entry's
+// Name to the full path instead of just the base filename.
+func TestVeleroKopiaVersionedObjectListing(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Step 1: Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Step 2: Enable versioning (required to reproduce the bug)
+	enableVersioning(t, client, bucketName)
+
+	// Step 3: Create objects matching Velero/Kopia's typical path structure
+	// These are the actual paths Kopia uses when storing repository data
+	testObjects := map[string]string{
+		"kopia/logpaste/kopia.blobcfg":    "blob-config-content",
+		"kopia/logpaste/kopia.repository": "repository-content",
+		"kopia/logpaste/.storageconfig":   "storage-config-content",
+		"backups/namespace1/backup.tar":   "backup-content",
+		"restic/locks/lock-001.json":      "lock-content",
+		"restic/index/index-001.json":     "index-content",
+		"data/a/b/c/deeply-nested.dat":    "nested-content",
+		"single-file.txt":                 "single-file-content",
+	}
+
+	// Upload all test objects
+	for key, content := range testObjects {
+		_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(key),
+			Body:   strings.NewReader(content),
+		})
+		require.NoError(t, err, "Failed to upload object %s", key)
+	}
+
+	// Step 4: Test listing with various prefixes (mimicking Kopia's ListObjects calls)
+	testCases := []struct {
+		name         string
+		prefix       string
+		expectedKeys []string
+		description  string
+	}{
+		{
+			name:   "kopia repository prefix",
+			prefix: "kopia/logpaste/",
+			expectedKeys: []string{
+				"kopia/logpaste/.storageconfig",
+				"kopia/logpaste/kopia.blobcfg",
+				"kopia/logpaste/kopia.repository",
+			},
+			description: "Kopia lists its repository directory to find config files",
+		},
+		{
+			name:         "exact config file",
+			prefix:       "kopia/logpaste/kopia.blobcfg",
+			expectedKeys: []string{"kopia/logpaste/kopia.blobcfg"},
+			description:  "Kopia checks for specific config files",
+		},
+		{
+			name:   "restic index directory",
+			prefix: "restic/index/",
+			expectedKeys: []string{
+				"restic/index/index-001.json",
+			},
+			description: "Restic lists index files (similar pattern to Kopia)",
+		},
+		{
+			name:   "deeply nested path",
+			prefix: "data/a/b/c/",
+			expectedKeys: []string{
+				"data/a/b/c/deeply-nested.dat",
+			},
+			description: "Tests deeply nested paths don't get doubled",
+		},
+		{
+			name:   "backup directory",
+			prefix: "backups/",
+			expectedKeys: []string{
+				"backups/namespace1/backup.tar",
+			},
+			description: "Velero lists backup directories",
+		},
+		{
+			name:   "root level single file",
+			prefix: "single-file",
+			expectedKeys: []string{
+				"single-file.txt",
+			},
+			description: "Files at bucket root should work correctly",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// List objects with prefix
+			listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+				Bucket: aws.String(bucketName),
+				Prefix: aws.String(tc.prefix),
+			})
+			require.NoError(t, err)
+
+			// Extract returned keys
+			var actualKeys []string
+			for _, obj := range listResp.Contents {
+				key := *obj.Key
+				actualKeys = append(actualKeys, key)
+
+				// Critical assertion: verify no path doubling
+				// The bug would cause "kopia/logpaste/kopia.blobcfg" to become
+				// "kopia/logpaste/kopia/logpaste/kopia.blobcfg"
+				assert.False(t, hasDoubledPath(key),
+					"Path doubling detected! Key '%s' should not have repeated path segments", key)
+
+				// Also verify we can actually GET the object using the listed key
+				getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+					Bucket: aws.String(bucketName),
+					Key:    aws.String(key),
+				})
+				require.NoError(t, err, "Should be able to GET object using listed key: %s", key)
+				getResp.Body.Close()
+			}
+
+			// Verify we got the expected keys
+			assert.ElementsMatch(t, tc.expectedKeys, actualKeys,
+				"%s: Listed keys should match expected", tc.description)
+		})
+	}
+}
+
+// TestVeleroKopiaGetAfterList simulates Kopia's pattern of listing and then getting objects
+// This specifically tests the bug where listed paths couldn't be retrieved
+func TestVeleroKopiaGetAfterList(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+	enableVersioning(t, client, bucketName)
+
+	// Create a nested object like Kopia does
+	objectKey := "kopia/appwrite/kopia.repository"
+	objectContent := "kopia-repository-config-data"
+
+	_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader(objectContent),
+	})
+	require.NoError(t, err)
+
+	// List objects (this is where the bug manifested - returned wrong Key)
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String("kopia/appwrite/"),
+	})
+	require.NoError(t, err)
+	require.Len(t, listResp.Contents, 1, "Should list exactly one object")
+
+	listedKey := *listResp.Contents[0].Key
+
+	// The bug caused listedKey to be "kopia/appwrite/kopia/appwrite/kopia.repository"
+	// instead of "kopia/appwrite/kopia.repository"
+	assert.Equal(t, objectKey, listedKey,
+		"Listed key should match the original key without path doubling")
+
+	// Now try to GET the object using the listed key
+	// With the bug, this would fail because the doubled path doesn't exist
+	getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(listedKey),
+	})
+	require.NoError(t, err, "Should be able to GET object using the key returned from ListObjects")
+	defer getResp.Body.Close()
+
+	// Verify content matches
+	body, err := io.ReadAll(getResp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, objectContent, string(body),
+		"Retrieved content should match original")
+}
+
+// TestVeleroMultipleVersionsWithNestedPaths tests listing when objects have multiple versions
+func TestVeleroMultipleVersionsWithNestedPaths(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+	enableVersioning(t, client, bucketName)
+
+	objectKey := "kopia/logpaste/kopia.blobcfg"
+
+	// Create multiple versions of the same object (Kopia updates config files)
+	versions := []string{"version-1", "version-2", "version-3"}
+	for _, content := range versions {
+		_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(objectKey),
+			Body:   strings.NewReader(content),
+		})
+		require.NoError(t, err)
+	}
+
+	// List objects - should return only the latest version in regular listing
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String("kopia/logpaste/"),
+	})
+	require.NoError(t, err)
+
+	// Should only see one object (the latest version)
+	require.Len(t, listResp.Contents, 1, "Should list only one object (latest version)")
+
+	listedKey := *listResp.Contents[0].Key
+	assert.Equal(t, objectKey, listedKey,
+		"Listed key should match original without path doubling")
+	assert.False(t, hasDoubledPath(listedKey),
+		"Path should not be doubled")
+
+	// Verify we can GET the latest version
+	getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(listedKey),
+	})
+	require.NoError(t, err)
+	defer getResp.Body.Close()
+
+	body, err := io.ReadAll(getResp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, "version-3", string(body),
+		"Should get the latest version content")
+}
+
+// TestVeleroListVersionsWithNestedPaths tests ListObjectVersions with nested paths
+func TestVeleroListVersionsWithNestedPaths(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+	enableVersioning(t, client, bucketName)
+
+	objectKey := "backups/velero/test-backup/backup.json"
+
+	// Create multiple versions
+	for i := 1; i <= 3; i++ {
+		_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(objectKey),
+			Body:   strings.NewReader(strings.Repeat("x", i*100)),
+		})
+		require.NoError(t, err)
+	}
+
+	// List all versions
+	listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String("backups/velero/"),
+	})
+	require.NoError(t, err)
+
+	// Should have 3 versions
+	require.Len(t, listResp.Versions, 3, "Should have 3 versions")
+
+	// Verify all version keys are correct (not doubled)
+	for _, version := range listResp.Versions {
+		key := *version.Key
+		assert.Equal(t, objectKey, key,
+			"Version key should match original")
+		assert.False(t, hasDoubledPath(key),
+			"Version key should not have doubled path: %s", key)
+
+		// Verify we can GET each version by its version ID
+		getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+			Bucket:    aws.String(bucketName),
+			Key:       aws.String(key),
+			VersionId: version.VersionId,
+		})
+		require.NoError(t, err, "Should be able to GET version %s", *version.VersionId)
+		func() {
+			defer getResp.Body.Close()
+		}()
+	}
+}
+
+// hasDoubledPath checks if a path has a repeated pair of segments, which indicates
+// the path doubling bug. For example: "kopia/logpaste/kopia/logpaste/file.txt" would return true.
+func hasDoubledPath(path string) bool {
+	parts := strings.Split(path, "/")
+	if len(parts) < 4 {
+		return false
+	}
+
+	// Check for repeated pairs of segments, e.g., a/b/.../a/b.
+	for i := 0; i < len(parts)-3; i++ {
+		for j := i + 2; j < len(parts)-1; j++ {
+			if parts[i] == parts[j] && parts[i+1] == parts[j+1] {
+				return true
+			}
+		}
+	}
+
+	return false
+}
@@ -0,0 +1,521 @@
+package s3api
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestMultipartUploadVersioningListETag tests that multipart uploaded objects
+// in versioned buckets have correct ETags when listed.
+// This covers a bug where synthetic entries for versioned objects didn't include
+// proper ETag handling for multipart uploads (ETags with format "<md5>-<parts>").
+func TestMultipartUploadVersioningListETag(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err, "Failed to enable versioning")
+
+	// Create multipart upload
+	objectKey := "multipart-test-object"
+	createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	uploadId := *createResp.UploadId
+
+	// Upload 2 parts (minimum 5MB per part except last)
+	partSize := 5 * 1024 * 1024 // 5MB
+	part1Data := bytes.Repeat([]byte("a"), partSize)
+	part2Data := bytes.Repeat([]byte("b"), partSize)
+
+	// Calculate MD5 for each part
+	part1MD5 := md5.Sum(part1Data)
+	part2MD5 := md5.Sum(part2Data)
+
+	// Upload part 1
+	uploadPart1Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   aws.String(uploadId),
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(part1Data),
+	})
+	require.NoError(t, err, "Failed to upload part 1")
+
+	// Upload part 2
+	uploadPart2Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   aws.String(uploadId),
+		PartNumber: aws.Int32(2),
+		Body:       bytes.NewReader(part2Data),
+	})
+	require.NoError(t, err, "Failed to upload part 2")
+
+	// Complete multipart upload
+	completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: aws.String(uploadId),
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: []types.CompletedPart{
+				{
+					ETag:       uploadPart1Resp.ETag,
+					PartNumber: aws.Int32(1),
+				},
+				{
+					ETag:       uploadPart2Resp.ETag,
+					PartNumber: aws.Int32(2),
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+
+	// Verify the ETag from CompleteMultipartUpload has the multipart format (md5-parts)
+	completeETag := strings.Trim(*completeResp.ETag, "\"")
+	assert.Contains(t, completeETag, "-", "Multipart ETag should contain '-' (format: md5-parts)")
+	assert.True(t, strings.HasSuffix(completeETag, "-2"), "Multipart ETag should end with '-2' for 2 parts")
+
+	t.Logf("CompleteMultipartUpload ETag: %s", completeETag)
+	t.Logf("Part 1 MD5: %x", part1MD5)
+	t.Logf("Part 2 MD5: %x", part2MD5)
+
+	// HeadObject should return the same ETag
+	headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to head object")
+
+	headETag := strings.Trim(*headResp.ETag, "\"")
+	assert.Equal(t, completeETag, headETag, "HeadObject ETag should match CompleteMultipartUpload ETag")
+
+	// ListObjectsV2 should return the same ETag
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects")
+	require.Len(t, listResp.Contents, 1, "Should have exactly one object")
+
+	listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
+	assert.Equal(t, completeETag, listETag, "ListObjectsV2 ETag should match CompleteMultipartUpload ETag")
+	assert.NotEmpty(t, listETag, "ListObjectsV2 ETag should not be empty")
+
+	t.Logf("ListObjectsV2 ETag: %s", listETag)
+
+	// ListObjectVersions should also return the correct ETag
+	versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list object versions")
+	require.Len(t, versionsResp.Versions, 1, "Should have exactly one version")
+
+	versionETag := strings.Trim(*versionsResp.Versions[0].ETag, "\"")
+	assert.Equal(t, completeETag, versionETag, "ListObjectVersions ETag should match CompleteMultipartUpload ETag")
+	assert.NotEmpty(t, versionETag, "ListObjectVersions ETag should not be empty")
+
+	t.Logf("ListObjectVersions ETag: %s", versionETag)
+}
+
+// TestMultipartUploadMultipleVersionsListETag tests that multiple versions
+// of multipart uploaded objects all have correct ETags when listed.
+func TestMultipartUploadMultipleVersionsListETag(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err, "Failed to enable versioning")
+
+	objectKey := "multipart-multi-version-object"
+	partSize := 5 * 1024 * 1024 // 5MB
+	var expectedETags []string
+
+	// Create 3 versions using multipart upload
+	for version := 1; version <= 3; version++ {
+		// Create multipart upload
+		createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(objectKey),
+		})
+		require.NoError(t, err, "Failed to create multipart upload for version %d", version)
+
+		uploadId := *createResp.UploadId
+
+		// Create unique data for each version
+		partData := bytes.Repeat([]byte(fmt.Sprintf("%d", version)), partSize)
+
+		// Upload single part (still results in multipart ETag format)
+		uploadPartResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+			Bucket:     aws.String(bucketName),
+			Key:        aws.String(objectKey),
+			UploadId:   aws.String(uploadId),
+			PartNumber: aws.Int32(1),
+			Body:       bytes.NewReader(partData),
+		})
+		require.NoError(t, err, "Failed to upload part for version %d", version)
+
+		// Complete multipart upload
+		completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+			Bucket:   aws.String(bucketName),
+			Key:      aws.String(objectKey),
+			UploadId: aws.String(uploadId),
+			MultipartUpload: &types.CompletedMultipartUpload{
+				Parts: []types.CompletedPart{
+					{
+						ETag:       uploadPartResp.ETag,
+						PartNumber: aws.Int32(1),
+					},
+				},
+			},
+		})
+		require.NoError(t, err, "Failed to complete multipart upload for version %d", version)
+
+		etag := strings.Trim(*completeResp.ETag, "\"")
+		expectedETags = append(expectedETags, etag)
+		t.Logf("Version %d ETag: %s", version, etag)
+	}
+
+	// ListObjectVersions should return all versions with correct ETags
+	versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list object versions")
+	require.Len(t, versionsResp.Versions, 3, "Should have exactly 3 versions")
+
+	// Collect ETags from the listing
+	var listedETags []string
+	for _, v := range versionsResp.Versions {
+		etag := strings.Trim(*v.ETag, "\"")
+		listedETags = append(listedETags, etag)
+		assert.NotEmpty(t, etag, "Version ETag should not be empty")
+		assert.Contains(t, etag, "-", "Multipart ETag should contain '-'")
+	}
+
+	t.Logf("Expected ETags: %v", expectedETags)
+	t.Logf("Listed ETags: %v", listedETags)
+
+	// Verify all expected ETags are present (order may differ due to version ordering)
+	assert.ElementsMatch(t, expectedETags, listedETags, "Listed ETags should match all expected ETags")
+
+	// Regular ListObjectsV2 should return only the latest version with correct ETag
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects")
+	require.Len(t, listResp.Contents, 1, "Should have exactly one object in regular listing")
+
+	listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
+	// The latest version (version 3) should be the one shown
+	assert.Equal(t, expectedETags[2], listETag, "ListObjectsV2 should show latest version's ETag")
+}
+
+// TestMixedSingleAndMultipartVersionsListETag tests that a mix of
+// single-part and multipart uploaded versions all have correct ETags.
+func TestMixedSingleAndMultipartVersionsListETag(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err, "Failed to enable versioning")
+
+	objectKey := "mixed-upload-versions"
+
+	// Version 1: Regular PutObject (single-part, pure MD5 ETag)
+	content1 := []byte("This is version 1 content - single part upload")
+	putResp1, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(content1),
+	})
+	require.NoError(t, err, "Failed to put version 1")
+	etag1 := strings.Trim(*putResp1.ETag, "\"")
+	assert.NotContains(t, etag1, "-", "Single-part ETag should not contain '-'")
+	t.Logf("Version 1 (PutObject) ETag: %s", etag1)
+
+	// Version 2: Multipart upload
+	partSize := 5 * 1024 * 1024
+	partData := bytes.Repeat([]byte("x"), partSize)
+
+	createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	uploadPartResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   createResp.UploadId,
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(partData),
+	})
+	require.NoError(t, err, "Failed to upload part")
+
+	completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: createResp.UploadId,
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: []types.CompletedPart{
+				{
+					ETag:       uploadPartResp.ETag,
+					PartNumber: aws.Int32(1),
+				},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+	etag2 := strings.Trim(*completeResp.ETag, "\"")
+	assert.Contains(t, etag2, "-", "Multipart ETag should contain '-'")
+	t.Logf("Version 2 (Multipart) ETag: %s", etag2)
+
+	// Version 3: Another regular PutObject
+	content3 := []byte("This is version 3 content - another single part upload")
+	putResp3, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(content3),
+	})
+	require.NoError(t, err, "Failed to put version 3")
+	etag3 := strings.Trim(*putResp3.ETag, "\"")
+	assert.NotContains(t, etag3, "-", "Single-part ETag should not contain '-'")
+	t.Logf("Version 3 (PutObject) ETag: %s", etag3)
+
+	// ListObjectVersions should return all 3 versions with correct ETags
+	versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list object versions")
+	require.Len(t, versionsResp.Versions, 3, "Should have exactly 3 versions")
+
+	var listedETags []string
+	for _, v := range versionsResp.Versions {
+		etag := strings.Trim(*v.ETag, "\"")
+		assert.NotEmpty(t, etag, "Version ETag should not be empty")
+		listedETags = append(listedETags, etag)
+		t.Logf("Listed version %s ETag: %s, IsLatest: %v", *v.VersionId, etag, *v.IsLatest)
+	}
+
+	// Verify all ETags were found
+	assert.ElementsMatch(t, []string{etag1, etag2, etag3}, listedETags, "Listed ETags should match all expected ETags")
+
+	// Regular ListObjectsV2 should return only the latest (version 3)
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects")
+	require.Len(t, listResp.Contents, 1, "Should have exactly one object")
+
+	listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
+	assert.Equal(t, etag3, listETag, "ListObjectsV2 should show latest version's ETag (version 3)")
+}
+
+// TestMultipartUploadDeleteMarkerListBehavior tests that delete markers work correctly
+// with multipart uploaded objects in versioned buckets.
+func TestMultipartUploadDeleteMarkerListBehavior(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err, "Failed to enable versioning")
+
+	objectKey := "multipart-delete-marker-test"
+	partSize := 5 * 1024 * 1024 // 5MB
+
+	// Create multipart upload
+	createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	// Upload 2 parts
+	part1Data := bytes.Repeat([]byte("a"), partSize)
+	part2Data := bytes.Repeat([]byte("b"), partSize)
+
+	uploadPart1Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   createResp.UploadId,
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(part1Data),
+	})
+	require.NoError(t, err, "Failed to upload part 1")
+
+	uploadPart2Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   createResp.UploadId,
+		PartNumber: aws.Int32(2),
+		Body:       bytes.NewReader(part2Data),
+	})
+	require.NoError(t, err, "Failed to upload part 2")
+
+	// Complete multipart upload
+	completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: createResp.UploadId,
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: []types.CompletedPart{
+				{ETag: uploadPart1Resp.ETag, PartNumber: aws.Int32(1)},
+				{ETag: uploadPart2Resp.ETag, PartNumber: aws.Int32(2)},
+			},
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+
+	multipartETag := strings.Trim(*completeResp.ETag, "\"")
+	multipartVersionId := *completeResp.VersionId
+	t.Logf("Multipart upload completed: ETag=%s, VersionId=%s", multipartETag, multipartVersionId)
+
+	// Verify object is visible in ListObjectsV2
+	listBeforeDelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects before delete")
+	require.Len(t, listBeforeDelete.Contents, 1, "Object should be visible before delete")
+	assert.Equal(t, multipartETag, strings.Trim(*listBeforeDelete.Contents[0].ETag, "\""),
+		"Listed ETag should match multipart ETag before delete")
+
+	// Delete object (creates delete marker)
+	deleteResp, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to delete object")
+	require.NotNil(t, deleteResp.DeleteMarker, "Should create delete marker")
+	assert.True(t, *deleteResp.DeleteMarker, "DeleteMarker should be true")
+	require.NotNil(t, deleteResp.VersionId, "Delete marker should have version ID")
+
+	deleteMarkerVersionId := *deleteResp.VersionId
+	t.Logf("Delete marker created: VersionId=%s", deleteMarkerVersionId)
+
+	// ListObjectsV2 should NOT show the object anymore
+	listAfterDelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects after delete")
+	assert.Empty(t, listAfterDelete.Contents, "Object should NOT be visible after delete marker")
+
+	// ListObjectVersions should show both the original version AND the delete marker
+	versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list object versions")
+
+	// Should have 1 version (the multipart object)
+	require.Len(t, versionsResp.Versions, 1, "Should have exactly 1 version (the multipart object)")
+	version := versionsResp.Versions[0]
+	assert.Equal(t, multipartVersionId, *version.VersionId, "Version ID should match")
+	assert.Equal(t, multipartETag, strings.Trim(*version.ETag, "\""), "Version ETag should match multipart ETag")
+	assert.False(t, *version.IsLatest, "Multipart version should NOT be latest (delete marker is latest)")
+
+	// Should have 1 delete marker
+	require.Len(t, versionsResp.DeleteMarkers, 1, "Should have exactly 1 delete marker")
+	deleteMarker := versionsResp.DeleteMarkers[0]
+	assert.Equal(t, deleteMarkerVersionId, *deleteMarker.VersionId, "Delete marker version ID should match")
+	assert.True(t, *deleteMarker.IsLatest, "Delete marker should be latest")
+
+	t.Logf("ListObjectVersions: 1 version (ETag=%s), 1 delete marker", multipartETag)
+
+	// Access the specific version by version ID - should still work
+	getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(multipartVersionId),
+	})
+	require.NoError(t, err, "Should be able to get object by version ID after delete marker")
+	defer getResp.Body.Close()
+
+	assert.Equal(t, multipartETag, strings.Trim(*getResp.ETag, "\""),
+		"GetObject with version ID should return correct ETag")
+	assert.Equal(t, int64(partSize*2), *getResp.ContentLength,
+		"GetObject with version ID should return correct size")
+
+	t.Logf("Successfully retrieved version %s after delete marker", multipartVersionId)
+
+	// Delete the delete marker to "undelete" the object
+	_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(deleteMarkerVersionId),
+	})
+	require.NoError(t, err, "Failed to delete the delete marker")
+
+	// ListObjectsV2 should show the object again
+	listAfterUndelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to list objects after undelete")
+	require.Len(t, listAfterUndelete.Contents, 1, "Object should be visible again after removing delete marker")
+	assert.Equal(t, multipartETag, strings.Trim(*listAfterUndelete.Contents[0].ETag, "\""),
+		"Undeleted object should have correct multipart ETag")
+
+	t.Logf("Object restored after delete marker removal, ETag=%s", multipartETag)
+}
+
@@ -0,0 +1,322 @@
+package s3api
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestVersioningPaginationOver1000Versions tests that ListObjectVersions correctly
+// handles objects with more than 1000 versions using pagination.
+// This is a stress test that creates 1500 versions and verifies all are listed.
+//
+// Run with: ENABLE_STRESS_TESTS=true go test -v -run TestVersioningPaginationOver1000Versions -timeout 30m
+func TestVersioningPaginationOver1000Versions(t *testing.T) {
+	if os.Getenv("ENABLE_STRESS_TESTS") != "true" {
+		t.Skip("Skipping stress test. Set ENABLE_STRESS_TESTS=true to run.")
+	}
+
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	// Create bucket
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	// Enable versioning
+	enableVersioning(t, client, bucketName)
+	checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
+
+	objectKey := "test-many-versions"
+	// Use 1100 for CI (tests pagination boundary), can override with env var for quick local testing
+	numVersions := 1100
+	if quickTest := os.Getenv("QUICK_TEST"); quickTest == "true" {
+		numVersions = 50
+	}
+	versionIds := make([]string, 0, numVersions)
+
+	t.Logf("Creating %d versions of object %s...", numVersions, objectKey)
+	startTime := time.Now()
+
+	// Create many versions of the same object
+	for i := 0; i < numVersions; i++ {
+		content := fmt.Sprintf("content-version-%d", i+1)
+		putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(objectKey),
+			Body:   strings.NewReader(content),
+		})
+		require.NoError(t, err, "Failed to create version %d", i+1)
+		require.NotNil(t, putResp.VersionId)
+		versionIds = append(versionIds, *putResp.VersionId)
+
+		if (i+1)%100 == 0 {
+			t.Logf("Created %d/%d versions...", i+1, numVersions)
+		}
+	}
+
+	createDuration := time.Since(startTime)
+	t.Logf("Created %d versions in %v", numVersions, createDuration)
+	t.Logf("Version IDs collected: %d", len(versionIds))
+
+	// Quick sanity check - list versions to see what we have
+	quickResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	t.Logf("Quick check: got %d versions, %d delete markers, truncated=%v",
+		len(quickResp.Versions), len(quickResp.DeleteMarkers),
+		quickResp.IsTruncated != nil && *quickResp.IsTruncated)
+
+	// Test 1: List all versions without pagination limit
+	t.Run("ListAllVersionsNoPagination", func(t *testing.T) {
+		allVersions := listAllVersions(t, client, bucketName, objectKey)
+		t.Logf("Got %d versions total", len(allVersions))
+		assert.Len(t, allVersions, numVersions, "Should have all %d versions", numVersions)
+
+		// Verify all version IDs are unique
+		versionIdSet := make(map[string]bool)
+		for _, v := range allVersions {
+			assert.False(t, versionIdSet[*v.VersionId], "Duplicate version ID: %s", *v.VersionId)
+			versionIdSet[*v.VersionId] = true
+		}
+
+		// Verify only one version is marked as latest
+		latestCount := 0
+		for _, v := range allVersions {
+			if *v.IsLatest {
+				latestCount++
+				assert.Equal(t, versionIds[numVersions-1], *v.VersionId, "Latest should be the last created")
+			}
+		}
+		assert.Equal(t, 1, latestCount, "Only one version should be marked as latest")
+	})
+
+	// Test 2: List with explicit small max-keys to force multiple pages
+	t.Run("ListWithSmallMaxKeys", func(t *testing.T) {
+		maxKeys := int32(100)
+		allVersions := make([]types.ObjectVersion, 0, numVersions)
+		var keyMarker, versionIdMarker *string
+		pageCount := 0
+
+		for {
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:          aws.String(bucketName),
+				Prefix:          aws.String(objectKey),
+				MaxKeys:         aws.Int32(maxKeys),
+				KeyMarker:       keyMarker,
+				VersionIdMarker: versionIdMarker,
+			})
+			require.NoError(t, err)
+			pageCount++
+
+			allVersions = append(allVersions, resp.Versions...)
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+			versionIdMarker = resp.NextVersionIdMarker
+		}
+
+		t.Logf("Listed %d versions in %d pages (max-keys=%d)", len(allVersions), pageCount, maxKeys)
+		assert.Len(t, allVersions, numVersions, "Should have all %d versions", numVersions)
+		if numVersions > int(maxKeys) {
+			assert.Greater(t, pageCount, 1, "Should require multiple pages when versions > max-keys")
+		}
+	})
+
+	// Test 3: List with max-keys=1000 (S3 default) to test the boundary
+	t.Run("ListWithDefaultMaxKeys", func(t *testing.T) {
+		maxKeys := int32(1000)
+		allVersions := make([]types.ObjectVersion, 0, numVersions)
+		var keyMarker, versionIdMarker *string
+		pageCount := 0
+
+		for {
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:          aws.String(bucketName),
+				Prefix:          aws.String(objectKey),
+				MaxKeys:         aws.Int32(maxKeys),
+				KeyMarker:       keyMarker,
+				VersionIdMarker: versionIdMarker,
+			})
+			require.NoError(t, err)
+			pageCount++
+
+			allVersions = append(allVersions, resp.Versions...)
+			t.Logf("Page %d: got %d versions, truncated=%v", pageCount, len(resp.Versions), resp.IsTruncated != nil && *resp.IsTruncated)
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+			versionIdMarker = resp.NextVersionIdMarker
+		}
+
+		t.Logf("Listed %d versions in %d pages (max-keys=%d)", len(allVersions), pageCount, maxKeys)
+		assert.Len(t, allVersions, numVersions, "Should have all %d versions", numVersions)
+		if numVersions > 1000 {
+			assert.GreaterOrEqual(t, pageCount, 2, "Should require at least 2 pages for >1000 versions with max-keys=1000")
+		}
+	})
+
+	// Test 4: Verify we can retrieve specific versions
+	t.Run("RetrieveSpecificVersions", func(t *testing.T) {
+		// Test first, middle, and last versions
+		testIndices := []int{0, numVersions / 2, numVersions - 1}
+		for _, idx := range testIndices {
+			expectedContent := fmt.Sprintf("content-version-%d", idx+1)
+			getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+				Bucket:    aws.String(bucketName),
+				Key:       aws.String(objectKey),
+				VersionId: aws.String(versionIds[idx]),
+			})
+			require.NoError(t, err, "Failed to get version at index %d", idx)
+
+			buf := make([]byte, len(expectedContent))
+			_, err = getResp.Body.Read(buf)
+			getResp.Body.Close()
+
+			assert.Equal(t, expectedContent, string(buf), "Content mismatch for version %d", idx+1)
+		}
+	})
+}
+
+// TestVersioningPaginationMultipleObjectsManyVersions tests pagination with multiple objects
+// each having many versions, ensuring correct handling of key-marker and version-id-marker.
+//
+// Run with: ENABLE_STRESS_TESTS=true go test -v -run TestVersioningPaginationMultipleObjectsManyVersions -timeout 30m
+func TestVersioningPaginationMultipleObjectsManyVersions(t *testing.T) {
+	if os.Getenv("ENABLE_STRESS_TESTS") != "true" {
+		t.Skip("Skipping stress test. Set ENABLE_STRESS_TESTS=true to run.")
+	}
+
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	enableVersioning(t, client, bucketName)
+	checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
+
+	numObjects := 5
+	versionsPerObject := 300
+	totalExpectedVersions := numObjects * versionsPerObject
+
+	t.Logf("Creating %d objects with %d versions each (%d total)...", numObjects, versionsPerObject, totalExpectedVersions)
+	startTime := time.Now()
+
+	// Create multiple objects with many versions each
+	for i := 0; i < numObjects; i++ {
+		objectKey := fmt.Sprintf("object-%03d", i)
+		for j := 0; j < versionsPerObject; j++ {
+			content := fmt.Sprintf("obj%d-ver%d", i, j+1)
+			_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+				Body:   strings.NewReader(content),
+			})
+			require.NoError(t, err)
+		}
+		t.Logf("Created object %s with %d versions", objectKey, versionsPerObject)
+	}
+
+	createDuration := time.Since(startTime)
+	t.Logf("Created %d total versions in %v", totalExpectedVersions, createDuration)
+
+	// List all versions using pagination
+	t.Run("ListAllWithPagination", func(t *testing.T) {
+		maxKeys := int32(500)
+		allVersions := make([]types.ObjectVersion, 0, totalExpectedVersions)
+		var keyMarker, versionIdMarker *string
+		pageCount := 0
+
+		for {
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:          aws.String(bucketName),
+				MaxKeys:         aws.Int32(maxKeys),
+				KeyMarker:       keyMarker,
+				VersionIdMarker: versionIdMarker,
+			})
+			require.NoError(t, err)
+			pageCount++
+
+			allVersions = append(allVersions, resp.Versions...)
+			t.Logf("Page %d: got %d versions, truncated=%v", pageCount, len(resp.Versions), resp.IsTruncated != nil && *resp.IsTruncated)
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+			versionIdMarker = resp.NextVersionIdMarker
+		}
+
+		t.Logf("Listed %d versions in %d pages", len(allVersions), pageCount)
+		assert.Len(t, allVersions, totalExpectedVersions, "Should have all versions")
+
+		// Verify each object has correct number of versions
+		versionsByKey := make(map[string]int)
+		for _, v := range allVersions {
+			versionsByKey[*v.Key]++
+		}
+		assert.Len(t, versionsByKey, numObjects, "Should have %d unique objects", numObjects)
+		for key, count := range versionsByKey {
+			assert.Equal(t, versionsPerObject, count, "Object %s should have %d versions", key, versionsPerObject)
+		}
+
+		// Verify each object has exactly one latest version
+		latestByKey := make(map[string]int)
+		for _, v := range allVersions {
+			if *v.IsLatest {
+				latestByKey[*v.Key]++
+			}
+		}
+		for key, count := range latestByKey {
+			assert.Equal(t, 1, count, "Object %s should have exactly one latest version", key)
+		}
+	})
+}
+
+// listAllVersions is a helper to list all versions of a specific object using pagination
+func listAllVersions(t *testing.T, client *s3.Client, bucketName, objectKey string) []types.ObjectVersion {
+	var allVersions []types.ObjectVersion
+	var keyMarker, versionIdMarker *string
+	pageCount := 0
+
+	for {
+		resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+			Bucket:          aws.String(bucketName),
+			Prefix:          aws.String(objectKey),
+			KeyMarker:       keyMarker,
+			VersionIdMarker: versionIdMarker,
+		})
+		require.NoError(t, err)
+		pageCount++
+
+		t.Logf("Page %d: got %d versions, truncated=%v", pageCount, len(resp.Versions), resp.IsTruncated != nil && *resp.IsTruncated)
+		allVersions = append(allVersions, resp.Versions...)
+
+		// Check if truncated (handle nil pointer)
+		if resp.IsTruncated == nil || !*resp.IsTruncated {
+			break
+		}
+		keyMarker = resp.NextKeyMarker
+		versionIdMarker = resp.NextVersionIdMarker
+	}
+
+	t.Logf("Total: %d versions in %d pages", len(allVersions), pageCount)
+	return allVersions
+}
+
@@ -90,3 +90,4 @@ To debug test failures:
   ```


+
@@ -35,3 +35,4 @@
 ]


+
@@ -0,0 +1,226 @@
+# Makefile for TUS Protocol Integration Tests
+# This Makefile provides targets for running TUS (resumable upload) integration tests
+
+# Default values
+SEAWEEDFS_BINARY ?= weed
+FILER_PORT ?= 18888
+VOLUME_PORT ?= 18080
+MASTER_PORT ?= 19333
+TEST_TIMEOUT ?= 10m
+VOLUME_MAX_SIZE_MB ?= 50
+VOLUME_MAX_COUNT ?= 100
+
+# Test directory
+TEST_DIR := $(shell pwd)
+SEAWEEDFS_ROOT := $(shell cd ../.. && pwd)
+
+# Colors for output
+RED := \033[0;31m
+GREEN := \033[0;32m
+YELLOW := \033[1;33m
+NC := \033[0m # No Color
+
+.PHONY: all test clean start-seaweedfs stop-seaweedfs check-binary build-weed help test-basic test-chunked test-resume test-errors test-with-server
+
+all: test
+
+# Build SeaweedFS binary
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd $(SEAWEEDFS_ROOT)/weed && go build -o weed
+	@echo "$(GREEN)SeaweedFS binary built successfully$(NC)"
+
+help:
+	@echo "SeaweedFS TUS Protocol Integration Tests"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  test              - Run all TUS integration tests"
+	@echo "  test-basic        - Run basic TUS upload tests"
+	@echo "  test-chunked      - Run chunked upload tests"
+	@echo "  test-resume       - Run upload resume tests"
+	@echo "  test-errors       - Run error handling tests"
+	@echo "  test-with-server  - Run tests with automatic server management"
+	@echo "  start-seaweedfs   - Start SeaweedFS server for testing"
+	@echo "  stop-seaweedfs    - Stop SeaweedFS server"
+	@echo "  clean             - Clean up test artifacts"
+	@echo "  check-binary      - Check if SeaweedFS binary exists"
+	@echo "  build-weed        - Build SeaweedFS binary"
+	@echo ""
+	@echo "Configuration:"
+	@echo "  SEAWEEDFS_BINARY=$(SEAWEEDFS_BINARY)"
+	@echo "  FILER_PORT=$(FILER_PORT)"
+	@echo "  VOLUME_PORT=$(VOLUME_PORT)"
+	@echo "  MASTER_PORT=$(MASTER_PORT)"
+	@echo "  TEST_TIMEOUT=$(TEST_TIMEOUT)"
+
+check-binary:
+	@if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1 && [ ! -f "$(SEAWEEDFS_ROOT)/weed/weed" ]; then \
+		echo "$(RED)Error: SeaweedFS binary not found$(NC)"; \
+		echo "Please build SeaweedFS first: make build-weed"; \
+		exit 1; \
+	fi
+	@echo "$(GREEN)SeaweedFS binary found$(NC)"
+
+start-seaweedfs: check-binary
+	@echo "$(YELLOW)Starting SeaweedFS server for TUS testing...$(NC)"
+	@# Clean up any existing processes on our test ports
+	@lsof -ti :$(MASTER_PORT) | grep -v '^$$' | xargs kill -TERM 2>/dev/null || true
+	@lsof -ti :$(VOLUME_PORT) | grep -v '^$$' | xargs kill -TERM 2>/dev/null || true
+	@lsof -ti :$(FILER_PORT) | grep -v '^$$' | xargs kill -TERM 2>/dev/null || true
+	@sleep 2
+	
+	# Create necessary directories
+	@mkdir -p /tmp/seaweedfs-test-tus-master
+	@mkdir -p /tmp/seaweedfs-test-tus-volume
+	@mkdir -p /tmp/seaweedfs-test-tus-filer
+	
+	# Start master server (use freshly built binary)
+	@echo "Starting master server..."
+	@nohup $(SEAWEEDFS_ROOT)/weed/weed master \
+		-port=$(MASTER_PORT) \
+		-mdir=/tmp/seaweedfs-test-tus-master \
+		-volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) \
+		-ip=127.0.0.1 \
+		> /tmp/seaweedfs-tus-master.log 2>&1 &
+	@sleep 3
+	
+	# Start volume server
+	@echo "Starting volume server..."
+	@nohup $(SEAWEEDFS_ROOT)/weed/weed volume \
+		-port=$(VOLUME_PORT) \
+		-mserver=127.0.0.1:$(MASTER_PORT) \
+		-dir=/tmp/seaweedfs-test-tus-volume \
+		-max=$(VOLUME_MAX_COUNT) \
+		-ip=127.0.0.1 \
+		> /tmp/seaweedfs-tus-volume.log 2>&1 &
+	@sleep 3
+	
+	# Start filer server with TUS enabled (default tusBasePath is .tus)
+	@echo "Starting filer server..."
+	@nohup $(SEAWEEDFS_ROOT)/weed/weed filer \
+		-port=$(FILER_PORT) \
+		-master=127.0.0.1:$(MASTER_PORT) \
+		-ip=127.0.0.1 \
+		> /tmp/seaweedfs-tus-filer.log 2>&1 &
+	@sleep 5
+	
+	# Wait for filer to be ready
+	@echo "$(YELLOW)Waiting for filer to be ready...$(NC)"
+	@for i in $$(seq 1 30); do \
+		if curl -s -f http://127.0.0.1:$(FILER_PORT)/ > /dev/null 2>&1; then \
+			echo "$(GREEN)Filer is ready$(NC)"; \
+			break; \
+		fi; \
+		if [ $$i -eq 30 ]; then \
+			echo "$(RED)Filer failed to start within 30 seconds$(NC)"; \
+			$(MAKE) debug-logs; \
+			exit 1; \
+		fi; \
+		echo "Waiting for filer... ($$i/30)"; \
+		sleep 1; \
+	done
+	
+	@echo "$(GREEN)SeaweedFS server started successfully for TUS testing$(NC)"
+	@echo "Master: http://localhost:$(MASTER_PORT)"
+	@echo "Volume: http://localhost:$(VOLUME_PORT)"
+	@echo "Filer: http://localhost:$(FILER_PORT)"
+	@echo "TUS Endpoint: http://localhost:$(FILER_PORT)/.tus/"
+
+stop-seaweedfs:
+	@echo "$(YELLOW)Stopping SeaweedFS server...$(NC)"
+	@lsof -ti :$(MASTER_PORT) | grep -v '^$$' | xargs kill -TERM 2>/dev/null || true
+	@lsof -ti :$(VOLUME_PORT) | grep -v '^$$' | xargs kill -TERM 2>/dev/null || true
+	@lsof -ti :$(FILER_PORT) | grep -v '^$$' | xargs kill -TERM 2>/dev/null || true
+	@sleep 2
+	@echo "$(GREEN)SeaweedFS server stopped$(NC)"
+
+clean:
+	@echo "$(YELLOW)Cleaning up TUS test artifacts...$(NC)"
+	@rm -rf /tmp/seaweedfs-test-tus-*
+	@rm -f /tmp/seaweedfs-tus-*.log
+	@echo "$(GREEN)TUS test cleanup completed$(NC)"
+
+# Run all tests
+test: check-binary
+	@echo "$(YELLOW)Running all TUS integration tests...$(NC)"
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/tus/...
+	@echo "$(GREEN)All TUS tests completed$(NC)"
+
+# Run basic upload tests
+test-basic: check-binary
+	@echo "$(YELLOW)Running basic TUS upload tests...$(NC)"
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusBasicUpload|TestTusOptionsHandler" ./test/tus/...
+	@echo "$(GREEN)Basic TUS tests completed$(NC)"
+
+# Run chunked upload tests
+test-chunked: check-binary
+	@echo "$(YELLOW)Running chunked TUS upload tests...$(NC)"
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusChunkedUpload" ./test/tus/...
+	@echo "$(GREEN)Chunked TUS tests completed$(NC)"
+
+# Run resume tests
+test-resume: check-binary
+	@echo "$(YELLOW)Running TUS upload resume tests...$(NC)"
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusResumeAfterInterruption|TestTusHeadRequest" ./test/tus/...
+	@echo "$(GREEN)TUS resume tests completed$(NC)"
+
+# Run error handling tests
+test-errors: check-binary
+	@echo "$(YELLOW)Running TUS error handling tests...$(NC)"
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusInvalidOffset|TestTusUploadNotFound|TestTusDeleteUpload" ./test/tus/...
+	@echo "$(GREEN)TUS error tests completed$(NC)"
+
+# Run tests with automatic server management
+test-with-server: build-weed
+	@echo "$(YELLOW)Running TUS tests with automatic server management...$(NC)"
+	@$(MAKE) -C $(TEST_DIR) start-seaweedfs && \
+	sleep 3 && \
+	cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/tus/...; \
+	TEST_RESULT=$$?; \
+	$(MAKE) -C $(TEST_DIR) stop-seaweedfs; \
+	$(MAKE) -C $(TEST_DIR) clean; \
+	if [ $$TEST_RESULT -eq 0 ]; then echo "$(GREEN)All TUS tests passed!$(NC)"; fi; \
+	exit $$TEST_RESULT
+
+# Debug targets
+debug-logs:
+	@echo "$(YELLOW)=== Master Log ===$(NC)"
+	@tail -n 50 /tmp/seaweedfs-tus-master.log 2>/dev/null || echo "No master log found"
+	@echo "$(YELLOW)=== Volume Log ===$(NC)"
+	@tail -n 50 /tmp/seaweedfs-tus-volume.log 2>/dev/null || echo "No volume log found"
+	@echo "$(YELLOW)=== Filer Log ===$(NC)"
+	@tail -n 50 /tmp/seaweedfs-tus-filer.log 2>/dev/null || echo "No filer log found"
+
+debug-status:
+	@echo "$(YELLOW)=== Process Status ===$(NC)"
+	@ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found"
+	@echo "$(YELLOW)=== Port Status ===$(NC)"
+	@lsof -i :$(MASTER_PORT) -i :$(VOLUME_PORT) -i :$(FILER_PORT) 2>/dev/null || echo "No ports in use"
+
+# Manual testing targets
+manual-start: start-seaweedfs
+	@echo "$(GREEN)SeaweedFS is now running for manual TUS testing$(NC)"
+	@echo ""
+	@echo "TUS Endpoints:"
+	@echo "  OPTIONS /.tus/           - Capability discovery"
+	@echo "  POST    /.tus/{path}     - Create upload"
+	@echo "  HEAD    /.tus/.uploads/{id} - Get offset"
+	@echo "  PATCH   /.tus/.uploads/{id} - Upload data"
+	@echo "  DELETE  /.tus/.uploads/{id} - Cancel upload"
+	@echo ""
+	@echo "Example curl commands:"
+	@echo "  curl -X OPTIONS http://localhost:$(FILER_PORT)/.tus/ -H 'Tus-Resumable: 1.0.0'"
+	@echo ""
+	@echo "Run 'make manual-stop' when finished"
+
+manual-stop: stop-seaweedfs clean
+
+# CI targets
+ci-test: test-with-server
+
+# Skip integration tests (short mode)
+test-short:
+	@echo "$(YELLOW)Running TUS tests in short mode (skipping integration tests)...$(NC)"
+	@cd $(SEAWEEDFS_ROOT) && go test -v -short ./test/tus/...
+	@echo "$(GREEN)Short tests completed$(NC)"
+
@@ -0,0 +1,241 @@
+# TUS Protocol Integration Tests
+
+This directory contains integration tests for the TUS (resumable upload) protocol support in SeaweedFS Filer.
+
+## Overview
+
+TUS is an open protocol for resumable file uploads over HTTP. It allows clients to upload files in chunks and resume uploads after network failures or interruptions.
+
+### Why TUS?
+
+- **Resumable uploads**: Resume interrupted uploads without re-sending data
+- **Chunked uploads**: Upload large files in smaller pieces
+- **Simple protocol**: Standard HTTP methods with custom headers
+- **Wide client support**: Libraries available for JavaScript, Python, Go, and more
+
+## TUS Protocol Endpoints
+
+| Method | Path | Description |
+|--------|------|-------------|
+| `OPTIONS` | `/.tus/` | Server capability discovery |
+| `POST` | `/.tus/{path}` | Create new upload session |
+| `HEAD` | `/.tus/.uploads/{id}` | Get current upload offset |
+| `PATCH` | `/.tus/.uploads/{id}` | Upload data at offset |
+| `DELETE` | `/.tus/.uploads/{id}` | Cancel upload |
+
+### TUS Headers
+
+**Request Headers:**
+- `Tus-Resumable: 1.0.0` - Protocol version (required)
+- `Upload-Length` - Total file size in bytes (required on POST)
+- `Upload-Offset` - Current byte offset (required on PATCH)
+- `Upload-Metadata` - Base64-encoded key-value pairs (optional)
+- `Content-Type: application/offset+octet-stream` (required on PATCH)
+
+**Response Headers:**
+- `Tus-Resumable` - Protocol version
+- `Tus-Version` - Supported versions
+- `Tus-Extension` - Supported extensions
+- `Tus-Max-Size` - Maximum upload size
+- `Upload-Offset` - Current byte offset
+- `Location` - Upload URL (on POST)
+
+## Enabling TUS
+
+TUS protocol support is enabled by default at `/.tus` path. You can customize the path using the `-tusBasePath` flag:
+
+```bash
+# Start filer with default TUS path (/.tus)
+weed filer -master=localhost:9333
+
+# Use a custom path (leading slash added automatically if missing)
+weed filer -master=localhost:9333 -tusBasePath=/.uploads/tus
+
+# Disable TUS by setting empty path
+weed filer -master=localhost:9333 -tusBasePath=
+```
+
+## Test Structure
+
+### Integration Tests
+
+The tests cover:
+
+1. **Basic Functionality**
+   - `TestTusOptionsHandler` - Capability discovery
+   - `TestTusBasicUpload` - Simple complete upload
+   - `TestTusCreationWithUpload` - Creation-with-upload extension
+
+2. **Chunked Uploads**
+   - `TestTusChunkedUpload` - Upload in multiple chunks
+
+3. **Resumable Uploads**
+   - `TestTusHeadRequest` - Offset tracking
+   - `TestTusResumeAfterInterruption` - Resume after failure
+
+4. **Error Handling**
+   - `TestTusInvalidOffset` - Offset mismatch (409 Conflict)
+   - `TestTusUploadNotFound` - Missing upload (404 Not Found)
+   - `TestTusDeleteUpload` - Upload cancellation
+
+## Running Tests
+
+### Prerequisites
+
+1. **Build SeaweedFS**:
+```bash
+make build-weed
+# or
+cd ../../weed && go build -o weed
+```
+
+### Using Makefile
+
+```bash
+# Show available targets
+make help
+
+# Run all tests with automatic server management
+make test-with-server
+
+# Run all tests (requires running server)
+make test
+
+# Run specific test categories
+make test-basic      # Basic upload tests
+make test-chunked    # Chunked upload tests
+make test-resume     # Resume/HEAD tests
+make test-errors     # Error handling tests
+
+# Manual testing
+make manual-start    # Start SeaweedFS for manual testing
+make manual-stop     # Stop and cleanup
+```
+
+### Using Go Test Directly
+
+```bash
+# Run all TUS tests
+go test -v ./test/tus/...
+
+# Run specific test
+go test -v ./test/tus -run TestTusBasicUpload
+
+# Skip integration tests (short mode)
+go test -v -short ./test/tus/...
+```
+
+### Debug
+
+```bash
+# View server logs
+make debug-logs
+
+# Check process and port status
+make debug-status
+```
+
+## Test Environment
+
+Each test run:
+1. Starts a SeaweedFS cluster (master, volume, filer)
+2. Creates uploads using TUS protocol
+3. Verifies files are stored correctly
+4. Cleans up test data
+
+### Default Ports
+
+| Service | Port |
+|---------|------|
+| Master | 19333 |
+| Volume | 18080 |
+| Filer | 18888 |
+
+### Configuration
+
+Override defaults via environment or Makefile variables:
+```bash
+FILER_PORT=8889 MASTER_PORT=9334 make test
+```
+
+## Example Usage
+
+### Create Upload
+
+```bash
+curl -X POST http://localhost:18888/.tus/mydir/file.txt \
+  -H "Tus-Resumable: 1.0.0" \
+  -H "Upload-Length: 1000" \
+  -H "Upload-Metadata: filename dGVzdC50eHQ="
+```
+
+### Upload Data
+
+```bash
+curl -X PATCH http://localhost:18888/.tus/.uploads/{upload-id} \
+  -H "Tus-Resumable: 1.0.0" \
+  -H "Upload-Offset: 0" \
+  -H "Content-Type: application/offset+octet-stream" \
+  --data-binary @file.txt
+```
+
+### Check Offset
+
+```bash
+curl -I http://localhost:18888/.tus/.uploads/{upload-id} \
+  -H "Tus-Resumable: 1.0.0"
+```
+
+### Cancel Upload
+
+```bash
+curl -X DELETE http://localhost:18888/.tus/.uploads/{upload-id} \
+  -H "Tus-Resumable: 1.0.0"
+```
+
+## TUS Extensions Supported
+
+- **creation**: Create new uploads with POST
+- **creation-with-upload**: Send data in creation request
+- **termination**: Cancel uploads with DELETE
+
+## Architecture
+
+```text
+Client                         Filer                      Volume Servers
+  |                              |                              |
+  |-- POST /.tus/path/file.mp4 ->|                              |
+  |                              |-- Create session dir ------->|
+  |<-- 201 Location: /.../{id} --|                              |
+  |                              |                              |
+  |-- PATCH /.tus/.uploads/{id} >|                              |
+  |   Upload-Offset: 0           |-- Assign volume ------------>|
+  |   [chunk data]               |-- Upload chunk ------------->|
+  |<-- 204 Upload-Offset: N -----|                              |
+  |                              |                              |
+  |   (network failure)          |                              |
+  |                              |                              |
+  |-- HEAD /.tus/.uploads/{id} ->|                              |
+  |<-- Upload-Offset: N ---------|                              |
+  |                              |                              |
+  |-- PATCH (resume) ----------->|-- Upload remaining -------->|
+  |<-- 204 (complete) -----------|-- Assemble final file ----->|
+```
+
+## Comparison with S3 Multipart
+
+| Feature | TUS | S3 Multipart |
+|---------|-----|--------------|
+| Protocol | Custom HTTP headers | S3 API |
+| Session Init | POST with Upload-Length | CreateMultipartUpload |
+| Upload Data | PATCH with offset | UploadPart with partNumber |
+| Resume | HEAD to get offset | ListParts |
+| Complete | Automatic at final offset | CompleteMultipartUpload |
+| Ordering | Sequential (offset-based) | Parallel (part numbers) |
+
+## Related Resources
+
+- [TUS Protocol Specification](https://tus.io/protocols/resumable-upload)
+- [tus-js-client](https://github.com/tus/tus-js-client) - JavaScript client
+- [go-tus](https://github.com/eventials/go-tus) - Go client
+- [SeaweedFS S3 API](../../weed/s3api) - Alternative multipart upload
@@ -0,0 +1,775 @@
+package tus
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	TusVersion     = "1.0.0"
+	testFilerPort  = "18888"
+	testMasterPort = "19333"
+	testVolumePort = "18080"
+)
+
+// TestCluster represents a running SeaweedFS cluster for testing
+type TestCluster struct {
+	masterCmd *exec.Cmd
+	volumeCmd *exec.Cmd
+	filerCmd  *exec.Cmd
+	dataDir   string
+}
+
+func (c *TestCluster) Stop() {
+	if c.filerCmd != nil && c.filerCmd.Process != nil {
+		c.filerCmd.Process.Signal(os.Interrupt)
+		c.filerCmd.Wait()
+	}
+	if c.volumeCmd != nil && c.volumeCmd.Process != nil {
+		c.volumeCmd.Process.Signal(os.Interrupt)
+		c.volumeCmd.Wait()
+	}
+	if c.masterCmd != nil && c.masterCmd.Process != nil {
+		c.masterCmd.Process.Signal(os.Interrupt)
+		c.masterCmd.Wait()
+	}
+}
+
+func (c *TestCluster) FilerURL() string {
+	return fmt.Sprintf("http://127.0.0.1:%s", testFilerPort)
+}
+
+func (c *TestCluster) TusURL() string {
+	return fmt.Sprintf("%s/.tus", c.FilerURL())
+}
+
+// FullURL converts a relative path to a full URL
+func (c *TestCluster) FullURL(path string) string {
+	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
+		return path
+	}
+	return fmt.Sprintf("http://127.0.0.1:%s%s", testFilerPort, path)
+}
+
+// startTestCluster starts a SeaweedFS cluster for testing
+func startTestCluster(t *testing.T, ctx context.Context) (*TestCluster, error) {
+	weedBinary := findWeedBinary()
+	if weedBinary == "" {
+		return nil, fmt.Errorf("weed binary not found - please build it first: cd weed && go build")
+	}
+
+	dataDir, err := os.MkdirTemp("", "seaweedfs_tus_test_")
+	if err != nil {
+		return nil, err
+	}
+
+	cluster := &TestCluster{dataDir: dataDir}
+
+	// Create subdirectories
+	masterDir := filepath.Join(dataDir, "master")
+	volumeDir := filepath.Join(dataDir, "volume")
+	filerDir := filepath.Join(dataDir, "filer")
+	os.MkdirAll(masterDir, 0755)
+	os.MkdirAll(volumeDir, 0755)
+	os.MkdirAll(filerDir, 0755)
+
+	// Start master
+	masterCmd := exec.CommandContext(ctx, weedBinary, "master",
+		"-port", testMasterPort,
+		"-mdir", masterDir,
+		"-ip", "127.0.0.1",
+	)
+	masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
+	if err != nil {
+		os.RemoveAll(dataDir)
+		return nil, fmt.Errorf("failed to create master log: %v", err)
+	}
+	masterCmd.Stdout = masterLogFile
+	masterCmd.Stderr = masterLogFile
+	if err := masterCmd.Start(); err != nil {
+		os.RemoveAll(dataDir)
+		return nil, fmt.Errorf("failed to start master: %v", err)
+	}
+	cluster.masterCmd = masterCmd
+
+	// Wait for master to be ready
+	if err := waitForHTTPServer("http://127.0.0.1:"+testMasterPort+"/dir/status", 30*time.Second); err != nil {
+		cluster.Stop()
+		os.RemoveAll(dataDir)
+		return nil, fmt.Errorf("master not ready: %v", err)
+	}
+
+	// Start volume server
+	volumeCmd := exec.CommandContext(ctx, weedBinary, "volume",
+		"-port", testVolumePort,
+		"-dir", volumeDir,
+		"-mserver", "127.0.0.1:"+testMasterPort,
+		"-ip", "127.0.0.1",
+	)
+	volumeLogFile, err := os.Create(filepath.Join(volumeDir, "volume.log"))
+	if err != nil {
+		cluster.Stop()
+		os.RemoveAll(dataDir)
+		return nil, fmt.Errorf("failed to create volume log: %v", err)
+	}
+	volumeCmd.Stdout = volumeLogFile
+	volumeCmd.Stderr = volumeLogFile
+	if err := volumeCmd.Start(); err != nil {
+		cluster.Stop()
+		os.RemoveAll(dataDir)
+		return nil, fmt.Errorf("failed to start volume server: %v", err)
+	}
+	cluster.volumeCmd = volumeCmd
+
+	// Wait for volume server to register with master
+	if err := waitForHTTPServer("http://127.0.0.1:"+testVolumePort+"/status", 30*time.Second); err != nil {
+		cluster.Stop()
+		os.RemoveAll(dataDir)
+		return nil, fmt.Errorf("volume server not ready: %v", err)
+	}
+
+	// Start filer with TUS enabled
+	filerCmd := exec.CommandContext(ctx, weedBinary, "filer",
+		"-port", testFilerPort,
+		"-master", "127.0.0.1:"+testMasterPort,
+		"-ip", "127.0.0.1",
+		"-defaultStoreDir", filerDir,
+		"-tusBasePath", "/.tus",
+	)
+	filerLogFile, err := os.Create(filepath.Join(filerDir, "filer.log"))
+	if err != nil {
+		cluster.Stop()
+		os.RemoveAll(dataDir)
+		return nil, fmt.Errorf("failed to create filer log: %v", err)
+	}
+	filerCmd.Stdout = filerLogFile
+	filerCmd.Stderr = filerLogFile
+	if err := filerCmd.Start(); err != nil {
+		cluster.Stop()
+		os.RemoveAll(dataDir)
+		return nil, fmt.Errorf("failed to start filer: %v", err)
+	}
+	cluster.filerCmd = filerCmd
+
+	// Wait for filer
+	if err := waitForHTTPServer("http://127.0.0.1:"+testFilerPort+"/", 30*time.Second); err != nil {
+		cluster.Stop()
+		os.RemoveAll(dataDir)
+		return nil, fmt.Errorf("filer not ready: %v", err)
+	}
+
+	// Wait a bit more for the cluster to fully stabilize
+	// Volumes are created lazily, and we need to ensure the master topology is ready
+	time.Sleep(5 * time.Second)
+
+	return cluster, nil
+}
+
+func findWeedBinary() string {
+	candidates := []string{
+		"../../weed/weed",
+		"../weed/weed",
+		"./weed/weed",
+		"weed",
+	}
+	for _, candidate := range candidates {
+		if _, err := os.Stat(candidate); err == nil {
+			return candidate
+		}
+	}
+	if path, err := exec.LookPath("weed"); err == nil {
+		return path
+	}
+	return ""
+}
+
+func waitForHTTPServer(url string, timeout time.Duration) error {
+	start := time.Now()
+	client := &http.Client{Timeout: 1 * time.Second}
+	for time.Since(start) < timeout {
+		resp, err := client.Get(url)
+		if err == nil {
+			resp.Body.Close()
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("timeout waiting for %s", url)
+}
+
+// encodeTusMetadata encodes key-value pairs for Upload-Metadata header
+func encodeTusMetadata(metadata map[string]string) string {
+	var parts []string
+	for k, v := range metadata {
+		encoded := base64.StdEncoding.EncodeToString([]byte(v))
+		parts = append(parts, fmt.Sprintf("%s %s", k, encoded))
+	}
+	return strings.Join(parts, ",")
+}
+
+// TestTusOptionsHandler tests the OPTIONS endpoint for capability discovery
+func TestTusOptionsHandler(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	cluster, err := startTestCluster(t, ctx)
+	require.NoError(t, err)
+	defer func() {
+		cluster.Stop()
+		os.RemoveAll(cluster.dataDir)
+	}()
+
+	// Test OPTIONS request
+	req, err := http.NewRequest(http.MethodOptions, cluster.TusURL()+"/", nil)
+	require.NoError(t, err)
+	req.Header.Set("Tus-Resumable", TusVersion)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	// Verify TUS headers
+	assert.Equal(t, http.StatusOK, resp.StatusCode, "OPTIONS should return 200 OK")
+	assert.Equal(t, TusVersion, resp.Header.Get("Tus-Resumable"), "Should return Tus-Resumable header")
+	assert.NotEmpty(t, resp.Header.Get("Tus-Version"), "Should return Tus-Version header")
+	assert.NotEmpty(t, resp.Header.Get("Tus-Extension"), "Should return Tus-Extension header")
+	assert.NotEmpty(t, resp.Header.Get("Tus-Max-Size"), "Should return Tus-Max-Size header")
+}
+
+// TestTusBasicUpload tests a simple complete upload
+func TestTusBasicUpload(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	cluster, err := startTestCluster(t, ctx)
+	require.NoError(t, err)
+	defer func() {
+		cluster.Stop()
+		os.RemoveAll(cluster.dataDir)
+	}()
+
+	testData := []byte("Hello, TUS Protocol! This is a test file.")
+	targetPath := "/testdir/testfile.txt"
+
+	// Step 1: Create upload (POST)
+	createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil)
+	require.NoError(t, err)
+	createReq.Header.Set("Tus-Resumable", TusVersion)
+	createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
+	createReq.Header.Set("Upload-Metadata", encodeTusMetadata(map[string]string{
+		"filename":     "testfile.txt",
+		"content-type": "text/plain",
+	}))
+
+	client := &http.Client{}
+	createResp, err := client.Do(createReq)
+	require.NoError(t, err)
+	defer createResp.Body.Close()
+
+	assert.Equal(t, http.StatusCreated, createResp.StatusCode, "POST should return 201 Created")
+	uploadLocation := createResp.Header.Get("Location")
+	assert.NotEmpty(t, uploadLocation, "Should return Location header with upload URL")
+	t.Logf("Upload location: %s", uploadLocation)
+
+	// Step 2: Upload data (PATCH)
+	patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData))
+	require.NoError(t, err)
+	patchReq.Header.Set("Tus-Resumable", TusVersion)
+	patchReq.Header.Set("Upload-Offset", "0")
+	patchReq.Header.Set("Content-Type", "application/offset+octet-stream")
+	patchReq.Header.Set("Content-Length", strconv.Itoa(len(testData)))
+
+	patchResp, err := client.Do(patchReq)
+	require.NoError(t, err)
+	defer patchResp.Body.Close()
+
+	assert.Equal(t, http.StatusNoContent, patchResp.StatusCode, "PATCH should return 204 No Content")
+	newOffset := patchResp.Header.Get("Upload-Offset")
+	assert.Equal(t, strconv.Itoa(len(testData)), newOffset, "Upload-Offset should equal total file size")
+
+	// Step 3: Verify the file was created
+	getResp, err := client.Get(cluster.FilerURL() + targetPath)
+	require.NoError(t, err)
+	defer getResp.Body.Close()
+
+	assert.Equal(t, http.StatusOK, getResp.StatusCode, "GET should return 200 OK")
+	body, err := io.ReadAll(getResp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, testData, body, "File content should match uploaded data")
+}
+
+// TestTusChunkedUpload tests uploading a file in multiple chunks
+func TestTusChunkedUpload(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	cluster, err := startTestCluster(t, ctx)
+	require.NoError(t, err)
+	defer func() {
+		cluster.Stop()
+		os.RemoveAll(cluster.dataDir)
+	}()
+
+	// Create test data (100KB)
+	testData := make([]byte, 100*1024)
+	for i := range testData {
+		testData[i] = byte(i % 256)
+	}
+	chunkSize := 32 * 1024 // 32KB chunks
+	targetPath := "/chunked/largefile.bin"
+
+	client := &http.Client{}
+
+	// Step 1: Create upload
+	createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil)
+	require.NoError(t, err)
+	createReq.Header.Set("Tus-Resumable", TusVersion)
+	createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
+
+	createResp, err := client.Do(createReq)
+	require.NoError(t, err)
+	defer createResp.Body.Close()
+
+	require.Equal(t, http.StatusCreated, createResp.StatusCode)
+	uploadLocation := createResp.Header.Get("Location")
+	require.NotEmpty(t, uploadLocation)
+	t.Logf("Upload location: %s", uploadLocation)
+
+	// Step 2: Upload in chunks
+	offset := 0
+	for offset < len(testData) {
+		end := offset + chunkSize
+		if end > len(testData) {
+			end = len(testData)
+		}
+		chunk := testData[offset:end]
+
+		patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(chunk))
+		require.NoError(t, err)
+		patchReq.Header.Set("Tus-Resumable", TusVersion)
+		patchReq.Header.Set("Upload-Offset", strconv.Itoa(offset))
+		patchReq.Header.Set("Content-Type", "application/offset+octet-stream")
+		patchReq.Header.Set("Content-Length", strconv.Itoa(len(chunk)))
+
+		patchResp, err := client.Do(patchReq)
+		require.NoError(t, err)
+		patchResp.Body.Close()
+
+		require.Equal(t, http.StatusNoContent, patchResp.StatusCode,
+			"PATCH chunk at offset %d should return 204", offset)
+		newOffset, err := strconv.Atoi(patchResp.Header.Get("Upload-Offset"))
+		require.NoError(t, err, "Upload-Offset header should be a valid integer")
+		require.Equal(t, end, newOffset, "New offset should be %d", end)
+
+		t.Logf("Uploaded chunk: offset=%d, size=%d, newOffset=%d", offset, len(chunk), newOffset)
+		offset = end
+	}
+
+	// Step 3: Verify the complete file
+	getResp, err := client.Get(cluster.FilerURL() + targetPath)
+	require.NoError(t, err)
+	defer getResp.Body.Close()
+
+	assert.Equal(t, http.StatusOK, getResp.StatusCode)
+	body, err := io.ReadAll(getResp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, testData, body, "File content should match uploaded data")
+}
+
+// TestTusHeadRequest tests the HEAD endpoint to get upload offset
+func TestTusHeadRequest(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	cluster, err := startTestCluster(t, ctx)
+	require.NoError(t, err)
+	defer func() {
+		cluster.Stop()
+		os.RemoveAll(cluster.dataDir)
+	}()
+
+	testData := []byte("Test data for HEAD request verification")
+	targetPath := "/headtest/file.txt"
+	client := &http.Client{}
+
+	// Create upload
+	createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil)
+	require.NoError(t, err)
+	createReq.Header.Set("Tus-Resumable", TusVersion)
+	createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
+
+	createResp, err := client.Do(createReq)
+	require.NoError(t, err)
+	defer createResp.Body.Close()
+	require.Equal(t, http.StatusCreated, createResp.StatusCode)
+	uploadLocation := createResp.Header.Get("Location")
+
+	// HEAD before any data uploaded - offset should be 0
+	headReq1, err := http.NewRequest(http.MethodHead, cluster.FullURL(uploadLocation), nil)
+	require.NoError(t, err)
+	headReq1.Header.Set("Tus-Resumable", TusVersion)
+
+	headResp1, err := client.Do(headReq1)
+	require.NoError(t, err)
+	defer headResp1.Body.Close()
+
+	assert.Equal(t, http.StatusOK, headResp1.StatusCode)
+	assert.Equal(t, "0", headResp1.Header.Get("Upload-Offset"), "Initial offset should be 0")
+	assert.Equal(t, strconv.Itoa(len(testData)), headResp1.Header.Get("Upload-Length"))
+
+	// Upload half the data
+	halfLen := len(testData) / 2
+	patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData[:halfLen]))
+	require.NoError(t, err)
+	patchReq.Header.Set("Tus-Resumable", TusVersion)
+	patchReq.Header.Set("Upload-Offset", "0")
+	patchReq.Header.Set("Content-Type", "application/offset+octet-stream")
+
+	patchResp, err := client.Do(patchReq)
+	require.NoError(t, err)
+	patchResp.Body.Close()
+	require.Equal(t, http.StatusNoContent, patchResp.StatusCode)
+
+	// HEAD after partial upload - offset should be halfLen
+	headReq2, err := http.NewRequest(http.MethodHead, cluster.FullURL(uploadLocation), nil)
+	require.NoError(t, err)
+	headReq2.Header.Set("Tus-Resumable", TusVersion)
+
+	headResp2, err := client.Do(headReq2)
+	require.NoError(t, err)
+	defer headResp2.Body.Close()
+
+	assert.Equal(t, http.StatusOK, headResp2.StatusCode)
+	assert.Equal(t, strconv.Itoa(halfLen), headResp2.Header.Get("Upload-Offset"),
+		"Offset should be %d after partial upload", halfLen)
+}
+
+// TestTusDeleteUpload tests canceling an in-progress upload
+func TestTusDeleteUpload(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	cluster, err := startTestCluster(t, ctx)
+	require.NoError(t, err)
+	defer func() {
+		cluster.Stop()
+		os.RemoveAll(cluster.dataDir)
+	}()
+
+	testData := []byte("Data to be deleted")
+	targetPath := "/deletetest/file.txt"
+	client := &http.Client{}
+
+	// Create upload
+	createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil)
+	require.NoError(t, err)
+	createReq.Header.Set("Tus-Resumable", TusVersion)
+	createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
+
+	createResp, err := client.Do(createReq)
+	require.NoError(t, err)
+	defer createResp.Body.Close()
+	require.Equal(t, http.StatusCreated, createResp.StatusCode)
+	uploadLocation := createResp.Header.Get("Location")
+
+	// Upload some data
+	patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData[:10]))
+	require.NoError(t, err)
+	patchReq.Header.Set("Tus-Resumable", TusVersion)
+	patchReq.Header.Set("Upload-Offset", "0")
+	patchReq.Header.Set("Content-Type", "application/offset+octet-stream")
+
+	patchResp, err := client.Do(patchReq)
+	require.NoError(t, err)
+	patchResp.Body.Close()
+
+	// Delete the upload
+	deleteReq, err := http.NewRequest(http.MethodDelete, cluster.FullURL(uploadLocation), nil)
+	require.NoError(t, err)
+	deleteReq.Header.Set("Tus-Resumable", TusVersion)
+
+	deleteResp, err := client.Do(deleteReq)
+	require.NoError(t, err)
+	defer deleteResp.Body.Close()
+
+	assert.Equal(t, http.StatusNoContent, deleteResp.StatusCode, "DELETE should return 204")
+
+	// Verify upload is gone - HEAD should return 404
+	headReq, err := http.NewRequest(http.MethodHead, cluster.FullURL(uploadLocation), nil)
+	require.NoError(t, err)
+	headReq.Header.Set("Tus-Resumable", TusVersion)
+
+	headResp, err := client.Do(headReq)
+	require.NoError(t, err)
+	defer headResp.Body.Close()
+
+	assert.Equal(t, http.StatusNotFound, headResp.StatusCode, "HEAD after DELETE should return 404")
+}
+
+// TestTusInvalidOffset tests error handling for mismatched offsets
+func TestTusInvalidOffset(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	cluster, err := startTestCluster(t, ctx)
+	require.NoError(t, err)
+	defer func() {
+		cluster.Stop()
+		os.RemoveAll(cluster.dataDir)
+	}()
+
+	testData := []byte("Test data for offset validation")
+	targetPath := "/offsettest/file.txt"
+	client := &http.Client{}
+
+	// Create upload
+	createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil)
+	require.NoError(t, err)
+	createReq.Header.Set("Tus-Resumable", TusVersion)
+	createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
+
+	createResp, err := client.Do(createReq)
+	require.NoError(t, err)
+	defer createResp.Body.Close()
+	require.Equal(t, http.StatusCreated, createResp.StatusCode)
+	uploadLocation := createResp.Header.Get("Location")
+
+	// Try to upload with wrong offset (should be 0, but we send 100)
+	patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData))
+	require.NoError(t, err)
+	patchReq.Header.Set("Tus-Resumable", TusVersion)
+	patchReq.Header.Set("Upload-Offset", "100") // Wrong offset!
+	patchReq.Header.Set("Content-Type", "application/offset+octet-stream")
+
+	patchResp, err := client.Do(patchReq)
+	require.NoError(t, err)
+	defer patchResp.Body.Close()
+
+	assert.Equal(t, http.StatusConflict, patchResp.StatusCode,
+		"PATCH with wrong offset should return 409 Conflict")
+}
+
+// TestTusUploadNotFound tests accessing a non-existent upload
+func TestTusUploadNotFound(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	cluster, err := startTestCluster(t, ctx)
+	require.NoError(t, err)
+	defer func() {
+		cluster.Stop()
+		os.RemoveAll(cluster.dataDir)
+	}()
+
+	client := &http.Client{}
+	fakeUploadURL := cluster.TusURL() + "/.uploads/nonexistent-upload-id"
+
+	// HEAD on non-existent upload
+	headReq, err := http.NewRequest(http.MethodHead, fakeUploadURL, nil)
+	require.NoError(t, err)
+	headReq.Header.Set("Tus-Resumable", TusVersion)
+
+	headResp, err := client.Do(headReq)
+	require.NoError(t, err)
+	defer headResp.Body.Close()
+
+	assert.Equal(t, http.StatusNotFound, headResp.StatusCode,
+		"HEAD on non-existent upload should return 404")
+
+	// PATCH on non-existent upload
+	patchReq, err := http.NewRequest(http.MethodPatch, fakeUploadURL, bytes.NewReader([]byte("data")))
+	require.NoError(t, err)
+	patchReq.Header.Set("Tus-Resumable", TusVersion)
+	patchReq.Header.Set("Upload-Offset", "0")
+	patchReq.Header.Set("Content-Type", "application/offset+octet-stream")
+
+	patchResp, err := client.Do(patchReq)
+	require.NoError(t, err)
+	defer patchResp.Body.Close()
+
+	assert.Equal(t, http.StatusNotFound, patchResp.StatusCode,
+		"PATCH on non-existent upload should return 404")
+}
+
+// TestTusCreationWithUpload tests the creation-with-upload extension
+func TestTusCreationWithUpload(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	cluster, err := startTestCluster(t, ctx)
+	require.NoError(t, err)
+	defer func() {
+		cluster.Stop()
+		os.RemoveAll(cluster.dataDir)
+	}()
+
+	testData := []byte("Small file uploaded in creation request")
+	targetPath := "/creationwithupload/smallfile.txt"
+	client := &http.Client{}
+
+	// Create upload with data in the same request
+	createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, bytes.NewReader(testData))
+	require.NoError(t, err)
+	createReq.Header.Set("Tus-Resumable", TusVersion)
+	createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
+	createReq.Header.Set("Content-Type", "application/offset+octet-stream")
+
+	createResp, err := client.Do(createReq)
+	require.NoError(t, err)
+	defer createResp.Body.Close()
+
+	assert.Equal(t, http.StatusCreated, createResp.StatusCode)
+	uploadLocation := createResp.Header.Get("Location")
+	assert.NotEmpty(t, uploadLocation)
+
+	// Check Upload-Offset header - should indicate all data was received
+	uploadOffset := createResp.Header.Get("Upload-Offset")
+	assert.Equal(t, strconv.Itoa(len(testData)), uploadOffset,
+		"Upload-Offset should equal file size for complete upload")
+
+	// Verify the file
+	getResp, err := client.Get(cluster.FilerURL() + targetPath)
+	require.NoError(t, err)
+	defer getResp.Body.Close()
+
+	assert.Equal(t, http.StatusOK, getResp.StatusCode)
+	body, err := io.ReadAll(getResp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, testData, body)
+}
+
+// TestTusResumeAfterInterruption simulates resuming an upload after failure
+func TestTusResumeAfterInterruption(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	cluster, err := startTestCluster(t, ctx)
+	require.NoError(t, err)
+	defer func() {
+		cluster.Stop()
+		os.RemoveAll(cluster.dataDir)
+	}()
+
+	// 50KB test data
+	testData := make([]byte, 50*1024)
+	for i := range testData {
+		testData[i] = byte(i % 256)
+	}
+	targetPath := "/resume/interrupted.bin"
+	client := &http.Client{}
+
+	// Create upload
+	createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil)
+	require.NoError(t, err)
+	createReq.Header.Set("Tus-Resumable", TusVersion)
+	createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
+
+	createResp, err := client.Do(createReq)
+	require.NoError(t, err)
+	defer createResp.Body.Close()
+	require.Equal(t, http.StatusCreated, createResp.StatusCode)
+	uploadLocation := createResp.Header.Get("Location")
+
+	// Upload first 20KB
+	firstChunkSize := 20 * 1024
+	patchReq1, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData[:firstChunkSize]))
+	require.NoError(t, err)
+	patchReq1.Header.Set("Tus-Resumable", TusVersion)
+	patchReq1.Header.Set("Upload-Offset", "0")
+	patchReq1.Header.Set("Content-Type", "application/offset+octet-stream")
+
+	patchResp1, err := client.Do(patchReq1)
+	require.NoError(t, err)
+	patchResp1.Body.Close()
+	require.Equal(t, http.StatusNoContent, patchResp1.StatusCode)
+
+	t.Log("Simulating network interruption...")
+
+	// Simulate resumption: Query current offset with HEAD
+	headReq, err := http.NewRequest(http.MethodHead, cluster.FullURL(uploadLocation), nil)
+	require.NoError(t, err)
+	headReq.Header.Set("Tus-Resumable", TusVersion)
+
+	headResp, err := client.Do(headReq)
+	require.NoError(t, err)
+	defer headResp.Body.Close()
+
+	require.Equal(t, http.StatusOK, headResp.StatusCode)
+	currentOffset, err := strconv.Atoi(headResp.Header.Get("Upload-Offset"))
+	require.NoError(t, err, "Upload-Offset header should be a valid integer")
+	t.Logf("Resumed upload at offset: %d", currentOffset)
+	require.Equal(t, firstChunkSize, currentOffset)
+
+	// Resume upload from current offset
+	patchReq2, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData[currentOffset:]))
+	require.NoError(t, err)
+	patchReq2.Header.Set("Tus-Resumable", TusVersion)
+	patchReq2.Header.Set("Upload-Offset", strconv.Itoa(currentOffset))
+	patchReq2.Header.Set("Content-Type", "application/offset+octet-stream")
+
+	patchResp2, err := client.Do(patchReq2)
+	require.NoError(t, err)
+	patchResp2.Body.Close()
+	require.Equal(t, http.StatusNoContent, patchResp2.StatusCode)
+
+	// Verify complete file
+	getResp, err := client.Get(cluster.FilerURL() + targetPath)
+	require.NoError(t, err)
+	defer getResp.Body.Close()
+
+	assert.Equal(t, http.StatusOK, getResp.StatusCode)
+	body, err := io.ReadAll(getResp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, testData, body, "Resumed upload should produce complete file")
+}
@@ -109,13 +109,6 @@ func (s *AdminServer) GetAdminData(username string) (AdminData, error) {
 		glog.Errorf("Failed to get cluster volume servers: %v", err)
 		return AdminData{}, err
 	}
-	// Sort the servers so they show up in consistent order after each reload
-	sort.Slice(volumeServersData.VolumeServers, func(i, j int) bool {
-		s1Name := volumeServersData.VolumeServers[i].GetDisplayAddress()
-		s2Name := volumeServersData.VolumeServers[j].GetDisplayAddress()
-
-		return s1Name < s2Name
-	})

 	// Get master nodes status
 	masterNodes := s.getMasterNodesStatus()
@@ -265,6 +258,11 @@ func (s *AdminServer) getFilerNodesStatus() []FilerNode {
 		return []FilerNode{}
 	}

+	// Sort filer nodes by address for consistent ordering on page refresh
+	sort.Slice(filerNodes, func(i, j int) bool {
+		return filerNodes[i].Address < filerNodes[j].Address
+	})
+
 	return filerNodes
 }

@@ -301,5 +299,10 @@ func (s *AdminServer) getMessageBrokerNodesStatus() []MessageBrokerNode {
 		return []MessageBrokerNode{}
 	}

+	// Sort message broker nodes by address for consistent ordering on page refresh
+	sort.Slice(messageBrokers, func(i, j int) bool {
+		return messageBrokers[i].Address < messageBrokers[j].Address
+	})
+
 	return messageBrokers
 }
@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"sort"
 	"strconv"
 	"time"

@@ -26,9 +27,46 @@ import (
 	"google.golang.org/grpc"

 	"github.com/seaweedfs/seaweedfs/weed/s3api"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
 )

+// FilerConfig holds filer configuration needed for bucket operations
+type FilerConfig struct {
+	BucketsPath string
+	FilerGroup  string
+}
+
+// getFilerConfig retrieves the filer configuration (buckets path and filer group)
+func (s *AdminServer) getFilerConfig() (*FilerConfig, error) {
+	config := &FilerConfig{
+		BucketsPath: s3_constants.DefaultBucketsPath,
+		FilerGroup:  "",
+	}
+
+	err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return fmt.Errorf("get filer configuration: %w", err)
+		}
+		if resp.DirBuckets != "" {
+			config.BucketsPath = resp.DirBuckets
+		}
+		config.FilerGroup = resp.FilerGroup
+		return nil
+	})
+
+	return config, err
+}
+
+// getCollectionName returns the collection name for a bucket, prefixed with filer group if configured
+func getCollectionName(filerGroup, bucketName string) string {
+	if filerGroup != "" {
+		return fmt.Sprintf("%s_%s", filerGroup, bucketName)
+	}
+	return bucketName
+}
+
 type AdminServer struct {
 	masterClient    *wdclient.MasterClient
 	templateFS      http.FileSystem
@@ -249,28 +287,17 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
 		return nil, fmt.Errorf("failed to get volume information: %w", err)
 	}

-	// Get filer configuration to determine FilerGroup
-	var filerGroup string
-	err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		configResp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
-		if err != nil {
-			glog.Warningf("Failed to get filer configuration: %v", err)
-			// Continue without filer group
-			return nil
-		}
-		filerGroup = configResp.FilerGroup
-		return nil
-	})
-
+	// Get filer configuration (buckets path and filer group)
+	filerConfig, err := s.getFilerConfig()
 	if err != nil {
-		return nil, fmt.Errorf("failed to get filer configuration: %w", err)
+		glog.Warningf("Failed to get filer configuration, using defaults: %v", err)
 	}

 	// Now list buckets from the filer and match with collection data
 	err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		// List buckets by looking at the /buckets directory
+		// List buckets by looking at the buckets directory
 		stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
-			Directory:          "/buckets",
+			Directory:          filerConfig.BucketsPath,
 			Prefix:             "",
 			StartFromFileName:  "",
 			InclusiveStartFrom: false,
@@ -293,12 +320,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
 				bucketName := resp.Entry.Name

 				// Determine collection name for this bucket
-				var collectionName string
-				if filerGroup != "" {
-					collectionName = fmt.Sprintf("%s_%s", filerGroup, bucketName)
-				} else {
-					collectionName = bucketName
-				}
+				collectionName := getCollectionName(filerConfig.FilerGroup, bucketName)

 				// Get size and object count from collection data
 				var size int64
@@ -317,11 +339,12 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
 					quotaEnabled = false
 				}

-				// Get versioning and object lock information from extended attributes
+				// Get versioning, object lock, and owner information from extended attributes
 				versioningEnabled := false
 				objectLockEnabled := false
 				objectLockMode := ""
 				var objectLockDuration int32 = 0
+				var owner string

 				if resp.Entry.Extended != nil {
 					// Use shared utility to extract versioning information
@@ -329,6 +352,11 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {

 					// Use shared utility to extract Object Lock information
 					objectLockEnabled, objectLockMode, objectLockDuration = extractObjectLockInfoFromEntry(resp.Entry)
+
+					// Extract owner information
+					if ownerBytes, ok := resp.Entry.Extended[s3_constants.AmzIdentityId]; ok {
+						owner = string(ownerBytes)
+					}
 				}

 				bucket := S3Bucket{
@@ -343,6 +371,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
 					ObjectLockEnabled:  objectLockEnabled,
 					ObjectLockMode:     objectLockMode,
 					ObjectLockDuration: objectLockDuration,
+					Owner:              owner,
 				}
 				buckets = append(buckets, bucket)
 			}
@@ -360,7 +389,13 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {

 // GetBucketDetails retrieves detailed information about a specific bucket
 func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error) {
-	bucketPath := fmt.Sprintf("/buckets/%s", bucketName)
+	// Get filer configuration (buckets path)
+	filerConfig, err := s.getFilerConfig()
+	if err != nil {
+		glog.Warningf("Failed to get filer configuration, using defaults: %v", err)
+	}
+
+	bucketPath := fmt.Sprintf("%s/%s", filerConfig.BucketsPath, bucketName)

 	details := &BucketDetails{
 		Bucket: S3Bucket{
@@ -370,10 +405,10 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
 		UpdatedAt: time.Now(),
 	}

-	err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 		// Get bucket info
 		bucketResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
-			Directory: "/buckets",
+			Directory: filerConfig.BucketsPath,
 			Name:      bucketName,
 		})
 		if err != nil {
@@ -394,11 +429,12 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
 		details.Bucket.Quota = quota
 		details.Bucket.QuotaEnabled = quotaEnabled

-		// Get versioning and object lock information from extended attributes
+		// Get versioning, object lock, and owner information from extended attributes
 		versioningEnabled := false
 		objectLockEnabled := false
 		objectLockMode := ""
 		var objectLockDuration int32 = 0
+		var owner string

 		if bucketResp.Entry.Extended != nil {
 			// Use shared utility to extract versioning information
@@ -406,15 +442,21 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error

 			// Use shared utility to extract Object Lock information
 			objectLockEnabled, objectLockMode, objectLockDuration = extractObjectLockInfoFromEntry(bucketResp.Entry)
+
+			// Extract owner information
+			if ownerBytes, ok := bucketResp.Entry.Extended[s3_constants.AmzIdentityId]; ok {
+				owner = string(ownerBytes)
+			}
 		}

 		details.Bucket.VersioningEnabled = versioningEnabled
 		details.Bucket.ObjectLockEnabled = objectLockEnabled
 		details.Bucket.ObjectLockMode = objectLockMode
 		details.Bucket.ObjectLockDuration = objectLockDuration
+		details.Bucket.Owner = owner

 		// List objects in bucket (recursively)
-		return s.listBucketObjects(client, bucketPath, "", details)
+		return s.listBucketObjects(client, bucketPath, bucketPath, "", details)
 	})

 	if err != nil {
@@ -425,7 +467,8 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
 }

 // listBucketObjects recursively lists all objects in a bucket
-func (s *AdminServer) listBucketObjects(client filer_pb.SeaweedFilerClient, directory, prefix string, details *BucketDetails) error {
+// bucketBasePath is the full path to the bucket (e.g., /buckets/mybucket)
+func (s *AdminServer) listBucketObjects(client filer_pb.SeaweedFilerClient, bucketBasePath, directory, prefix string, details *BucketDetails) error {
 	stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
 		Directory:          directory,
 		Prefix:             prefix,
@@ -450,16 +493,16 @@ func (s *AdminServer) listBucketObjects(client filer_pb.SeaweedFilerClient, dire
 		if entry.IsDirectory {
 			// Recursively list subdirectories
 			subDir := fmt.Sprintf("%s/%s", directory, entry.Name)
-			err := s.listBucketObjects(client, subDir, "", details)
+			err := s.listBucketObjects(client, bucketBasePath, subDir, "", details)
 			if err != nil {
 				return err
 			}
 		} else {
 			// Add file object
 			objectKey := entry.Name
-			if directory != fmt.Sprintf("/buckets/%s", details.Bucket.Name) {
+			if directory != bucketBasePath {
 				// Remove bucket prefix to get relative path
-				relativePath := directory[len(fmt.Sprintf("/buckets/%s", details.Bucket.Name))+1:]
+				relativePath := directory[len(bucketBasePath)+1:]
 				objectKey = fmt.Sprintf("%s/%s", relativePath, entry.Name)
 			}

@@ -491,14 +534,45 @@ func (s *AdminServer) CreateS3Bucket(bucketName string) error {

 // DeleteS3Bucket deletes an S3 bucket and all its contents
 func (s *AdminServer) DeleteS3Bucket(bucketName string) error {
+	ctx := context.Background()
+
+	// Get filer configuration (buckets path and filer group)
+	filerConfig, err := s.getFilerConfig()
+	if err != nil {
+		return fmt.Errorf("failed to get filer configuration: %w", err)
+	}
+
+	// Check if bucket has Object Lock enabled and if there are locked objects
+	err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		return s3api.CheckBucketForLockedObjects(ctx, client, filerConfig.BucketsPath, bucketName)
+	})
+	if err != nil {
+		return err
+	}
+
+	// Delete the collection first (same as s3.bucket.delete shell command)
+	// This ensures volume data is cleaned up properly
+	// Collection name must be prefixed with filer group if configured
+	collectionName := getCollectionName(filerConfig.FilerGroup, bucketName)
+	err = s.WithMasterClient(func(client master_pb.SeaweedClient) error {
+		_, err := client.CollectionDelete(ctx, &master_pb.CollectionDeleteRequest{
+			Name: collectionName,
+		})
+		return err
+	})
+	if err != nil {
+		return fmt.Errorf("failed to delete collection %s: %w", collectionName, err)
+	}
+
+	// Then delete bucket directory recursively from filer
+	// Use same parameters as s3.bucket.delete shell command and S3 API
 	return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		// Delete bucket directory recursively
-		_, err := client.DeleteEntry(context.Background(), &filer_pb.DeleteEntryRequest{
-			Directory:            "/buckets",
+		_, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{
+			Directory:            filerConfig.BucketsPath,
 			Name:                 bucketName,
-			IsDeleteData:         true,
+			IsDeleteData:         false, // Collection already deleted, just remove metadata
 			IsRecursive:          true,
-			IgnoreRecursiveError: false,
+			IgnoreRecursiveError: true, // Same as S3 API and shell command
 		})
 		if err != nil {
 			return fmt.Errorf("failed to delete bucket: %w", err)
@@ -647,6 +721,11 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 		masters = append(masters, *masterInfo)
 	}

+	// Sort masters by address for consistent ordering on page refresh
+	sort.Slice(masters, func(i, j int) bool {
+		return masters[i].Address < masters[j].Address
+	})
+
 	// If no masters found at all, add the current master as fallback
 	if len(masters) == 0 {
 		currentMaster := s.masterClient.GetMaster(context.Background())
@@ -703,6 +782,11 @@ func (s *AdminServer) GetClusterFilers() (*ClusterFilersData, error) {
 		return nil, fmt.Errorf("failed to get filer nodes from master: %w", err)
 	}

+	// Sort filers by address for consistent ordering on page refresh
+	sort.Slice(filers, func(i, j int) bool {
+		return filers[i].Address < filers[j].Address
+	})
+
 	return &ClusterFilersData{
 		Filers:      filers,
 		TotalFilers: len(filers),
@@ -745,6 +829,11 @@ func (s *AdminServer) GetClusterBrokers() (*ClusterBrokersData, error) {
 		return nil, fmt.Errorf("failed to get broker nodes from master: %w", err)
 	}

+	// Sort brokers by address for consistent ordering on page refresh
+	sort.Slice(brokers, func(i, j int) bool {
+		return brokers[i].Address < brokers[j].Address
+	})
+
 	return &ClusterBrokersData{
 		Brokers:      brokers,
 		TotalBrokers: len(brokers),
@@ -11,8 +11,14 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 )

+// MaxOwnerNameLength is the maximum allowed length for bucket owner identity names.
+// This is a reasonable limit to prevent abuse; AWS IAM user names are limited to 64 chars,
+// but we use 256 to allow for more complex identity formats (e.g., email addresses).
+const MaxOwnerNameLength = 256
+
 // S3 Bucket management data structures for templates
 type S3BucketsData struct {
 	Username     string     `json:"username"`
@@ -33,6 +39,7 @@ type CreateBucketRequest struct {
 	ObjectLockMode      string `json:"object_lock_mode"`      // Object lock mode: "GOVERNANCE" or "COMPLIANCE"
 	SetDefaultRetention bool   `json:"set_default_retention"` // Whether to set default retention
 	ObjectLockDuration  int32  `json:"object_lock_duration"`  // Default retention duration in days
+	Owner               string `json:"owner"`                 // Bucket owner identity (for S3 IAM authentication)
 }

 // S3 Bucket Management Handlers
@@ -118,7 +125,14 @@ func (s *AdminServer) CreateBucket(c *gin.Context) {
 	// Convert quota to bytes
 	quotaBytes := convertQuotaToBytes(req.QuotaSize, req.QuotaUnit)

-	err := s.CreateS3BucketWithObjectLock(req.Name, quotaBytes, req.QuotaEnabled, req.VersioningEnabled, req.ObjectLockEnabled, req.ObjectLockMode, req.SetDefaultRetention, req.ObjectLockDuration)
+	// Sanitize owner: trim whitespace and enforce max length
+	owner := strings.TrimSpace(req.Owner)
+	if len(owner) > MaxOwnerNameLength {
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength)})
+		return
+	}
+
+	err := s.CreateS3BucketWithObjectLock(req.Name, quotaBytes, req.QuotaEnabled, req.VersioningEnabled, req.ObjectLockEnabled, req.ObjectLockMode, req.SetDefaultRetention, req.ObjectLockDuration, owner)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create bucket: " + err.Error()})
 		return
@@ -134,6 +148,7 @@ func (s *AdminServer) CreateBucket(c *gin.Context) {
 		"object_lock_enabled":  req.ObjectLockEnabled,
 		"object_lock_mode":     req.ObjectLockMode,
 		"object_lock_duration": req.ObjectLockDuration,
+		"owner":                owner,
 	})
 }

@@ -193,6 +208,88 @@ func (s *AdminServer) DeleteBucket(c *gin.Context) {
 	})
 }

+// UpdateBucketOwner updates the owner of an S3 bucket
+func (s *AdminServer) UpdateBucketOwner(c *gin.Context) {
+	bucketName := c.Param("bucket")
+	if bucketName == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Bucket name is required"})
+		return
+	}
+
+	// Use pointer to detect if owner field was explicitly provided
+	var req struct {
+		Owner *string `json:"owner"`
+	}
+	if err := c.ShouldBindJSON(&req); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request: " + err.Error()})
+		return
+	}
+
+	// Require owner field to be explicitly provided
+	if req.Owner == nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Owner field is required (use empty string to clear owner)"})
+		return
+	}
+
+	// Trim and validate owner
+	owner := strings.TrimSpace(*req.Owner)
+	if len(owner) > MaxOwnerNameLength {
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength)})
+		return
+	}
+
+	err := s.SetBucketOwner(bucketName, owner)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update bucket owner: " + err.Error()})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"message": "Bucket owner updated successfully",
+		"bucket":  bucketName,
+		"owner":   owner,
+	})
+}
+
+// SetBucketOwner sets the owner of a bucket
+func (s *AdminServer) SetBucketOwner(bucketName string, owner string) error {
+	return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		// Get the current bucket entry
+		lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+			Directory: "/buckets",
+			Name:      bucketName,
+		})
+		if err != nil {
+			return fmt.Errorf("lookup bucket %s: %w", bucketName, err)
+		}
+
+		bucketEntry := lookupResp.Entry
+
+		// Initialize Extended map if nil
+		if bucketEntry.Extended == nil {
+			bucketEntry.Extended = make(map[string][]byte)
+		}
+
+		// Set or remove the owner
+		if owner == "" {
+			delete(bucketEntry.Extended, s3_constants.AmzIdentityId)
+		} else {
+			bucketEntry.Extended[s3_constants.AmzIdentityId] = []byte(owner)
+		}
+
+		// Update the entry
+		_, err = client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+			Directory: "/buckets",
+			Entry:     bucketEntry,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to update bucket owner: %w", err)
+		}
+
+		return nil
+	})
+}
+
 // ListBucketsAPI returns the list of buckets as JSON
 func (s *AdminServer) ListBucketsAPI(c *gin.Context) {
 	buckets, err := s.GetS3Buckets()
@@ -288,11 +385,11 @@ func (s *AdminServer) SetBucketQuota(bucketName string, quotaBytes int64, quotaE

 // CreateS3BucketWithQuota creates a new S3 bucket with quota settings
 func (s *AdminServer) CreateS3BucketWithQuota(bucketName string, quotaBytes int64, quotaEnabled bool) error {
-	return s.CreateS3BucketWithObjectLock(bucketName, quotaBytes, quotaEnabled, false, false, "", false, 0)
+	return s.CreateS3BucketWithObjectLock(bucketName, quotaBytes, quotaEnabled, false, false, "", false, 0, "")
 }

-// CreateS3BucketWithObjectLock creates a new S3 bucket with quota, versioning, and object lock settings
-func (s *AdminServer) CreateS3BucketWithObjectLock(bucketName string, quotaBytes int64, quotaEnabled, versioningEnabled, objectLockEnabled bool, objectLockMode string, setDefaultRetention bool, objectLockDuration int32) error {
+// CreateS3BucketWithObjectLock creates a new S3 bucket with quota, versioning, object lock settings, and owner
+func (s *AdminServer) CreateS3BucketWithObjectLock(bucketName string, quotaBytes int64, quotaEnabled, versioningEnabled, objectLockEnabled bool, objectLockMode string, setDefaultRetention bool, objectLockDuration int32, owner string) error {
 	return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 		// First ensure /buckets directory exists
 		_, err := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
@@ -344,9 +441,14 @@ func (s *AdminServer) CreateS3BucketWithObjectLock(bucketName string, quotaBytes
 			TtlSec:   0,
 		}

-		// Create extended attributes map for versioning
+		// Create extended attributes map for versioning and owner
 		extended := make(map[string][]byte)

+		// Set bucket owner if specified
+		if owner != "" {
+			extended[s3_constants.AmzIdentityId] = []byte(owner)
+		}
+
 		// Create bucket entry
 		bucketEntry := &filer_pb.Entry{
 			Name:        bucketName,
@@ -82,6 +82,7 @@ type S3Bucket struct {
 	ObjectLockEnabled  bool      `json:"object_lock_enabled"`  // Whether object lock is enabled
 	ObjectLockMode     string    `json:"object_lock_mode"`     // Object lock mode: "GOVERNANCE" or "COMPLIANCE"
 	ObjectLockDuration int32     `json:"object_lock_duration"` // Default retention duration in days
+	Owner              string    `json:"owner,omitempty"`      // Bucket owner identity; empty means admin-only access
 }

 type S3Object struct {
@@ -544,6 +544,11 @@ func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, erro
 		volumeServers = append(volumeServers, *vs)
 	}

+	// Sort volume servers by address for consistent ordering on page refresh
+	sort.Slice(volumeServers, func(i, j int) bool {
+		return volumeServers[i].GetDisplayAddress() < volumeServers[j].GetDisplayAddress()
+	})
+
 	var totalCapacity int64
 	var totalVolumes int
 	for _, vs := range volumeServers {
@@ -117,7 +117,7 @@ func (s *WorkerGrpcServer) Stop() error {
 	s.connMutex.Lock()
 	for _, conn := range s.connections {
 		conn.cancel()
-		close(conn.outgoing)
+		s.safeCloseOutgoingChannel(conn, "Stop")
 	}
 	s.connections = make(map[string]*WorkerConnection)
 	s.connMutex.Unlock()
@@ -182,15 +182,27 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
 	}
 	conn.capabilities = capabilities

-	// Register connection
+	// Register connection - clean up old connection if worker is reconnecting
 	s.connMutex.Lock()
+	if oldConn, exists := s.connections[workerID]; exists {
+		glog.Infof("Worker %s reconnected, cleaning up old connection", workerID)
+		// Cancel old connection to stop its goroutines
+		oldConn.cancel()
+		// Don't close oldConn.outgoing here as it may cause panic in handleOutgoingMessages
+		// Let the goroutine exit naturally when it detects context cancellation
+	}
 	s.connections[workerID] = conn
 	s.connMutex.Unlock()

 	// Register worker with maintenance manager
 	s.registerWorkerWithManager(conn)

-	// Send registration response
+	// IMPORTANT: Start outgoing message handler BEFORE sending registration response
+	// This ensures the handler is ready to process messages and prevents race conditions
+	// where the worker might send requests before we're ready to respond
+	go s.handleOutgoingMessages(conn)
+
+	// Send registration response (after handler is started)
 	regResponse := &worker_pb.AdminMessage{
 		Timestamp: time.Now().Unix(),
 		Message: &worker_pb.AdminMessage_RegistrationResponse{
@@ -203,13 +215,11 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr

 	select {
 	case conn.outgoing <- regResponse:
+		glog.V(1).Infof("Registration response sent to worker %s", workerID)
 	case <-time.After(5 * time.Second):
 		glog.Errorf("Failed to send registration response to worker %s", workerID)
 	}

-	// Start outgoing message handler
-	go s.handleOutgoingMessages(conn)
-
 	// Handle incoming messages
 	for {
 		select {
@@ -235,7 +245,9 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
 			return err
 		}

+		s.connMutex.Lock()
 		conn.lastSeen = time.Now()
+		s.connMutex.Unlock()
 		s.handleWorkerMessage(conn, msg)
 	}
 }
@@ -440,16 +452,36 @@ func (s *WorkerGrpcServer) handleTaskLogResponse(conn *WorkerConnection, respons
 	s.logRequestsMutex.Unlock()
 }

+// safeCloseOutgoingChannel safely closes the outgoing channel for a worker connection.
+func (s *WorkerGrpcServer) safeCloseOutgoingChannel(conn *WorkerConnection, source string) {
+	defer func() {
+		if r := recover(); r != nil {
+			glog.V(1).Infof("%s: recovered from panic closing outgoing channel for worker %s: %v", source, conn.workerID, r)
+		}
+	}()
+	close(conn.outgoing)
+}
+
 // unregisterWorker removes a worker connection
 func (s *WorkerGrpcServer) unregisterWorker(workerID string) {
 	s.connMutex.Lock()
-	if conn, exists := s.connections[workerID]; exists {
-		conn.cancel()
-		close(conn.outgoing)
-		delete(s.connections, workerID)
+	conn, exists := s.connections[workerID]
+	if !exists {
+		s.connMutex.Unlock()
+		glog.V(2).Infof("unregisterWorker: worker %s not found in connections map (already unregistered)", workerID)
+		return
 	}
+
+	// Remove from map first to prevent duplicate cleanup attempts
+	delete(s.connections, workerID)
 	s.connMutex.Unlock()

+	// Cancel context to signal goroutines to stop
+	conn.cancel()
+
+	// Safely close the outgoing channel with recover to handle potential double-close
+	s.safeCloseOutgoingChannel(conn, "unregisterWorker")
+
 	glog.V(1).Infof("Unregistered worker %s", workerID)
 }

@@ -479,7 +511,7 @@ func (s *WorkerGrpcServer) cleanupStaleConnections() {
 		if conn.lastSeen.Before(cutoff) {
 			glog.Warningf("Cleaning up stale worker connection: %s", workerID)
 			conn.cancel()
-			close(conn.outgoing)
+			s.safeCloseOutgoingChannel(conn, "cleanupStaleConnections")
 			delete(s.connections, workerID)
 		}
 	}
@@ -119,6 +119,7 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
 				s3Api.DELETE("/buckets/:bucket", h.adminServer.DeleteBucket)
 				s3Api.GET("/buckets/:bucket", h.adminServer.ShowBucketDetails)
 				s3Api.PUT("/buckets/:bucket/quota", h.adminServer.UpdateBucketQuota)
+				s3Api.PUT("/buckets/:bucket/owner", h.adminServer.UpdateBucketOwner)
 			}

 			// User management API routes
@@ -245,6 +246,7 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
 				s3Api.DELETE("/buckets/:bucket", h.adminServer.DeleteBucket)
 				s3Api.GET("/buckets/:bucket", h.adminServer.ShowBucketDetails)
 				s3Api.PUT("/buckets/:bucket/quota", h.adminServer.UpdateBucketQuota)
+				s3Api.PUT("/buckets/:bucket/owner", h.adminServer.UpdateBucketOwner)
 			}

 			// User management API routes
@@ -266,7 +266,26 @@ func (s *MaintenanceIntegration) ScanWithTaskDetectors(volumeMetrics []*types.Vo

 // UpdateTopologyInfo updates the volume shard tracker with topology information for empty servers
 func (s *MaintenanceIntegration) UpdateTopologyInfo(topologyInfo *master_pb.TopologyInfo) error {
-	return s.activeTopology.UpdateTopology(topologyInfo)
+	// Log topology details before update for diagnostics
+	if topologyInfo != nil {
+		dcCount, nodeCount, diskCount := topology.CountTopologyResources(topologyInfo)
+		glog.V(2).Infof("UpdateTopologyInfo: received topology with %d datacenters, %d nodes, %d disks",
+			dcCount, nodeCount, diskCount)
+	} else {
+		glog.Warningf("UpdateTopologyInfo: received nil topologyInfo")
+	}
+
+	err := s.activeTopology.UpdateTopology(topologyInfo)
+
+	if err != nil {
+		glog.Errorf("UpdateTopologyInfo: topology update failed: %v", err)
+	} else {
+		// Log success with current disk count
+		currentDiskCount := s.activeTopology.GetDiskCount()
+		glog.V(1).Infof("UpdateTopologyInfo: topology update successful, active topology now has %d disks", currentDiskCount)
+	}
+
+	return err
 }

 // convertToExistingFormat converts task results to existing system format using dynamic mapping
@@ -137,6 +137,7 @@ func (mm *MaintenanceManager) Start() error {
 	// Start background processes
 	go mm.scanLoop()
 	go mm.cleanupLoop()
+	go mm.topologyStatusLoop() // Periodic diagnostic logging

 	glog.Infof("Maintenance manager started with scan interval %ds", mm.config.ScanIntervalSeconds)
 	return nil
@@ -255,6 +256,54 @@ func (mm *MaintenanceManager) cleanupLoop() {
 	}
 }

+// topologyStatusLoop periodically logs topology status for diagnostics
+func (mm *MaintenanceManager) topologyStatusLoop() {
+	// Log topology status every 5 minutes for diagnostic purposes
+	statusInterval := 5 * time.Minute
+	ticker := time.NewTicker(statusInterval)
+	defer ticker.Stop()
+
+	for mm.running {
+		select {
+		case <-mm.stopChan:
+			return
+		case <-ticker.C:
+			mm.logTopologyStatus()
+		}
+	}
+}
+
+// logTopologyStatus logs current topology and worker status for diagnostics
+func (mm *MaintenanceManager) logTopologyStatus() {
+	if mm.scanner == nil || mm.scanner.integration == nil {
+		glog.V(2).Infof("Topology status: scanner/integration not available")
+		return
+	}
+
+	activeTopology := mm.scanner.integration.GetActiveTopology()
+	if activeTopology == nil {
+		glog.V(1).Infof("Topology status: ActiveTopology is nil")
+		return
+	}
+
+	diskCount := activeTopology.GetDiskCount()
+	nodeCount := len(activeTopology.GetAllNodes())
+
+	// Get queue stats
+	stats := mm.queue.GetStats()
+	workerCount := len(mm.queue.GetWorkers())
+
+	mm.mutex.RLock()
+	errorCount := mm.errorCount
+	mm.mutex.RUnlock()
+
+	glog.V(0).Infof("Topology status: %d nodes, %d disks, %d workers, %d pending tasks, %d running tasks, errors: %d",
+		nodeCount, diskCount, workerCount,
+		stats.TasksByStatus[TaskStatusPending],
+		stats.TasksByStatus[TaskStatusInProgress]+stats.TasksByStatus[TaskStatusAssigned],
+		errorCount)
+}
+
 // performScan executes a maintenance scan with error handling and backoff
 func (mm *MaintenanceManager) performScan() {
 	defer func() {
@@ -8,11 +8,53 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 )

+// CountTopologyResources counts datacenters, nodes, and disks in topology info
+func CountTopologyResources(topologyInfo *master_pb.TopologyInfo) (dcCount, nodeCount, diskCount int) {
+	if topologyInfo == nil {
+		return 0, 0, 0
+	}
+	dcCount = len(topologyInfo.DataCenterInfos)
+	for _, dc := range topologyInfo.DataCenterInfos {
+		for _, rack := range dc.RackInfos {
+			nodeCount += len(rack.DataNodeInfos)
+			for _, node := range rack.DataNodeInfos {
+				diskCount += len(node.DiskInfos)
+			}
+		}
+	}
+	return
+}
+
 // UpdateTopology updates the topology information from master
 func (at *ActiveTopology) UpdateTopology(topologyInfo *master_pb.TopologyInfo) error {
 	at.mutex.Lock()
 	defer at.mutex.Unlock()

+	// Validate topology updates to prevent clearing disk maps with invalid data
+	if topologyInfo == nil {
+		glog.Warningf("UpdateTopology received nil topologyInfo, preserving last-known-good topology")
+		return fmt.Errorf("rejected invalid topology update: nil topologyInfo")
+	}
+
+	if len(topologyInfo.DataCenterInfos) == 0 {
+		glog.Warningf("UpdateTopology received empty DataCenterInfos, preserving last-known-good topology (had %d nodes, %d disks)",
+			len(at.nodes), len(at.disks))
+		return fmt.Errorf("rejected invalid topology update: empty DataCenterInfos (had %d nodes, %d disks)", len(at.nodes), len(at.disks))
+	}
+
+	// Count incoming topology for validation logging
+	dcCount, incomingNodes, incomingDisks := CountTopologyResources(topologyInfo)
+
+	// Reject updates that would wipe out a valid topology with an empty one (e.g. during master restart)
+	if incomingNodes == 0 && len(at.nodes) > 0 {
+		glog.Warningf("UpdateTopology received topology with 0 nodes, preserving last-known-good topology (had %d nodes, %d disks)",
+			len(at.nodes), len(at.disks))
+		return fmt.Errorf("rejected invalid topology update: 0 nodes (had %d nodes, %d disks)", len(at.nodes), len(at.disks))
+	}
+
+	glog.V(2).Infof("UpdateTopology: validating update with %d datacenters, %d nodes, %d disks (current: %d nodes, %d disks)",
+		dcCount, incomingNodes, incomingDisks, len(at.nodes), len(at.disks))
+
 	at.topologyInfo = topologyInfo
 	at.lastUpdated = time.Now()

@@ -142,8 +184,21 @@ func (at *ActiveTopology) GetNodeDisks(nodeID string) []*DiskInfo {
 	return disks
 }

+// GetDiskCount returns the total number of disks in the active topology
+func (at *ActiveTopology) GetDiskCount() int {
+	at.mutex.RLock()
+	defer at.mutex.RUnlock()
+	return len(at.disks)
+}
+
 // rebuildIndexes rebuilds the volume and EC shard indexes for O(1) lookups
 func (at *ActiveTopology) rebuildIndexes() {
+	// Nil-safety guard: return early if topology is not valid
+	if at.topologyInfo == nil || at.topologyInfo.DataCenterInfos == nil {
+		glog.V(1).Infof("rebuildIndexes: skipping rebuild due to nil topology or DataCenterInfos")
+		return
+	}
+
 	// Clear existing indexes
 	at.volumeIndex = make(map[uint32][]string)
 	at.ecShardIndex = make(map[uint32][]string)
@@ -2,330 +2,337 @@ package app

 import (
 	"fmt"
+	"sort"
 	"strings"
 	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 )

 templ ClusterEcVolumes(data dash.ClusterEcVolumesData) {
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <title>EC Volumes - SeaweedFS</title>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
-    <link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css" rel="stylesheet">
-</head>
-<body>
-    <div class="container-fluid">
-        <div class="row">
-            <div class="col-12">
-                <h2 class="mb-4">
-                    <i class="fas fa-database me-2"></i>EC Volumes
-                    <small class="text-muted">({fmt.Sprintf("%d", data.TotalVolumes)} volumes)</small>
-                </h2>
-            </div>
-        </div>
-
-        <!-- Statistics Cards -->
-        <div class="row mb-4">
-            <div class="col-md-3">
-                <div class="card text-bg-primary">
-                    <div class="card-body">
-                        <div class="d-flex justify-content-between">
-                            <div>
-                                <h6 class="card-title">Total Volumes</h6>
-                                <h4 class="mb-0">{fmt.Sprintf("%d", data.TotalVolumes)}</h4>
-                                <small>EC encoded volumes</small>
-                            </div>
-                            <div class="align-self-center">
-                                <i class="fas fa-cubes fa-2x"></i>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            <div class="col-md-3">
-                <div class="card text-bg-info">
-                    <div class="card-body">
-                        <div class="d-flex justify-content-between">
-                            <div>
-                                <h6 class="card-title">Total Shards</h6>
-                                <h4 class="mb-0">{fmt.Sprintf("%d", data.TotalShards)}</h4>
-                                <small>Distributed shards</small>
-                            </div>
-                            <div class="align-self-center">
-                                <i class="fas fa-puzzle-piece fa-2x"></i>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            <div class="col-md-3">
-                <div class="card text-bg-success">
-                    <div class="card-body">
-                        <div class="d-flex justify-content-between">
-                            <div>
-                                <h6 class="card-title">Complete Volumes</h6>
-                                <h4 class="mb-0">{fmt.Sprintf("%d", data.CompleteVolumes)}</h4>
-                                <small>All shards present</small>
-                            </div>
-                            <div class="align-self-center">
-                                <i class="fas fa-check-circle fa-2x"></i>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            <div class="col-md-3">
-                <div class="card text-bg-warning">
-                    <div class="card-body">
-                        <div class="d-flex justify-content-between">
-                            <div>
-                                <h6 class="card-title">Incomplete Volumes</h6>
-                                <h4 class="mb-0">{fmt.Sprintf("%d", data.IncompleteVolumes)}</h4>
-                                <small>Missing shards</small>
-                            </div>
-                            <div class="align-self-center">
-                                <i class="fas fa-exclamation-triangle fa-2x"></i>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-        </div>
-
-        <!-- EC Storage Information Note -->
-        <div class="alert alert-info mb-4" role="alert">
-            <i class="fas fa-info-circle me-2"></i>
-            <strong>EC Storage Note:</strong> 
-            EC volumes use erasure coding ({ fmt.Sprintf("%d+%d", erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount) }) which stores data across { fmt.Sprintf("%d", erasure_coding.TotalShardsCount) } shards with redundancy. 
-            Physical storage is approximately { fmt.Sprintf("%.1fx", float64(erasure_coding.TotalShardsCount)/float64(erasure_coding.DataShardsCount)) } the original logical data size due to { fmt.Sprintf("%d", erasure_coding.ParityShardsCount) } parity shards.
-        </div>
-
-        <!-- Volumes Table -->
-        <div class="d-flex justify-content-between align-items-center mb-3">
-            <div class="d-flex align-items-center">
-                <span class="me-3">
-                    Showing {fmt.Sprintf("%d", (data.Page-1)*data.PageSize + 1)} to {fmt.Sprintf("%d", func() int {
-                        end := data.Page * data.PageSize
-                        if end > data.TotalVolumes {
-                            return data.TotalVolumes
-                        }
-                        return end
-                    }())} of {fmt.Sprintf("%d", data.TotalVolumes)} volumes
-                </span>
-                
-                <div class="d-flex align-items-center">
-                    <label for="pageSize" class="form-label me-2 mb-0">Show:</label>
-                    <select id="pageSize" class="form-select form-select-sm" style="width: auto;" onchange="changePageSize(this.value)">
-                        <option value="5" if data.PageSize == 5 { selected }>5</option>
-                        <option value="10" if data.PageSize == 10 { selected }>10</option>
-                        <option value="25" if data.PageSize == 25 { selected }>25</option>
-                        <option value="50" if data.PageSize == 50 { selected }>50</option>
-                        <option value="100" if data.PageSize == 100 { selected }>100</option>
-                    </select>
-                    <span class="ms-2">per page</span>
-                </div>
-            </div>
-            
+    <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
+        <div>
+            <h1 class="h2">
+                <i class="fas fa-database me-2"></i>EC Volumes
+            </h1>
            if data.Collection != "" {
-                <div>
+                <div class="d-flex align-items-center mt-2">
                    if data.Collection == "default" {
-                        <span class="badge bg-secondary text-white">Collection: default</span>
+                        <span class="badge bg-secondary text-white me-2">
+                            <i class="fas fa-filter me-1"></i>Collection: default
+                        </span>
                    } else {
-                        <span class="badge bg-info text-white">Collection: {data.Collection}</span>
+                        <span class="badge bg-info text-white me-2">
+                            <i class="fas fa-filter me-1"></i>Collection: {data.Collection}
+                        </span>
                    }
-                    <a href="/cluster/ec-shards" class="btn btn-sm btn-outline-secondary ms-2">Clear Filter</a>
+                    <a href="/cluster/ec-volumes" class="btn btn-sm btn-outline-secondary">
+                        <i class="fas fa-times me-1"></i>Clear Filter
+                    </a>
                </div>
            }
        </div>
-
-        <div class="table-responsive">
-            <table class="table table-striped table-hover" id="ecVolumesTable">
-                <thead>
-                    <tr>
-                        <th>
-                            <a href="#" onclick="sortBy('volume_id')" class="text-dark text-decoration-none">
-                                Volume ID
-                                if data.SortBy == "volume_id" {
-                                    if data.SortOrder == "asc" {
-                                        <i class="fas fa-sort-up ms-1"></i>
-                                    } else {
-                                        <i class="fas fa-sort-down ms-1"></i>
-                                    }
-                                } else {
-                                    <i class="fas fa-sort ms-1 text-muted"></i>
-                                }
-                            </a>
-                        </th>
-                        if data.ShowCollectionColumn {
-                            <th>
-                                <a href="#" onclick="sortBy('collection')" class="text-dark text-decoration-none">
-                                    Collection
-                                    if data.SortBy == "collection" {
-                                        if data.SortOrder == "asc" {
-                                            <i class="fas fa-sort-up ms-1"></i>
-                                        } else {
-                                            <i class="fas fa-sort-down ms-1"></i>
-                                        }
-                                    } else {
-                                        <i class="fas fa-sort ms-1 text-muted"></i>
-                                    }
-                                </a>
-                            </th>
-                        }
-                        <th>
-                            <a href="#" onclick="sortBy('total_shards')" class="text-dark text-decoration-none">
-                                Shard Count
-                                if data.SortBy == "total_shards" {
-                                    if data.SortOrder == "asc" {
-                                        <i class="fas fa-sort-up ms-1"></i>
-                                    } else {
-                                        <i class="fas fa-sort-down ms-1"></i>
-                                    }
-                                } else {
-                                    <i class="fas fa-sort ms-1 text-muted"></i>
-                                }
-                            </a>
-                        </th>
-                        <th class="text-dark">Shard Size</th>
-                        <th class="text-dark">Shard Locations</th>
-                        <th>
-                            <a href="#" onclick="sortBy('completeness')" class="text-dark text-decoration-none">
-                                Status
-                                if data.SortBy == "completeness" {
-                                    if data.SortOrder == "asc" {
-                                        <i class="fas fa-sort-up ms-1"></i>
-                                    } else {
-                                        <i class="fas fa-sort-down ms-1"></i>
-                                    }
-                                } else {
-                                    <i class="fas fa-sort ms-1 text-muted"></i>
-                                }
-                            </a>
-                        </th>
-                        if data.ShowDataCenterColumn {
-                            <th class="text-dark">Data Centers</th>
-                        }
-                        <th class="text-dark">Actions</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    for _, volume := range data.EcVolumes {
-                        <tr>
-                            <td>
-                                <strong>{fmt.Sprintf("%d", volume.VolumeID)}</strong>
-                            </td>
-                            if data.ShowCollectionColumn {
-                                <td>
-                                    if volume.Collection != "" {
-                                        <a href="/cluster/ec-shards?collection={volume.Collection}" class="text-decoration-none">
-                                            <span class="badge bg-info text-white">{volume.Collection}</span>
-                                        </a>
-                                    } else {
-                                        <a href="/cluster/ec-shards?collection=default" class="text-decoration-none">
-                                            <span class="badge bg-secondary text-white">default</span>
-                                        </a>
-                                    }
-                                </td>
-                            }
-                            <td>
-                                <span class="badge bg-primary">{fmt.Sprintf("%d/14", volume.TotalShards)}</span>
-                            </td>
-                            <td>
-                                @displayShardSizes(volume.ShardSizes)
-                            </td>
-                            <td>
-                                @displayVolumeDistribution(volume)
-                            </td>
-                            <td>
-                                @displayEcVolumeStatus(volume)
-                            </td>
-                            if data.ShowDataCenterColumn {
-                                <td>
-                                    for i, dc := range volume.DataCenters {
-                                        if i > 0 {
-                                            <span>, </span>
-                                        }
-                                        <span class="badge bg-primary text-white">{dc}</span>
-                                    }
-                                </td>
-                            }
-                            <td>
-                                <div class="btn-group" role="group">
-                                    <button type="button" class="btn btn-sm btn-outline-primary" 
-                                            onclick="showVolumeDetails(event)" 
-                                            data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
-                                            title="View EC volume details">
-                                        <i class="fas fa-info-circle"></i>
-                                    </button>
-                                    if !volume.IsComplete {
-                                        <button type="button" class="btn btn-sm btn-outline-warning" 
-                                                onclick="repairVolume(event)" 
-                                                data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
-                                                title="Repair missing shards">
-                                            <i class="fas fa-wrench"></i>
-                                        </button>
-                                    }
-                                </div>
-                            </td>
-                        </tr>
-                    }
-                </tbody>
-            </table>
+        <div class="btn-toolbar mb-2 mb-md-0">
+            <div class="btn-group me-2">
+                <select class="form-select form-select-sm me-2" id="pageSizeSelect" onchange="changePageSize(this.value)" style="width: auto;">
+                    <option value="5" if data.PageSize == 5 { selected="selected" }>5 per page</option>
+                    <option value="10" if data.PageSize == 10 { selected="selected" }>10 per page</option>
+                    <option value="25" if data.PageSize == 25 { selected="selected" }>25 per page</option>
+                    <option value="50" if data.PageSize == 50 { selected="selected" }>50 per page</option>
+                    <option value="100" if data.PageSize == 100 { selected="selected" }>100 per page</option>
+                </select>
+                <button type="button" class="btn btn-sm btn-outline-primary" onclick="window.location.reload()">
+                    <i class="fas fa-refresh me-1"></i>Refresh
+                </button>
+            </div>
        </div>
-
-        <!-- Pagination -->
-        if data.TotalPages > 1 {
-            <nav aria-label="EC Volumes pagination">
-                <ul class="pagination justify-content-center">
-                    if data.Page > 1 {
-                        <li class="page-item">
-                            <a class="page-link" href="#" onclick="goToPage(event)" data-page="1">First</a>
-                        </li>
-                        <li class="page-item">
-                            <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page-1) }>Previous</a>
-                        </li>
-                    }
-                    
-                    for i := 1; i <= data.TotalPages; i++ {
-                        if i == data.Page {
-                            <li class="page-item active">
-                                <span class="page-link">{fmt.Sprintf("%d", i)}</span>
-                            </li>
-                        } else if i <= 3 || i > data.TotalPages-3 || (i >= data.Page-2 && i <= data.Page+2) {
-                            <li class="page-item">
-                                <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", i) }>{fmt.Sprintf("%d", i)}</a>
-                            </li>
-                        } else if i == 4 && data.Page > 6 {
-                            <li class="page-item disabled">
-                                <span class="page-link">...</span>
-                            </li>
-                        } else if i == data.TotalPages-3 && data.Page < data.TotalPages-5 {
-                            <li class="page-item disabled">
-                                <span class="page-link">...</span>
-                            </li>
-                        }
-                    }
-                    
-                    if data.Page < data.TotalPages {
-                        <li class="page-item">
-                            <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page+1) }>Next</a>
-                        </li>
-                        <li class="page-item">
-                            <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.TotalPages) }>Last</a>
-                        </li>
-                    }
-                </ul>
-            </nav>
-        }
    </div>

-    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
+    <!-- Statistics Cards -->
+    <div class="row mb-4">
+        <div class="col-md-3">
+            <div class="card text-bg-primary">
+                <div class="card-body">
+                    <div class="d-flex justify-content-between">
+                        <div>
+                            <h6 class="card-title">Total Volumes</h6>
+                            <h4 class="mb-0">{fmt.Sprintf("%d", data.TotalVolumes)}</h4>
+                        </div>
+                        <div class="align-self-center">
+                            <i class="fas fa-cubes fa-2x"></i>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="col-md-3">
+            <div class="card text-bg-info">
+                <div class="card-body">
+                    <div class="d-flex justify-content-between">
+                        <div>
+                            <h6 class="card-title">Total Shards</h6>
+                            <h4 class="mb-0">{fmt.Sprintf("%d", data.TotalShards)}</h4>
+                        </div>
+                        <div class="align-self-center">
+                            <i class="fas fa-puzzle-piece fa-2x"></i>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="col-md-3">
+            <div class="card text-bg-success">
+                <div class="card-body">
+                    <div class="d-flex justify-content-between">
+                        <div>
+                            <h6 class="card-title">Healthy Volumes</h6>
+                            <h4 class="mb-0">{fmt.Sprintf("%d", data.CompleteVolumes)}</h4>
+                            <small>All { fmt.Sprintf("%d", erasure_coding.TotalShardsCount) } shards present</small>
+                        </div>
+                        <div class="align-self-center">
+                            <i class="fas fa-check-circle fa-2x"></i>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="col-md-3">
+            <div class="card text-bg-warning">
+                <div class="card-body">
+                    <div class="d-flex justify-content-between">
+                        <div>
+                            <h6 class="card-title">Degraded Volumes</h6>
+                            <h4 class="mb-0">{fmt.Sprintf("%d", data.IncompleteVolumes)}</h4>
+                            <small>Incomplete/Critical</small>
+                        </div>
+                        <div class="align-self-center">
+                            <i class="fas fa-exclamation-triangle fa-2x"></i>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <!-- EC Storage Information Note -->
+    <div class="alert alert-info mb-4" role="alert">
+        <i class="fas fa-info-circle me-2"></i>
+        <strong>EC Storage Note:</strong> 
+        EC volumes use erasure coding ({ fmt.Sprintf("%d+%d", erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount) }) which stores data across { fmt.Sprintf("%d", erasure_coding.TotalShardsCount) } shards with redundancy. 
+        Physical storage is approximately { fmt.Sprintf("%.1fx", float64(erasure_coding.TotalShardsCount)/float64(erasure_coding.DataShardsCount)) } the original logical data size due to { fmt.Sprintf("%d", erasure_coding.ParityShardsCount) } parity shards.
+    </div>
+
+    <!-- Volumes Table -->
+    <div class="table-responsive">
+        <table class="table table-striped table-hover" id="ecVolumesTable">
+            <thead>
+                <tr>
+                    <th>
+                        <a href="#" onclick="sortBy('volume_id')" class="text-dark text-decoration-none">
+                            Volume ID
+                            if data.SortBy == "volume_id" {
+                                if data.SortOrder == "asc" {
+                                    <i class="fas fa-sort-up ms-1"></i>
+                                } else {
+                                    <i class="fas fa-sort-down ms-1"></i>
+                                }
+                            } else {
+                                <i class="fas fa-sort ms-1 text-muted"></i>
+                            }
+                        </a>
+                    </th>
+                    if data.ShowCollectionColumn {
+                        <th>
+                            <a href="#" onclick="sortBy('collection')" class="text-dark text-decoration-none">
+                                Collection
+                                if data.SortBy == "collection" {
+                                    if data.SortOrder == "asc" {
+                                        <i class="fas fa-sort-up ms-1"></i>
+                                    } else {
+                                        <i class="fas fa-sort-down ms-1"></i>
+                                    }
+                                } else {
+                                    <i class="fas fa-sort ms-1 text-muted"></i>
+                                }
+                            </a>
+                        </th>
+                    }
+                    <th>
+                        <a href="#" onclick="sortBy('total_shards')" class="text-dark text-decoration-none">
+                            Shard Count
+                            if data.SortBy == "total_shards" {
+                                if data.SortOrder == "asc" {
+                                    <i class="fas fa-sort-up ms-1"></i>
+                                } else {
+                                    <i class="fas fa-sort-down ms-1"></i>
+                                }
+                            } else {
+                                <i class="fas fa-sort ms-1 text-muted"></i>
+                            }
+                        </a>
+                    </th>
+                    <th class="text-dark">Shard Size</th>
+                    <th class="text-dark">Shard Locations</th>
+                    <th>
+                        <a href="#" onclick="sortBy('completeness')" class="text-dark text-decoration-none">
+                            Status
+                            if data.SortBy == "completeness" {
+                                if data.SortOrder == "asc" {
+                                    <i class="fas fa-sort-up ms-1"></i>
+                                } else {
+                                    <i class="fas fa-sort-down ms-1"></i>
+                                }
+                            } else {
+                                <i class="fas fa-sort ms-1 text-muted"></i>
+                            }
+                        </a>
+                    </th>
+                    if data.ShowDataCenterColumn {
+                        <th class="text-dark">Data Centers</th>
+                    }
+                    <th class="text-dark">Actions</th>
+                </tr>
+            </thead>
+            <tbody>
+                for _, volume := range data.EcVolumes {
+                    <tr>
+                        <td>
+                            <span class="fw-bold">{fmt.Sprintf("%d", volume.VolumeID)}</span>
+                        </td>
+                        if data.ShowCollectionColumn {
+                            <td>
+                                if volume.Collection != "" {
+                                    <a href={ templ.URL(fmt.Sprintf("/cluster/ec-shards?collection=%s", volume.Collection)) } class="text-decoration-none">
+                                        <span class="badge bg-info text-white">{volume.Collection}</span>
+                                    </a>
+                                } else {
+                                    <a href={ templ.URL("/cluster/ec-shards?collection=default") } class="text-decoration-none">
+                                        <span class="badge bg-secondary text-white">default</span>
+                                    </a>
+                                }
+                            </td>
+                        }
+                        <td>
+                            <span class="badge bg-primary">{fmt.Sprintf("%d/%d", volume.TotalShards, erasure_coding.TotalShardsCount)}</span>
+                        </td>
+                        <td>
+                            @displayShardSizes(volume.ShardSizes)
+                        </td>
+                        <td>
+                            @displayVolumeDistribution(volume)
+                        </td>
+                        <td>
+                            @displayEcVolumeStatus(volume)
+                        </td>
+                        if data.ShowDataCenterColumn {
+                            <td>
+                                for i, dc := range volume.DataCenters {
+                                    if i > 0 {
+                                        <span>, </span>
+                                    }
+                                    <span class="badge bg-primary text-white">{dc}</span>
+                                }
+                            </td>
+                        }
+                        <td>
+                            <div class="btn-group" role="group">
+                                <button type="button" class="btn btn-sm btn-outline-primary" 
+                                        onclick="showVolumeDetails(event)" 
+                                        data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
+                                        title="View EC volume details">
+                                    <i class="fas fa-info-circle"></i>
+                                </button>
+                                if !volume.IsComplete {
+                                    <button type="button" class="btn btn-sm btn-outline-warning" 
+                                            onclick="repairVolume(event)" 
+                                            data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
+                                            title="Repair missing shards">
+                                        <i class="fas fa-wrench"></i>
+                                    </button>
+                                }
+                            </div>
+                        </td>
+                    </tr>
+                }
+            </tbody>
+        </table>
+    </div>
+
+    <!-- Pagination -->
+    if data.TotalPages > 1 {
+        <nav aria-label="EC Volumes pagination">
+            <ul class="pagination justify-content-center">
+                if data.Page > 1 {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page="1">
+                            <i class="fas fa-angle-double-left"></i>
+                        </a>
+                    </li>
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page-1) }>
+                            <i class="fas fa-chevron-left"></i>
+                        </a>
+                    </li>
+                }
+                
+                <!-- First page -->
+                if data.Page > 3 {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page="1">1</a>
+                    </li>
+                    if data.Page > 4 {
+                        <li class="page-item disabled">
+                            <span class="page-link">...</span>
+                        </li>
+                    }
+                }
+                
+                <!-- Current page neighbors -->
+                if data.Page > 1 {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page-1) }>{fmt.Sprintf("%d", data.Page-1)}</a>
+                    </li>
+                }
+                
+                <li class="page-item active">
+                    <span class="page-link">{fmt.Sprintf("%d", data.Page)}</span>
+                </li>
+                
+                if data.Page < data.TotalPages {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page+1) }>{fmt.Sprintf("%d", data.Page+1)}</a>
+                    </li>
+                }
+                
+                <!-- Last page -->
+                if data.Page < data.TotalPages-2 {
+                    if data.Page < data.TotalPages-3 {
+                        <li class="page-item disabled">
+                            <span class="page-link">...</span>
+                        </li>
+                    }
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.TotalPages) }>{fmt.Sprintf("%d", data.TotalPages)}</a>
+                    </li>
+                }
+                
+                if data.Page < data.TotalPages {
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page+1) }>
+                            <i class="fas fa-chevron-right"></i>
+                        </a>
+                    </li>
+                    <li class="page-item">
+                        <a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.TotalPages) }>
+                            <i class="fas fa-angle-double-right"></i>
+                        </a>
+                    </li>
+                }
+            </ul>
+        </nav>
+    }
+
+    <!-- JavaScript -->
    <script>
-        // Sorting functionality
        function sortBy(field) {
            const currentSort = new URLSearchParams(window.location.search).get('sort_by');
            const currentOrder = new URLSearchParams(window.location.search).get('sort_order') || 'asc';
@@ -335,65 +342,69 @@ templ ClusterEcVolumes(data dash.ClusterEcVolumesData) {
                newOrder = 'desc';
            }
            
-            const url = new URL(window.location);
-            url.searchParams.set('sort_by', field);
-            url.searchParams.set('sort_order', newOrder);
-            url.searchParams.set('page', '1'); // Reset to first page
-            window.location.href = url.toString();
+            updateUrl({
+                sort_by: field,
+                sort_order: newOrder,
+                page: 1
+            });
        }

-        // Pagination functionality
        function goToPage(event) {
            event.preventDefault();
            const page = event.target.closest('a').getAttribute('data-page');
-            const url = new URL(window.location);
-            url.searchParams.set('page', page);
-            window.location.href = url.toString();
+            updateUrl({ page: page });
        }

-        // Page size functionality
        function changePageSize(newPageSize) {
+            updateUrl({ page_size: newPageSize, page: 1 });
+        }
+
+        function updateUrl(params) {
            const url = new URL(window.location);
-            url.searchParams.set('page_size', newPageSize);
-            url.searchParams.set('page', '1'); // Reset to first page when changing page size
+            Object.keys(params).forEach(key => {
+                if (params[key] != null) {
+                    url.searchParams.set(key, params[key]);
+                } else {
+                    url.searchParams.delete(key);
+                }
+            });
            window.location.href = url.toString();
        }

-        // Volume details
        function showVolumeDetails(event) {
            const volumeId = event.target.closest('button').getAttribute('data-volume-id');
            window.location.href = `/cluster/ec-volumes/${volumeId}`;
        }

-        // Repair volume
        function repairVolume(event) {
            const volumeId = event.target.closest('button').getAttribute('data-volume-id');
            if (confirm(`Are you sure you want to repair missing shards for volume ${volumeId}?`)) {
-                // TODO: Implement repair functionality
-                alert('Repair functionality will be implemented soon.');
+                fetch(`/api/cluster/ec-volumes/${volumeId}/repair`, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                    }
+                })
+                .then(response => {
+                    if (!response.ok) {
+                        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                    }
+                    return response.json();
+                })
+                .then(data => {
+                    if (data && data.success) {
+                        alert('Repair initiated successfully');
+                        location.reload();
+                    } else {
+                        alert('Failed to initiate repair: ' + (data && data.error ? data.error : 'Unknown error'));
+                    }
+                })
+                .catch(error => {
+                    alert('Error: ' + error.message);
+                });
            }
        }
    </script>
-</body>
-</html>
-}
-
-// displayShardLocationsHTML renders shard locations as proper HTML
-templ displayShardLocationsHTML(shardLocations map[int]string) {
-	if len(shardLocations) == 0 {
-		<span class="text-muted">No shards</span>
-	} else {
-		for i, serverInfo := range groupShardsByServer(shardLocations) {
-			if i > 0 {
-				<br/>
-			}
-			<strong>
-				<a href={ templ.URL("/cluster/volume-servers/" + serverInfo.Server) } class="text-primary text-decoration-none">
-					{ serverInfo.Server }
-				</a>:
-			</strong> { serverInfo.ShardRanges }
-		}
-	}
 }

 // displayShardSizes renders shard sizes in a compact format
@@ -439,13 +450,7 @@ func groupShardsByServer(shardLocations map[int]string) []ServerShardInfo {
 	var serverInfos []ServerShardInfo
 	for server, shards := range serverShards {
 		// Sort shards for each server
-		for i := 0; i < len(shards); i++ {
-			for j := i + 1; j < len(shards); j++ {
-				if shards[i] > shards[j] {
-					shards[i], shards[j] = shards[j], shards[i]
-				}
-			}
-		}
+		sort.Ints(shards)
 		
 		// Format shard ranges compactly
 		shardRanges := formatShardRanges(shards)
@@ -456,56 +461,9 @@ func groupShardsByServer(shardLocations map[int]string) []ServerShardInfo {
 	}

 	// Sort by server name
-	for i := 0; i < len(serverInfos); i++ {
-		for j := i + 1; j < len(serverInfos); j++ {
-			if serverInfos[i].Server > serverInfos[j].Server {
-				serverInfos[i], serverInfos[j] = serverInfos[j], serverInfos[i]
-			}
-		}
-	}
-
-	return serverInfos
-}
-
-// groupShardsByServerWithSizes groups shards by server and formats ranges with sizes
-func groupShardsByServerWithSizes(shardLocations map[int]string, shardSizes map[int]int64) []ServerShardInfo {
-	if len(shardLocations) == 0 {
-		return []ServerShardInfo{}
-	}
-
-	// Group shards by server
-	serverShards := make(map[string][]int)
-	for shardId, server := range shardLocations {
-		serverShards[server] = append(serverShards[server], shardId)
-	}
-
-	var serverInfos []ServerShardInfo
-	for server, shards := range serverShards {
-		// Sort shards for each server
-		for i := 0; i < len(shards); i++ {
-			for j := i + 1; j < len(shards); j++ {
-				if shards[i] > shards[j] {
-					shards[i], shards[j] = shards[j], shards[i]
-				}
-			}
-		}
-		
-		// Format shard ranges compactly with sizes
-		shardRanges := formatShardRangesWithSizes(shards, shardSizes)
-		serverInfos = append(serverInfos, ServerShardInfo{
-			Server:      server,
-			ShardRanges: shardRanges,
-		})
-	}
-
-	// Sort by server name
-	for i := 0; i < len(serverInfos); i++ {
-		for j := i + 1; j < len(serverInfos); j++ {
-			if serverInfos[i].Server > serverInfos[j].Server {
-				serverInfos[i], serverInfos[j] = serverInfos[j], serverInfos[i]
-			}
-		}
-	}
+	sort.Slice(serverInfos, func(i, j int) bool {
+		return serverInfos[i].Server < serverInfos[j].Server
+	})

 	return serverInfos
 }
@@ -544,72 +502,6 @@ func formatShardRanges(shards []int) string {
 	return strings.Join(ranges, ",")
 }

-// Helper function to format shard ranges with sizes (e.g., "0(1.2MB),1-3(2.5MB),7(800KB)")
-func formatShardRangesWithSizes(shards []int, shardSizes map[int]int64) string {
-	if len(shards) == 0 {
-		return ""
-	}
-
-	var ranges []string
-	start := shards[0]
-	end := shards[0]
-	var totalSize int64
-
-	for i := 1; i < len(shards); i++ {
-		if shards[i] == end+1 {
-			end = shards[i]
-			totalSize += shardSizes[shards[i]]
-		} else {
-			// Add current range with size
-			if start == end {
-				size := shardSizes[start]
-				if size > 0 {
-					ranges = append(ranges, fmt.Sprintf("%d(%s)", start, bytesToHumanReadable(size)))
-				} else {
-					ranges = append(ranges, fmt.Sprintf("%d", start))
-				}
-			} else {
-				// Calculate total size for the range
-				rangeSize := shardSizes[start]
-				for j := start + 1; j <= end; j++ {
-					rangeSize += shardSizes[j]
-				}
-				if rangeSize > 0 {
-					ranges = append(ranges, fmt.Sprintf("%d-%d(%s)", start, end, bytesToHumanReadable(rangeSize)))
-				} else {
-					ranges = append(ranges, fmt.Sprintf("%d-%d", start, end))
-				}
-			}
-			start = shards[i]
-			end = shards[i]
-			totalSize = shardSizes[shards[i]]
-		}
-	}
-
-	// Add the last range
-	if start == end {
-		size := shardSizes[start]
-		if size > 0 {
-			ranges = append(ranges, fmt.Sprintf("%d(%s)", start, bytesToHumanReadable(size)))
-		} else {
-			ranges = append(ranges, fmt.Sprintf("%d", start))
-		}
-	} else {
-		// Calculate total size for the range
-		rangeSize := shardSizes[start]
-		for j := start + 1; j <= end; j++ {
-			rangeSize += shardSizes[j]
-		}
-		if rangeSize > 0 {
-			ranges = append(ranges, fmt.Sprintf("%d-%d(%s)", start, end, bytesToHumanReadable(rangeSize)))
-		} else {
-			ranges = append(ranges, fmt.Sprintf("%d-%d", start, end))
-		}
-	}
-
-	return strings.Join(ranges, ",")
-}
-
 // Helper function to convert bytes to human readable format
 func bytesToHumanReadable(bytes int64) string {
 	const unit = 1024
@@ -680,13 +572,7 @@ func formatIndividualShardSizes(shardSizes map[int]int64) string {
 		var groupStrs []string
 		for size, shardIds := range sizeGroups {
 			// Sort shard IDs
-			for i := 0; i < len(shardIds); i++ {
-				for j := i + 1; j < len(shardIds); j++ {
-					if shardIds[i] > shardIds[j] {
-						shardIds[i], shardIds[j] = shardIds[j], shardIds[i]
-					}
-				}
-			}
+			sort.Ints(shardIds)
 			
 			var idRanges []string
 			if len(shardIds) <= erasure_coding.ParityShardsCount {
@@ -715,18 +601,27 @@ templ displayVolumeDistribution(volume dash.EcVolumeWithShards) {
    </div>
 }

-// displayEcVolumeStatus shows an improved status display for EC volumes
+// displayEcVolumeStatus shows an improved status display for EC volumes.
+// Status thresholds are based on EC recovery capability:
+// - Critical: More than DataShardsCount missing (data is unrecoverable)
+// - Degraded: More than half of DataShardsCount missing (high risk)
+// - Incomplete: More than half of ParityShardsCount missing (reduced redundancy)
+// - Minor Issues: Few shards missing (still fully recoverable)
 templ displayEcVolumeStatus(volume dash.EcVolumeWithShards) {
    if volume.IsComplete {
        <span class="badge bg-success"><i class="fas fa-check me-1"></i>Complete</span>
    } else {
        if len(volume.MissingShards) > erasure_coding.DataShardsCount {
+            // Unrecoverable: more shards missing than EC can reconstruct
            <span class="badge bg-danger"><i class="fas fa-skull me-1"></i>Critical ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
        } else if len(volume.MissingShards) > (erasure_coding.DataShardsCount/2) {
+            // High risk: approaching unrecoverable state
            <span class="badge bg-warning"><i class="fas fa-exclamation-triangle me-1"></i>Degraded ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
        } else if len(volume.MissingShards) > (erasure_coding.ParityShardsCount/2) {
+            // Reduced redundancy but still recoverable
            <span class="badge bg-warning"><i class="fas fa-info-circle me-1"></i>Incomplete ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
        } else {
+            // Minor: few shards missing, fully recoverable with good margin
            <span class="badge bg-info"><i class="fas fa-info-circle me-1"></i>Minor Issues ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
        }
    }
@@ -113,6 +113,7 @@ templ S3Buckets(data dash.S3BucketsData) {
                                <thead>
                                    <tr>
                                        <th>Name</th>
+                                        <th>Owner</th>
                                        <th>Created</th>
                                        <th>Objects</th>
                                        <th>Size</th>
@@ -126,12 +127,21 @@ templ S3Buckets(data dash.S3BucketsData) {
                                    for _, bucket := range data.Buckets {
                                        <tr>
                                            <td>
-                                                <a href={templ.SafeURL(fmt.Sprintf("/s3/buckets/%s", bucket.Name))} 
+                                                <a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))} 
                                                   class="text-decoration-none">
                                                    <i class="fas fa-cube me-2"></i>
                                                    {bucket.Name}
                                                </a>
                                            </td>
+                                            <td>
+                                                if bucket.Owner != "" {
+                                                    <span class="badge bg-info">
+                                                        <i class="fas fa-user me-1"></i>{bucket.Owner}
+                                                    </span>
+                                                } else {
+                                                    <span class="text-muted small">No owner</span>
+                                                }
+                                            </td>
                                            <td>{bucket.CreatedAt.Format("2006-01-02 15:04")}</td>
                                            <td>{fmt.Sprintf("%d", bucket.ObjectCount)}</td>
                                            <td>{formatBytes(bucket.Size)}</td>
@@ -193,6 +203,13 @@ templ S3Buckets(data dash.S3BucketsData) {
                                                            title="View Details">
                                                        <i class="fas fa-eye"></i>
                                                    </button>
+                                                    <button type="button" 
+                                                            class="btn btn-outline-info btn-sm owner-btn"
+                                                            data-bucket-name={bucket.Name}
+                                                            data-current-owner={bucket.Owner}
+                                                            title="Manage Owner">
+                                                        <i class="fas fa-user-edit"></i>
+                                                    </button>
                                                    <button type="button" 
                                                            class="btn btn-outline-warning btn-sm quota-btn"
                                                            data-bucket-name={bucket.Name}
@@ -213,7 +230,7 @@ templ S3Buckets(data dash.S3BucketsData) {
                                    }
                                    if len(data.Buckets) == 0 {
                                        <tr>
-                                            <td colspan="8" class="text-center text-muted py-4">
+                                            <td colspan="9" class="text-center text-muted py-4">
                                                <i class="fas fa-cube fa-3x mb-3 text-muted"></i>
                                                <div>
                                                    					<h5>No Object Store buckets found</h5>
@@ -268,6 +285,17 @@ templ S3Buckets(data dash.S3BucketsData) {
                                Bucket names must be between 3 and 63 characters, contain only lowercase letters, numbers, dots, and hyphens.
                            </div>
                        </div>
+
+                        <div class="mb-3">
+                            <label for="bucketOwner" class="form-label">Owner (Optional)</label>
+                            <select class="form-select" id="bucketOwner" name="owner">
+                                <option value="">No owner (admin-only access)</option>
+                                <!-- Options will be populated dynamically when modal opens -->
+                            </select>
+                            <div class="form-text">
+                                The S3 identity that owns this bucket. Non-admin users can only access buckets they own.
+                            </div>
+                        </div>
                        
                        <div class="mb-3">
                            <div class="form-check">
@@ -481,9 +509,68 @@ templ S3Buckets(data dash.S3BucketsData) {
        </div>
    </div>

+    <!-- Manage Owner Modal -->
+    <div class="modal fade" id="manageOwnerModal" tabindex="-1" aria-labelledby="manageOwnerModalLabel" aria-hidden="true">
+        <div class="modal-dialog">
+            <div class="modal-content">
+                <div class="modal-header">
+                    <h5 class="modal-title" id="manageOwnerModalLabel">
+                        <i class="fas fa-user-edit me-2"></i>Manage Bucket Owner
+                    </h5>
+                    <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+                </div>
+                <form id="ownerForm">
+                    <div class="modal-body">
+                        <div class="mb-3">
+                            <label class="form-label">Bucket Name</label>
+                            <input type="text" class="form-control" id="ownerBucketName" readonly>
+                        </div>
+                        
+                        <div class="mb-3">
+                            <label for="bucketOwnerSelect" class="form-label">Owner</label>
+                            <select class="form-select" id="bucketOwnerSelect" name="owner">
+                                <option value="">No owner (admin-only access)</option>
+                                <!-- Options will be populated dynamically -->
+                            </select>
+                            <div class="form-text">
+                                Select the S3 identity that owns this bucket. Non-admin users can only access buckets they own.
+                            </div>
+                        </div>
+
+                        <div id="ownerLoadingSpinner" class="text-center py-2" style="display: none;">
+                            <div class="spinner-border spinner-border-sm text-primary" role="status">
+                                <span class="visually-hidden">Loading users...</span>
+                            </div>
+                            <span class="ms-2">Loading users...</span>
+                        </div>
+                    </div>
+                    <div class="modal-footer">
+                        <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
+                        <button type="submit" class="btn btn-info">
+                            <i class="fas fa-save me-1"></i>Update Owner
+                        </button>
+                    </div>
+                </form>
+            </div>
+        </div>
+    </div>
+
    <!-- JavaScript for bucket management -->
    <script>
+    // Global state (shared between DOMContentLoaded handlers and global functions)
+    let deleteModalInstance = null;
+    let quotaModalInstance = null;
+    let ownerModalInstance = null;
+    let detailsModalInstance = null;
+    let cachedUsers = null;
+
    document.addEventListener('DOMContentLoaded', function() {
+        // Initialize modal instances once (reuse with show/hide)
+        deleteModalInstance = new bootstrap.Modal(document.getElementById('deleteBucketModal'));
+        quotaModalInstance = new bootstrap.Modal(document.getElementById('manageQuotaModal'));
+        ownerModalInstance = new bootstrap.Modal(document.getElementById('manageOwnerModal'));
+        detailsModalInstance = new bootstrap.Modal(document.getElementById('bucketDetailsModal'));
+
        const quotaCheckbox = document.getElementById('enableQuota');
        const quotaSettings = document.getElementById('quotaSettings');
        const versioningCheckbox = document.getElementById('enableVersioning');
@@ -517,6 +604,32 @@ templ S3Buckets(data dash.S3BucketsData) {
            defaultRetentionSettings.style.display = this.checked ? 'block' : 'none';
        });

+        // Populate owner dropdown when create bucket modal opens
+        document.getElementById('createBucketModal').addEventListener('show.bs.modal', async function() {
+            const ownerSelect = document.getElementById('bucketOwner');
+            
+            // Only fetch if not already populated
+            if (ownerSelect.options.length <= 1) {
+                try {
+                    const response = await fetch('/api/users');
+                    const data = await response.json();
+                    const users = data.users || [];
+                    
+                    users.forEach(user => {
+                        const option = document.createElement('option');
+                        option.value = user.username;
+                        option.textContent = user.username;
+                        ownerSelect.appendChild(option);
+                    });
+                } catch (error) {
+                    console.error('Error fetching users for owner dropdown:', error);
+                    // Reset to default state on error - user can still create bucket without owner
+                    ownerSelect.innerHTML = '<option value="">No owner (admin-only access)</option>';
+                    ownerSelect.selectedIndex = 0;
+                }
+            }
+        });
+
        // Handle form submission
        createBucketForm.addEventListener('submit', function(e) {
            e.preventDefault();
@@ -524,6 +637,7 @@ templ S3Buckets(data dash.S3BucketsData) {
            const formData = new FormData(this);
            const data = {
                name: formData.get('name'),
+                owner: formData.get('owner') || '',
                region: formData.get('region') || '',
                quota_size: quotaCheckbox.checked ? parseInt(formData.get('quota_size')) || 0 : 0,
                quota_unit: formData.get('quota_unit') || 'MB',
@@ -569,41 +683,19 @@ templ S3Buckets(data dash.S3BucketsData) {
        });

        // Handle delete bucket
-        let deleteModalInstance = null;
+        const deleteForm = document.getElementById('deleteBucketModal');
        document.querySelectorAll('.delete-bucket-btn').forEach(button => {
            button.addEventListener('click', function() {
                const bucketName = this.dataset.bucketName;
                document.getElementById('deleteBucketName').textContent = bucketName;
-                window.currentBucketToDelete = bucketName;
-                
-                // Dispose of existing modal instance if it exists
-                if (deleteModalInstance) {
-                    deleteModalInstance.dispose();
-                }
-                
-                // Create new modal instance
-                deleteModalInstance = new bootstrap.Modal(document.getElementById('deleteBucketModal'));
+                // Store bucket name on modal element instead of global window property
+                deleteForm.dataset.bucketName = bucketName;
                deleteModalInstance.show();
            });
        });

-        // Add event listener to properly dispose of delete modal when hidden
-        document.getElementById('deleteBucketModal').addEventListener('hidden.bs.modal', function() {
-            if (deleteModalInstance) {
-                deleteModalInstance.dispose();
-                deleteModalInstance = null;
-            }
-            // Force remove any remaining backdrops
-            document.querySelectorAll('.modal-backdrop').forEach(backdrop => {
-                backdrop.remove();
-            });
-            // Ensure body classes are removed
-            document.body.classList.remove('modal-open');
-            document.body.style.removeProperty('padding-right');
-        });
-
        // Handle quota management
-        let quotaModalInstance = null;
+        const quotaForm = document.getElementById('quotaForm');
        document.querySelectorAll('.quota-btn').forEach(button => {
            button.addEventListener('click', function() {
                const bucketName = this.dataset.bucketName;
@@ -617,15 +709,8 @@ templ S3Buckets(data dash.S3BucketsData) {
                // Toggle quota size settings
                document.getElementById('quotaSizeSettings').style.display = quotaEnabled ? 'block' : 'none';
                
-                window.currentBucketToUpdate = bucketName;
-                
-                // Dispose of existing modal instance if it exists
-                if (quotaModalInstance) {
-                    quotaModalInstance.dispose();
-                }
-                
-                // Create new modal instance
-                quotaModalInstance = new bootstrap.Modal(document.getElementById('manageQuotaModal'));
+                // Store bucket name on form element instead of global window property
+                quotaForm.dataset.bucketName = bucketName;
                quotaModalInstance.show();
            });
        });
@@ -649,6 +734,9 @@ templ S3Buckets(data dash.S3BucketsData) {
        document.getElementById('quotaForm').addEventListener('submit', function(e) {
            e.preventDefault();
            
+            const bucketName = this.dataset.bucketName;
+            if (!bucketName) return;
+
            const formData = new FormData(this);
            const enabled = document.getElementById('quotaEnabled').checked;
            const data = {
@@ -657,7 +745,7 @@ templ S3Buckets(data dash.S3BucketsData) {
                quota_enabled: enabled
            };

-            fetch(`/api/s3/buckets/${window.currentBucketToUpdate}/quota`, {
+            fetch(`/api/s3/buckets/${bucketName}/quota`, {
                method: 'PUT',
                headers: {
                    'Content-Type': 'application/json',
@@ -688,8 +776,97 @@ templ S3Buckets(data dash.S3BucketsData) {
            document.getElementById('quotaSizeSettings').style.display = this.checked ? 'block' : 'none';
        });

+        // Handle owner management
+        const ownerForm = document.getElementById('ownerForm');
+        document.querySelectorAll('.owner-btn').forEach(button => {
+            button.addEventListener('click', async function() {
+                const bucketName = this.dataset.bucketName;
+                const currentOwner = this.dataset.currentOwner || '';
+                
+                document.getElementById('ownerBucketName').value = bucketName;
+                // Store bucket name on form element instead of global window property
+                ownerForm.dataset.bucketName = bucketName;
+                
+                // Show loading spinner
+                document.getElementById('ownerLoadingSpinner').style.display = 'block';
+                document.getElementById('bucketOwnerSelect').disabled = true;
+                
+                ownerModalInstance.show();
+                
+                // Fetch users if not cached
+                try {
+                    if (!cachedUsers) {
+                        const response = await fetch('/api/users');
+                        const data = await response.json();
+                        cachedUsers = data.users || [];
+                    }
+                    
+                    // Populate the select dropdown
+                    const select = document.getElementById('bucketOwnerSelect');
+                    select.innerHTML = '<option value="">No owner (admin-only access)</option>';
+                    
+                    cachedUsers.forEach(user => {
+                        const option = document.createElement('option');
+                        option.value = user.username;
+                        option.textContent = user.username;
+                        if (user.username === currentOwner) {
+                            option.selected = true;
+                        }
+                        select.appendChild(option);
+                    });
+                    
+                    select.disabled = false;
+                } catch (error) {
+                    console.error('Error fetching users:', error);
+                    alert('Error loading users: ' + error.message);
+                    // Re-enable select and reset to default on error
+                    const select = document.getElementById('bucketOwnerSelect');
+                    select.innerHTML = '<option value="">No owner (admin-only access)</option>';
+                    select.selectedIndex = 0;
+                    select.disabled = false;
+                } finally {
+                    document.getElementById('ownerLoadingSpinner').style.display = 'none';
+                }
+            });
+        });
+
+        // Handle owner form submission
+        document.getElementById('ownerForm').addEventListener('submit', function(e) {
+            e.preventDefault();
+            
+            const bucketName = this.dataset.bucketName;
+            if (!bucketName) return;
+
+            const owner = document.getElementById('bucketOwnerSelect').value;
+            const data = { owner: owner };
+
+            fetch(`/api/s3/buckets/${bucketName}/owner`, {
+                method: 'PUT',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(data)
+            })
+            .then(response => response.json())
+            .then(data => {
+                if (data.error) {
+                    alert('Error updating owner: ' + data.error);
+                } else {
+                    alert('Bucket owner updated successfully!');
+                    // Properly close the modal before reloading
+                    if (ownerModalInstance) {
+                        ownerModalInstance.hide();
+                    }
+                    setTimeout(() => location.reload(), 500);
+                }
+            })
+            .catch(error => {
+                console.error('Error:', error);
+                alert('Error updating owner: ' + error.message);
+            });
+        });
+
        // Handle view details button
-        let detailsModalInstance = null;
        document.querySelectorAll('.view-details-btn').forEach(button => {
            button.addEventListener('click', function() {
                const bucketName = this.dataset.bucketName;
@@ -707,13 +884,6 @@ templ S3Buckets(data dash.S3BucketsData) {
                    '<div class="mt-2">Loading bucket details...</div>' +
                    '</div>';
                
-                // Dispose of existing modal instance if it exists
-                if (detailsModalInstance) {
-                    detailsModalInstance.dispose();
-                }
-                
-                // Create new modal instance
-                detailsModalInstance = new bootstrap.Modal(document.getElementById('bucketDetailsModal'));
                detailsModalInstance.show();
                
                // Fetch bucket details
@@ -740,25 +910,10 @@ templ S3Buckets(data dash.S3BucketsData) {
                    });
            });
        });
-
-        // Add event listener to properly dispose of details modal when hidden
-        document.getElementById('bucketDetailsModal').addEventListener('hidden.bs.modal', function() {
-            if (detailsModalInstance) {
-                detailsModalInstance.dispose();
-                detailsModalInstance = null;
-            }
-            // Force remove any remaining backdrops
-            document.querySelectorAll('.modal-backdrop').forEach(backdrop => {
-                backdrop.remove();
-            });
-            // Ensure body classes are removed
-            document.body.classList.remove('modal-open');
-            document.body.style.removeProperty('padding-right');
-        });
    });

    function deleteBucket() {
-        const bucketName = window.currentBucketToDelete;
+        const bucketName = document.getElementById('deleteBucketModal').dataset.bucketName;
        if (!bucketName) return;

        fetch(`/api/s3/buckets/${bucketName}`, {
@@ -786,6 +941,16 @@ templ S3Buckets(data dash.S3BucketsData) {
    function displayBucketDetails(data) {
        const bucket = data.bucket;
        const objects = data.objects || [];
+
+        // Helper function to escape HTML to prevent XSS
+        function escapeHtml(v) {
+            return String(v ?? '')
+                .replace(/&/g, '&amp;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;');
+        }
        
        // Helper function to format bytes
        function formatBytes(bytes) {
@@ -818,10 +983,10 @@ templ S3Buckets(data dash.S3BucketsData) {
                '<tbody>' +
                objects.map(obj => 
                    '<tr>' +
-                    '<td><i class="fas fa-file me-1"></i>' + obj.key + '</td>' +
+                    '<td><i class="fas fa-file me-1"></i>' + escapeHtml(obj.key) + '</td>' +
                    '<td>' + formatBytes(obj.size) + '</td>' +
                    '<td>' + formatDate(obj.last_modified) + '</td>' +
-                    '<td><span class="badge bg-primary">' + obj.storage_class + '</span></td>' +
+                    '<td><span class="badge bg-primary">' + escapeHtml(obj.storage_class) + '</span></td>' +
                    '</tr>'
                ).join('') +
                '</tbody>' +
@@ -840,7 +1005,11 @@ templ S3Buckets(data dash.S3BucketsData) {
            '<table class="table table-sm">' +
            '<tr>' +
            '<td><strong>Name:</strong></td>' +
-            '<td>' + bucket.name + '</td>' +
+            '<td>' + escapeHtml(bucket.name) + '</td>' +
+            '</tr>' +
+            '<tr>' +
+            '<td><strong>Owner:</strong></td>' +
+            '<td>' + (bucket.owner ? '<span class="badge bg-info"><i class="fas fa-user me-1"></i>' + escapeHtml(bucket.owner) + '</span>' : '<span class="text-muted">No owner (admin-only)</span>') + '</td>' +
            '</tr>' +
            '<tr>' +
            '<td><strong>Created:</strong></td>' +
@@ -886,7 +1055,7 @@ templ S3Buckets(data dash.S3BucketsData) {
            '<td>' +
            (bucket.object_lock_enabled ? 
                '<span class="badge bg-warning"><i class="fas fa-lock me-1"></i>Enabled</span>' +
-                '<br><small class="text-muted">' + bucket.object_lock_mode + ' • ' + bucket.object_lock_duration + ' days</small>' : 
+                '<br><small class="text-muted">' + escapeHtml(bucket.object_lock_mode) + ' • ' + bucket.object_lock_duration + ' days</small>' : 
                '<span class="badge bg-secondary"><i class="fas fa-unlock me-1"></i>Disabled</span>'
            ) +
            '</td>' +
@@ -906,26 +1075,45 @@ templ S3Buckets(data dash.S3BucketsData) {
    }

    function exportBucketList() {
-        // Simple CSV export
+        // RFC 4180 compliant CSV escaping: escape double quotes by doubling them
+        function escapeCsvField(value) {
+            const str = String(value ?? '');
+            // If the field contains comma, double quote, or newline, wrap in quotes and escape internal quotes
+            if (str.includes(',') || str.includes('"') || str.includes('\n') || str.includes('\r')) {
+                return '"' + str.replace(/"/g, '""') + '"';
+            }
+            return '"' + str + '"';
+        }
+
        const buckets = Array.from(document.querySelectorAll('#bucketsTable tbody tr')).map(row => {
            const cells = row.querySelectorAll('td');
            if (cells.length > 1) {
                return {
                    name: cells[0].textContent.trim(),
-                    created: cells[1].textContent.trim(),
-                    objects: cells[2].textContent.trim(),
-                    size: cells[3].textContent.trim(),
-                    quota: cells[4].textContent.trim(),
-                    versioning: cells[5].textContent.trim(),
-                    objectLock: cells[6].textContent.trim()
+                    owner: cells[1].textContent.trim(),
+                    created: cells[2].textContent.trim(),
+                    objects: cells[3].textContent.trim(),
+                    size: cells[4].textContent.trim(),
+                    quota: cells[5].textContent.trim(),
+                    versioning: cells[6].textContent.trim(),
+                    objectLock: cells[7].textContent.trim()
                };
            }
            return null;
        }).filter(bucket => bucket !== null);

        const csvContent = "data:text/csv;charset=utf-8," + 
-            "Name,Created,Objects,Size,Quota,Versioning,Object Lock\n" +
-            buckets.map(b => '"' + b.name + '","' + b.created + '","' + b.objects + '","' + b.size + '","' + b.quota + '","' + b.versioning + '","' + b.objectLock + '"').join("\n");
+            "Name,Owner,Created,Objects,Size,Quota,Versioning,Object Lock\n" +
+            buckets.map(b => [
+                escapeCsvField(b.name),
+                escapeCsvField(b.owner),
+                escapeCsvField(b.created),
+                escapeCsvField(b.objects),
+                escapeCsvField(b.size),
+                escapeCsvField(b.quota),
+                escapeCsvField(b.versioning),
+                escapeCsvField(b.objectLock)
+            ].join(',')).join("\n");

        const encodedUri = encodeURI(csvContent);
        const link = document.createElement("a");
@@ -3,6 +3,8 @@ package cluster
 import (
 	"context"
 	"fmt"
+	"strings"
+	"sync/atomic"
 	"time"

 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
@@ -36,10 +38,11 @@ type LiveLock struct {
 	hostFiler      pb.ServerAddress
 	cancelCh       chan struct{}
 	grpcDialOption grpc.DialOption
-	isLocked       bool
+	isLocked       int32 // 0 = unlocked, 1 = locked; use atomic operations
 	self           string
 	lc             *LockClient
 	owner          string
+	lockTTL        time.Duration
 }

 // NewShortLivedLock creates a lock with a 5-second duration
@@ -58,17 +61,24 @@ func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLoc
 }

 // StartLongLivedLock starts a goroutine to lock the key and returns immediately.
-func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerChange func(newLockOwner string)) (lock *LiveLock) {
+// lockTTL specifies how long the lock should be held. The renewal interval is
+// automatically derived as lockTTL / 2 to ensure timely renewals.
+func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerChange func(newLockOwner string), lockTTL time.Duration) (lock *LiveLock) {
 	lock = &LiveLock{
 		key:            key,
 		hostFiler:      lc.seedFiler,
 		cancelCh:       make(chan struct{}),
-		expireAtNs:     time.Now().Add(lock_manager.LiveLockTTL).UnixNano(),
+		expireAtNs:     time.Now().Add(lockTTL).UnixNano(),
 		grpcDialOption: lc.grpcDialOption,
 		self:           owner,
 		lc:             lc,
+		lockTTL:        lockTTL,
+	}
+	if lock.lockTTL == 0 {
+		lock.lockTTL = lock_manager.LiveLockTTL
 	}
 	go func() {
+		renewInterval := lock.lockTTL / 2
 		isLocked := false
 		lockOwner := ""
 		for {
@@ -81,13 +91,15 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
 			}

 			if isLocked {
-				if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err != nil {
+				if err := lock.AttemptToLock(lock.lockTTL); err != nil {
 					glog.V(0).Infof("Lost lock %s: %v", key, err)
 					isLocked = false
+					atomic.StoreInt32(&lock.isLocked, 0)
 				}
 			} else {
-				if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err == nil {
+				if err := lock.AttemptToLock(lock.lockTTL); err == nil {
 					isLocked = true
+					// Note: AttemptToLock already sets lock.isLocked atomically on success
 				}
 			}
 			if lockOwner != lock.LockOwner() && lock.LockOwner() != "" {
@@ -99,7 +111,11 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
 			case <-lock.cancelCh:
 				return
 			default:
-				time.Sleep(lock_manager.RenewInterval)
+				if isLocked {
+					time.Sleep(renewInterval)
+				} else {
+					time.Sleep(5 * renewInterval)
+				}
 			}
 		}
 	}()
@@ -126,24 +142,28 @@ func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {
 		return err
 	}
 	if errorMessage != "" {
-		glog.V(1).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
+		if strings.Contains(errorMessage, "lock already owned") {
+			glog.V(3).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
+		} else {
+			glog.V(2).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
+		}
 		time.Sleep(time.Second)
 		return fmt.Errorf("%v", errorMessage)
 	}
-	if !lock.isLocked {
+	if atomic.LoadInt32(&lock.isLocked) == 0 {
 		// Only log when transitioning from unlocked to locked
 		glog.V(1).Infof("LOCK: Successfully acquired key=%s owner=%s", lock.key, lock.self)
 	}
-	lock.isLocked = true
+	atomic.StoreInt32(&lock.isLocked, 1)
 	return nil
 }

 func (lock *LiveLock) StopShortLivedLock() error {
-	if !lock.isLocked {
+	if atomic.LoadInt32(&lock.isLocked) == 0 {
 		return nil
 	}
 	defer func() {
-		lock.isLocked = false
+		atomic.StoreInt32(&lock.isLocked, 0)
 	}()
 	return pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		_, err := client.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
@@ -203,7 +223,7 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 			errorMessage = resp.Error
 			if resp.LockHostMovedTo != "" && resp.LockHostMovedTo != string(previousHostFiler) {
 				// Only log if the host actually changed
-				glog.V(1).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
+				glog.V(2).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
 				lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
 				lock.lc.seedFiler = lock.hostFiler
 			} else if resp.LockHostMovedTo != "" {
@@ -211,12 +231,12 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 			}
 			if resp.LockOwner != "" && resp.LockOwner != previousOwner {
 				// Only log if the owner actually changed
-				glog.V(1).Infof("LOCK: Owner changed from %s to %s for key=%s", previousOwner, resp.LockOwner, lock.key)
+				glog.V(2).Infof("LOCK: Owner changed from %s to %s for key=%s", previousOwner, resp.LockOwner, lock.key)
 				lock.owner = resp.LockOwner
 			} else if resp.LockOwner != "" {
 				lock.owner = resp.LockOwner
 			} else if previousOwner != "" {
-				glog.V(1).Infof("LOCK: Owner cleared for key=%s", lock.key)
+				glog.V(2).Infof("LOCK: Owner cleared for key=%s", lock.key)
 				lock.owner = ""
 			}
 		}
@@ -228,3 +248,8 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 func (lock *LiveLock) LockOwner() string {
 	return lock.owner
 }
+
+// IsLocked returns true if this instance currently holds the lock
+func (lock *LiveLock) IsLocked() bool {
+	return atomic.LoadInt32(&lock.isLocked) == 1
+}
@@ -32,6 +32,7 @@ var Commands = []*Command{
 	cmdIam,
 	cmdMaster,
 	cmdMasterFollower,
+	cmdMini,
 	cmdMount,
 	cmdMqAgent,
 	cmdMqBroker,
@@ -42,39 +42,40 @@ var (
 )

 type FilerOptions struct {
-	masters                      *pb.ServerDiscovery
-	mastersString                *string
-	ip                           *string
-	bindIp                       *string
-	port                         *int
-	portGrpc                     *int
-	publicPort                   *int
-	filerGroup                   *string
-	collection                   *string
-	defaultReplicaPlacement      *string
-	disableDirListing            *bool
-	maxMB                        *int
-	dirListingLimit              *int
-	dataCenter                   *string
-	rack                         *string
-	enableNotification           *bool
-	disableHttp                  *bool
-	cipher                       *bool
-	metricsHttpPort              *int
-	metricsHttpIp                *string
-	saveToFilerLimit             *int
-	defaultLevelDbDirectory      *string
-	concurrentUploadLimitMB      *int
-	concurrentFileUploadLimit    *int
-	debug                        *bool
-	debugPort                    *int
-	localSocket                  *string
-	showUIDirectoryDelete        *bool
-	downloadMaxMBps              *int
-	diskType                     *string
-	allowedOrigins               *string
-	exposeDirectoryData          *bool
-	certProvider                 certprovider.Provider
+	masters                   *pb.ServerDiscovery
+	mastersString             *string
+	ip                        *string
+	bindIp                    *string
+	port                      *int
+	portGrpc                  *int
+	publicPort                *int
+	filerGroup                *string
+	collection                *string
+	defaultReplicaPlacement   *string
+	disableDirListing         *bool
+	maxMB                     *int
+	dirListingLimit           *int
+	dataCenter                *string
+	rack                      *string
+	enableNotification        *bool
+	disableHttp               *bool
+	cipher                    *bool
+	metricsHttpPort           *int
+	metricsHttpIp             *string
+	saveToFilerLimit          *int
+	defaultLevelDbDirectory   *string
+	concurrentUploadLimitMB   *int
+	concurrentFileUploadLimit *int
+	debug                     *bool
+	debugPort                 *int
+	localSocket               *string
+	showUIDirectoryDelete     *bool
+	downloadMaxMBps           *int
+	diskType                  *string
+	allowedOrigins            *string
+	exposeDirectoryData       *bool
+	tusBasePath               *string
+	certProvider              certprovider.Provider
 }

 func init() {
@@ -109,6 +110,7 @@ func init() {
 	f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")
 	f.exposeDirectoryData = cmdFiler.Flag.Bool("exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI")
+	f.tusBasePath = cmdFiler.Flag.String("tusBasePath", "/.tus", "TUS resumable upload endpoint base path (e.g., /.tus)")

 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -134,6 +136,7 @@ func init() {
 	filerS3Options.idleTimeout = cmdFiler.Flag.Int("s3.idleTimeout", 120, "connection idle seconds")
 	filerS3Options.concurrentUploadLimitMB = cmdFiler.Flag.Int("s3.concurrentUploadLimitMB", 0, "limit total concurrent upload size for S3, 0 means unlimited")
 	filerS3Options.concurrentFileUploadLimit = cmdFiler.Flag.Int("s3.concurrentFileUploadLimit", 0, "limit number of concurrent file uploads for S3, 0 means unlimited")
+	filerS3Options.enableIam = cmdFiler.Flag.Bool("s3.iam", true, "enable embedded IAM API on the same S3 port")

 	// start webdav on filer
 	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
@@ -341,6 +344,7 @@ func (fo *FilerOptions) startFiler() {
 		DownloadMaxBytesPs:        int64(*fo.downloadMaxMBps) * 1024 * 1024,
 		DiskType:                  *fo.diskType,
 		AllowedOrigins:            strings.Split(*fo.allowedOrigins, ","),
+		TusBasePath:               *fo.tusBasePath,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)
@@ -53,6 +53,8 @@ type SyncOptions struct {
 	bDoDeleteFiles  *bool
 	clientId        int32
 	clientEpoch     atomic.Int32
+	debug           *bool
+	debugPort       *int
 }

 const (
@@ -60,10 +62,22 @@ const (
 	DefaultConcurrencyLimit = 32
 )

+// syncState tracks the current sync state for graceful shutdown checkpoint saving
+type syncState struct {
+	processor            *MetadataProcessor
+	grpcDialOption       grpc.DialOption
+	targetFiler          pb.ServerAddress
+	sourcePath           string
+	sourceFilerSignature int32
+}
+
 var (
 	syncOptions    SyncOptions
 	syncCpuProfile *string
 	syncMemProfile *string
+	// atomic pointers to current sync states for graceful shutdown
+	syncStateA2B atomic.Pointer[syncState]
+	syncStateB2A atomic.Pointer[syncState]
 )

 func init() {
@@ -96,6 +110,8 @@ func init() {
 	syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
 	syncOptions.aDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("a.doDeleteFiles", true, "delete and update files when synchronizing on filer A")
 	syncOptions.bDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("b.doDeleteFiles", true, "delete and update files when synchronizing on filer B")
+	syncOptions.debug = cmdFilerSynchronize.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	syncOptions.debugPort = cmdFilerSynchronize.Flag.Int("debug.port", 6060, "http port for debugging")
 	syncOptions.clientId = util.RandomInt32()
 }

@@ -118,6 +134,9 @@ var cmdFilerSynchronize = &Command{
 }

 func runFilerSynchronize(cmd *Command, args []string) bool {
+	if *syncOptions.debug {
+		grace.StartDebugServer(*syncOptions.debugPort)
+	}

 	util.LoadSecurityConfiguration()
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
@@ -143,6 +162,27 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 		return true
 	}

+	// register graceful shutdown hook to save checkpoints
+	grace.OnInterrupt(func() {
+		saveCheckpoint := func(name string, state *syncState) {
+			if state == nil || state.processor == nil {
+				return
+			}
+			offsetTsNs := state.processor.processedTsWatermark.Load()
+			if offsetTsNs == 0 {
+				return
+			}
+			if err := setOffset(state.grpcDialOption, state.targetFiler, getSignaturePrefixByPath(state.sourcePath), state.sourceFilerSignature, offsetTsNs); err != nil {
+				glog.Errorf("failed to save checkpoint for %s on shutdown: %v", name, err)
+			} else {
+				glog.V(0).Infof("saved checkpoint for %s on shutdown: %v", name, time.Unix(0, offsetTsNs))
+			}
+		}
+
+		saveCheckpoint("A->B", syncStateA2B.Load())
+		saveCheckpoint("B->A", syncStateB2A.Load())
+	})
+
 	go func() {
 		// a->b
 		// set synchronization start timestamp to offset
@@ -172,7 +212,8 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 				*syncOptions.concurrency,
 				*syncOptions.bDoDeleteFiles,
 				aFilerSignature,
-				bFilerSignature)
+				bFilerSignature,
+				&syncStateA2B)
 			if err != nil {
 				glog.Errorf("sync from %s to %s: %v", *syncOptions.filerA, *syncOptions.filerB, err)
 				time.Sleep(1747 * time.Millisecond)
@@ -210,7 +251,8 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 					*syncOptions.concurrency,
 					*syncOptions.aDoDeleteFiles,
 					bFilerSignature,
-					aFilerSignature)
+					aFilerSignature,
+					&syncStateB2A)
 				if err != nil {
 					glog.Errorf("sync from %s to %s: %v", *syncOptions.filerB, *syncOptions.filerA, err)
 					time.Sleep(2147 * time.Millisecond)
@@ -241,7 +283,7 @@ func initOffsetFromTsMs(grpcDialOption grpc.DialOption, targetFiler pb.ServerAdd
 }

 func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceExcludePaths []string, sourceReadChunkFromFiler bool, targetFiler pb.ServerAddress, targetPath string,
-	replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, concurrency int, doDeleteFiles bool, sourceFilerSignature int32, targetFilerSignature int32) error {
+	replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, concurrency int, doDeleteFiles bool, sourceFilerSignature int32, targetFilerSignature int32, statePtr *atomic.Pointer[syncState]) error {

 	// if first time, start from now
 	// if has previously synced, resume from that point of time
@@ -278,6 +320,17 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 	}
 	processor := NewMetadataProcessor(processEventFn, concurrency, sourceFilerOffsetTsNs)

+	// update sync state for graceful shutdown checkpoint saving
+	if statePtr != nil {
+		statePtr.Store(&syncState{
+			processor:            processor,
+			grpcDialOption:       grpcDialOption,
+			targetFiler:          targetFiler,
+			sourcePath:           sourcePath,
+			sourceFilerSignature: sourceFilerSignature,
+		})
+	}
+
 	var lastLogTsNs = time.Now().UnixNano()
 	var clientName = fmt.Sprintf("syncFrom_%s_To_%s", string(sourceFiler), string(targetFiler))
 	processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
@@ -44,13 +44,34 @@ func init() {

 var cmdIam = &Command{
 	UsageLine: "iam [-port=8111] [-filer=<ip:port>[,<ip:port>]...] [-master=<ip:port>,<ip:port>]",
-	Short:     "start a iam API compatible server",
-	Long:      `start a iam API compatible server.
+	Short:     "[DEPRECATED] start a standalone iam API compatible server",
+	Long: `[DEPRECATED] start a standalone iam API compatible server.
+
+	DEPRECATION NOTICE:
+	The standalone 'weed iam' command is deprecated and will be removed in a future release.
+	
+	The IAM API is now embedded in the S3 server by default. Simply use 'weed s3' instead,
+	which provides both S3 and IAM APIs on the same port (enabled by default with -iam=true).
+	
+	This simplifies deployment by running a single server instead of two separate servers,
+	following the pattern used by MinIO and Ceph RGW.
+	
+	To use the embedded IAM API:
+	  weed s3 -port=8333          # IAM API is available on the same port
+	
+	To disable the embedded IAM API (if you prefer the old behavior):
+	  weed s3 -iam=false          # Run S3 without IAM
+	  weed iam -port=8111         # Run IAM separately (deprecated)

 	Multiple filer addresses can be specified for high availability, separated by commas.`,
 }

 func runIam(cmd *Command, args []string) bool {
+	glog.Warningf("================================================================================")
+	glog.Warningf("DEPRECATION WARNING: 'weed iam' is deprecated and will be removed in a future release.")
+	glog.Warningf("The IAM API is now embedded in 'weed s3' by default (use -iam=true, which is the default).")
+	glog.Warningf("Please migrate to using 'weed s3' which provides both S3 and IAM APIs on the same port.")
+	glog.Warningf("================================================================================")
 	return iamStandaloneOptions.startIamServer()
 }

@@ -89,7 +110,7 @@ func (iamopt *IamOptions) startIamServer() bool {
 	if iamApiServer_err != nil {
 		glog.Fatalf("IAM API Server startup error: %v", iamApiServer_err)
 	}
-	
+
 	// Register shutdown handler to prevent goroutine leak
 	grace.OnInterrupt(func() {
 		iamApiServer.Shutdown()
--- a/Show More
+++ b/Show More
				`@@ -224,3 +224,4 @@ jobs:`
				`echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"`
				`@@ -91,3 +91,4 @@ jobs:`
				`echo "\| readonly \| /public \| Read-only \|" >> $GITHUB_STEP_SUMMARY`