s3api: skip object lock check for buckets without object lock

Optimize DELETE and PUT operations by skipping the object lock
protection check for buckets that don't have object lock enabled.

Previously, enforceObjectLockProtections was always called, which
performed an expensive entry lookup even for buckets without object lock.

This optimization:
- Adds isObjectLockEnabled() helper that uses cached bucket config
- Skips enforceObjectLockProtections for buckets without object lock
- Applies to both versioned and non-versioned DELETE paths
- Applies to PUT operations on non-versioned buckets

Performance impact: For buckets without object lock (the common case),
this eliminates one filer lookup per DELETE/PUT operation.

.github/workflows/container_latest.yml

@@ -26,7 +26,7 @@ jobs:
             qemu: true
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Free Disk Space
         run: |
           echo "Available disk space before cleanup:"
@@ -83,7 +83,7 @@ jobs:
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       - name: Build ${{ matrix.platform }}
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         env:
           DOCKER_BUILDKIT: 1
         with:
@@ -113,7 +113,7 @@ jobs:
     if: github.event_name != 'pull_request'
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Docker meta
         id: docker_meta
         uses: docker/metadata-action@v5

.github/workflows/container_release1.yml

@@ -29,7 +29,7 @@ jobs:
             qemu: true
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Free Disk Space
         run: |
           echo "Available disk space before cleanup:"
@@ -73,7 +73,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Build ${{ matrix.platform }}
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         env:
           DOCKER_BUILDKIT: 1
         with:
@@ -102,7 +102,7 @@ jobs:
     if: github.event_name != 'pull_request'
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Docker meta
         id: docker_meta
         uses: docker/metadata-action@v5

.github/workflows/container_release2.yml

@@ -29,7 +29,7 @@ jobs:
             qemu: true
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Free Disk Space
         run: |
           echo "Available disk space before cleanup:"
@@ -73,7 +73,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Build ${{ matrix.platform }}
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         env:
           DOCKER_BUILDKIT: 1
         with:
@@ -103,7 +103,7 @@ jobs:
     if: github.event_name != 'pull_request'
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Docker meta
         id: docker_meta
         uses: docker/metadata-action@v5

.github/workflows/container_release3.yml

@@ -20,7 +20,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       -
         name: Free Disk Space
         run: |
@@ -59,7 +59,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release4.yml

@@ -19,7 +19,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       -
         name: Free Disk Space
         run: |
@@ -59,7 +59,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release5.yml

@@ -19,7 +19,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       -
         name: Free Disk Space
         run: |
@@ -49,7 +49,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release_unified.yml

@@ -59,7 +59,7 @@ jobs:
             
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         
       - name: Free Disk Space
         run: |
@@ -120,7 +120,7 @@ jobs:
           password: ${{ secrets.GHCR_TOKEN }}
           
       - name: Build and push ${{ matrix.variant }}
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         env:
           DOCKER_BUILDKIT: 1
         with:
@@ -223,3 +223,4 @@ jobs:
           
           echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
 
+

.github/workflows/ec-integration-tests.yml

@@ -0,0 +1,41 @@
+name: "EC Integration Tests"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+permissions:
+  contents: read
+
+jobs:
+  ec-integration-tests:
+    name: EC Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v6
+      with:
+        go-version: ^1.24
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v4
+
+    - name: Build weed binary
+      run: |
+        cd weed && go build -o weed .
+
+    - name: Run EC Integration Tests
+      working-directory: test/erasure_coding
+      run: |
+        go test -v
+
+    - name: Archive logs
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: ec-integration-test-logs
+        path: test/erasure_coding

.github/workflows/helm_ci.yml

@@ -44,6 +44,80 @@ jobs:
       - name: Run chart-testing (lint)
         run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts
 
+      - name: Verify template rendering
+        run: |
+          set -e
+          CHART_DIR="k8s/charts/seaweedfs"
+          
+          echo "=== Testing default configuration ==="
+          helm template test $CHART_DIR > /tmp/default.yaml
+          echo "✓ Default configuration renders successfully"
+          
+          echo "=== Testing with S3 enabled ==="
+          helm template test $CHART_DIR --set s3.enabled=true > /tmp/s3.yaml
+          grep -q "kind: Deployment" /tmp/s3.yaml && grep -q "seaweedfs-s3" /tmp/s3.yaml
+          echo "✓ S3 deployment renders correctly"
+          
+          echo "=== Testing with all-in-one mode ==="
+          helm template test $CHART_DIR --set allInOne.enabled=true > /tmp/allinone.yaml
+          grep -q "seaweedfs-all-in-one" /tmp/allinone.yaml
+          echo "✓ All-in-one deployment renders correctly"
+          
+          echo "=== Testing with security enabled ==="
+          helm template test $CHART_DIR --set global.enableSecurity=true > /tmp/security.yaml
+          grep -q "security-config" /tmp/security.yaml
+          echo "✓ Security configuration renders correctly"
+          
+          echo "=== Testing with monitoring enabled ==="
+          helm template test $CHART_DIR \
+            --set global.monitoring.enabled=true \
+            --set global.monitoring.gatewayHost=prometheus \
+            --set global.monitoring.gatewayPort=9091 > /tmp/monitoring.yaml
+          echo "✓ Monitoring configuration renders correctly"
+          
+          echo "=== Testing with PVC storage ==="
+          helm template test $CHART_DIR \
+            --set master.data.type=persistentVolumeClaim \
+            --set master.data.size=10Gi \
+            --set master.data.storageClass=standard > /tmp/pvc.yaml
+          grep -q "PersistentVolumeClaim" /tmp/pvc.yaml
+          echo "✓ PVC configuration renders correctly"
+          
+          echo "=== Testing with custom replicas ==="
+          helm template test $CHART_DIR \
+            --set master.replicas=3 \
+            --set filer.replicas=2 \
+            --set volume.replicas=3 > /tmp/replicas.yaml
+          echo "✓ Custom replicas configuration renders correctly"
+          
+          echo "=== Testing filer with S3 gateway ==="
+          helm template test $CHART_DIR \
+            --set filer.s3.enabled=true \
+            --set filer.s3.enableAuth=true > /tmp/filer-s3.yaml
+          echo "✓ Filer S3 gateway renders correctly"
+          
+          echo "=== Testing SFTP enabled ==="
+          helm template test $CHART_DIR --set sftp.enabled=true > /tmp/sftp.yaml
+          grep -q "seaweedfs-sftp" /tmp/sftp.yaml
+          echo "✓ SFTP deployment renders correctly"
+          
+          echo "=== Testing ingress configurations ==="
+          helm template test $CHART_DIR \
+            --set master.ingress.enabled=true \
+            --set filer.ingress.enabled=true \
+            --set s3.enabled=true \
+            --set s3.ingress.enabled=true > /tmp/ingress.yaml
+          grep -q "kind: Ingress" /tmp/ingress.yaml
+          echo "✓ Ingress configurations render correctly"
+          
+          echo "=== Testing COSI driver ==="
+          helm template test $CHART_DIR --set cosi.enabled=true > /tmp/cosi.yaml
+          grep -q "seaweedfs-cosi" /tmp/cosi.yaml
+          echo "✓ COSI driver renders correctly"
+          
+          echo ""
+          echo "✅ All template rendering tests passed!"
+
       - name: Create kind cluster
         uses: helm/kind-action@v1.13.0
 

.github/workflows/s3-go-tests.yml

@@ -411,4 +411,58 @@ jobs:
           path: test/s3/versioning/weed-test*.log
           retention-days: 7
 
+  s3-tagging-tests:
+    # CI job for S3 object tagging tests
+    name: S3 Tagging Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false
+
+      - name: Run S3 Tagging Tests
+        timeout-minutes: 15
+        working-directory: test/s3/tagging
+        run: |
+          set -x
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          
+          # Set environment variables for the test
+          export S3_ENDPOINT="http://localhost:8006"
+          export S3_ACCESS_KEY="0555b35654ad1656d804"
+          export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
+          
+          # Run the specific test that is equivalent to AWS S3 tagging behavior
+          make test-with-server || {
+            echo "❌ Test failed, checking logs..."
+            if [ -f weed-test.log ]; then
+              echo "=== Server logs ==="
+              tail -100 weed-test.log
+            fi
+            echo "=== Process information ==="
+            ps aux | grep -E "(weed|test)" || true
+            exit 1
+          }
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v5
+        with:
+          name: s3-tagging-test-logs
+          path: test/s3/tagging/weed-test*.log
+          retention-days: 3
+
   # Removed SSE-C integration tests and compatibility job

.github/workflows/sftp-tests.yml

@@ -0,0 +1,93 @@
+name: "SFTP Integration Tests"
+
+on:
+  push:
+    branches: [ master, main ]
+    paths:
+      - 'weed/sftpd/**'
+      - 'weed/command/sftp.go'
+      - 'test/sftp/**'
+      - '.github/workflows/sftp-tests.yml'
+  pull_request:
+    branches: [ master, main ]
+    paths:
+      - 'weed/sftpd/**'
+      - 'weed/command/sftp.go'
+      - 'test/sftp/**'
+      - '.github/workflows/sftp-tests.yml'
+
+concurrency:
+  group: ${{ github.head_ref }}/sftp-tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  GO_VERSION: '1.24'
+  TEST_TIMEOUT: '15m'
+
+jobs:
+  sftp-integration:
+    name: SFTP Integration Testing
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      
+    - name: Set up Go ${{ env.GO_VERSION }}
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ env.GO_VERSION }}
+        
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y openssh-client
+        
+    - name: Build SeaweedFS
+      run: |
+        cd weed
+        go build -o weed .
+        chmod +x weed
+        ./weed version
+        
+    - name: Run SFTP Integration Tests
+      run: |
+        cd test/sftp
+        
+        echo "🧪 Running SFTP integration tests..."
+        echo "============================================"
+        
+        # Install test dependencies
+        go mod download
+        
+        # Run all SFTP tests
+        go test -v -timeout=${{ env.TEST_TIMEOUT }} ./...
+        
+        echo "============================================"
+        echo "✅ SFTP integration tests completed"
+        
+    - name: Test Summary
+      if: always()
+      run: |
+        echo "## 🔐 SFTP Integration Test Summary" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
+        echo "- ✅ **HomeDir Path Translation**: User home directory mapping (fixes #7470)" >> $GITHUB_STEP_SUMMARY
+        echo "- ✅ **File Operations**: Upload, download, delete" >> $GITHUB_STEP_SUMMARY
+        echo "- ✅ **Directory Operations**: Create, list, remove" >> $GITHUB_STEP_SUMMARY
+        echo "- ✅ **Large File Handling**: 1MB+ file support" >> $GITHUB_STEP_SUMMARY
+        echo "- ✅ **Path Edge Cases**: Unicode, trailing slashes, .. paths" >> $GITHUB_STEP_SUMMARY
+        echo "- ✅ **Admin Access**: Root user verification" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "### Test Configuration" >> $GITHUB_STEP_SUMMARY
+        echo "| User | HomeDir | Permissions |" >> $GITHUB_STEP_SUMMARY
+        echo "|------|---------|-------------|" >> $GITHUB_STEP_SUMMARY
+        echo "| admin | / | Full access |" >> $GITHUB_STEP_SUMMARY
+        echo "| testuser | /sftp/testuser | Home directory only |" >> $GITHUB_STEP_SUMMARY
+        echo "| readonly | /public | Read-only |" >> $GITHUB_STEP_SUMMARY
+
+

.github/workflows/spark-integration-tests.yml

@@ -34,7 +34,7 @@ jobs:
       uses: actions/checkout@v6
 
     - name: Set up JDK 11
-      uses: actions/setup-java@v4
+      uses: actions/setup-java@v5
       with:
         java-version: '11'
         distribution: 'temurin'
@@ -125,7 +125,7 @@ jobs:
 
     - name: Upload test results
       if: always()
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v5
       with:
         name: spark-test-results
         path: test/java/spark/target/surefire-reports/
@@ -133,7 +133,7 @@ jobs:
 
     - name: Publish test report
       if: always()
-      uses: dorny/test-reporter@v1
+      uses: dorny/test-reporter@v2
       with:
         name: Spark Test Results
         path: test/java/spark/target/surefire-reports/*.xml

README.md

@@ -592,65 +592,22 @@ Percentage of the requests served within a certain time (ms)
 
 ```
 make benchmark
-warp: Benchmark data written to "warp-mixed-2023-10-16[102354]-l70a.csv.zst"                                                                                                                                                                                               
+warp: Benchmark data written to "warp-mixed-2025-12-05[194844]-kBpU.csv.zst"
+
 Mixed operations.
-Operation: DELETE, 10%, Concurrency: 20, Ran 4m59s.
- * Throughput: 6.19 obj/s
+Operation: DELETE, 10%, Concurrency: 20, Ran 42s.
+ * Throughput: 55.13 obj/s
 
-Operation: GET, 45%, Concurrency: 20, Ran 5m0s.
- * Throughput: 279.85 MiB/s, 27.99 obj/s
+Operation: GET, 45%, Concurrency: 20, Ran 42s.
+ * Throughput: 2477.45 MiB/s, 247.75 obj/s
 
-Operation: PUT, 15%, Concurrency: 20, Ran 5m0s.
- * Throughput: 89.86 MiB/s, 8.99 obj/s
+Operation: PUT, 15%, Concurrency: 20, Ran 42s.
+ * Throughput: 825.85 MiB/s, 82.59 obj/s
 
-Operation: STAT, 30%, Concurrency: 20, Ran 5m0s.
- * Throughput: 18.63 obj/s
+Operation: STAT, 30%, Concurrency: 20, Ran 42s.
+ * Throughput: 165.27 obj/s
 
-Cluster Total: 369.74 MiB/s, 61.79 obj/s, 0 errors over 5m0s.
-```
-
-To see segmented request statistics, use the --analyze.v parameter.
-```
-warp analyze --analyze.v warp-mixed-2023-10-16[102354]-l70a.csv.zst
-18642 operations loaded... Done!
-Mixed operations.
-----------------------------------------
-Operation: DELETE - total: 1854, 10.0%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
- * Throughput: 6.19 obj/s
-
-Requests considered: 1855:
- * Avg: 104ms, 50%: 30ms, 90%: 207ms, 99%: 1.355s, Fastest: 1ms, Slowest: 4.613s, StdDev: 320ms
-
-----------------------------------------
-Operation: GET - total: 8388, 45.3%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.12 +0500 +05
- * Throughput: 279.77 MiB/s, 27.98 obj/s
-
-Requests considered: 8389:
- * Avg: 221ms, 50%: 106ms, 90%: 492ms, 99%: 1.739s, Fastest: 8ms, Slowest: 8.633s, StdDev: 383ms
- * TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 171ms, 99th: 669ms, Worst: 4.783s StdDev: 163ms
- * First Access: Avg: 240ms, 50%: 105ms, 90%: 511ms, 99%: 2.08s, Fastest: 12ms, Slowest: 8.633s, StdDev: 480ms
- * First Access TTFB: Avg: 88ms, Best: 2ms, 25th: 24ms, Median: 38ms, 75th: 64ms, 90th: 179ms, 99th: 919ms, Worst: 4.783s StdDev: 199ms
- * Last Access: Avg: 219ms, 50%: 106ms, 90%: 463ms, 99%: 1.782s, Fastest: 9ms, Slowest: 8.633s, StdDev: 416ms
- * Last Access TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 161ms, 99th: 657ms, Worst: 4.783s StdDev: 176ms
-
-----------------------------------------
-Operation: PUT - total: 2688, 14.5%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
- * Throughput: 89.83 MiB/s, 8.98 obj/s
-
-Requests considered: 2689:
- * Avg: 1.165s, 50%: 878ms, 90%: 2.015s, 99%: 5.74s, Fastest: 99ms, Slowest: 8.264s, StdDev: 968ms
-
-----------------------------------------
-Operation: STAT - total: 5586, 30.2%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.113 +0500 +05
- * Throughput: 18.63 obj/s
-
-Requests considered: 5587:
- * Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 80ms, Fastest: 0s, Slowest: 245ms, StdDev: 17ms
- * First Access: Avg: 14ms, 50%: 10ms, 90%: 33ms, 99%: 69ms, Fastest: 0s, Slowest: 203ms, StdDev: 16ms
- * Last Access: Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 74ms, Fastest: 0s, Slowest: 203ms, StdDev: 17ms
-
-Cluster Total: 369.64 MiB/s, 61.77 obj/s, 0 errors over 5m0s.
-Total Errors:0.
+Cluster Total: 3302.88 MiB/s, 550.51 obj/s over 43s.
 ```
 
 [Back to TOC](#table-of-contents)

docker/Dockerfile.go_build

@@ -23,6 +23,9 @@ RUN mkdir -p /etc/seaweedfs
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
 
+# FIPS 140-3 mode is ON by default (Go 1.24+)
+# To disable: docker run -e GODEBUG=fips140=off ...
+
 # Install dependencies and create non-root user
 RUN apk add --no-cache fuse su-exec && \
     addgroup -g 1000 seaweed && \

docker/entrypoint.sh

@@ -1,5 +1,9 @@
 #!/bin/sh
 
+# Enable FIPS 140-3 mode by default (Go 1.24+)
+# To disable: docker run -e GODEBUG=fips140=off ...
+export GODEBUG="${GODEBUG:+$GODEBUG,}fips140=on"
+
 # Fix permissions for mounted volumes
 # If /data is mounted from host, it might have different ownership
 # Fix this by ensuring seaweed user owns the directory

go.mod

@@ -50,7 +50,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
-	github.com/klauspost/compress v1.18.1
+	github.com/klauspost/compress v1.18.2
 	github.com/klauspost/reedsolomon v1.12.5
 	github.com/kurin/blazer v0.5.3
 	github.com/linxGnu/grocksdb v1.10.3
@@ -67,7 +67,7 @@ require (
 	github.com/prometheus/client_golang v1.23.2
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect
-	github.com/prometheus/procfs v0.19.1
+	github.com/prometheus/procfs v0.19.2
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
@@ -124,7 +124,7 @@ require (
 	github.com/ThreeDotsLabs/watermill v1.5.1
 	github.com/a-h/templ v0.3.943
 	github.com/apple/foundationdb/bindings/go v0.0.0-20240515141816-262c6fe778ad
-	github.com/arangodb/go-driver v1.6.7
+	github.com/arangodb/go-driver v1.6.9
 	github.com/armon/go-metrics v0.4.1
 	github.com/aws/aws-sdk-go-v2 v1.40.0
 	github.com/aws/aws-sdk-go-v2/config v1.31.3
@@ -154,12 +154,12 @@ require (
 	github.com/rdleal/intervalst v1.5.0
 	github.com/redis/go-redis/v9 v9.14.1
 	github.com/schollz/progressbar/v3 v3.18.0
-	github.com/shirou/gopsutil/v4 v4.25.10
+	github.com/shirou/gopsutil/v4 v4.25.11
 	github.com/tarantool/go-tarantool/v2 v2.4.1
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.118.2
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0
 	go.etcd.io/etcd/client/pkg/v3 v3.6.6
 	go.uber.org/atomic v1.11.0
 	golang.org/x/sync v0.18.0
@@ -298,7 +298,7 @@ require (
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
-	github.com/ebitengine/purego v0.9.0 // indirect
+	github.com/ebitengine/purego v0.9.1 // indirect
 	github.com/elastic/gosigar v0.14.3 // indirect
 	github.com/emersion/go-message v0.18.2 // indirect
 	github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
@@ -415,8 +415,8 @@ require (
 	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
 	github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
 	github.com/tinylib/msgp v1.3.0 // indirect
-	github.com/tklauser/go-sysconf v0.3.15 // indirect
-	github.com/tklauser/numcpus v0.10.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.16 // indirect
+	github.com/tklauser/numcpus v0.11.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
 	github.com/ugorji/go/codec v1.3.0 // indirect
@@ -425,7 +425,7 @@ require (
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20250911135631-b3beddd517d9 // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb // indirect
 	github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect

go.sum

@@ -158,8 +158,6 @@ cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZ
 cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
-cloud.google.com/go/compute/metadata v0.8.0 h1:HxMRIbao8w17ZX6wBnjhcDkW6lTFpgcaobyVfZWqRLA=
-cloud.google.com/go/compute/metadata v0.8.0/go.mod h1:sYOGTp851OV9bOFJ9CH7elVvyzopvWQFNNghtDQ/Biw=
 cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
 cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
 cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY=
@@ -578,8 +576,6 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
 github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/Files-com/files-sdk-go/v3 v3.2.218 h1:tIvcbHXNY/bq+Sno6vajOJOxhe5XbU59Fa1ohOybK+s=
 github.com/Files-com/files-sdk-go/v3 v3.2.218/go.mod h1:E0BaGQbcMUcql+AfubCR/iasWKBxX5UZPivnQGC2z0M=
-github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4=
-github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM=
@@ -659,8 +655,8 @@ github.com/apple/foundationdb/bindings/go v0.0.0-20240515141816-262c6fe778ad h1:
 github.com/apple/foundationdb/bindings/go v0.0.0-20240515141816-262c6fe778ad/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
 github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc h1:LoL75er+LKDHDUfU5tRvFwxH0LjPpZN8OoG8Ll+liGU=
 github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc/go.mod h1:w648aMHEgFYS6xb0KVMMtZ2uMeemhiKCuD2vj6gY52A=
-github.com/arangodb/go-driver v1.6.7 h1:9FBUsH60cKu7DjFGozTsaqWMy+3UeEplplqUn4yEcg4=
-github.com/arangodb/go-driver v1.6.7/go.mod h1:H6uhiKUD/ki7fS9dNDK6xzMX/D5ibj5kGN1bGKd37Ho=
+github.com/arangodb/go-driver v1.6.9 h1:zckB+xuA16NmHUuYOX7INCJTIyIkoBQjAGqNpiyf2HQ=
+github.com/arangodb/go-driver v1.6.9/go.mod h1:eAM/drVZw39hTGFdkxvbVv0uJsDGFaUpqQHVZMSoALc=
 github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g=
 github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho=
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
@@ -800,8 +796,6 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls=
-github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
 github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
 github.com/cockroachdb/apd/v3 v3.1.0 h1:MK3Ow7LH0W8zkd5GMKA1PvS9qG3bWFI95WaVNfyZJ/w=
@@ -865,8 +859,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A
 github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/ebitengine/purego v0.9.0 h1:mh0zpKBIXDceC63hpvPuGLiJ8ZAa3DfrFTudmfi8A4k=
-github.com/ebitengine/purego v0.9.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
+github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
 github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
@@ -884,11 +878,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34=
 github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI=
-github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M=
-github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA=
 github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
-github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A=
-github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw=
+github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
 github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
 github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
 github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
@@ -957,8 +948,6 @@ github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmn
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI=
-github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA=
 github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
 github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -1335,8 +1324,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
-github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
+github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
+github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
@@ -1570,8 +1559,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.19.1 h1:QVtROpTkphuXuNlnCv3m1ut3JytkXHtQ3xvck/YmzMM=
-github.com/prometheus/procfs v0.19.1/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
+github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
+github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
 github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
@@ -1635,8 +1624,8 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
 github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/shirou/gopsutil/v4 v4.25.10 h1:at8lk/5T1OgtuCp+AwrDofFRjnvosn0nkN2OLQ6g8tA=
-github.com/shirou/gopsutil/v4 v4.25.10/go.mod h1:+kSwyC8DRUD9XXEHCAFjK+0nuArFJM0lva+StQAcskM=
+github.com/shirou/gopsutil/v4 v4.25.11 h1:X53gB7muL9Gnwwo2evPSE+SfOrltMoR6V3xJAXZILTY=
+github.com/shirou/gopsutil/v4 v4.25.11/go.mod h1:EivAfP5x2EhLp2ovdpKSozecVXn1TmuG7SMzs/Wh4PU=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -1673,8 +1662,6 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
 github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
 github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
-github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE=
-github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g=
 github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
 github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -1731,10 +1718,10 @@ github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1/go.mod h1:3cTcfo8GR
 github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
 github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
 github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
-github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4=
-github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4=
-github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso=
-github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ=
+github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
+github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
+github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
+github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
 github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 h1:6iRwZdrFUzbcVYZwa8dXTIILGIxmmhjyUPJEcwzPGaU=
 github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365/go.mod h1:zj0GJHGvyf1ed3Jm/Tb4830c/ZKDq+YoLsCt2rGQuT0=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
@@ -1788,14 +1775,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD
 github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
-github.com/ydb-platform/ydb-go-genproto v0.0.0-20250911135631-b3beddd517d9 h1:SKqSRP6/ocY2Z4twOqKEKxpmawVTHTvQiom7hrU6jt0=
-github.com/ydb-platform/ydb-go-genproto v0.0.0-20250911135631-b3beddd517d9/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0 h1:/NyPd9KnCJgzrEXCArqk1ThqCH2Dh31uUwl88o/VkuM=
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.118.2 h1:+R4RDTDZ+k/tFRIflg2ynQ7t6bZBg9t5vKQU3PvahQg=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.118.2/go.mod h1:UEMMk+JMunUveo2j+zlJEJ5I7ntf2+MbimciVNJYnNs=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0 h1:Ml4WMsaRY+/+Wh3DCp1A+/zjM1aMdQg3irjuYFwWVHQ=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0/go.mod h1:/LjMxb/rXmoGAAnImoqAFIlhO5ampHacbvDetQitCk4=
 github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
 github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
 github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
@@ -1845,20 +1832,14 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
-go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/contrib/detectors/gcp v1.37.0 h1:B+WbN9RPsvobe6q4vP6KgM8/9plR/HNjgGBrfcOlweA=
-go.opentelemetry.io/contrib/detectors/gcp v1.37.0/go.mod h1:K5zQ3TT7p2ru9Qkzk0bKtCql0RGkPj9pRjpXgZJZ+rU=
 go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs=
 go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 h1:rbRJ8BBoVMsQShESYZ0FkvcITu8X8QNwJogcLUmDNNw=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0/go.mod h1:ru6KHrNtNHxM4nD/vd6QrLVWgKhxPYgblq4VAtNawTQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY=
-go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
-go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
 go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
 go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
@@ -1869,20 +1850,12 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.37.0 h1:6VjV6Et+1Hd2iL
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.37.0/go.mod h1:u8hcp8ji5gaM/RfcOo8z9NMnf1pVLfVY7lBY2VOGuUU=
 go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
 go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
-go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
-go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
 go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
 go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
-go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
-go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
 go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
 go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
-go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
-go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
 go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
 go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
-go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
-go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
 go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
 go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
@@ -2113,8 +2086,6 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
 golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
 golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
-golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
-golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
 golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY=
 golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -2586,12 +2557,8 @@ google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q
 google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
 google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 h1:Nt6z9UHqSlIdIGJdz6KhTIs2VRx/iOsA5iE8bmQNcxs=
 google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79/go.mod h1:kTmlBHMPqR5uCZPBvwa2B18mvubkjyY3CRLI0c6fj0s=
-google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c h1:AtEkQdl5b6zsybXcbz00j1LwNodDuH6hVifIaNqk7NQ=
-google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c/go.mod h1:ea2MjsO70ssTfCjiwHgI0ZFqcw45Ksuk2ckf9G468GA=
 google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4=
 google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c h1:qXWI/sQtv5UKboZ/zUk7h+mrf/lXORyI+n9DKDAusdg=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@@ -2634,14 +2601,11 @@ google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsA
 google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
 google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
 google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
-google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI=
-google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
 google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
 google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
-google.golang.org/grpc/examples v0.0.0-20230224211313-3775f633ce20 h1:MLBCGN1O7GzIx+cBiwfYPwtmZ41U3Mn/cotLJciaArI=
-google.golang.org/grpc/examples v0.0.0-20230224211313-3775f633ce20/go.mod h1:Nr5H8+MlGWr5+xX/STzdoEqJrO+YteqFbMyCsrb6mH0=
 google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds=
+google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8=
 google.golang.org/grpc/security/advancedtls v1.0.0 h1:/KQ7VP/1bs53/aopk9QhuPyFAp9Dm9Ejix3lzYkCrDA=
 google.golang.org/grpc/security/advancedtls v1.0.0/go.mod h1:o+s4go+e1PJ2AjuQMY5hU82W7lDlefjJA6FqEHRVHWk=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -2660,8 +2624,6 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
-google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
 google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
 google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=

k8s/charts/seaweedfs/Chart.yaml

@@ -3,4 +3,4 @@ description: SeaweedFS
 name: seaweedfs
 appVersion: "4.01"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.401
+version: 4.0.401

k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml

@@ -352,7 +352,7 @@ spec:
             httpGet:
               path: {{ .Values.allInOne.readinessProbe.httpGet.path }}
               port: {{ .Values.master.port }}
-              scheme: {{ .Values.allInOne.readinessProbe.scheme }}
+              scheme: {{ .Values.allInOne.readinessProbe.httpGet.scheme }}
             initialDelaySeconds: {{ .Values.allInOne.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.allInOne.readinessProbe.periodSeconds }}
             successThreshold: {{ .Values.allInOne.readinessProbe.successThreshold }}
@@ -364,7 +364,7 @@ spec:
             httpGet:
               path: {{ .Values.allInOne.livenessProbe.httpGet.path }}
               port: {{ .Values.master.port }}
-              scheme: {{ .Values.allInOne.livenessProbe.scheme }}
+              scheme: {{ .Values.allInOne.livenessProbe.httpGet.scheme }}
             initialDelaySeconds: {{ .Values.allInOne.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.allInOne.livenessProbe.periodSeconds }}
             successThreshold: {{ .Values.allInOne.livenessProbe.successThreshold }}

k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml

@@ -289,7 +289,7 @@ spec:
             httpGet:
               path: {{ .Values.filer.readinessProbe.httpGet.path }}
               port: {{ .Values.filer.port }}
-              scheme: {{ .Values.filer.readinessProbe.scheme }}
+              scheme: {{ .Values.filer.readinessProbe.httpGet.scheme }}
             initialDelaySeconds: {{ .Values.filer.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.filer.readinessProbe.periodSeconds }}
             successThreshold: {{ .Values.filer.readinessProbe.successThreshold }}
@@ -301,7 +301,7 @@ spec:
             httpGet:
               path: {{ .Values.filer.livenessProbe.httpGet.path }}
               port: {{ .Values.filer.port }}
-              scheme: {{ .Values.filer.livenessProbe.scheme }}
+              scheme: {{ .Values.filer.livenessProbe.httpGet.scheme }}
             initialDelaySeconds: {{ .Values.filer.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.filer.livenessProbe.periodSeconds }}
             successThreshold: {{ .Values.filer.livenessProbe.successThreshold }}

k8s/charts/seaweedfs/templates/master/master-statefulset.yaml

@@ -235,7 +235,7 @@ spec:
             httpGet:
               path: {{ .Values.master.readinessProbe.httpGet.path }}
               port: {{ .Values.master.port }}
-              scheme: {{ .Values.master.readinessProbe.scheme }}
+              scheme: {{ .Values.master.readinessProbe.httpGet.scheme }}
             initialDelaySeconds: {{ .Values.master.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.master.readinessProbe.periodSeconds }}
             successThreshold: {{ .Values.master.readinessProbe.successThreshold }}
@@ -247,7 +247,7 @@ spec:
             httpGet:
               path: {{ .Values.master.livenessProbe.httpGet.path }}
               port: {{ .Values.master.port }}
-              scheme: {{ .Values.master.livenessProbe.scheme }}
+              scheme: {{ .Values.master.livenessProbe.httpGet.scheme }}
             initialDelaySeconds: {{ .Values.master.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.master.livenessProbe.periodSeconds }}
             successThreshold: {{ .Values.master.livenessProbe.successThreshold }}

k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml

@@ -204,7 +204,7 @@ spec:
             httpGet:
               path: {{ .Values.s3.readinessProbe.httpGet.path }}
               port: {{ .Values.s3.port }}
-              scheme: {{ .Values.s3.readinessProbe.scheme }}
+              scheme: {{ .Values.s3.readinessProbe.httpGet.scheme }}
             initialDelaySeconds: {{ .Values.s3.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.s3.readinessProbe.periodSeconds }}
             successThreshold: {{ .Values.s3.readinessProbe.successThreshold }}
@@ -216,7 +216,7 @@ spec:
             httpGet:
               path: {{ .Values.s3.livenessProbe.httpGet.path }}
               port: {{ .Values.s3.port }}
-              scheme: {{ .Values.s3.livenessProbe.scheme }}
+              scheme: {{ .Values.s3.livenessProbe.httpGet.scheme }}
             initialDelaySeconds: {{ .Values.s3.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.s3.livenessProbe.periodSeconds }}
             successThreshold: {{ .Values.s3.livenessProbe.successThreshold }}

k8s/charts/seaweedfs/templates/volume/volume-statefulset.yaml

@@ -251,7 +251,7 @@ spec:
             httpGet:
               path: {{ $volume.readinessProbe.httpGet.path }}
               port: {{ $volume.port }}
-              scheme: {{ $volume.readinessProbe.scheme }}
+              scheme: {{ $volume.readinessProbe.httpGet.scheme }}
             initialDelaySeconds: {{ $volume.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ $volume.readinessProbe.periodSeconds }}
             successThreshold: {{ $volume.readinessProbe.successThreshold }}
@@ -263,7 +263,7 @@ spec:
             httpGet:
               path: {{ $volume.livenessProbe.httpGet.path }}
               port: {{ $volume.port }}
-              scheme: {{ $volume.livenessProbe.scheme }}
+              scheme: {{ $volume.livenessProbe.httpGet.scheme }}
             initialDelaySeconds: {{ $volume.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ $volume.livenessProbe.periodSeconds }}
             successThreshold: {{ $volume.livenessProbe.successThreshold }}

k8s/charts/seaweedfs/values.yaml

@@ -979,9 +979,9 @@ s3:
   extraEnvironmentVars:
 
   # Custom command line arguments to add to the s3 command
-  # Example to fix connection idle seconds:
-  extraArgs: ["-idleTimeout=30"]
-  # extraArgs: []
+  # Default idleTimeout is 120 seconds. Example to customize:
+  # extraArgs: ["-idleTimeout=300"]
+  extraArgs: []
 
   # used to configure livenessProbe on s3 containers
   #
@@ -1133,7 +1133,7 @@ allInOne:
     httpGet:
       path: /cluster/status
       port: 9333
-    scheme: HTTP
+      scheme: HTTP
     initialDelaySeconds: 10
     periodSeconds: 15
     successThreshold: 1
@@ -1145,7 +1145,7 @@ allInOne:
     httpGet:
       path: /cluster/status
       port: 9333
-    scheme: HTTP
+      scheme: HTTP
     initialDelaySeconds: 20
     periodSeconds: 30
     successThreshold: 1

test/erasure_coding/ec_integration_test.go

@@ -5,9 +5,11 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -139,6 +141,15 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
 			t.Logf("EC encoding completed successfully")
 		}
 
+		// Add detailed logging for EC encoding command
+		t.Logf("Debug: Executing EC encoding command for volume %d", volumeId)
+		t.Logf("Debug: Command arguments: %v", args)
+		if err != nil {
+			t.Logf("Debug: EC encoding command failed with error: %v", err)
+		} else {
+			t.Logf("Debug: EC encoding command completed successfully")
+		}
+
 		// The key test: check if the fix prevents the timing issue
 		if contains(outputStr, "Collecting volume locations") && contains(outputStr, "before EC encoding") {
 			t.Logf("FIX DETECTED: Volume locations collected BEFORE EC encoding (timing bug prevented)")
@@ -526,7 +537,8 @@ func uploadTestData(data []byte, masterAddress string) (needle.VolumeId, error)
 
 func getVolumeLocations(commandEnv *shell.CommandEnv, volumeId needle.VolumeId) ([]string, error) {
 	// Retry mechanism to handle timing issues with volume registration
-	for i := 0; i < 10; i++ {
+	// Increase retry attempts for volume location retrieval
+	for i := 0; i < 20; i++ { // Increased from 10 to 20 retries
 		locations, ok := commandEnv.MasterClient.GetLocationsClone(uint32(volumeId))
 		if ok {
 			var result []string
@@ -646,3 +658,427 @@ func TestECEncodingRegressionPrevention(t *testing.T) {
 		t.Log("Timing pattern regression test passed")
 	})
 }
+
+// TestDiskAwareECRebalancing tests EC shard placement across multiple disks per server
+// This verifies the disk-aware EC rebalancing feature works correctly
+func TestDiskAwareECRebalancing(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping disk-aware integration test in short mode")
+	}
+
+	testDir, err := os.MkdirTemp("", "seaweedfs_disk_aware_ec_test_")
+	require.NoError(t, err)
+	defer os.RemoveAll(testDir)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
+	defer cancel()
+
+	// Start cluster with MULTIPLE DISKS per volume server
+	cluster, err := startMultiDiskCluster(ctx, testDir)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	// Wait for servers to be ready
+	require.NoError(t, waitForServer("127.0.0.1:9334", 30*time.Second))
+	for i := 0; i < 3; i++ {
+		require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:809%d", i), 30*time.Second))
+	}
+
+	// Wait longer for volume servers to register with master and create volumes
+	t.Log("Waiting for volume servers to register with master...")
+	time.Sleep(10 * time.Second)
+
+	// Create command environment
+	options := &shell.ShellOptions{
+		Masters:        stringPtr("127.0.0.1:9334"),
+		GrpcDialOption: grpc.WithInsecure(),
+		FilerGroup:     stringPtr("default"),
+	}
+	commandEnv := shell.NewCommandEnv(options)
+
+	// Connect to master with longer timeout
+	ctx2, cancel2 := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel2()
+	go commandEnv.MasterClient.KeepConnectedToMaster(ctx2)
+	commandEnv.MasterClient.WaitUntilConnected(ctx2)
+
+	// Wait for master client to fully sync
+	time.Sleep(5 * time.Second)
+
+	// Upload test data to create a volume - retry if volumes not ready
+	var volumeId needle.VolumeId
+	testData := []byte("Disk-aware EC rebalancing test data - this needs to be large enough to create a volume")
+	for retry := 0; retry < 5; retry++ {
+		volumeId, err = uploadTestDataToMaster(testData, "127.0.0.1:9334")
+		if err == nil {
+			break
+		}
+		t.Logf("Upload attempt %d failed: %v, retrying...", retry+1, err)
+		time.Sleep(3 * time.Second)
+	}
+	require.NoError(t, err, "Failed to upload test data after retries")
+	t.Logf("Created volume %d for disk-aware EC test", volumeId)
+
+	// Wait for volume to be registered
+	time.Sleep(3 * time.Second)
+
+	t.Run("verify_multi_disk_setup", func(t *testing.T) {
+		// Verify that each server has multiple disk directories
+		for server := 0; server < 3; server++ {
+			diskCount := 0
+			for disk := 0; disk < 4; disk++ {
+				diskDir := filepath.Join(testDir, fmt.Sprintf("server%d_disk%d", server, disk))
+				if _, err := os.Stat(diskDir); err == nil {
+					diskCount++
+				}
+			}
+			assert.Equal(t, 4, diskCount, "Server %d should have 4 disk directories", server)
+			t.Logf("Server %d has %d disk directories", server, diskCount)
+		}
+	})
+
+	t.Run("ec_encode_with_disk_awareness", func(t *testing.T) {
+		// Get lock first
+		lockCmd := shell.Commands[findCommandIndex("lock")]
+		var lockOutput bytes.Buffer
+		err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
+		if err != nil {
+			t.Logf("Lock command failed: %v", err)
+		}
+
+		// Execute EC encoding
+		var output bytes.Buffer
+		ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
+		args := []string{"-volumeId", fmt.Sprintf("%d", volumeId), "-collection", "test", "-force"}
+
+		// Capture output
+		oldStdout := os.Stdout
+		oldStderr := os.Stderr
+		r, w, _ := os.Pipe()
+		os.Stdout = w
+		os.Stderr = w
+
+		err = ecEncodeCmd.Do(args, commandEnv, &output)
+
+		w.Close()
+		os.Stdout = oldStdout
+		os.Stderr = oldStderr
+
+		capturedOutput, _ := io.ReadAll(r)
+		outputStr := string(capturedOutput) + output.String()
+
+		t.Logf("EC encode output:\n%s", outputStr)
+
+		if err != nil {
+			t.Logf("EC encoding completed with error: %v", err)
+		} else {
+			t.Logf("EC encoding completed successfully")
+		}
+	})
+
+	t.Run("verify_disk_level_shard_distribution", func(t *testing.T) {
+		// Wait for shards to be distributed
+		time.Sleep(2 * time.Second)
+
+		// Count shards on each disk of each server
+		diskDistribution := countShardsPerDisk(testDir, uint32(volumeId))
+
+		totalShards := 0
+		disksWithShards := 0
+		maxShardsOnSingleDisk := 0
+
+		t.Logf("Disk-level shard distribution for volume %d:", volumeId)
+		for server, disks := range diskDistribution {
+			for diskId, shardCount := range disks {
+				if shardCount > 0 {
+					t.Logf("  %s disk %d: %d shards", server, diskId, shardCount)
+					totalShards += shardCount
+					disksWithShards++
+					if shardCount > maxShardsOnSingleDisk {
+						maxShardsOnSingleDisk = shardCount
+					}
+				}
+			}
+		}
+
+		t.Logf("Summary: %d total shards across %d disks (max %d on single disk)",
+			totalShards, disksWithShards, maxShardsOnSingleDisk)
+
+		// EC creates 14 shards (10 data + 4 parity), plus .ecx and .ecj files
+		// We should see shards distributed across multiple disks
+		if disksWithShards > 1 {
+			t.Logf("PASS: Shards distributed across %d disks", disksWithShards)
+		} else {
+			t.Logf("INFO: Shards on %d disk(s) - may be expected if volume was on single disk", disksWithShards)
+		}
+	})
+
+	t.Run("test_ec_balance_disk_awareness", func(t *testing.T) {
+		// Calculate initial disk balance variance
+		initialDistribution := countShardsPerDisk(testDir, uint32(volumeId))
+		initialVariance := calculateDiskShardVariance(initialDistribution)
+		t.Logf("Initial disk shard variance: %.2f", initialVariance)
+
+		// Run ec.balance command
+		var output bytes.Buffer
+		ecBalanceCmd := shell.Commands[findCommandIndex("ec.balance")]
+
+		oldStdout := os.Stdout
+		oldStderr := os.Stderr
+		r, w, _ := os.Pipe()
+		os.Stdout = w
+		os.Stderr = w
+
+		err := ecBalanceCmd.Do([]string{"-force"}, commandEnv, &output)
+
+		w.Close()
+		os.Stdout = oldStdout
+		os.Stderr = oldStderr
+
+		capturedOutput, _ := io.ReadAll(r)
+		outputStr := string(capturedOutput) + output.String()
+
+		if err != nil {
+			t.Logf("ec.balance error: %v", err)
+		}
+		t.Logf("ec.balance output:\n%s", outputStr)
+
+		// Wait for balance to complete
+		time.Sleep(2 * time.Second)
+
+		// Calculate final disk balance variance
+		finalDistribution := countShardsPerDisk(testDir, uint32(volumeId))
+		finalVariance := calculateDiskShardVariance(finalDistribution)
+		t.Logf("Final disk shard variance: %.2f", finalVariance)
+
+		t.Logf("Variance change: %.2f -> %.2f", initialVariance, finalVariance)
+	})
+
+	t.Run("verify_no_disk_overload", func(t *testing.T) {
+		// Verify that no single disk has too many shards of the same volume
+		diskDistribution := countShardsPerDisk(testDir, uint32(volumeId))
+
+		for server, disks := range diskDistribution {
+			for diskId, shardCount := range disks {
+				// With 14 EC shards and 12 disks (3 servers x 4 disks), ideally ~1-2 shards per disk
+				// Allow up to 4 shards per disk as a reasonable threshold
+				if shardCount > 4 {
+					t.Logf("WARNING: %s disk %d has %d shards (may indicate imbalance)",
+						server, diskId, shardCount)
+				}
+			}
+		}
+	})
+}
+
+// MultiDiskCluster represents a test cluster with multiple disks per volume server
+type MultiDiskCluster struct {
+	masterCmd     *exec.Cmd
+	volumeServers []*exec.Cmd
+	testDir       string
+}
+
+func (c *MultiDiskCluster) Stop() {
+	// Stop volume servers first
+	for _, cmd := range c.volumeServers {
+		if cmd != nil && cmd.Process != nil {
+			cmd.Process.Kill()
+			cmd.Wait()
+		}
+	}
+
+	// Stop master server
+	if c.masterCmd != nil && c.masterCmd.Process != nil {
+		c.masterCmd.Process.Kill()
+		c.masterCmd.Wait()
+	}
+}
+
+// startMultiDiskCluster starts a SeaweedFS cluster with multiple disks per volume server
+func startMultiDiskCluster(ctx context.Context, dataDir string) (*MultiDiskCluster, error) {
+	weedBinary := findWeedBinary()
+	if weedBinary == "" {
+		return nil, fmt.Errorf("weed binary not found")
+	}
+
+	cluster := &MultiDiskCluster{testDir: dataDir}
+
+	// Create master directory
+	masterDir := filepath.Join(dataDir, "master")
+	os.MkdirAll(masterDir, 0755)
+
+	// Start master server on a different port to avoid conflict
+	masterCmd := exec.CommandContext(ctx, weedBinary, "master",
+		"-port", "9334",
+		"-mdir", masterDir,
+		"-volumeSizeLimitMB", "10",
+		"-ip", "127.0.0.1",
+	)
+
+	masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create master log file: %v", err)
+	}
+	masterCmd.Stdout = masterLogFile
+	masterCmd.Stderr = masterLogFile
+
+	if err := masterCmd.Start(); err != nil {
+		return nil, fmt.Errorf("failed to start master server: %v", err)
+	}
+	cluster.masterCmd = masterCmd
+
+	// Wait for master to be ready
+	time.Sleep(2 * time.Second)
+
+	// Start 3 volume servers, each with 4 disks
+	const numServers = 3
+	const disksPerServer = 4
+
+	for i := 0; i < numServers; i++ {
+		// Create 4 disk directories per server
+		var diskDirs []string
+		var maxVolumes []string
+
+		for d := 0; d < disksPerServer; d++ {
+			diskDir := filepath.Join(dataDir, fmt.Sprintf("server%d_disk%d", i, d))
+			if err := os.MkdirAll(diskDir, 0755); err != nil {
+				cluster.Stop()
+				return nil, fmt.Errorf("failed to create disk dir: %v", err)
+			}
+			diskDirs = append(diskDirs, diskDir)
+			maxVolumes = append(maxVolumes, "5")
+		}
+
+		port := fmt.Sprintf("809%d", i)
+		rack := fmt.Sprintf("rack%d", i)
+
+		volumeCmd := exec.CommandContext(ctx, weedBinary, "volume",
+			"-port", port,
+			"-dir", strings.Join(diskDirs, ","),
+			"-max", strings.Join(maxVolumes, ","),
+			"-mserver", "127.0.0.1:9334",
+			"-ip", "127.0.0.1",
+			"-dataCenter", "dc1",
+			"-rack", rack,
+		)
+
+		// Create log file for this volume server
+		logDir := filepath.Join(dataDir, fmt.Sprintf("server%d_logs", i))
+		os.MkdirAll(logDir, 0755)
+		volumeLogFile, err := os.Create(filepath.Join(logDir, "volume.log"))
+		if err != nil {
+			cluster.Stop()
+			return nil, fmt.Errorf("failed to create volume log file: %v", err)
+		}
+		volumeCmd.Stdout = volumeLogFile
+		volumeCmd.Stderr = volumeLogFile
+
+		if err := volumeCmd.Start(); err != nil {
+			cluster.Stop()
+			return nil, fmt.Errorf("failed to start volume server %d: %v", i, err)
+		}
+		cluster.volumeServers = append(cluster.volumeServers, volumeCmd)
+	}
+
+	// Wait for volume servers to register with master
+	// Multi-disk servers may take longer to initialize
+	time.Sleep(8 * time.Second)
+
+	return cluster, nil
+}
+
+// uploadTestDataToMaster uploads test data to a specific master address
+func uploadTestDataToMaster(data []byte, masterAddress string) (needle.VolumeId, error) {
+	assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
+		return pb.ServerAddress(masterAddress)
+	}, grpc.WithInsecure(), &operation.VolumeAssignRequest{
+		Count:       1,
+		Collection:  "test",
+		Replication: "000",
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	uploader, err := operation.NewUploader()
+	if err != nil {
+		return 0, err
+	}
+
+	uploadResult, err, _ := uploader.Upload(context.Background(), bytes.NewReader(data), &operation.UploadOption{
+		UploadUrl: "http://" + assignResult.Url + "/" + assignResult.Fid,
+		Filename:  "testfile.txt",
+		MimeType:  "text/plain",
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	if uploadResult.Error != "" {
+		return 0, fmt.Errorf("upload error: %s", uploadResult.Error)
+	}
+
+	fid, err := needle.ParseFileIdFromString(assignResult.Fid)
+	if err != nil {
+		return 0, err
+	}
+
+	return fid.VolumeId, nil
+}
+
+// countShardsPerDisk counts EC shards on each disk of each server
+// Returns map: "serverN" -> map[diskId]shardCount
+func countShardsPerDisk(testDir string, volumeId uint32) map[string]map[int]int {
+	result := make(map[string]map[int]int)
+
+	const numServers = 3
+	const disksPerServer = 4
+
+	for server := 0; server < numServers; server++ {
+		serverKey := fmt.Sprintf("server%d", server)
+		result[serverKey] = make(map[int]int)
+
+		for disk := 0; disk < disksPerServer; disk++ {
+			diskDir := filepath.Join(testDir, fmt.Sprintf("server%d_disk%d", server, disk))
+			count, err := countECShardFiles(diskDir, volumeId)
+			if err == nil && count > 0 {
+				result[serverKey][disk] = count
+			}
+		}
+	}
+
+	return result
+}
+
+// calculateDiskShardVariance measures how evenly shards are distributed across disks
+// Lower variance means better distribution
+func calculateDiskShardVariance(distribution map[string]map[int]int) float64 {
+	var counts []float64
+
+	for _, disks := range distribution {
+		for _, count := range disks {
+			if count > 0 {
+				counts = append(counts, float64(count))
+			}
+		}
+	}
+
+	if len(counts) == 0 {
+		return 0
+	}
+
+	// Calculate mean
+	mean := 0.0
+	for _, c := range counts {
+		mean += c
+	}
+	mean /= float64(len(counts))
+
+	// Calculate variance
+	variance := 0.0
+	for _, c := range counts {
+		variance += (c - mean) * (c - mean)
+	}
+
+	return math.Sqrt(variance / float64(len(counts)))
+}

test/kafka/go.mod

@@ -20,12 +20,12 @@ require (
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.2 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
 	github.com/Files-com/files-sdk-go/v3 v3.2.218 // indirect
 	github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
@@ -43,7 +43,7 @@ require (
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go v1.55.8 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.39.5 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.40.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.31.3 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.18.20 // indirect
@@ -61,7 +61,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect
-	github.com/aws/smithy-go v1.23.1 // indirect
+	github.com/aws/smithy-go v1.23.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
@@ -85,7 +85,7 @@ require (
 	github.com/eapache/go-resiliency v1.7.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
-	github.com/ebitengine/purego v0.9.0 // indirect
+	github.com/ebitengine/purego v0.9.1 // indirect
 	github.com/emersion/go-message v0.18.2 // indirect
 	github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -139,7 +139,7 @@ require (
 	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
 	github.com/karlseguin/ccache/v2 v2.0.8 // indirect
-	github.com/klauspost/compress v1.18.1 // indirect
+	github.com/klauspost/compress v1.18.2 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/klauspost/reedsolomon v1.12.5 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
@@ -176,7 +176,7 @@ require (
 	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect
-	github.com/prometheus/procfs v0.19.1 // indirect
+	github.com/prometheus/procfs v0.19.2 // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
 	github.com/rclone/rclone v1.71.2 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
@@ -189,7 +189,7 @@ require (
 	github.com/sagikazarmark/locafero v0.11.0 // indirect
 	github.com/samber/lo v1.51.0 // indirect
 	github.com/seaweedfs/goexif v1.0.3 // indirect
-	github.com/shirou/gopsutil/v4 v4.25.10 // indirect
+	github.com/shirou/gopsutil/v4 v4.25.11 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/smarty/assertions v1.16.0 // indirect
@@ -204,8 +204,8 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect
 	github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c // indirect
-	github.com/tklauser/go-sysconf v0.3.15 // indirect
-	github.com/tklauser/numcpus v0.10.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.16 // indirect
+	github.com/tklauser/numcpus v0.11.0 // indirect
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 // indirect
 	github.com/unknwon/goconfig v1.0.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect

test/kafka/go.sum

@@ -39,8 +39,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 h1:KpMC6LFL7mqpExyMC9jVOYRiVhLmamjeZfRsUpB7l4s=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0/go.mod h1:J7MUC/wtRpfGVbQ5sIItY5/FuVWmvzlY21WAOfQnq/I=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
@@ -55,8 +55,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 h1:XkkQbfMyuH2jTSjQjSoihryI8GINRcs4xp8lNawg0FI=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Files-com/files-sdk-go/v3 v3.2.218 h1:tIvcbHXNY/bq+Sno6vajOJOxhe5XbU59Fa1ohOybK+s=
@@ -102,8 +102,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.39.5 h1:e/SXuia3rkFtapghJROrydtQpfQaaUgd1cUvyO1mp2w=
-github.com/aws/aws-sdk-go-v2 v1.39.5/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM=
+github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
+github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
 github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
@@ -138,8 +138,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3z
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA=
 github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY=
 github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs=
-github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M=
-github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
+github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo=
@@ -208,8 +208,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A
 github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/ebitengine/purego v0.9.0 h1:mh0zpKBIXDceC63hpvPuGLiJ8ZAa3DfrFTudmfi8A4k=
-github.com/ebitengine/purego v0.9.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
+github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
 github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
 github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg=
@@ -429,8 +429,8 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt
 github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
-github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
+github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
+github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/klauspost/reedsolomon v1.12.5 h1:4cJuyH926If33BeDgiZpI5OU0pE+wUHZvMSyNGqN73Y=
@@ -532,8 +532,8 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw
 github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
 github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
 github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
-github.com/prometheus/procfs v0.19.1 h1:QVtROpTkphuXuNlnCv3m1ut3JytkXHtQ3xvck/YmzMM=
-github.com/prometheus/procfs v0.19.1/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
+github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
+github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
 github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
@@ -569,8 +569,8 @@ github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9
 github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
 github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/shirou/gopsutil/v4 v4.25.10 h1:at8lk/5T1OgtuCp+AwrDofFRjnvosn0nkN2OLQ6g8tA=
-github.com/shirou/gopsutil/v4 v4.25.10/go.mod h1:+kSwyC8DRUD9XXEHCAFjK+0nuArFJM0lva+StQAcskM=
+github.com/shirou/gopsutil/v4 v4.25.11 h1:X53gB7muL9Gnwwo2evPSE+SfOrltMoR6V3xJAXZILTY=
+github.com/shirou/gopsutil/v4 v4.25.11/go.mod h1:EivAfP5x2EhLp2ovdpKSozecVXn1TmuG7SMzs/Wh4PU=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
@@ -624,10 +624,10 @@ github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJ
 github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c h1:BLopNCyqewbE8+BtlIp/Juzu8AJGxz0gHdGADnsblVc=
 github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c/go.mod h1:ykucQyiE9Q2qx1wLlEtZkkNn1IURib/2O+Mvd25i1Fo=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
-github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4=
-github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4=
-github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso=
-github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ=
+github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
+github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
+github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
+github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 h1:QEePdg0ty2r0t1+qwfZmQ4OOl/MB2UXIeJSpIZv56lg=

test/s3/sse/s3_sse_integration_test.go

@@ -2082,6 +2082,78 @@ func TestCopyToBucketDefaultEncryptedRegression(t *testing.T) {
 		require.NoError(t, err, "Failed to read object")
 		assertDataEqual(t, testData, data, "Data mismatch")
 	})
+
+	t.Run("LargeFileCopyEncrypted_ToTemp_ToEncrypted", func(t *testing.T) {
+		// Test with large file (1MB) to exercise chunk-by-chunk copy path
+		// This verifies consistent behavior with SSE-C and SSE-KMS
+		largeTestData := generateTestData(1024 * 1024) // 1MB
+		objectKey := "large-file-test.bin"
+
+		// Step 1: Upload large object to source bucket (will be automatically encrypted)
+		_, err = client.PutObject(ctx, &s3.PutObjectInput{
+			Bucket: aws.String(srcBucket),
+			Key:    aws.String(objectKey),
+			Body:   bytes.NewReader(largeTestData),
+		})
+		require.NoError(t, err, "Failed to upload large file to source bucket")
+
+		// Verify source object is encrypted
+		srcHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+			Bucket: aws.String(srcBucket),
+			Key:    aws.String(objectKey),
+		})
+		require.NoError(t, err, "Failed to HEAD source object")
+		assert.Equal(t, types.ServerSideEncryptionAes256, srcHead.ServerSideEncryption,
+			"Source object should be SSE-S3 encrypted")
+
+		// Step 2: Copy to temp bucket (unencrypted) - exercises chunk-by-chunk decrypt
+		_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
+			Bucket:     aws.String(tempBucket),
+			Key:        aws.String(objectKey),
+			CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)),
+		})
+		require.NoError(t, err, "Failed to copy large file to temp bucket")
+
+		// Verify temp object is unencrypted and data is correct
+		tempGet, err := client.GetObject(ctx, &s3.GetObjectInput{
+			Bucket: aws.String(tempBucket),
+			Key:    aws.String(objectKey),
+		})
+		require.NoError(t, err, "Failed to GET temp object")
+		tempData, err := io.ReadAll(tempGet.Body)
+		tempGet.Body.Close()
+		require.NoError(t, err, "Failed to read temp object")
+		assertDataEqual(t, largeTestData, tempData, "Temp object data mismatch after decrypt")
+
+		// Step 3: Copy from temp bucket to dest bucket (with default encryption)
+		// This exercises chunk-by-chunk encrypt copy
+		_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
+			Bucket:     aws.String(dstBucket),
+			Key:        aws.String(objectKey),
+			CopySource: aws.String(fmt.Sprintf("%s/%s", tempBucket, objectKey)),
+		})
+		require.NoError(t, err, "Failed to copy large file to destination bucket")
+
+		// Verify destination object is encrypted
+		dstHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+			Bucket: aws.String(dstBucket),
+			Key:    aws.String(objectKey),
+		})
+		require.NoError(t, err, "Failed to HEAD destination object")
+		assert.Equal(t, types.ServerSideEncryptionAes256, dstHead.ServerSideEncryption,
+			"Destination object should be SSE-S3 encrypted via bucket default")
+
+		// Verify destination object content is correct after re-encryption
+		dstGet, err := client.GetObject(ctx, &s3.GetObjectInput{
+			Bucket: aws.String(dstBucket),
+			Key:    aws.String(objectKey),
+		})
+		require.NoError(t, err, "Failed to GET destination object")
+		dstData, err := io.ReadAll(dstGet.Body)
+		dstGet.Body.Close()
+		require.NoError(t, err, "Failed to read destination object")
+		assertDataEqual(t, largeTestData, dstData, "Large file data mismatch after re-encryption")
+	})
 }
 
 // REGRESSION TESTS FOR CRITICAL BUGS FIXED

test/s3/tagging/Makefile

@@ -0,0 +1,321 @@
+# S3 Object Tagging Tests
+# Tests for GitHub issue #7589: S3 object Tags query comes back empty
+
+.PHONY: help build-weed setup-server start-server stop-server test-tagging test-tagging-quick test-tagging-comprehensive test-all clean logs check-deps test-with-server
+
+# Configuration
+WEED_BINARY := ../../../weed/weed_binary
+S3_PORT := 8006
+MASTER_PORT := 9338
+VOLUME_PORT := 8085
+FILER_PORT := 8893
+TEST_TIMEOUT := 10m
+TEST_PATTERN := TestObjectTaggingOnUpload|TestPutObjectTaggingAPI|TestDeleteObjectTagging|TestTag
+
+# Default target
+help:
+	@echo "S3 Object Tagging Tests Makefile"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  help                       - Show this help message"
+	@echo "  build-weed                - Build the SeaweedFS binary"
+	@echo "  check-deps                - Check dependencies and build binary if needed"
+	@echo "  start-server              - Start SeaweedFS server for testing"
+	@echo "  stop-server               - Stop SeaweedFS server"
+	@echo "  test-tagging              - Run all tagging tests"
+	@echo "  test-tagging-quick        - Run core tagging tests only"
+	@echo "  test-tagging-comprehensive - Run comprehensive tagging tests"
+	@echo "  test-with-server          - Start server, run tests, stop server"
+	@echo "  logs                      - Show server logs"
+	@echo "  clean                     - Clean up test artifacts and stop server"
+	@echo "  health-check              - Check if server is accessible"
+	@echo ""
+	@echo "Configuration:"
+	@echo "  S3_PORT=${S3_PORT}"
+	@echo "  TEST_TIMEOUT=${TEST_TIMEOUT}"
+
+# Build the SeaweedFS binary
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd ../../../weed && go build -o weed_binary .
+	@chmod +x $(WEED_BINARY)
+	@echo "✅ SeaweedFS binary built at $(WEED_BINARY)"
+
+check-deps: build-weed
+	@echo "Checking dependencies..."
+	@echo "🔍 DEBUG: Checking Go installation..."
+	@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
+	@echo "🔍 DEBUG: Go version: $$(go version)"
+	@echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)..."
+	@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
+	@echo "🔍 DEBUG: Binary size: $$(ls -lh $(WEED_BINARY) | awk '{print $$5}')"
+	@echo "🔍 DEBUG: Binary permissions: $$(ls -la $(WEED_BINARY) | awk '{print $$1}')"
+	@echo "🔍 DEBUG: Checking Go module dependencies..."
+	@go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1)
+	@go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1)
+	@echo "✅ All dependencies are available"
+
+# Start SeaweedFS server for testing
+start-server: check-deps
+	@echo "Starting SeaweedFS server..."
+	@echo "🔍 DEBUG: Current working directory: $$(pwd)"
+	@echo "🔍 DEBUG: Checking for existing weed processes..."
+	@ps aux | grep weed | grep -v grep || echo "No existing weed processes found"
+	@echo "🔍 DEBUG: Cleaning up any existing PID file..."
+	@rm -f weed-server.pid
+	@echo "🔍 DEBUG: Checking for port conflicts..."
+	@if netstat -tlnp 2>/dev/null | grep $(S3_PORT) >/dev/null; then \
+		echo "⚠️  Port $(S3_PORT) is already in use, trying to find the process..."; \
+		netstat -tlnp 2>/dev/null | grep $(S3_PORT) || true; \
+	else \
+		echo "✅ Port $(S3_PORT) is available"; \
+	fi
+	@echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)"
+	@ls -la $(WEED_BINARY) || (echo "❌ Binary not found!" && exit 1)
+	@echo "🔍 DEBUG: Checking config file at ../../../docker/compose/s3.json"
+	@ls -la ../../../docker/compose/s3.json || echo "⚠️  Config file not found, continuing without it"
+	@echo "🔍 DEBUG: Creating volume directory..."
+	@mkdir -p ./test-volume-data
+	@echo "🔍 DEBUG: Launching SeaweedFS server in background..."
+	@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 -dir=./test-volume-data -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 -volume.max=100 -volume.preStopSeconds=1 -master.port=$(MASTER_PORT) -volume.port=$(VOLUME_PORT) -filer.port=$(FILER_PORT) -s3.port=$(S3_PORT) -metricsPort=9329 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -master.peers=none"
+	@$(WEED_BINARY) server \
+		-filer \
+		-filer.maxMB=64 \
+		-s3 \
+		-ip.bind 0.0.0.0 \
+		-dir=./test-volume-data \
+		-master.raftHashicorp \
+		-master.electionTimeout 1s \
+		-master.volumeSizeLimitMB=100 \
+		-volume.max=100 \
+		-volume.preStopSeconds=1 \
+		-master.port=$(MASTER_PORT) \
+		-volume.port=$(VOLUME_PORT) \
+		-filer.port=$(FILER_PORT) \
+		-s3.port=$(S3_PORT) \
+		-metricsPort=9329 \
+		-s3.allowEmptyFolder=false \
+		-s3.allowDeleteBucketNotEmpty=true \
+		-s3.config=../../../docker/compose/s3.json \
+		-master.peers=none \
+		> weed-test.log 2>&1 & echo $$! > weed-server.pid
+	@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
+	@echo "🔍 DEBUG: Checking if PID is still running..."
+	@sleep 2
+	@if [ -f weed-server.pid ]; then \
+		SERVER_PID=$$(cat weed-server.pid); \
+		ps -p $$SERVER_PID || echo "⚠️  Server PID $$SERVER_PID not found after 2 seconds"; \
+	else \
+		echo "⚠️  PID file not found"; \
+	fi
+	@echo "🔍 DEBUG: Waiting for server to start (up to 90 seconds)..."
+	@for i in $$(seq 1 90); do \
+		echo "🔍 DEBUG: Attempt $$i/90 - checking port $(S3_PORT)"; \
+		if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
+			echo "✅ SeaweedFS server started successfully on port $(S3_PORT) after $$i seconds"; \
+			exit 0; \
+		fi; \
+		if [ $$i -eq 5 ]; then \
+			echo "🔍 DEBUG: After 5 seconds, checking process and logs..."; \
+			ps aux | grep weed | grep -v grep || echo "No weed processes found"; \
+			if [ -f weed-test.log ]; then \
+				echo "=== First server logs ==="; \
+				head -20 weed-test.log; \
+			fi; \
+		fi; \
+		if [ $$i -eq 15 ]; then \
+			echo "🔍 DEBUG: After 15 seconds, checking port bindings..."; \
+			netstat -tlnp 2>/dev/null | grep $(S3_PORT) || echo "Port $(S3_PORT) not bound"; \
+			netstat -tlnp 2>/dev/null | grep $(MASTER_PORT) || echo "Port $(MASTER_PORT) not bound"; \
+			netstat -tlnp 2>/dev/null | grep $(VOLUME_PORT) || echo "Port $(VOLUME_PORT) not bound"; \
+		fi; \
+		if [ $$i -eq 30 ]; then \
+			echo "⚠️  Server taking longer than expected (30s), checking logs..."; \
+			if [ -f weed-test.log ]; then \
+				echo "=== Recent server logs ==="; \
+				tail -20 weed-test.log; \
+			fi; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "❌ Server failed to start within 90 seconds"; \
+	echo "🔍 DEBUG: Final process check:"; \
+	ps aux | grep weed | grep -v grep || echo "No weed processes found"; \
+	echo "🔍 DEBUG: Final port check:"; \
+	netstat -tlnp 2>/dev/null | grep -E "($(S3_PORT)|$(MASTER_PORT)|$(VOLUME_PORT))" || echo "No ports bound"; \
+	echo "=== Full server logs ==="; \
+	if [ -f weed-test.log ]; then \
+		cat weed-test.log; \
+	else \
+		echo "No log file found"; \
+	fi; \
+	exit 1
+
+# Stop SeaweedFS server
+stop-server:
+	@echo "Stopping SeaweedFS server..."
+	@if [ -f weed-server.pid ]; then \
+		SERVER_PID=$$(cat weed-server.pid); \
+		echo "Killing server PID $$SERVER_PID"; \
+		if ps -p $$SERVER_PID >/dev/null 2>&1; then \
+			kill -TERM $$SERVER_PID 2>/dev/null || true; \
+			sleep 2; \
+			if ps -p $$SERVER_PID >/dev/null 2>&1; then \
+				echo "Process still running, sending KILL signal..."; \
+				kill -KILL $$SERVER_PID 2>/dev/null || true; \
+				sleep 1; \
+			fi; \
+		else \
+			echo "Process $$SERVER_PID not found (already stopped)"; \
+		fi; \
+		rm -f weed-server.pid; \
+	else \
+		echo "No PID file found, checking for running processes..."; \
+		echo "⚠️  Skipping automatic process cleanup to avoid CI issues"; \
+		echo "Note: Any remaining weed processes should be cleaned up by the CI environment"; \
+	fi
+	@echo "✅ SeaweedFS server stopped"
+
+# Show server logs
+logs:
+	@if test -f weed-test.log; then \
+		echo "=== SeaweedFS Server Logs ==="; \
+		tail -f weed-test.log; \
+	else \
+		echo "No log file found. Server may not be running."; \
+	fi
+
+# Core tagging tests (basic functionality)
+test-tagging-quick: check-deps
+	@echo "Running core tagging tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTaggingOnUpload|TestPutObjectTaggingAPI" .
+	@echo "✅ Core tagging tests completed"
+
+# All tagging tests (comprehensive)
+test-tagging: check-deps
+	@echo "Running all tagging tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" .
+	@echo "✅ All tagging tests completed"
+
+# Comprehensive tagging tests (all features)
+test-tagging-comprehensive: check-deps
+	@echo "Running comprehensive tagging tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) .
+	@echo "✅ Comprehensive tagging tests completed"
+
+# All tests without server management
+test-tagging-simple: check-deps
+	@echo "Running tagging tests (assuming server is already running)..."
+	@go test -v -timeout=$(TEST_TIMEOUT) .
+	@echo "✅ All tagging tests completed"
+
+# Start server, run tests, stop server
+test-with-server: start-server
+	@echo "Running tagging tests with managed server..."
+	@sleep 5  # Give server time to fully start
+	@make test-tagging-comprehensive || (echo "Tests failed, stopping server..." && make stop-server && exit 1)
+	@make stop-server
+	@echo "✅ All tests completed with managed server"
+
+# Health check
+health-check:
+	@echo "Checking server health..."
+	@if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
+		echo "✅ Server is accessible on port $(S3_PORT)"; \
+	else \
+		echo "❌ Server is not accessible on port $(S3_PORT)"; \
+		exit 1; \
+	fi
+
+# Clean up
+clean:
+	@echo "Cleaning up test artifacts..."
+	@make stop-server
+	@rm -f weed-test.log
+	@rm -f weed-server.pid
+	@rm -rf ./test-volume-data
+	@rm -f tagging.test
+	@go clean -testcache
+	@echo "✅ Cleanup completed"
+
+# Individual test targets for specific functionality
+test-upload:
+	@echo "Running upload tagging tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTaggingOnUpload" .
+
+test-special-chars:
+	@echo "Running special characters tagging tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTaggingOnUploadWithSpecialCharacters" .
+
+test-api:
+	@echo "Running API tagging tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestPutObjectTaggingAPI" .
+
+test-get:
+	@echo "Running get tagging tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestGetObjectTaggingAPI" .
+
+test-delete:
+	@echo "Running delete tagging tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestDeleteObjectTaggingAPI" .
+
+# Development targets
+dev-start: start-server
+	@echo "Development server started. Access S3 API at http://localhost:$(S3_PORT)"
+	@echo "To stop: make stop-server"
+
+dev-test: check-deps
+	@echo "Running tests in development mode..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTaggingOnUpload" .
+
+# CI targets
+ci-test: check-deps
+	@echo "Running tests in CI mode..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -race .
+
+# All targets
+test-all: test-tagging test-tagging-comprehensive
+	@echo "✅ All tagging tests completed"
+
+# Benchmark targets
+benchmark-tagging:
+	@echo "Running tagging performance benchmarks..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -bench=. -benchmem .
+
+# Coverage targets
+coverage:
+	@echo "Running tests with coverage..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -coverprofile=coverage.out .
+	@go tool cover -html=coverage.out -o coverage.html
+	@echo "Coverage report generated: coverage.html"
+
+# Format and lint
+fmt:
+	@echo "Formatting Go code..."
+	@go fmt .
+
+lint:
+	@echo "Running linter..."
+	@golint . || echo "golint not available, skipping..."
+
+# Install dependencies for development
+install-deps:
+	@echo "Installing Go dependencies..."
+	@go mod tidy
+	@go mod download
+
+# Show current configuration
+show-config:
+	@echo "Current configuration:"
+	@echo "  WEED_BINARY: $(WEED_BINARY)"
+	@echo "  S3_PORT: $(S3_PORT)"
+	@echo "  TEST_TIMEOUT: $(TEST_TIMEOUT)"
+	@echo "  TEST_PATTERN: $(TEST_PATTERN)"
+
+# Legacy targets for backward compatibility
+test: test-with-server
+test-verbose: test-tagging-comprehensive
+test-single: test-upload
+test-clean: clean
+build: check-deps
+setup: check-deps

test/s3/tagging/README.md

@@ -0,0 +1,53 @@
+# S3 Object Tagging Tests
+
+This directory contains tests for S3 object tagging functionality.
+
+## Issue Reference
+
+These tests were created to verify the fix for [GitHub Issue #7589](https://github.com/seaweedfs/seaweedfs/issues/7589):
+**S3 object Tags query comes back empty**
+
+## Problem Description
+
+When uploading an object with tags using the `X-Amz-Tagging` header, the tags were not being stored. 
+When querying the object tagging with `GetObjectTagging`, the response was empty.
+
+This was a regression between SeaweedFS 4.00 and 4.01.
+
+## Root Cause
+
+The `putToFiler` function in `s3api_object_handlers_put.go` was not parsing the `X-Amz-Tagging` header
+and storing the tags in the entry's Extended metadata. The code was only copying user metadata 
+(headers starting with `X-Amz-Meta-`) but not object tags.
+
+## Fix
+
+Added tag parsing logic to `putToFiler` that:
+1. Reads the `X-Amz-Tagging` header
+2. Parses it using `url.ParseQuery()` for proper URL decoding
+3. Stores each tag with the prefix `X-Amz-Tagging-` in the entry's Extended metadata
+
+## Running Tests
+
+```bash
+# Run all tagging tests
+cd test/s3/tagging
+make test
+
+# Run specific test
+make test-upload
+
+# Or using go test directly
+go test -v ./...
+```
+
+## Test Cases
+
+1. **TestObjectTaggingOnUpload** - Basic test for tags sent during object upload
+2. **TestObjectTaggingOnUploadWithSpecialCharacters** - Tests URL-encoded tag values
+3. **TestObjectTaggingOnUploadWithEmptyValue** - Tests tags with empty values
+4. **TestPutObjectTaggingAPI** - Tests the PutObjectTagging API separately
+5. **TestDeleteObjectTagging** - Tests tag deletion
+6. **TestTagsNotPreservedAfterObjectOverwrite** - Verifies AWS S3 behavior on overwrite
+7. **TestMaximumNumberOfTags** - Tests storing the maximum 10 tags
+8. **TestTagCountHeader** - Tests the x-amz-tagging-count header in HeadObject

test/s3/tagging/s3_tagging_test.go

@@ -0,0 +1,446 @@
+package tagging
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// S3TestConfig holds configuration for S3 tests
+type S3TestConfig struct {
+	Endpoint      string
+	AccessKey     string
+	SecretKey     string
+	Region        string
+	BucketPrefix  string
+	UseSSL        bool
+	SkipVerifySSL bool
+}
+
+// getDefaultConfig returns a fresh instance of the default test configuration
+func getDefaultConfig() *S3TestConfig {
+	endpoint := os.Getenv("S3_ENDPOINT")
+	if endpoint == "" {
+		endpoint = "http://localhost:8333" // Default SeaweedFS S3 port
+	}
+	accessKey := os.Getenv("S3_ACCESS_KEY")
+	if accessKey == "" {
+		accessKey = "some_access_key1"
+	}
+	secretKey := os.Getenv("S3_SECRET_KEY")
+	if secretKey == "" {
+		secretKey = "some_secret_key1"
+	}
+	return &S3TestConfig{
+		Endpoint:      endpoint,
+		AccessKey:     accessKey,
+		SecretKey:     secretKey,
+		Region:        "us-east-1",
+		BucketPrefix:  "test-tagging-",
+		UseSSL:        false,
+		SkipVerifySSL: true,
+	}
+}
+
+// getS3Client creates an AWS S3 client for testing
+func getS3Client(t *testing.T) *s3.Client {
+	defaultConfig := getDefaultConfig()
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(defaultConfig.Region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			defaultConfig.AccessKey,
+			defaultConfig.SecretKey,
+			"",
+		)),
+		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
+			func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+				return aws.Endpoint{
+					URL:           defaultConfig.Endpoint,
+					SigningRegion: defaultConfig.Region,
+				}, nil
+			})),
+	)
+	require.NoError(t, err)
+
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+	return client
+}
+
+// createTestBucket creates a test bucket with a unique name
+func createTestBucket(t *testing.T, client *s3.Client) string {
+	defaultConfig := getDefaultConfig()
+	bucketName := fmt.Sprintf("%s%d", defaultConfig.BucketPrefix, time.Now().UnixNano())
+
+	_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+
+	// Wait for bucket metadata to be fully processed
+	time.Sleep(50 * time.Millisecond)
+
+	return bucketName
+}
+
+// cleanupTestBucket removes the test bucket and all its contents
+func cleanupTestBucket(t *testing.T, client *s3.Client, bucketName string) {
+	// First, delete all objects in the bucket
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+	})
+	if err == nil {
+		for _, obj := range listResp.Contents {
+			_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    obj.Key,
+			})
+			if err != nil {
+				t.Logf("Warning: failed to delete object %s: %v", *obj.Key, err)
+			}
+		}
+	}
+
+	// Then delete the bucket
+	_, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		t.Logf("Warning: failed to delete bucket %s: %v", bucketName, err)
+	}
+}
+
+// TestObjectTaggingOnUpload tests that tags sent during object upload (via X-Amz-Tagging header)
+// are properly stored and can be retrieved. This is the fix for GitHub issue #7589.
+func TestObjectTaggingOnUpload(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "test-object-with-tags"
+	objectContent := "Hello, World!"
+
+	// Put object with tags using the Tagging parameter (X-Amz-Tagging header)
+	_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket:  aws.String(bucketName),
+		Key:     aws.String(objectKey),
+		Body:    strings.NewReader(objectContent),
+		Tagging: aws.String("env=production&team=platform"),
+	})
+	require.NoError(t, err, "Should be able to put object with tags")
+
+	// Get the tags back
+	tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Should be able to get object tags")
+
+	// Verify tags were stored correctly
+	require.Len(t, tagResp.TagSet, 2, "Should have 2 tags")
+
+	// Build a map for easier assertion
+	tagMap := make(map[string]string)
+	for _, tag := range tagResp.TagSet {
+		tagMap[*tag.Key] = *tag.Value
+	}
+
+	assert.Equal(t, "production", tagMap["env"], "env tag should be 'production'")
+	assert.Equal(t, "platform", tagMap["team"], "team tag should be 'platform'")
+}
+
+// TestObjectTaggingOnUploadWithSpecialCharacters tests tags with URL-encoded characters
+func TestObjectTaggingOnUploadWithSpecialCharacters(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "test-object-with-special-tags"
+	objectContent := "Hello, World!"
+
+	// Put object with tags containing special characters
+	// AWS SDK will URL-encode these automatically
+	_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket:  aws.String(bucketName),
+		Key:     aws.String(objectKey),
+		Body:    strings.NewReader(objectContent),
+		Tagging: aws.String("timestamp=2025-07-16 14:40:39&path=/tmp/file.txt"),
+	})
+	require.NoError(t, err, "Should be able to put object with special character tags")
+
+	// Get the tags back
+	tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Should be able to get object tags")
+
+	// Verify tags were stored and URL-decoded correctly
+	require.Len(t, tagResp.TagSet, 2, "Should have 2 tags")
+
+	tagMap := make(map[string]string)
+	for _, tag := range tagResp.TagSet {
+		tagMap[*tag.Key] = *tag.Value
+	}
+
+	assert.Equal(t, "2025-07-16 14:40:39", tagMap["timestamp"], "timestamp tag should be decoded correctly")
+	assert.Equal(t, "/tmp/file.txt", tagMap["path"], "path tag should be decoded correctly")
+}
+
+// TestObjectTaggingOnUploadWithEmptyValue tests tags with empty values
+func TestObjectTaggingOnUploadWithEmptyValue(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "test-object-with-empty-tag"
+	objectContent := "Hello, World!"
+
+	// Put object with a tag that has an empty value
+	_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket:  aws.String(bucketName),
+		Key:     aws.String(objectKey),
+		Body:    strings.NewReader(objectContent),
+		Tagging: aws.String("marker=&env=dev"),
+	})
+	require.NoError(t, err, "Should be able to put object with empty tag value")
+
+	// Get the tags back
+	tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Should be able to get object tags")
+
+	// Verify tags were stored correctly
+	require.Len(t, tagResp.TagSet, 2, "Should have 2 tags")
+
+	tagMap := make(map[string]string)
+	for _, tag := range tagResp.TagSet {
+		tagMap[*tag.Key] = *tag.Value
+	}
+
+	assert.Equal(t, "", tagMap["marker"], "marker tag should have empty value")
+	assert.Equal(t, "dev", tagMap["env"], "env tag should be 'dev'")
+}
+
+// TestPutObjectTaggingAPI tests the PutObjectTagging API separately from upload
+func TestPutObjectTaggingAPI(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "test-object-for-tagging-api"
+	objectContent := "Hello, World!"
+
+	// First, put object without tags
+	_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader(objectContent),
+	})
+	require.NoError(t, err, "Should be able to put object without tags")
+
+	// Get tags - should be empty
+	tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Should be able to get object tags")
+	assert.Len(t, tagResp.TagSet, 0, "Should have no tags initially")
+
+	// Now add tags using PutObjectTagging API
+	_, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Tagging: &types.Tagging{
+			TagSet: []types.Tag{
+				{Key: aws.String("env"), Value: aws.String("staging")},
+				{Key: aws.String("version"), Value: aws.String("1.0")},
+			},
+		},
+	})
+	require.NoError(t, err, "Should be able to put object tags via API")
+
+	// Get tags - should now have the tags
+	tagResp, err = client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Should be able to get object tags after PutObjectTagging")
+	require.Len(t, tagResp.TagSet, 2, "Should have 2 tags")
+
+	tagMap := make(map[string]string)
+	for _, tag := range tagResp.TagSet {
+		tagMap[*tag.Key] = *tag.Value
+	}
+
+	assert.Equal(t, "staging", tagMap["env"], "env tag should be 'staging'")
+	assert.Equal(t, "1.0", tagMap["version"], "version tag should be '1.0'")
+}
+
+// TestDeleteObjectTagging tests the DeleteObjectTagging API
+func TestDeleteObjectTagging(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "test-object-for-delete-tags"
+	objectContent := "Hello, World!"
+
+	// Put object with tags
+	_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket:  aws.String(bucketName),
+		Key:     aws.String(objectKey),
+		Body:    strings.NewReader(objectContent),
+		Tagging: aws.String("env=production"),
+	})
+	require.NoError(t, err, "Should be able to put object with tags")
+
+	// Verify tags exist
+	tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Should be able to get object tags")
+	require.Len(t, tagResp.TagSet, 1, "Should have 1 tag")
+
+	// Delete tags
+	_, err = client.DeleteObjectTagging(context.TODO(), &s3.DeleteObjectTaggingInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Should be able to delete object tags")
+
+	// Verify tags are deleted
+	tagResp, err = client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Should be able to get object tags after deletion")
+	assert.Len(t, tagResp.TagSet, 0, "Should have no tags after deletion")
+}
+
+// TestTagsNotPreservedAfterObjectOverwrite tests that tags are NOT preserved when an object is overwritten
+// This matches AWS S3 behavior where overwriting an object replaces all metadata including tags
+func TestTagsNotPreservedAfterObjectOverwrite(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "test-object-overwrite-tags"
+	objectContent := "Original content"
+
+	// Put object with tags
+	_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket:  aws.String(bucketName),
+		Key:     aws.String(objectKey),
+		Body:    strings.NewReader(objectContent),
+		Tagging: aws.String("original=true"),
+	})
+	require.NoError(t, err, "Should be able to put object with tags")
+
+	// Verify original tags exist
+	tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Should be able to get object tags")
+	require.Len(t, tagResp.TagSet, 1, "Should have 1 tag")
+	assert.Equal(t, "original", *tagResp.TagSet[0].Key)
+
+	// Overwrite the object WITHOUT tags
+	_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader("New content"),
+	})
+	require.NoError(t, err, "Should be able to overwrite object")
+
+	// Tags should be gone after overwrite (matches AWS S3 behavior)
+	tagResp, err = client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Should be able to get object tags after overwrite")
+	assert.Len(t, tagResp.TagSet, 0, "Tags should be cleared after object overwrite")
+}
+
+// TestMaximumNumberOfTags tests that we can store the maximum 10 tags per object
+func TestMaximumNumberOfTags(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "test-object-max-tags"
+	objectContent := "Hello, World!"
+
+	// Build 10 tags (S3 max)
+	tags := []string{}
+	for i := 1; i <= 10; i++ {
+		tags = append(tags, fmt.Sprintf("key%d=value%d", i, i))
+	}
+	tagging := strings.Join(tags, "&")
+
+	// Put object with 10 tags
+	_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket:  aws.String(bucketName),
+		Key:     aws.String(objectKey),
+		Body:    strings.NewReader(objectContent),
+		Tagging: aws.String(tagging),
+	})
+	require.NoError(t, err, "Should be able to put object with 10 tags")
+
+	// Get the tags back
+	tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Should be able to get object tags")
+	assert.Len(t, tagResp.TagSet, 10, "Should have 10 tags")
+}
+
+// TestTagCountHeader tests that the x-amz-tagging-count header is returned in HeadObject
+func TestTagCountHeader(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "test-object-tag-count"
+	objectContent := "Hello, World!"
+
+	// Put object with tags
+	_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket:  aws.String(bucketName),
+		Key:     aws.String(objectKey),
+		Body:    strings.NewReader(objectContent),
+		Tagging: aws.String("env=prod&team=backend&version=2.0"),
+	})
+	require.NoError(t, err, "Should be able to put object with tags")
+
+	// Head object to get tag count
+	headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Should be able to head object")
+
+	// Check tag count header
+	if headResp.TagCount != nil {
+		assert.Equal(t, int32(3), *headResp.TagCount, "Tag count should be 3")
+	} else {
+		t.Log("Warning: TagCount header not returned - this may be expected depending on implementation")
+	}
+}

test/sftp/Makefile

@@ -0,0 +1,41 @@
+.PHONY: all build test test-verbose test-short test-homedir test-debug clean deps tidy
+
+all: build test
+
+# Build the weed binary first
+build:
+	cd ../../weed && go build -o weed .
+
+# Install test dependencies
+deps:
+	go mod download
+
+# Run all tests
+test: build deps
+	go test -timeout 5m ./...
+
+# Run tests with verbose output
+test-verbose: build deps
+	go test -v -timeout 5m ./...
+
+# Run quick tests only (skip integration tests)
+test-short: deps
+	go test -short -v ./...
+
+# Run specific test
+test-homedir: build deps
+	go test -v -timeout 5m -run TestHomeDirPathTranslation ./...
+
+# Run tests with debug output from SeaweedFS
+test-debug: build deps
+	go test -v -timeout 5m ./... 2>&1 | tee test.log
+
+# Clean up test artifacts
+clean:
+	rm -f test.log
+	go clean -testcache
+
+# Update go.sum
+tidy:
+	go mod tidy
+

test/sftp/README.md

@@ -0,0 +1,92 @@
+# SeaweedFS SFTP Integration Tests
+
+This directory contains integration tests for the SeaweedFS SFTP server.
+
+## Prerequisites
+
+1. Build the SeaweedFS binary:
+   ```bash
+   cd ../../weed
+   go build -o weed .
+   ```
+
+2. Ensure `ssh-keygen` is available (for generating test SSH host keys)
+
+## Running Tests
+
+### Run all tests
+```bash
+make test
+```
+
+### Run tests with verbose output
+```bash
+make test-verbose
+```
+
+### Run a specific test
+```bash
+go test -v -run TestHomeDirPathTranslation
+```
+
+### Skip long-running tests
+```bash
+go test -short ./...
+```
+
+## Test Structure
+
+- `framework.go` - Test framework that starts SeaweedFS cluster with SFTP
+- `basic_test.go` - Basic SFTP operation tests including:
+  - HomeDir path translation (fixes issue #7470)
+  - File upload/download
+  - Directory operations
+  - Large file handling
+  - Edge cases
+
+## Test Configuration
+
+Tests use `testdata/userstore.json` which defines test users:
+
+| Username | Password | HomeDir | Permissions |
+|----------|----------|---------|-------------|
+| admin | adminpassword | / | Full access |
+| testuser | testuserpassword | /sftp/testuser | Full access to home |
+| readonly | readonlypassword | /public | Read-only |
+
+## Key Tests
+
+### TestHomeDirPathTranslation
+
+Tests the fix for [issue #7470](https://github.com/seaweedfs/seaweedfs/issues/7470) where
+users with a non-root HomeDir (e.g., `/sftp/testuser`) could not upload files to `/`
+because the path wasn't being translated to their home directory.
+
+The test verifies:
+- Uploading to `/` correctly maps to the user's HomeDir
+- Creating directories at `/` works
+- Listing `/` shows the user's home directory contents
+- All path operations respect the HomeDir translation
+
+## Debugging
+
+To debug test failures:
+
+1. Enable verbose output:
+   ```bash
+   go test -v -run TestName
+   ```
+
+2. Keep test artifacts (don't cleanup):
+   ```go
+   config := DefaultTestConfig()
+   config.SkipCleanup = true
+   ```
+
+3. Enable debug logging:
+   ```go
+   config := DefaultTestConfig()
+   config.EnableDebug = true
+   ```
+
+

test/sftp/basic_test.go

@@ -0,0 +1,652 @@
+package sftp
+
+import (
+	"bytes"
+	"io"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestHomeDirPathTranslation tests that SFTP operations correctly translate
+// paths relative to the user's HomeDir.
+// This is the fix for https://github.com/seaweedfs/seaweedfs/issues/7470
+func TestHomeDirPathTranslation(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	config := DefaultTestConfig()
+	config.EnableDebug = testing.Verbose()
+
+	fw := NewSftpTestFramework(t, config)
+	err := fw.Setup(config)
+	require.NoError(t, err, "failed to setup test framework")
+	defer fw.Cleanup()
+
+	// Test with user "testuser" who has HomeDir="/sftp/testuser"
+	// When they upload to "/", it should actually go to "/sftp/testuser"
+	sftpClient, sshConn, err := fw.ConnectSFTP("testuser", "testuserpassword")
+	require.NoError(t, err, "failed to connect as testuser")
+	defer sshConn.Close()
+	defer sftpClient.Close()
+
+	// Test 1: Upload file to "/" (should map to /sftp/testuser/)
+	t.Run("UploadToRoot", func(t *testing.T) {
+		testContent := []byte("Hello from SFTP test!")
+		filename := "test_upload.txt"
+
+		// Create file at "/" from user's perspective
+		file, err := sftpClient.Create("/" + filename)
+		require.NoError(t, err, "should be able to create file at /")
+
+		_, err = file.Write(testContent)
+		require.NoError(t, err, "should be able to write to file")
+		err = file.Close()
+		require.NoError(t, err, "should be able to close file")
+
+		// Verify file exists and has correct content
+		readFile, err := sftpClient.Open("/" + filename)
+		require.NoError(t, err, "should be able to open file")
+		defer readFile.Close()
+
+		content, err := io.ReadAll(readFile)
+		require.NoError(t, err, "should be able to read file")
+		require.Equal(t, testContent, content, "file content should match")
+
+		// Clean up
+		err = sftpClient.Remove("/" + filename)
+		require.NoError(t, err, "should be able to remove file")
+	})
+
+	// Test 2: Create directory at "/" (should map to /sftp/testuser/)
+	t.Run("CreateDirAtRoot", func(t *testing.T) {
+		dirname := "test_dir"
+
+		err := sftpClient.Mkdir("/" + dirname)
+		require.NoError(t, err, "should be able to create directory at /")
+
+		// Verify directory exists
+		info, err := sftpClient.Stat("/" + dirname)
+		require.NoError(t, err, "should be able to stat directory")
+		require.True(t, info.IsDir(), "should be a directory")
+
+		// Clean up
+		err = sftpClient.RemoveDirectory("/" + dirname)
+		require.NoError(t, err, "should be able to remove directory")
+	})
+
+	// Test 3: List directory at "/" (should list /sftp/testuser/)
+	t.Run("ListRoot", func(t *testing.T) {
+		// Create a test file first
+		testContent := []byte("list test content")
+		filename := "list_test.txt"
+
+		file, err := sftpClient.Create("/" + filename)
+		require.NoError(t, err)
+		_, err = file.Write(testContent)
+		require.NoError(t, err)
+		file.Close()
+
+		// List root directory
+		files, err := sftpClient.ReadDir("/")
+		require.NoError(t, err, "should be able to list root directory")
+
+		// Should find our test file
+		found := false
+		for _, f := range files {
+			if f.Name() == filename {
+				found = true
+				break
+			}
+		}
+		require.True(t, found, "should find test file in listing")
+
+		// Clean up
+		err = sftpClient.Remove("/" + filename)
+		require.NoError(t, err)
+	})
+
+	// Test 4: Nested directory operations
+	t.Run("NestedOperations", func(t *testing.T) {
+		// Create nested directory structure
+		err := sftpClient.MkdirAll("/nested/dir/structure")
+		require.NoError(t, err, "should be able to create nested directories")
+
+		// Create file in nested directory
+		testContent := []byte("nested file content")
+		file, err := sftpClient.Create("/nested/dir/structure/file.txt")
+		require.NoError(t, err)
+		_, err = file.Write(testContent)
+		require.NoError(t, err)
+		file.Close()
+
+		// Verify file exists
+		readFile, err := sftpClient.Open("/nested/dir/structure/file.txt")
+		require.NoError(t, err)
+		content, err := io.ReadAll(readFile)
+		require.NoError(t, err)
+		readFile.Close()
+		require.Equal(t, testContent, content)
+
+		// Clean up
+		err = sftpClient.Remove("/nested/dir/structure/file.txt")
+		require.NoError(t, err)
+		err = sftpClient.RemoveDirectory("/nested/dir/structure")
+		require.NoError(t, err)
+		err = sftpClient.RemoveDirectory("/nested/dir")
+		require.NoError(t, err)
+		err = sftpClient.RemoveDirectory("/nested")
+		require.NoError(t, err)
+	})
+
+	// Test 5: Rename operation
+	t.Run("RenameFile", func(t *testing.T) {
+		testContent := []byte("rename test content")
+
+		file, err := sftpClient.Create("/original.txt")
+		require.NoError(t, err)
+		_, err = file.Write(testContent)
+		require.NoError(t, err)
+		file.Close()
+
+		// Rename file
+		err = sftpClient.Rename("/original.txt", "/renamed.txt")
+		require.NoError(t, err, "should be able to rename file")
+
+		// Verify old file doesn't exist
+		_, err = sftpClient.Stat("/original.txt")
+		require.Error(t, err, "original file should not exist")
+
+		// Verify new file exists with correct content
+		readFile, err := sftpClient.Open("/renamed.txt")
+		require.NoError(t, err, "renamed file should exist")
+		content, err := io.ReadAll(readFile)
+		require.NoError(t, err)
+		readFile.Close()
+		require.Equal(t, testContent, content)
+
+		// Clean up
+		err = sftpClient.Remove("/renamed.txt")
+		require.NoError(t, err)
+	})
+}
+
+// TestAdminRootAccess tests that admin user with HomeDir="/" can access everything
+func TestAdminRootAccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	config := DefaultTestConfig()
+	config.EnableDebug = testing.Verbose()
+
+	fw := NewSftpTestFramework(t, config)
+	err := fw.Setup(config)
+	require.NoError(t, err, "failed to setup test framework")
+	defer fw.Cleanup()
+
+	// Connect as admin with HomeDir="/"
+	sftpClient, sshConn, err := fw.ConnectSFTP("admin", "adminpassword")
+	require.NoError(t, err, "failed to connect as admin")
+	defer sshConn.Close()
+	defer sftpClient.Close()
+
+	// Admin should be able to create directories anywhere
+	t.Run("CreateAnyDirectory", func(t *testing.T) {
+		// Create the user's home directory structure
+		err := sftpClient.MkdirAll("/sftp/testuser")
+		require.NoError(t, err, "admin should be able to create any directory")
+
+		// Create file in that directory
+		testContent := []byte("admin created this")
+		file, err := sftpClient.Create("/sftp/testuser/admin_file.txt")
+		require.NoError(t, err)
+		_, err = file.Write(testContent)
+		require.NoError(t, err)
+		file.Close()
+
+		// Verify file exists
+		info, err := sftpClient.Stat("/sftp/testuser/admin_file.txt")
+		require.NoError(t, err)
+		require.False(t, info.IsDir())
+
+		// Clean up
+		err = sftpClient.Remove("/sftp/testuser/admin_file.txt")
+		require.NoError(t, err)
+	})
+}
+
+// TestLargeFileUpload tests uploading larger files through SFTP
+func TestLargeFileUpload(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	config := DefaultTestConfig()
+	config.EnableDebug = testing.Verbose()
+
+	fw := NewSftpTestFramework(t, config)
+	err := fw.Setup(config)
+	require.NoError(t, err, "failed to setup test framework")
+	defer fw.Cleanup()
+
+	sftpClient, sshConn, err := fw.ConnectSFTP("testuser", "testuserpassword")
+	require.NoError(t, err, "failed to connect as testuser")
+	defer sshConn.Close()
+	defer sftpClient.Close()
+
+	// Create a 1MB file
+	t.Run("Upload1MB", func(t *testing.T) {
+		size := 1024 * 1024 // 1MB
+		testData := bytes.Repeat([]byte("A"), size)
+
+		file, err := sftpClient.Create("/large_file.bin")
+		require.NoError(t, err)
+		n, err := file.Write(testData)
+		require.NoError(t, err)
+		require.Equal(t, size, n)
+		file.Close()
+
+		// Verify file size
+		info, err := sftpClient.Stat("/large_file.bin")
+		require.NoError(t, err)
+		require.Equal(t, int64(size), info.Size())
+
+		// Verify content
+		readFile, err := sftpClient.Open("/large_file.bin")
+		require.NoError(t, err)
+		content, err := io.ReadAll(readFile)
+		require.NoError(t, err)
+		readFile.Close()
+		require.Equal(t, testData, content)
+
+		// Clean up
+		err = sftpClient.Remove("/large_file.bin")
+		require.NoError(t, err)
+	})
+}
+
+// TestStatOperations tests Stat and Lstat operations
+func TestStatOperations(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	config := DefaultTestConfig()
+	config.EnableDebug = testing.Verbose()
+
+	fw := NewSftpTestFramework(t, config)
+	err := fw.Setup(config)
+	require.NoError(t, err, "failed to setup test framework")
+	defer fw.Cleanup()
+
+	sftpClient, sshConn, err := fw.ConnectSFTP("testuser", "testuserpassword")
+	require.NoError(t, err, "failed to connect as testuser")
+	defer sshConn.Close()
+	defer sftpClient.Close()
+
+	// Create a test file
+	testContent := []byte("stat test content")
+	file, err := sftpClient.Create("/stat_test.txt")
+	require.NoError(t, err)
+	_, err = file.Write(testContent)
+	require.NoError(t, err)
+	file.Close()
+
+	t.Run("StatFile", func(t *testing.T) {
+		info, err := sftpClient.Stat("/stat_test.txt")
+		require.NoError(t, err)
+		require.Equal(t, "stat_test.txt", info.Name())
+		require.Equal(t, int64(len(testContent)), info.Size())
+		require.False(t, info.IsDir())
+	})
+
+	t.Run("StatDirectory", func(t *testing.T) {
+		err := sftpClient.Mkdir("/stat_dir")
+		require.NoError(t, err)
+
+		info, err := sftpClient.Stat("/stat_dir")
+		require.NoError(t, err)
+		require.Equal(t, "stat_dir", info.Name())
+		require.True(t, info.IsDir())
+
+		// Clean up
+		err = sftpClient.RemoveDirectory("/stat_dir")
+		require.NoError(t, err)
+	})
+
+	t.Run("StatRoot", func(t *testing.T) {
+		// Should be able to stat "/" which maps to user's home directory
+		info, err := sftpClient.Stat("/")
+		require.NoError(t, err, "should be able to stat root (home) directory")
+		require.True(t, info.IsDir(), "root should be a directory")
+	})
+
+	// Clean up
+	err = sftpClient.Remove("/stat_test.txt")
+	require.NoError(t, err)
+}
+
+// TestWalk tests walking directory trees
+func TestWalk(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	config := DefaultTestConfig()
+	config.EnableDebug = testing.Verbose()
+
+	fw := NewSftpTestFramework(t, config)
+	err := fw.Setup(config)
+	require.NoError(t, err, "failed to setup test framework")
+	defer fw.Cleanup()
+
+	sftpClient, sshConn, err := fw.ConnectSFTP("testuser", "testuserpassword")
+	require.NoError(t, err, "failed to connect as testuser")
+	defer sshConn.Close()
+	defer sftpClient.Close()
+
+	// Create directory structure
+	err = sftpClient.MkdirAll("/walk/a/b")
+	require.NoError(t, err)
+	err = sftpClient.MkdirAll("/walk/c")
+	require.NoError(t, err)
+
+	// Create files
+	for _, p := range []string{"/walk/file1.txt", "/walk/a/file2.txt", "/walk/a/b/file3.txt", "/walk/c/file4.txt"} {
+		file, err := sftpClient.Create(p)
+		require.NoError(t, err)
+		file.Write([]byte("test"))
+		file.Close()
+	}
+
+	t.Run("WalkEntireTree", func(t *testing.T) {
+		var paths []string
+		walker := sftpClient.Walk("/walk")
+		for walker.Step() {
+			if walker.Err() != nil {
+				continue
+			}
+			paths = append(paths, walker.Path())
+		}
+
+		// Should find all directories and files
+		require.Contains(t, paths, "/walk")
+		require.Contains(t, paths, "/walk/a")
+		require.Contains(t, paths, "/walk/a/b")
+		require.Contains(t, paths, "/walk/c")
+	})
+
+	// Clean up
+	for _, p := range []string{"/walk/file1.txt", "/walk/a/file2.txt", "/walk/a/b/file3.txt", "/walk/c/file4.txt"} {
+		require.NoError(t, sftpClient.Remove(p))
+	}
+	for _, p := range []string{"/walk/a/b", "/walk/a", "/walk/c", "/walk"} {
+		require.NoError(t, sftpClient.RemoveDirectory(p))
+	}
+}
+
+// TestCurrentWorkingDirectory tests that Getwd and Chdir work correctly
+func TestCurrentWorkingDirectory(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	config := DefaultTestConfig()
+	config.EnableDebug = testing.Verbose()
+
+	fw := NewSftpTestFramework(t, config)
+	err := fw.Setup(config)
+	require.NoError(t, err, "failed to setup test framework")
+	defer fw.Cleanup()
+
+	sftpClient, sshConn, err := fw.ConnectSFTP("testuser", "testuserpassword")
+	require.NoError(t, err, "failed to connect as testuser")
+	defer sshConn.Close()
+	defer sftpClient.Close()
+
+	// Create test directory
+	err = sftpClient.Mkdir("/cwd_test")
+	require.NoError(t, err)
+
+	t.Run("GetCurrentDir", func(t *testing.T) {
+		cwd, err := sftpClient.Getwd()
+		require.NoError(t, err)
+		// The initial working directory should be the user's home directory
+		// which from the user's perspective is "/"
+		require.Equal(t, "/", cwd, "initial working directory should be the virtual root")
+	})
+
+	t.Run("ChangeAndCreate", func(t *testing.T) {
+		// Create file in subdirectory using relative path after chdir
+		// Note: pkg/sftp doesn't support Chdir, so we test using absolute paths
+		file, err := sftpClient.Create("/cwd_test/relative_file.txt")
+		require.NoError(t, err)
+		file.Write([]byte("test"))
+		file.Close()
+
+		// Verify using absolute path
+		_, err = sftpClient.Stat("/cwd_test/relative_file.txt")
+		require.NoError(t, err)
+
+		// Clean up
+		sftpClient.Remove("/cwd_test/relative_file.txt")
+	})
+
+	// Clean up
+	err = sftpClient.RemoveDirectory("/cwd_test")
+	require.NoError(t, err)
+}
+
+// TestPathEdgeCases tests various edge cases in path handling
+func TestPathEdgeCases(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	config := DefaultTestConfig()
+	config.EnableDebug = testing.Verbose()
+
+	fw := NewSftpTestFramework(t, config)
+	err := fw.Setup(config)
+	require.NoError(t, err, "failed to setup test framework")
+	defer fw.Cleanup()
+
+	sftpClient, sshConn, err := fw.ConnectSFTP("testuser", "testuserpassword")
+	require.NoError(t, err, "failed to connect as testuser")
+	defer sshConn.Close()
+	defer sftpClient.Close()
+
+	t.Run("PathWithDotDot", func(t *testing.T) {
+		// Create directory structure
+		err := sftpClient.MkdirAll("/edge/subdir")
+		require.NoError(t, err)
+
+		// Create file using path with ..
+		file, err := sftpClient.Create("/edge/subdir/../file.txt")
+		require.NoError(t, err)
+		file.Write([]byte("test"))
+		file.Close()
+
+		// Verify file was created in /edge
+		_, err = sftpClient.Stat("/edge/file.txt")
+		require.NoError(t, err, "file should be created in parent directory")
+
+		// Clean up
+		sftpClient.Remove("/edge/file.txt")
+		sftpClient.RemoveDirectory("/edge/subdir")
+		sftpClient.RemoveDirectory("/edge")
+	})
+
+	t.Run("PathWithTrailingSlash", func(t *testing.T) {
+		err := sftpClient.Mkdir("/trailing")
+		require.NoError(t, err)
+
+		// Stat with trailing slash
+		info, err := sftpClient.Stat("/trailing/")
+		require.NoError(t, err)
+		require.True(t, info.IsDir())
+
+		// Clean up
+		sftpClient.RemoveDirectory("/trailing")
+	})
+
+	t.Run("CreateFileAtRootPath", func(t *testing.T) {
+		// This is the exact scenario from issue #7470
+		// User with HomeDir="/sftp/testuser" uploads to "/"
+		file, err := sftpClient.Create("/issue7470.txt")
+		require.NoError(t, err, "should be able to create file at / (issue #7470)")
+		file.Write([]byte("This tests the fix for issue #7470"))
+		file.Close()
+
+		// Verify
+		_, err = sftpClient.Stat("/issue7470.txt")
+		require.NoError(t, err)
+
+		// Clean up
+		sftpClient.Remove("/issue7470.txt")
+	})
+
+	// Security test: path traversal attacks should be blocked
+	t.Run("PathTraversalPrevention", func(t *testing.T) {
+		// User's HomeDir is "/sftp/testuser"
+		// Attempting to escape via "../.." should NOT create files outside home directory
+
+		// First, create a valid file to ensure we can write
+		validFile, err := sftpClient.Create("/valid.txt")
+		require.NoError(t, err)
+		validFile.Write([]byte("valid"))
+		validFile.Close()
+
+		// Try various path traversal attempts
+		// These should either:
+		// 1. Be blocked (error returned), OR
+		// 2. Be safely resolved to stay within home directory
+
+		traversalPaths := []string{
+			"/../escape.txt",
+			"/../../escape.txt",
+			"/../../../escape.txt",
+			"/subdir/../../escape.txt",
+			"/./../../escape.txt",
+		}
+
+		for _, traversalPath := range traversalPaths {
+			t.Run(traversalPath, func(t *testing.T) {
+				// Note: The pkg/sftp client sanitizes paths locally before sending them to the server.
+				// So "/../escape.txt" becomes "/escape.txt" on the wire.
+				// Therefore, we cannot trigger the server-side path traversal block with this client.
+				// Instead, we verify that the file is created successfully within the jail (contained).
+				// The server-side protection logic is verified in unit tests (sftpd/sftp_server_test.go).
+				
+				file, err := sftpClient.Create(traversalPath)
+				require.NoError(t, err, "creation should succeed because client sanitizes path")
+				file.Close()
+				
+				// Clean up
+				err = sftpClient.Remove(traversalPath)
+				require.NoError(t, err)
+			})
+		}
+
+		// Clean up
+		sftpClient.Remove("/valid.txt")
+	})
+}
+
+// TestFileContent tests reading and writing file content correctly
+func TestFileContent(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	config := DefaultTestConfig()
+	config.EnableDebug = testing.Verbose()
+
+	fw := NewSftpTestFramework(t, config)
+	err := fw.Setup(config)
+	require.NoError(t, err, "failed to setup test framework")
+	defer fw.Cleanup()
+
+	sftpClient, sshConn, err := fw.ConnectSFTP("testuser", "testuserpassword")
+	require.NoError(t, err, "failed to connect as testuser")
+	defer sshConn.Close()
+	defer sftpClient.Close()
+
+	t.Run("BinaryContent", func(t *testing.T) {
+		// Create binary data with all byte values
+		data := make([]byte, 256)
+		for i := 0; i < 256; i++ {
+			data[i] = byte(i)
+		}
+
+		file, err := sftpClient.Create("/binary.bin")
+		require.NoError(t, err)
+		n, err := file.Write(data)
+		require.NoError(t, err)
+		require.Equal(t, 256, n)
+		file.Close()
+
+		// Read back
+		readFile, err := sftpClient.Open("/binary.bin")
+		require.NoError(t, err)
+		content, err := io.ReadAll(readFile)
+		require.NoError(t, err)
+		readFile.Close()
+
+		require.Equal(t, data, content, "binary content should match")
+
+		// Clean up
+		sftpClient.Remove("/binary.bin")
+	})
+
+	t.Run("EmptyFile", func(t *testing.T) {
+		file, err := sftpClient.Create("/empty.txt")
+		require.NoError(t, err)
+		file.Close()
+
+		info, err := sftpClient.Stat("/empty.txt")
+		require.NoError(t, err)
+		require.Equal(t, int64(0), info.Size())
+
+		// Clean up
+		sftpClient.Remove("/empty.txt")
+	})
+
+	t.Run("UnicodeFilename", func(t *testing.T) {
+		filename := "/文件名.txt"
+		content := []byte("Unicode content: 你好世界")
+
+		file, err := sftpClient.Create(filename)
+		require.NoError(t, err)
+		file.Write(content)
+		file.Close()
+
+		// Read back
+		readFile, err := sftpClient.Open(filename)
+		require.NoError(t, err)
+		readContent, err := io.ReadAll(readFile)
+		require.NoError(t, err)
+		readFile.Close()
+
+		require.Equal(t, content, readContent)
+
+		// Verify in listing
+		files, err := sftpClient.ReadDir("/")
+		require.NoError(t, err)
+		found := false
+		for _, f := range files {
+			if f.Name() == path.Base(filename) {
+				found = true
+				break
+			}
+		}
+		require.True(t, found, "should find unicode filename in listing")
+
+		// Clean up
+		sftpClient.Remove(filename)
+	})
+}
+

test/sftp/framework.go

@@ -0,0 +1,423 @@
+package sftp
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/pkg/sftp"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/crypto/ssh"
+)
+
+// SftpTestFramework provides utilities for SFTP integration testing
+type SftpTestFramework struct {
+	t             *testing.T
+	tempDir       string
+	dataDir       string
+	masterProcess *os.Process
+	volumeProcess *os.Process
+	filerProcess  *os.Process
+	sftpProcess   *os.Process
+	masterAddr    string
+	volumeAddr    string
+	filerAddr     string
+	sftpAddr      string
+	weedBinary    string
+	userStoreFile string
+	hostKeyFile   string
+	isSetup       bool
+	skipCleanup   bool
+}
+
+// TestConfig holds configuration for SFTP tests
+type TestConfig struct {
+	NumVolumes    int
+	EnableDebug   bool
+	SkipCleanup   bool // for debugging failed tests
+	UserStoreFile string
+}
+
+// DefaultTestConfig returns a default configuration for SFTP tests
+func DefaultTestConfig() *TestConfig {
+	return &TestConfig{
+		NumVolumes:    3,
+		EnableDebug:   false,
+		SkipCleanup:   false,
+		UserStoreFile: "",
+	}
+}
+
+// NewSftpTestFramework creates a new SFTP testing framework
+func NewSftpTestFramework(t *testing.T, config *TestConfig) *SftpTestFramework {
+	if config == nil {
+		config = DefaultTestConfig()
+	}
+
+	tempDir, err := os.MkdirTemp("", "seaweedfs_sftp_test_")
+	require.NoError(t, err)
+
+	// Generate SSH host key for SFTP server
+	hostKeyFile := filepath.Join(tempDir, "ssh_host_key")
+	cmd := exec.Command("ssh-keygen", "-t", "ed25519", "-f", hostKeyFile, "-N", "")
+	err = cmd.Run()
+	require.NoError(t, err, "failed to generate SSH host key")
+
+	// Use provided userstore or copy the test one
+	userStoreFile := config.UserStoreFile
+	if userStoreFile == "" {
+		// Copy test userstore to temp dir
+		userStoreFile = filepath.Join(tempDir, "userstore.json")
+		testDataPath := findTestDataPath()
+		input, err := os.ReadFile(filepath.Join(testDataPath, "userstore.json"))
+		require.NoError(t, err, "failed to read test userstore.json")
+		err = os.WriteFile(userStoreFile, input, 0644)
+		require.NoError(t, err, "failed to write userstore.json")
+	}
+
+	return &SftpTestFramework{
+		t:             t,
+		tempDir:       tempDir,
+		dataDir:       filepath.Join(tempDir, "data"),
+		masterAddr:    "127.0.0.1:19333",
+		volumeAddr:    "127.0.0.1:18080",
+		filerAddr:     "127.0.0.1:18888",
+		sftpAddr:      "127.0.0.1:12022",
+		weedBinary:    findWeedBinary(),
+		userStoreFile: userStoreFile,
+		hostKeyFile:   hostKeyFile,
+		isSetup:       false,
+	}
+}
+
+// Setup starts SeaweedFS cluster with SFTP server
+func (f *SftpTestFramework) Setup(config *TestConfig) error {
+	if f.isSetup {
+		return fmt.Errorf("framework already setup")
+	}
+
+	// Create all data directories
+	dirs := []string{
+		f.dataDir,
+		filepath.Join(f.dataDir, "master"),
+		filepath.Join(f.dataDir, "volume"),
+	}
+	for _, dir := range dirs {
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return fmt.Errorf("failed to create directory %s: %v", dir, err)
+		}
+	}
+
+	// Start master
+	if err := f.startMaster(config); err != nil {
+		return fmt.Errorf("failed to start master: %v", err)
+	}
+
+	// Wait for master to be ready
+	if err := f.waitForService(f.masterAddr, 30*time.Second); err != nil {
+		return fmt.Errorf("master not ready: %v", err)
+	}
+
+	// Start volume server
+	if err := f.startVolumeServer(config); err != nil {
+		return fmt.Errorf("failed to start volume server: %v", err)
+	}
+
+	// Wait for volume server to be ready
+	if err := f.waitForService(f.volumeAddr, 30*time.Second); err != nil {
+		return fmt.Errorf("volume server not ready: %v", err)
+	}
+
+	// Start filer
+	if err := f.startFiler(config); err != nil {
+		return fmt.Errorf("failed to start filer: %v", err)
+	}
+
+	// Wait for filer to be ready
+	if err := f.waitForService(f.filerAddr, 30*time.Second); err != nil {
+		return fmt.Errorf("filer not ready: %v", err)
+	}
+
+	// Start SFTP server
+	if err := f.startSftpServer(config); err != nil {
+		return fmt.Errorf("failed to start SFTP server: %v", err)
+	}
+
+	// Wait for SFTP server to be ready
+	if err := f.waitForService(f.sftpAddr, 30*time.Second); err != nil {
+		return fmt.Errorf("SFTP server not ready: %v", err)
+	}
+
+	// Additional wait for all services to stabilize (gRPC endpoints)
+	time.Sleep(500 * time.Millisecond)
+
+	f.skipCleanup = config.SkipCleanup
+	f.isSetup = true
+	return nil
+}
+
+// Cleanup stops all processes and removes temporary files
+func (f *SftpTestFramework) Cleanup() {
+	// Stop processes in reverse order
+	processes := []*os.Process{f.sftpProcess, f.filerProcess, f.volumeProcess, f.masterProcess}
+	for _, proc := range processes {
+		if proc != nil {
+			proc.Signal(syscall.SIGTERM)
+			proc.Wait()
+		}
+	}
+
+	// Remove temp directory
+	if !f.skipCleanup {
+		os.RemoveAll(f.tempDir)
+	}
+}
+
+// GetSftpAddr returns the SFTP server address
+func (f *SftpTestFramework) GetSftpAddr() string {
+	return f.sftpAddr
+}
+
+// GetFilerAddr returns the filer address
+func (f *SftpTestFramework) GetFilerAddr() string {
+	return f.filerAddr
+}
+
+// ConnectSFTP creates an SFTP client connection with the given credentials
+func (f *SftpTestFramework) ConnectSFTP(username, password string) (*sftp.Client, *ssh.Client, error) {
+	// Load the known host public key for verification
+	hostKeyCallback, err := f.getHostKeyCallback()
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to get host key callback: %v", err)
+	}
+
+	config := &ssh.ClientConfig{
+		User: username,
+		Auth: []ssh.AuthMethod{
+			ssh.Password(password),
+		},
+		HostKeyCallback: hostKeyCallback,
+		Timeout:         5 * time.Second,
+	}
+
+	sshConn, err := ssh.Dial("tcp", f.sftpAddr, config)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to connect SSH: %v", err)
+	}
+
+	sftpClient, err := sftp.NewClient(sshConn)
+	if err != nil {
+		sshConn.Close()
+		return nil, nil, fmt.Errorf("failed to create SFTP client: %v", err)
+	}
+
+	return sftpClient, sshConn, nil
+}
+
+// getHostKeyCallback returns a callback that verifies the server's host key
+// matches the known test server key we generated
+func (f *SftpTestFramework) getHostKeyCallback() (ssh.HostKeyCallback, error) {
+	// Read the public key file generated alongside the private key
+	pubKeyFile := f.hostKeyFile + ".pub"
+	pubKeyBytes, err := os.ReadFile(pubKeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read host public key: %v", err)
+	}
+
+	// Parse the public key
+	pubKey, _, _, _, err := ssh.ParseAuthorizedKey(pubKeyBytes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse host public key: %v", err)
+	}
+
+	// Return a callback that verifies the server key matches our known key
+	return ssh.FixedHostKey(pubKey), nil
+}
+
+// startMaster starts the SeaweedFS master server
+func (f *SftpTestFramework) startMaster(config *TestConfig) error {
+	args := []string{
+		"master",
+		"-ip=127.0.0.1",
+		"-port=19333",
+		"-mdir=" + filepath.Join(f.dataDir, "master"),
+		"-raftBootstrap",
+		"-peers=none",
+	}
+
+	cmd := exec.Command(f.weedBinary, args...)
+	cmd.Dir = f.tempDir
+	if config.EnableDebug {
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+	}
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	f.masterProcess = cmd.Process
+	return nil
+}
+
+// startVolumeServer starts SeaweedFS volume server
+func (f *SftpTestFramework) startVolumeServer(config *TestConfig) error {
+	args := []string{
+		"volume",
+		"-mserver=" + f.masterAddr,
+		"-ip=127.0.0.1",
+		"-port=18080",
+		"-dir=" + filepath.Join(f.dataDir, "volume"),
+		fmt.Sprintf("-max=%d", config.NumVolumes),
+	}
+
+	cmd := exec.Command(f.weedBinary, args...)
+	cmd.Dir = f.tempDir
+	if config.EnableDebug {
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+	}
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	f.volumeProcess = cmd.Process
+	return nil
+}
+
+// startFiler starts the SeaweedFS filer server
+func (f *SftpTestFramework) startFiler(config *TestConfig) error {
+	args := []string{
+		"filer",
+		"-master=" + f.masterAddr,
+		"-ip=127.0.0.1",
+		"-port=18888",
+	}
+
+	cmd := exec.Command(f.weedBinary, args...)
+	cmd.Dir = f.tempDir
+	if config.EnableDebug {
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+	}
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	f.filerProcess = cmd.Process
+	return nil
+}
+
+// startSftpServer starts the SeaweedFS SFTP server
+func (f *SftpTestFramework) startSftpServer(config *TestConfig) error {
+	args := []string{
+		"sftp",
+		"-filer=" + f.filerAddr,
+		"-ip.bind=127.0.0.1",
+		"-port=12022",
+		"-sshPrivateKey=" + f.hostKeyFile,
+		"-userStoreFile=" + f.userStoreFile,
+	}
+
+	cmd := exec.Command(f.weedBinary, args...)
+	cmd.Dir = f.tempDir
+	if config.EnableDebug {
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+	}
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	f.sftpProcess = cmd.Process
+	return nil
+}
+
+// waitForService waits for a service to be available
+func (f *SftpTestFramework) waitForService(addr string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, 1*time.Second)
+		if err == nil {
+			conn.Close()
+			return nil
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return fmt.Errorf("service at %s not ready within timeout", addr)
+}
+
+// findWeedBinary locates the weed binary
+// Prefers local build over system-installed weed to ensure we test the latest code
+func findWeedBinary() string {
+	// Get the directory where this source file is located
+	// This ensures we find the locally built weed binary first
+	_, thisFile, _, ok := runtime.Caller(0)
+	if ok {
+		thisDir := filepath.Dir(thisFile)
+		// From test/sftp/, the weed binary should be at ../../weed/weed
+		candidates := []string{
+			filepath.Join(thisDir, "../../weed/weed"),
+			filepath.Join(thisDir, "../weed/weed"),
+		}
+		for _, candidate := range candidates {
+			if _, err := os.Stat(candidate); err == nil {
+				abs, _ := filepath.Abs(candidate)
+				return abs
+			}
+		}
+	}
+
+	// Try relative paths from current working directory
+	cwd, _ := os.Getwd()
+	candidates := []string{
+		filepath.Join(cwd, "../../weed/weed"),
+		filepath.Join(cwd, "../weed/weed"),
+		filepath.Join(cwd, "./weed"),
+	}
+	for _, candidate := range candidates {
+		if _, err := os.Stat(candidate); err == nil {
+			abs, _ := filepath.Abs(candidate)
+			return abs
+		}
+	}
+
+	// Fallback to PATH only if local build not found
+	if path, err := exec.LookPath("weed"); err == nil {
+		return path
+	}
+
+	// Default fallback
+	return "weed"
+}
+
+// findTestDataPath locates the testdata directory
+func findTestDataPath() string {
+	// Get the directory where this source file is located
+	_, thisFile, _, ok := runtime.Caller(0)
+	if ok {
+		thisDir := filepath.Dir(thisFile)
+		testDataPath := filepath.Join(thisDir, "testdata")
+		if _, err := os.Stat(testDataPath); err == nil {
+			return testDataPath
+		}
+	}
+
+	// Try relative paths from current working directory
+	cwd, _ := os.Getwd()
+	candidates := []string{
+		filepath.Join(cwd, "testdata"),
+		filepath.Join(cwd, "../sftp/testdata"),
+		filepath.Join(cwd, "test/sftp/testdata"),
+	}
+
+	for _, candidate := range candidates {
+		if _, err := os.Stat(candidate); err == nil {
+			return candidate
+		}
+	}
+
+	return "./testdata"
+}
+

test/sftp/go.mod

@@ -0,0 +1,17 @@
+module seaweedfs-sftp-tests
+
+go 1.24.0
+
+require (
+	github.com/pkg/sftp v1.13.7
+	github.com/stretchr/testify v1.10.0
+	golang.org/x/crypto v0.45.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

test/sftp/go.sum

@@ -0,0 +1,64 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM=
+github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
+golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
+golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

test/sftp/testdata/userstore.json

@@ -0,0 +1,37 @@
+[
+  {
+    "Username": "admin",
+    "Password": "adminpassword",
+    "PublicKeys": [],
+    "HomeDir": "/",
+    "Permissions": {
+      "/": ["*"]
+    },
+    "Uid": 0,
+    "Gid": 0
+  },
+  {
+    "Username": "testuser",
+    "Password": "testuserpassword",
+    "PublicKeys": [],
+    "HomeDir": "/sftp/testuser",
+    "Permissions": {
+      "/sftp/testuser": ["*"]
+    },
+    "Uid": 1001,
+    "Gid": 1001
+  },
+  {
+    "Username": "readonly",
+    "Password": "readonlypassword",
+    "PublicKeys": [],
+    "HomeDir": "/public",
+    "Permissions": {
+      "/public": ["read", "list"]
+    },
+    "Uid": 1002,
+    "Gid": 1002
+  }
+]
+
+

weed/admin/dash/admin_server.go

@@ -98,33 +98,24 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string)
 		glog.Warningf("Failed to initialize credential manager: %v", err)
 		// Continue without credential manager - will fall back to legacy approach
 	} else {
-		// For stores that need filer client details, set them
-		if store := credentialManager.GetStore(); store != nil {
-			if filerClientSetter, ok := store.(interface {
-				SetFilerClient(string, grpc.DialOption)
-			}); ok {
-				// We'll set the filer client later when we discover filers
-				// For now, just store the credential manager
-				server.credentialManager = credentialManager
+		server.credentialManager = credentialManager
+		glog.V(0).Infof("Credential manager initialized with store type: %s", credentialManager.GetStore().GetName())
 
-				// Set up a goroutine to set filer client once we discover filers
-				go func() {
-					for {
-						filerAddr := server.GetFilerAddress()
-						if filerAddr != "" {
-							filerClientSetter.SetFilerClient(filerAddr, server.grpcDialOption)
-							glog.V(1).Infof("Set filer client for credential manager: %s", filerAddr)
-							break
-						}
-						glog.V(1).Infof("Waiting for filer discovery for credential manager...")
-						time.Sleep(5 * time.Second) // Retry every 5 seconds
-					}
-				}()
+		// For stores that need filer address function, configure them
+		if store := credentialManager.GetStore(); store != nil {
+			if filerFuncSetter, ok := store.(interface {
+				SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption)
+			}); ok {
+				// Configure the filer address function to dynamically return the current active filer
+				// This function will be called each time credentials need to be loaded/saved,
+				// so it will automatically use whatever filer is currently available (HA-aware)
+				filerFuncSetter.SetFilerAddressFunc(func() pb.ServerAddress {
+					return pb.ServerAddress(server.GetFilerAddress())
+				}, server.grpcDialOption)
+				glog.V(0).Infof("Credential store configured with dynamic filer address function")
 			} else {
-				server.credentialManager = credentialManager
+				glog.V(0).Infof("Credential store %s does not support filer address function", store.GetName())
 			}
-		} else {
-			server.credentialManager = credentialManager
 		}
 	}
 

weed/admin/dash/user_management_test.go

@@ -0,0 +1,109 @@
+package dash
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/credential"
+	_ "github.com/seaweedfs/seaweedfs/weed/credential/filer_etc" // Import to register filer_etc store
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"google.golang.org/grpc"
+)
+
+// TestFilerAddressFunctionInterface tests that the filer_etc store
+// implements the correct SetFilerAddressFunc interface (issue #7575)
+func TestFilerAddressFunctionInterface(t *testing.T) {
+	// Create credential manager with filer_etc store
+	credentialManager, err := credential.NewCredentialManagerWithDefaults("")
+	if err != nil {
+		t.Fatalf("Failed to initialize credential manager: %v", err)
+	}
+
+	store := credentialManager.GetStore()
+	if store == nil {
+		t.Fatal("Credential store is nil")
+	}
+
+	// Check if store is filer_etc type
+	if store.GetName() != credential.StoreTypeFilerEtc {
+		t.Skipf("Skipping test - store is not filer_etc (got: %s)", store.GetName())
+	}
+
+	// Check if store implements SetFilerAddressFunc interface
+	// This is the critical check for bug #7575
+	filerFuncSetter, ok := store.(interface {
+		SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption)
+	})
+	if !ok {
+		t.Fatal("FilerEtcStore does not implement SetFilerAddressFunc interface - bug #7575")
+	}
+
+	// Verify we can call the method without panic
+	mockFilerAddress := pb.ServerAddress("localhost:8888")
+	filerFuncSetter.SetFilerAddressFunc(func() pb.ServerAddress {
+		return mockFilerAddress
+	}, grpc.WithInsecure())
+
+	t.Log("FilerEtcStore correctly implements SetFilerAddressFunc interface")
+}
+
+// TestGenerateAccessKey tests the access key generation function
+func TestGenerateAccessKey(t *testing.T) {
+	key1 := generateAccessKey()
+	key2 := generateAccessKey()
+
+	// Check length
+	if len(key1) != 20 {
+		t.Errorf("Expected access key length 20, got %d", len(key1))
+	}
+
+	// Check uniqueness
+	if key1 == key2 {
+		t.Error("Generated access keys should be unique")
+	}
+
+	// Check character set
+	for _, c := range key1 {
+		if !((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) {
+			t.Errorf("Access key contains invalid character: %c", c)
+		}
+	}
+}
+
+// TestGenerateSecretKey tests the secret key generation function
+func TestGenerateSecretKey(t *testing.T) {
+	key1 := generateSecretKey()
+	key2 := generateSecretKey()
+
+	// Check length (base64 encoding of 30 bytes = 40 characters)
+	if len(key1) != 40 {
+		t.Errorf("Expected secret key length 40, got %d", len(key1))
+	}
+
+	// Check uniqueness
+	if key1 == key2 {
+		t.Error("Generated secret keys should be unique")
+	}
+}
+
+// TestGenerateAccountId tests the account ID generation function
+func TestGenerateAccountId(t *testing.T) {
+	id1 := generateAccountId()
+	id2 := generateAccountId()
+
+	// Check length
+	if len(id1) != 12 {
+		t.Errorf("Expected account ID length 12, got %d", len(id1))
+	}
+
+	// Check that it's a number
+	for _, c := range id1 {
+		if c < '0' || c > '9' {
+			t.Errorf("Account ID contains non-digit character: %c", c)
+		}
+	}
+
+	// Check uniqueness (they should usually be different)
+	if id1 == id2 {
+		t.Log("Warning: Generated account IDs are the same (rare but possible)")
+	}
+}

weed/admin/handlers/file_browser_handlers.go

@@ -5,10 +5,12 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"mime"
 	"mime/multipart"
 	"net"
 	"net/http"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -20,15 +22,37 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/http/client"
 )
 
 type FileBrowserHandlers struct {
 	adminServer *dash.AdminServer
+	httpClient  *client.HTTPClient
 }
 
 func NewFileBrowserHandlers(adminServer *dash.AdminServer) *FileBrowserHandlers {
+	// Create HTTP client with TLS support from https.client configuration
+	// The client is created without a timeout - each operation will set its own timeout
+	// If TLS is enabled but misconfigured, fail fast to alert the operator immediately
+	// rather than silently falling back to HTTP and causing confusing runtime errors
+	httpClient, err := client.NewHttpClient(client.Client)
+	if err != nil {
+		glog.Fatalf("Failed to create HTTPS client for file browser: %v", err)
+	}
+
 	return &FileBrowserHandlers{
 		adminServer: adminServer,
+		httpClient:  httpClient,
+	}
+}
+
+// newClientWithTimeout creates a temporary http.Client with the specified timeout,
+// reusing the TLS transport from the shared httpClient.
+func (h *FileBrowserHandlers) newClientWithTimeout(timeout time.Duration) http.Client {
+	return http.Client{
+		Transport: h.httpClient.Client.Transport,
+		Timeout:   timeout,
 	}
 }
 
@@ -245,8 +269,12 @@ func (h *FileBrowserHandlers) UploadFile(c *gin.Context) {
 			continue
 		}
 
-		// Create full path for the file
-		fullPath := filepath.Join(currentPath, fileName)
+		// Normalize Windows-style backslashes to forward slashes
+		fileName = util.CleanWindowsPath(fileName)
+
+		// Create full path for the file using path.Join for URL path semantics
+		// path.Join handles double slashes and is not OS-specific like filepath.Join
+		fullPath := path.Join(currentPath, fileName)
 		if !strings.HasPrefix(fullPath, "/") {
 			fullPath = "/" + fullPath
 		}
@@ -327,8 +355,10 @@ func (h *FileBrowserHandlers) uploadFileToFiler(filePath string, fileHeader *mul
 	var body bytes.Buffer
 	writer := multipart.NewWriter(&body)
 
-	// Create form file field
-	part, err := writer.CreateFormFile("file", fileHeader.Filename)
+	// Create form file field with normalized base filename
+	// Use path.Base (not filepath.Base) since cleanFilePath uses URL path semantics
+	baseFileName := path.Base(cleanFilePath)
+	part, err := writer.CreateFormFile("file", baseFileName)
 	if err != nil {
 		return fmt.Errorf("failed to create form file: %w", err)
 	}
@@ -345,8 +375,15 @@ func (h *FileBrowserHandlers) uploadFileToFiler(filePath string, fileHeader *mul
 		return fmt.Errorf("failed to close multipart writer: %w", err)
 	}
 
-	// Create the upload URL with validated components
-	uploadURL := fmt.Sprintf("http://%s%s", filerAddress, cleanFilePath)
+	// Create the upload URL - the httpClient will normalize to the correct scheme (http/https)
+	// based on the https.client configuration in security.toml
+	uploadURL := fmt.Sprintf("%s%s", filerAddress, cleanFilePath)
+
+	// Normalize the URL scheme based on TLS configuration
+	uploadURL, err = h.httpClient.NormalizeHttpScheme(uploadURL)
+	if err != nil {
+		return fmt.Errorf("failed to normalize URL scheme: %w", err)
+	}
 
 	// Create HTTP request
 	req, err := http.NewRequest("POST", uploadURL, &body)
@@ -357,11 +394,11 @@ func (h *FileBrowserHandlers) uploadFileToFiler(filePath string, fileHeader *mul
 	// Set content type with boundary
 	req.Header.Set("Content-Type", writer.FormDataContentType())
 
-	// Send request
-	client := &http.Client{Timeout: 60 * time.Second} // Increased timeout for larger files
+	// Send request using TLS-aware HTTP client with 60s timeout for large file uploads
 	// lgtm[go/ssrf]
 	// Safe: filerAddress validated by validateFilerAddress() to match configured filer
 	// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
+	client := h.newClientWithTimeout(60 * time.Second)
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to upload file: %w", err)
@@ -423,8 +460,12 @@ func (h *FileBrowserHandlers) validateAndCleanFilePath(filePath string) (string,
 		return "", fmt.Errorf("file path cannot be empty")
 	}
 
+	// Normalize Windows-style backslashes to forward slashes
+	filePath = util.CleanWindowsPath(filePath)
+
 	// Clean the path to remove any .. or . components
-	cleanPath := filepath.Clean(filePath)
+	// Use path.Clean (not filepath.Clean) since this is a URL path
+	cleanPath := path.Clean(filePath)
 
 	// Ensure the path starts with /
 	if !strings.HasPrefix(cleanPath, "/") {
@@ -444,7 +485,57 @@ func (h *FileBrowserHandlers) validateAndCleanFilePath(filePath string) (string,
 	return cleanPath, nil
 }
 
-// DownloadFile handles file download requests
+// fetchFileContent fetches file content from the filer and returns the content or an error.
+func (h *FileBrowserHandlers) fetchFileContent(filePath string, timeout time.Duration) (string, error) {
+	filerAddress := h.adminServer.GetFilerAddress()
+	if filerAddress == "" {
+		return "", fmt.Errorf("filer address not configured")
+	}
+
+	if err := h.validateFilerAddress(filerAddress); err != nil {
+		return "", fmt.Errorf("invalid filer address configuration: %w", err)
+	}
+
+	cleanFilePath, err := h.validateAndCleanFilePath(filePath)
+	if err != nil {
+		return "", err
+	}
+
+	// Create the file URL with proper scheme based on TLS configuration
+	fileURL := fmt.Sprintf("%s%s", filerAddress, cleanFilePath)
+	fileURL, err = h.httpClient.NormalizeHttpScheme(fileURL)
+	if err != nil {
+		return "", fmt.Errorf("failed to construct file URL: %w", err)
+	}
+
+	// lgtm[go/ssrf]
+	// Safe: filerAddress validated by validateFilerAddress() to match configured filer
+	// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
+	client := h.newClientWithTimeout(timeout)
+	resp, err := client.Get(fileURL)
+	if err != nil {
+		return "", fmt.Errorf("failed to fetch file from filer: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return "", fmt.Errorf("filer returned status %d but failed to read response body: %w", resp.StatusCode, err)
+		}
+		return "", fmt.Errorf("filer returned status %d: %s", resp.StatusCode, string(body))
+	}
+
+	contentBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read file content: %w", err)
+	}
+
+	return string(contentBytes), nil
+}
+
+// DownloadFile handles file download requests by proxying through the Admin UI server
+// This ensures mTLS works correctly since the Admin UI server has the client certificates
 func (h *FileBrowserHandlers) DownloadFile(c *gin.Context) {
 	filePath := c.Query("path")
 	if filePath == "" {
@@ -459,6 +550,12 @@ func (h *FileBrowserHandlers) DownloadFile(c *gin.Context) {
 		return
 	}
 
+	// Validate filer address to prevent SSRF
+	if err := h.validateFilerAddress(filerAddress); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Invalid filer address configuration"})
+		return
+	}
+
 	// Validate and sanitize the file path
 	cleanFilePath, err := h.validateAndCleanFilePath(filePath)
 	if err != nil {
@@ -466,16 +563,66 @@ func (h *FileBrowserHandlers) DownloadFile(c *gin.Context) {
 		return
 	}
 
-	// Create the download URL
-	downloadURL := fmt.Sprintf("http://%s%s", filerAddress, cleanFilePath)
+	// Create the download URL with proper scheme based on TLS configuration
+	downloadURL := fmt.Sprintf("%s%s", filerAddress, cleanFilePath)
+	downloadURL, err = h.httpClient.NormalizeHttpScheme(downloadURL)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to construct download URL: " + err.Error()})
+		return
+	}
+
+	// Proxy the download through the Admin UI server to support mTLS
+	// lgtm[go/ssrf]
+	// Safe: filerAddress validated by validateFilerAddress() to match configured filer
+	// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
+	// Use request context so download is cancelled when client disconnects
+	req, err := http.NewRequestWithContext(c.Request.Context(), "GET", downloadURL, nil)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create request: " + err.Error()})
+		return
+	}
+	client := h.newClientWithTimeout(5 * time.Minute) // Longer timeout for large file downloads
+	resp, err := client.Do(req)
+	if err != nil {
+		c.JSON(http.StatusBadGateway, gin.H{"error": "Failed to fetch file from filer: " + err.Error()})
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			c.JSON(resp.StatusCode, gin.H{"error": fmt.Sprintf("Filer returned status %d but failed to read response body: %v", resp.StatusCode, err)})
+			return
+		}
+		c.JSON(resp.StatusCode, gin.H{"error": fmt.Sprintf("Filer returned status %d: %s", resp.StatusCode, string(body))})
+		return
+	}
 
 	// Set headers for file download
 	fileName := filepath.Base(cleanFilePath)
-	c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName))
-	c.Header("Content-Type", "application/octet-stream")
+	// Use mime.FormatMediaType for RFC 6266 compliant Content-Disposition,
+	// properly handling non-ASCII characters and special characters
+	c.Header("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName}))
 
-	// Proxy the request to filer
-	c.Redirect(http.StatusFound, downloadURL)
+	// Use content type from filer response, or default to octet-stream
+	contentType := resp.Header.Get("Content-Type")
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	c.Header("Content-Type", contentType)
+
+	// Set content length if available
+	if resp.ContentLength > 0 {
+		c.Header("Content-Length", fmt.Sprintf("%d", resp.ContentLength))
+	}
+
+	// Stream the response body to the client
+	c.Status(http.StatusOK)
+	_, err = io.Copy(c.Writer, resp.Body)
+	if err != nil {
+		glog.Errorf("Error streaming file download: %v", err)
+	}
 }
 
 // ViewFile handles file viewing requests (for text files, images, etc.)
@@ -559,46 +706,13 @@ func (h *FileBrowserHandlers) ViewFile(c *gin.Context) {
 			viewable = false
 			reason = "File too large for viewing (>1MB)"
 		} else {
-			// Get file content from filer
-			filerAddress := h.adminServer.GetFilerAddress()
-			if filerAddress != "" {
-				// Validate filer address to prevent SSRF
-				if err := h.validateFilerAddress(filerAddress); err != nil {
-					viewable = false
-					reason = "Invalid filer address configuration"
-				} else {
-					cleanFilePath, err := h.validateAndCleanFilePath(filePath)
-					if err == nil {
-						fileURL := fmt.Sprintf("http://%s%s", filerAddress, cleanFilePath)
-
-						client := &http.Client{Timeout: 30 * time.Second}
-						// lgtm[go/ssrf]
-						// Safe: filerAddress validated by validateFilerAddress() to match configured filer
-						// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
-						resp, err := client.Get(fileURL)
-						if err == nil && resp.StatusCode == http.StatusOK {
-							defer resp.Body.Close()
-							contentBytes, err := io.ReadAll(resp.Body)
-							if err == nil {
-								content = string(contentBytes)
-								viewable = true
-							} else {
-								viewable = false
-								reason = "Failed to read file content"
-							}
-						} else {
-							viewable = false
-							reason = "Failed to fetch file from filer"
-						}
-					} else {
-						viewable = false
-						reason = "Invalid file path"
-					}
-				}
-			} else {
-				viewable = false
-				reason = "Filer address not configured"
+			// Fetch file content from filer
+			var err error
+			content, err = h.fetchFileContent(filePath, 30*time.Second)
+			if err != nil {
+				reason = err.Error()
 			}
+			viewable = (err == nil)
 		}
 	} else {
 		// Not a text file, but might be viewable as image or PDF
@@ -893,18 +1007,28 @@ func (h *FileBrowserHandlers) isLikelyTextFile(filePath string, maxCheckSize int
 		return false
 	}
 
-	fileURL := fmt.Sprintf("http://%s%s", filerAddress, cleanFilePath)
+	// Create the file URL with proper scheme based on TLS configuration
+	fileURL := fmt.Sprintf("%s%s", filerAddress, cleanFilePath)
+	fileURL, err = h.httpClient.NormalizeHttpScheme(fileURL)
+	if err != nil {
+		glog.Errorf("Failed to normalize URL scheme: %v", err)
+		return false
+	}
 
-	client := &http.Client{Timeout: 10 * time.Second}
 	// lgtm[go/ssrf]
 	// Safe: filerAddress validated by validateFilerAddress() to match configured filer
 	// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
+	client := h.newClientWithTimeout(10 * time.Second)
 	resp, err := client.Get(fileURL)
-	if err != nil || resp.StatusCode != http.StatusOK {
+	if err != nil {
 		return false
 	}
 	defer resp.Body.Close()
 
+	if resp.StatusCode != http.StatusOK {
+		return false
+	}
+
 	// Read first few bytes to check if it's text
 	buffer := make([]byte, min(maxCheckSize, 512))
 	n, err := resp.Body.Read(buffer)

weed/command/admin.go

@@ -22,6 +22,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/admin"
 	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
 	"github.com/seaweedfs/seaweedfs/weed/admin/handlers"
+	_ "github.com/seaweedfs/seaweedfs/weed/credential/filer_etc" // Register filer_etc credential store
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"

weed/command/filer.go

@@ -128,7 +128,7 @@ func init() {
 	filerS3Options.tlsCACertificate = cmdFiler.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")
 	filerS3Options.tlsVerifyClientCert = cmdFiler.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate")
 	filerS3Options.bindIp = cmdFiler.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")
-	filerS3Options.idleTimeout = cmdFiler.Flag.Int("s3.idleTimeout", 10, "connection idle seconds")
+	filerS3Options.idleTimeout = cmdFiler.Flag.Int("s3.idleTimeout", 120, "connection idle seconds")
 	filerS3Options.concurrentUploadLimitMB = cmdFiler.Flag.Int("s3.concurrentUploadLimitMB", 128, "limit total concurrent upload size for S3")
 	filerS3Options.concurrentFileUploadLimit = cmdFiler.Flag.Int("s3.concurrentFileUploadLimit", 0, "limit number of concurrent file uploads for S3, 0 means unlimited")
 

weed/command/s3.go

@@ -84,7 +84,7 @@ func init() {
 	s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 	s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path")
 	s3StandaloneOptions.localSocket = cmdS3.Flag.String("localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
-	s3StandaloneOptions.idleTimeout = cmdS3.Flag.Int("idleTimeout", 10, "connection idle seconds")
+	s3StandaloneOptions.idleTimeout = cmdS3.Flag.Int("idleTimeout", 120, "connection idle seconds")
 	s3StandaloneOptions.concurrentUploadLimitMB = cmdS3.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
 	s3StandaloneOptions.concurrentFileUploadLimit = cmdS3.Flag.Int("concurrentFileUploadLimit", 0, "limit number of concurrent file uploads, 0 means unlimited")
 }

weed/command/scaffold/filer.toml

@@ -201,8 +201,11 @@ table = "seaweedfs"
 [redis2]
 enabled = false
 address = "localhost:6379"
+username = ""
 password = ""
 database = 0
+# prefix for filer redis keys
+keyPrefix = ""
 enable_tls = false
 ca_cert_path = ""
 client_cert_path = ""
@@ -217,6 +220,8 @@ masterName = "master"
 username = ""
 password = ""
 database = 0
+# prefix for filer redis keys
+keyPrefix = ""
 enable_tls = false
 ca_cert_path = ""
 client_cert_path = ""
@@ -232,7 +237,10 @@ addresses = [
     "localhost:30005",
     "localhost:30006",
 ]
+username = ""
 password = ""
+# prefix for filer redis keys
+keyPrefix = ""
 enable_tls = false
 ca_cert_path = ""
 client_cert_path = ""
@@ -248,8 +256,11 @@ superLargeDirectories = []
 [redis_lua]
 enabled = false
 address = "localhost:6379"
+username = ""
 password = ""
 database = 0
+# prefix for filer redis keys
+keyPrefix = ""
 enable_tls = false
 ca_cert_path = ""
 client_cert_path = ""
@@ -264,6 +275,8 @@ masterName = "master"
 username = ""
 password = ""
 database = 0
+# prefix for filer redis keys
+keyPrefix = ""
 enable_tls = false
 ca_cert_path = ""
 client_cert_path = ""
@@ -279,7 +292,10 @@ addresses = [
     "localhost:30005",
     "localhost:30006",
 ]
+username = ""
 password = ""
+# prefix for filer redis keys
+keyPrefix = ""
 enable_tls = false
 ca_cert_path = ""
 client_cert_path = ""
@@ -373,8 +389,10 @@ dialTimeOut = 10
 enabled = false
 location = "/tmp/"
 address = "localhost:6379"
+username = ""
 password = ""
 database = 1
+keyPrefix = ""
 
 [tikv]
 enabled = false

weed/command/server.go

@@ -133,11 +133,13 @@ func init() {
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
 	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
+	serverOptions.v.id = cmdServer.Flag.String("volume.id", "", "volume server id. If empty, default to ip:port")
 	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	serverOptions.v.diskType = cmdServer.Flag.String("volume.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	serverOptions.v.readMode = cmdServer.Flag.String("volume.readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|read in remote node|redirect volume location'.")
 	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
+	serverOptions.v.maintenanceMBPerSecond = cmdServer.Flag.Int("volume.maintenanceMBps", 0, "limit maintenance (replication / balance) IO rate in MB/s. Unset is 0, no limitation.")
 	serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
 	serverOptions.v.ldbTimeout = cmdServer.Flag.Int64("volume.index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
 	serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
@@ -168,7 +170,7 @@ func init() {
 	s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 	s3Options.localSocket = cmdServer.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
 	s3Options.bindIp = cmdServer.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")
-	s3Options.idleTimeout = cmdServer.Flag.Int("s3.idleTimeout", 10, "connection idle seconds")
+	s3Options.idleTimeout = cmdServer.Flag.Int("s3.idleTimeout", 120, "connection idle seconds")
 	s3Options.concurrentUploadLimitMB = cmdServer.Flag.Int("s3.concurrentUploadLimitMB", 128, "limit total concurrent upload size for S3")
 	s3Options.concurrentFileUploadLimit = cmdServer.Flag.Int("s3.concurrentFileUploadLimit", 0, "limit number of concurrent file uploads for S3, 0 means unlimited")
 

weed/command/volume.go

@@ -41,6 +41,7 @@ type VolumeServerOptions struct {
 	folderMaxLimits           []int32
 	idxFolder                 *string
 	ip                        *string
+	id                        *string
 	publicUrl                 *string
 	bindIp                    *string
 	mastersString             *string
@@ -57,6 +58,7 @@ type VolumeServerOptions struct {
 	cpuProfile                *string
 	memProfile                *string
 	compactionMBPerSecond     *int
+	maintenanceMBPerSecond    *int
 	fileSizeLimitMB           *int
 	concurrentUploadLimitMB   *int
 	concurrentDownloadLimitMB *int
@@ -78,6 +80,7 @@ func init() {
 	v.portGrpc = cmdVolume.Flag.Int("port.grpc", 0, "grpc listen port")
 	v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
 	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
+	v.id = cmdVolume.Flag.String("id", "", "volume server id. If empty, default to ip:port")
 	v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
 	v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
 	v.mastersString = cmdVolume.Flag.String("master", "localhost:9333", "comma-separated master servers")
@@ -94,6 +97,7 @@ func init() {
 	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
 	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
 	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
+	v.maintenanceMBPerSecond = cmdVolume.Flag.Int("maintenanceMBps", 0, "limit maintenance (replication / balance) IO rate in MB/s. Unset is 0, no limitation.")
 	v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
 	v.ldbTimeout = cmdVolume.Flag.Int64("index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
 	v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 256, "limit total concurrent upload size")
@@ -253,8 +257,11 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
 	}
 
+	// Determine volume server ID: if not specified, use ip:port
+	volumeServerId := util.GetVolumeServerId(*v.id, *v.ip, *v.port)
+
 	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
-		*v.ip, *v.port, *v.portGrpc, *v.publicUrl,
+		*v.ip, *v.port, *v.portGrpc, *v.publicUrl, volumeServerId,
 		v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
 		*v.idxFolder,
 		volumeNeedleMapKind,
@@ -262,6 +269,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		v.whiteList,
 		*v.fixJpgOrientation, *v.readMode,
 		*v.compactionMBPerSecond,
+		*v.maintenanceMBPerSecond,
 		*v.fileSizeLimitMB,
 		int64(*v.concurrentUploadLimitMB)*1024*1024,
 		int64(*v.concurrentDownloadLimitMB)*1024*1024,

weed/credential/filer_etc/filer_etc_store.go

@@ -58,7 +58,7 @@ func (store *FilerEtcStore) withFilerClient(fn func(client filer_pb.SeaweedFiler
 	store.mu.RLock()
 	if store.filerAddressFunc == nil {
 		store.mu.RUnlock()
-		return fmt.Errorf("filer_etc: filer address function not configured")
+		return fmt.Errorf("filer_etc: filer not yet available - please wait for filer discovery to complete and try again")
 	}
 
 	filerAddress := store.filerAddressFunc()
@@ -66,7 +66,7 @@ func (store *FilerEtcStore) withFilerClient(fn func(client filer_pb.SeaweedFiler
 	store.mu.RUnlock()
 	
 	if filerAddress == "" {
-		return fmt.Errorf("filer_etc: filer address is empty")
+		return fmt.Errorf("filer_etc: no filer discovered yet - please ensure a filer is running and accessible")
 	}
 
 	// Use the pb.WithGrpcFilerClient helper similar to existing code

weed/filer/empty_folder_cleanup/cleanup_queue.go

@@ -0,0 +1,207 @@
+package empty_folder_cleanup
+
+import (
+	"container/list"
+	"sync"
+	"time"
+)
+
+// CleanupQueue manages a deduplicated queue of folders pending cleanup.
+// It uses a doubly-linked list ordered by event time (oldest at front) and a map for O(1) deduplication.
+// Processing is triggered when:
+// - Queue size reaches maxSize, OR
+// - Oldest item exceeds maxAge
+type CleanupQueue struct {
+	mu       sync.Mutex
+	items    *list.List               // Linked list of *queueItem ordered by time (front = oldest)
+	itemsMap map[string]*list.Element // folder -> list element for O(1) lookup
+	maxSize  int                      // Max queue size before triggering cleanup
+	maxAge   time.Duration            // Max age before triggering cleanup
+}
+
+// queueItem represents an item in the cleanup queue
+type queueItem struct {
+	folder    string
+	queueTime time.Time
+}
+
+// NewCleanupQueue creates a new CleanupQueue with the specified limits
+func NewCleanupQueue(maxSize int, maxAge time.Duration) *CleanupQueue {
+	return &CleanupQueue{
+		items:    list.New(),
+		itemsMap: make(map[string]*list.Element),
+		maxSize:  maxSize,
+		maxAge:   maxAge,
+	}
+}
+
+// Add adds a folder to the queue with the specified event time.
+// The item is inserted in time-sorted order (oldest at front) to handle out-of-order events.
+// If folder already exists with an older time, the time is updated and position adjusted.
+// Returns true if the folder was newly added, false if it was updated.
+func (q *CleanupQueue) Add(folder string, eventTime time.Time) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	// Check if folder already exists
+	if elem, exists := q.itemsMap[folder]; exists {
+		existingItem := elem.Value.(*queueItem)
+		// Only update if new event is later
+		if eventTime.After(existingItem.queueTime) {
+			// Remove from current position
+			q.items.Remove(elem)
+			// Re-insert with new time in sorted position
+			newElem := q.insertSorted(folder, eventTime)
+			q.itemsMap[folder] = newElem
+		}
+		return false
+	}
+
+	// Insert new folder in sorted position
+	elem := q.insertSorted(folder, eventTime)
+	q.itemsMap[folder] = elem
+	return true
+}
+
+// insertSorted inserts an item in the correct position to maintain time ordering (oldest at front)
+func (q *CleanupQueue) insertSorted(folder string, eventTime time.Time) *list.Element {
+	item := &queueItem{
+		folder:    folder,
+		queueTime: eventTime,
+	}
+
+	// Find the correct position (insert before the first item with a later time)
+	for elem := q.items.Back(); elem != nil; elem = elem.Prev() {
+		existingItem := elem.Value.(*queueItem)
+		if !eventTime.Before(existingItem.queueTime) {
+			// Insert after this element
+			return q.items.InsertAfter(item, elem)
+		}
+	}
+
+	// This item is the oldest, insert at front
+	return q.items.PushFront(item)
+}
+
+// Remove removes a specific folder from the queue (e.g., when a file is created).
+// Returns true if the folder was found and removed.
+func (q *CleanupQueue) Remove(folder string) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	elem, exists := q.itemsMap[folder]
+	if !exists {
+		return false
+	}
+
+	q.items.Remove(elem)
+	delete(q.itemsMap, folder)
+	return true
+}
+
+// ShouldProcess returns true if the queue should be processed.
+// This is true when:
+// - Queue size >= maxSize, OR
+// - Oldest item age > maxAge
+func (q *CleanupQueue) ShouldProcess() bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	return q.shouldProcessLocked()
+}
+
+// shouldProcessLocked checks if processing is needed (caller must hold lock)
+func (q *CleanupQueue) shouldProcessLocked() bool {
+	if q.items.Len() == 0 {
+		return false
+	}
+
+	// Check if queue is full
+	if q.items.Len() >= q.maxSize {
+		return true
+	}
+
+	// Check if oldest item exceeds max age
+	front := q.items.Front()
+	if front != nil {
+		item := front.Value.(*queueItem)
+		if time.Since(item.queueTime) > q.maxAge {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Pop removes and returns the oldest folder from the queue.
+// Returns the folder and true if an item was available, or empty string and false if queue is empty.
+func (q *CleanupQueue) Pop() (string, bool) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	front := q.items.Front()
+	if front == nil {
+		return "", false
+	}
+
+	item := front.Value.(*queueItem)
+	q.items.Remove(front)
+	delete(q.itemsMap, item.folder)
+
+	return item.folder, true
+}
+
+// Peek returns the oldest folder without removing it.
+// Returns the folder and queue time if available, or empty values if queue is empty.
+func (q *CleanupQueue) Peek() (folder string, queueTime time.Time, ok bool) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	front := q.items.Front()
+	if front == nil {
+		return "", time.Time{}, false
+	}
+
+	item := front.Value.(*queueItem)
+	return item.folder, item.queueTime, true
+}
+
+// Len returns the current queue size.
+func (q *CleanupQueue) Len() int {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return q.items.Len()
+}
+
+// Contains checks if a folder is in the queue.
+func (q *CleanupQueue) Contains(folder string) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	_, exists := q.itemsMap[folder]
+	return exists
+}
+
+// Clear removes all items from the queue.
+func (q *CleanupQueue) Clear() {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	q.items.Init()
+	q.itemsMap = make(map[string]*list.Element)
+}
+
+// OldestAge returns the age of the oldest item in the queue, or 0 if empty.
+func (q *CleanupQueue) OldestAge() time.Duration {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	front := q.items.Front()
+	if front == nil {
+		return 0
+	}
+
+	item := front.Value.(*queueItem)
+	return time.Since(item.queueTime)
+}
+
+

weed/filer/empty_folder_cleanup/cleanup_queue_test.go

@@ -0,0 +1,371 @@
+package empty_folder_cleanup
+
+import (
+	"testing"
+	"time"
+)
+
+func TestCleanupQueue_Add(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	now := time.Now()
+
+	// Add first item
+	if !q.Add("/buckets/b1/folder1", now) {
+		t.Error("expected Add to return true for new item")
+	}
+	if q.Len() != 1 {
+		t.Errorf("expected len 1, got %d", q.Len())
+	}
+
+	// Add second item with later time
+	if !q.Add("/buckets/b1/folder2", now.Add(1*time.Second)) {
+		t.Error("expected Add to return true for new item")
+	}
+	if q.Len() != 2 {
+		t.Errorf("expected len 2, got %d", q.Len())
+	}
+
+	// Add duplicate with newer time - should update and reposition
+	if q.Add("/buckets/b1/folder1", now.Add(2*time.Second)) {
+		t.Error("expected Add to return false for existing item")
+	}
+	if q.Len() != 2 {
+		t.Errorf("expected len 2 after duplicate, got %d", q.Len())
+	}
+
+	// folder1 should now be at the back (newer time) - verify by popping
+	folder1, _ := q.Pop()
+	folder2, _ := q.Pop()
+	if folder1 != "/buckets/b1/folder2" || folder2 != "/buckets/b1/folder1" {
+		t.Errorf("expected folder1 to be moved to back, got %s, %s", folder1, folder2)
+	}
+}
+
+func TestCleanupQueue_Add_OutOfOrder(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	baseTime := time.Now()
+
+	// Add items out of order
+	q.Add("/buckets/b1/folder3", baseTime.Add(3*time.Second))
+	q.Add("/buckets/b1/folder1", baseTime.Add(1*time.Second))
+	q.Add("/buckets/b1/folder2", baseTime.Add(2*time.Second))
+
+	// Items should be in time order (oldest first) - verify by popping
+	expected := []string{"/buckets/b1/folder1", "/buckets/b1/folder2", "/buckets/b1/folder3"}
+	for i, exp := range expected {
+		folder, ok := q.Pop()
+		if !ok || folder != exp {
+			t.Errorf("at index %d: expected %s, got %s", i, exp, folder)
+		}
+	}
+}
+
+func TestCleanupQueue_Add_DuplicateWithOlderTime(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	baseTime := time.Now()
+
+	// Add folder at t=5
+	q.Add("/buckets/b1/folder1", baseTime.Add(5*time.Second))
+
+	// Try to add same folder with older time - should NOT update
+	q.Add("/buckets/b1/folder1", baseTime.Add(2*time.Second))
+
+	// Time should remain at t=5
+	_, queueTime, _ := q.Peek()
+	if queueTime != baseTime.Add(5*time.Second) {
+		t.Errorf("expected time to remain unchanged, got %v", queueTime)
+	}
+}
+
+func TestCleanupQueue_Remove(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	now := time.Now()
+
+	q.Add("/buckets/b1/folder1", now)
+	q.Add("/buckets/b1/folder2", now.Add(1*time.Second))
+	q.Add("/buckets/b1/folder3", now.Add(2*time.Second))
+
+	// Remove middle item
+	if !q.Remove("/buckets/b1/folder2") {
+		t.Error("expected Remove to return true for existing item")
+	}
+	if q.Len() != 2 {
+		t.Errorf("expected len 2, got %d", q.Len())
+	}
+	if q.Contains("/buckets/b1/folder2") {
+		t.Error("removed item should not be in queue")
+	}
+
+	// Remove non-existent item
+	if q.Remove("/buckets/b1/nonexistent") {
+		t.Error("expected Remove to return false for non-existent item")
+	}
+
+	// Verify order is preserved by popping
+	folder1, _ := q.Pop()
+	folder3, _ := q.Pop()
+	if folder1 != "/buckets/b1/folder1" || folder3 != "/buckets/b1/folder3" {
+		t.Errorf("unexpected order: %s, %s", folder1, folder3)
+	}
+}
+
+func TestCleanupQueue_Pop(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	now := time.Now()
+
+	// Pop from empty queue
+	folder, ok := q.Pop()
+	if ok {
+		t.Error("expected Pop to return false for empty queue")
+	}
+	if folder != "" {
+		t.Errorf("expected empty folder, got %s", folder)
+	}
+
+	// Add items and pop in order
+	q.Add("/buckets/b1/folder1", now)
+	q.Add("/buckets/b1/folder2", now.Add(1*time.Second))
+	q.Add("/buckets/b1/folder3", now.Add(2*time.Second))
+
+	folder, ok = q.Pop()
+	if !ok || folder != "/buckets/b1/folder1" {
+		t.Errorf("expected folder1, got %s (ok=%v)", folder, ok)
+	}
+
+	folder, ok = q.Pop()
+	if !ok || folder != "/buckets/b1/folder2" {
+		t.Errorf("expected folder2, got %s (ok=%v)", folder, ok)
+	}
+
+	folder, ok = q.Pop()
+	if !ok || folder != "/buckets/b1/folder3" {
+		t.Errorf("expected folder3, got %s (ok=%v)", folder, ok)
+	}
+
+	// Queue should be empty now
+	if q.Len() != 0 {
+		t.Errorf("expected empty queue, got len %d", q.Len())
+	}
+}
+
+func TestCleanupQueue_Peek(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	now := time.Now()
+
+	// Peek empty queue
+	folder, _, ok := q.Peek()
+	if ok {
+		t.Error("expected Peek to return false for empty queue")
+	}
+
+	// Add item and peek
+	q.Add("/buckets/b1/folder1", now)
+	folder, queueTime, ok := q.Peek()
+	if !ok || folder != "/buckets/b1/folder1" {
+		t.Errorf("expected folder1, got %s (ok=%v)", folder, ok)
+	}
+	if queueTime != now {
+		t.Errorf("expected queue time %v, got %v", now, queueTime)
+	}
+
+	// Peek should not remove item
+	if q.Len() != 1 {
+		t.Errorf("Peek should not remove item, len=%d", q.Len())
+	}
+}
+
+func TestCleanupQueue_Contains(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	now := time.Now()
+
+	q.Add("/buckets/b1/folder1", now)
+
+	if !q.Contains("/buckets/b1/folder1") {
+		t.Error("expected Contains to return true")
+	}
+	if q.Contains("/buckets/b1/folder2") {
+		t.Error("expected Contains to return false for non-existent")
+	}
+}
+
+func TestCleanupQueue_ShouldProcess_MaxSize(t *testing.T) {
+	q := NewCleanupQueue(3, 10*time.Minute)
+	now := time.Now()
+
+	// Empty queue
+	if q.ShouldProcess() {
+		t.Error("empty queue should not need processing")
+	}
+
+	// Add items below max
+	q.Add("/buckets/b1/folder1", now)
+	q.Add("/buckets/b1/folder2", now.Add(1*time.Second))
+	if q.ShouldProcess() {
+		t.Error("queue below max should not need processing")
+	}
+
+	// Add item to reach max
+	q.Add("/buckets/b1/folder3", now.Add(2*time.Second))
+	if !q.ShouldProcess() {
+		t.Error("queue at max should need processing")
+	}
+}
+
+func TestCleanupQueue_ShouldProcess_MaxAge(t *testing.T) {
+	q := NewCleanupQueue(100, 100*time.Millisecond) // Short max age for testing
+
+	// Add item with old event time
+	oldTime := time.Now().Add(-1 * time.Second) // 1 second ago
+	q.Add("/buckets/b1/folder1", oldTime)
+
+	// Item is older than maxAge, should need processing
+	if !q.ShouldProcess() {
+		t.Error("old item should trigger processing")
+	}
+
+	// Clear and add fresh item
+	q.Clear()
+	q.Add("/buckets/b1/folder2", time.Now())
+
+	// Fresh item should not trigger processing
+	if q.ShouldProcess() {
+		t.Error("fresh item should not trigger processing")
+	}
+}
+
+func TestCleanupQueue_Clear(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	now := time.Now()
+
+	q.Add("/buckets/b1/folder1", now)
+	q.Add("/buckets/b1/folder2", now.Add(1*time.Second))
+	q.Add("/buckets/b1/folder3", now.Add(2*time.Second))
+
+	q.Clear()
+
+	if q.Len() != 0 {
+		t.Errorf("expected empty queue after Clear, got len %d", q.Len())
+	}
+	if q.Contains("/buckets/b1/folder1") {
+		t.Error("queue should not contain items after Clear")
+	}
+}
+
+func TestCleanupQueue_OldestAge(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+
+	// Empty queue
+	if q.OldestAge() != 0 {
+		t.Error("empty queue should have zero oldest age")
+	}
+
+	// Add item with time in the past
+	oldTime := time.Now().Add(-5 * time.Minute)
+	q.Add("/buckets/b1/folder1", oldTime)
+
+	// Age should be approximately 5 minutes
+	age := q.OldestAge()
+	if age < 4*time.Minute || age > 6*time.Minute {
+		t.Errorf("expected ~5m age, got %v", age)
+	}
+}
+
+func TestCleanupQueue_TimeOrder(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	baseTime := time.Now()
+
+	// Add items in order
+	items := []string{
+		"/buckets/b1/a",
+		"/buckets/b1/b",
+		"/buckets/b1/c",
+		"/buckets/b1/d",
+		"/buckets/b1/e",
+	}
+	for i, item := range items {
+		q.Add(item, baseTime.Add(time.Duration(i)*time.Second))
+	}
+
+	// Pop should return in time order
+	for i, expected := range items {
+		got, ok := q.Pop()
+		if !ok {
+			t.Errorf("Pop %d: expected item, got empty", i)
+		}
+		if got != expected {
+			t.Errorf("Pop %d: expected %s, got %s", i, expected, got)
+		}
+	}
+}
+
+func TestCleanupQueue_DuplicateWithNewerTime(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	baseTime := time.Now()
+
+	// Add items
+	q.Add("/buckets/b1/folder1", baseTime)
+	q.Add("/buckets/b1/folder2", baseTime.Add(1*time.Second))
+	q.Add("/buckets/b1/folder3", baseTime.Add(2*time.Second))
+
+	// Add duplicate with newer time - should update and reposition
+	q.Add("/buckets/b1/folder1", baseTime.Add(3*time.Second))
+
+	// folder1 should now be at the back (newest time) - verify by popping
+	expected := []string{"/buckets/b1/folder2", "/buckets/b1/folder3", "/buckets/b1/folder1"}
+	for i, exp := range expected {
+		folder, ok := q.Pop()
+		if !ok || folder != exp {
+			t.Errorf("at index %d: expected %s, got %s", i, exp, folder)
+		}
+	}
+}
+
+func TestCleanupQueue_Concurrent(t *testing.T) {
+	q := NewCleanupQueue(1000, 10*time.Minute)
+	done := make(chan bool)
+	now := time.Now()
+
+	// Concurrent adds
+	go func() {
+		for i := 0; i < 100; i++ {
+			q.Add("/buckets/b1/folder"+string(rune('A'+i%26)), now.Add(time.Duration(i)*time.Millisecond))
+		}
+		done <- true
+	}()
+
+	// Concurrent removes
+	go func() {
+		for i := 0; i < 50; i++ {
+			q.Remove("/buckets/b1/folder" + string(rune('A'+i%26)))
+		}
+		done <- true
+	}()
+
+	// Concurrent pops
+	go func() {
+		for i := 0; i < 30; i++ {
+			q.Pop()
+		}
+		done <- true
+	}()
+
+	// Concurrent reads
+	go func() {
+		for i := 0; i < 100; i++ {
+			q.Len()
+			q.Contains("/buckets/b1/folderA")
+			q.ShouldProcess()
+		}
+		done <- true
+	}()
+
+	// Wait for all goroutines
+	for i := 0; i < 4; i++ {
+		<-done
+	}
+
+	// Just verify no panic occurred and queue is in consistent state
+	_ = q.Len()
+}
+
+

weed/filer/empty_folder_cleanup/empty_folder_cleaner.go

@@ -0,0 +1,436 @@
+package empty_folder_cleanup
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	DefaultMaxCountCheck  = 1000
+	DefaultCacheExpiry    = 5 * time.Minute
+	DefaultQueueMaxSize   = 1000
+	DefaultQueueMaxAge    = 10 * time.Minute
+	DefaultProcessorSleep = 10 * time.Second // How often to check queue
+)
+
+// FilerOperations defines the filer operations needed by EmptyFolderCleaner
+type FilerOperations interface {
+	CountDirectoryEntries(ctx context.Context, dirPath util.FullPath, limit int) (count int, err error)
+	DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32, ifNotModifiedAfter int64) error
+}
+
+// folderState tracks the state of a folder for empty folder cleanup
+type folderState struct {
+	roughCount  int       // Cached rough count (up to maxCountCheck)
+	lastAddTime time.Time // Last time an item was added
+	lastDelTime time.Time // Last time an item was deleted
+	lastCheck   time.Time // Last time we checked the actual count
+}
+
+// EmptyFolderCleaner handles asynchronous cleanup of empty folders
+// Each filer owns specific folders via consistent hashing based on the peer filer list
+type EmptyFolderCleaner struct {
+	filer    FilerOperations
+	lockRing *lock_manager.LockRing
+	host     pb.ServerAddress
+
+	// Folder state tracking
+	mu           sync.RWMutex
+	folderCounts map[string]*folderState // Rough count cache
+
+	// Cleanup queue (thread-safe, has its own lock)
+	cleanupQueue *CleanupQueue
+
+	// Configuration
+	maxCountCheck  int           // Max items to count (1000)
+	cacheExpiry    time.Duration // How long to keep cache entries
+	processorSleep time.Duration // How often processor checks queue
+	bucketPath     string        // e.g., "/buckets"
+
+	// Control
+	enabled bool
+	stopCh  chan struct{}
+}
+
+// NewEmptyFolderCleaner creates a new EmptyFolderCleaner
+func NewEmptyFolderCleaner(filer FilerOperations, lockRing *lock_manager.LockRing, host pb.ServerAddress, bucketPath string) *EmptyFolderCleaner {
+	efc := &EmptyFolderCleaner{
+		filer:          filer,
+		lockRing:       lockRing,
+		host:           host,
+		folderCounts:   make(map[string]*folderState),
+		cleanupQueue:   NewCleanupQueue(DefaultQueueMaxSize, DefaultQueueMaxAge),
+		maxCountCheck:  DefaultMaxCountCheck,
+		cacheExpiry:    DefaultCacheExpiry,
+		processorSleep: DefaultProcessorSleep,
+		bucketPath:     bucketPath,
+		enabled:        true,
+		stopCh:         make(chan struct{}),
+	}
+	go efc.cacheEvictionLoop()
+	go efc.cleanupProcessor()
+	return efc
+}
+
+// SetEnabled enables or disables the cleaner
+func (efc *EmptyFolderCleaner) SetEnabled(enabled bool) {
+	efc.mu.Lock()
+	defer efc.mu.Unlock()
+	efc.enabled = enabled
+}
+
+// IsEnabled returns whether the cleaner is enabled
+func (efc *EmptyFolderCleaner) IsEnabled() bool {
+	efc.mu.RLock()
+	defer efc.mu.RUnlock()
+	return efc.enabled
+}
+
+// ownsFolder checks if this filer owns the folder via consistent hashing
+func (efc *EmptyFolderCleaner) ownsFolder(folder string) bool {
+	servers := efc.lockRing.GetSnapshot()
+	if len(servers) <= 1 {
+		return true // Single filer case
+	}
+	return efc.hashKeyToServer(folder, servers) == efc.host
+}
+
+// hashKeyToServer uses consistent hashing to map a folder to a server
+func (efc *EmptyFolderCleaner) hashKeyToServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
+	if len(servers) == 0 {
+		return ""
+	}
+	x := util.HashStringToLong(key)
+	if x < 0 {
+		x = -x
+	}
+	x = x % int64(len(servers))
+	return servers[x]
+}
+
+// OnDeleteEvent is called when a file or directory is deleted
+// Both file and directory deletions count towards making the parent folder empty
+// eventTime is the time when the delete event occurred (for proper ordering)
+func (efc *EmptyFolderCleaner) OnDeleteEvent(directory string, entryName string, isDirectory bool, eventTime time.Time) {
+	// Skip if not under bucket path (must be at least /buckets/<bucket>/...)
+	if efc.bucketPath != "" && !isUnderBucketPath(directory, efc.bucketPath) {
+		return
+	}
+
+	// Check if we own this folder
+	if !efc.ownsFolder(directory) {
+		glog.V(4).Infof("EmptyFolderCleaner: not owner of %s, skipping", directory)
+		return
+	}
+
+	efc.mu.Lock()
+	defer efc.mu.Unlock()
+
+	// Check enabled inside lock to avoid race with Stop()
+	if !efc.enabled {
+		return
+	}
+
+	glog.V(3).Infof("EmptyFolderCleaner: delete event in %s/%s (isDir=%v)", directory, entryName, isDirectory)
+
+	// Update cached count (create entry if needed)
+	state, exists := efc.folderCounts[directory]
+	if !exists {
+		state = &folderState{}
+		efc.folderCounts[directory] = state
+	}
+	if state.roughCount > 0 {
+		state.roughCount--
+	}
+	state.lastDelTime = eventTime
+
+	// Only add to cleanup queue if roughCount suggests folder might be empty
+	if state.roughCount > 0 {
+		glog.V(3).Infof("EmptyFolderCleaner: skipping queue for %s, roughCount=%d", directory, state.roughCount)
+		return
+	}
+
+	// Add to cleanup queue with event time (handles out-of-order events)
+	if efc.cleanupQueue.Add(directory, eventTime) {
+		glog.V(3).Infof("EmptyFolderCleaner: queued %s for cleanup", directory)
+	}
+}
+
+// OnCreateEvent is called when a file or directory is created
+// Both file and directory creations cancel pending cleanup for the parent folder
+func (efc *EmptyFolderCleaner) OnCreateEvent(directory string, entryName string, isDirectory bool) {
+	// Skip if not under bucket path (must be at least /buckets/<bucket>/...)
+	if efc.bucketPath != "" && !isUnderBucketPath(directory, efc.bucketPath) {
+		return
+	}
+
+	efc.mu.Lock()
+	defer efc.mu.Unlock()
+
+	// Check enabled inside lock to avoid race with Stop()
+	if !efc.enabled {
+		return
+	}
+
+	// Update cached count only if already tracked (no need to track new folders)
+	if state, exists := efc.folderCounts[directory]; exists {
+		state.roughCount++
+		state.lastAddTime = time.Now()
+	}
+
+	// Remove from cleanup queue (cancel pending cleanup)
+	if efc.cleanupQueue.Remove(directory) {
+		glog.V(3).Infof("EmptyFolderCleaner: cancelled cleanup for %s due to new entry", directory)
+	}
+}
+
+// cleanupProcessor runs in background and processes the cleanup queue
+func (efc *EmptyFolderCleaner) cleanupProcessor() {
+	ticker := time.NewTicker(efc.processorSleep)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-efc.stopCh:
+			return
+		case <-ticker.C:
+			efc.processCleanupQueue()
+		}
+	}
+}
+
+// processCleanupQueue processes items from the cleanup queue
+func (efc *EmptyFolderCleaner) processCleanupQueue() {
+	// Check if we should process
+	if !efc.cleanupQueue.ShouldProcess() {
+		return
+	}
+
+	glog.V(3).Infof("EmptyFolderCleaner: processing cleanup queue (len=%d, age=%v)",
+		efc.cleanupQueue.Len(), efc.cleanupQueue.OldestAge())
+
+	// Process all items that are ready
+	for efc.cleanupQueue.Len() > 0 {
+		// Check if still enabled
+		if !efc.IsEnabled() {
+			return
+		}
+
+		// Pop the oldest item
+		folder, ok := efc.cleanupQueue.Pop()
+		if !ok {
+			break
+		}
+
+		// Execute cleanup for this folder
+		efc.executeCleanup(folder)
+
+		// If queue is no longer full and oldest item is not old enough, stop processing
+		if !efc.cleanupQueue.ShouldProcess() {
+			break
+		}
+	}
+}
+
+// executeCleanup performs the actual cleanup of an empty folder
+func (efc *EmptyFolderCleaner) executeCleanup(folder string) {
+	efc.mu.Lock()
+
+	// Quick check: if we have cached count and it's > 0, skip
+	if state, exists := efc.folderCounts[folder]; exists {
+		if state.roughCount > 0 {
+			glog.V(3).Infof("EmptyFolderCleaner: skipping %s, cached count=%d", folder, state.roughCount)
+			efc.mu.Unlock()
+			return
+		}
+		// If there was an add after our delete, skip
+		if !state.lastAddTime.IsZero() && state.lastAddTime.After(state.lastDelTime) {
+			glog.V(3).Infof("EmptyFolderCleaner: skipping %s, add happened after delete", folder)
+			efc.mu.Unlock()
+			return
+		}
+	}
+	efc.mu.Unlock()
+
+	// Re-check ownership (topology might have changed)
+	if !efc.ownsFolder(folder) {
+		glog.V(3).Infof("EmptyFolderCleaner: no longer owner of %s, skipping", folder)
+		return
+	}
+
+	// Check if folder is actually empty (count up to maxCountCheck)
+	ctx := context.Background()
+	count, err := efc.countItems(ctx, folder)
+	if err != nil {
+		glog.V(2).Infof("EmptyFolderCleaner: error counting items in %s: %v", folder, err)
+		return
+	}
+
+	efc.mu.Lock()
+	// Update cache
+	if _, exists := efc.folderCounts[folder]; !exists {
+		efc.folderCounts[folder] = &folderState{}
+	}
+	efc.folderCounts[folder].roughCount = count
+	efc.folderCounts[folder].lastCheck = time.Now()
+	efc.mu.Unlock()
+
+	if count > 0 {
+		glog.V(3).Infof("EmptyFolderCleaner: folder %s has %d items, not empty", folder, count)
+		return
+	}
+
+	// Delete the empty folder
+	glog.V(2).Infof("EmptyFolderCleaner: deleting empty folder %s", folder)
+	if err := efc.deleteFolder(ctx, folder); err != nil {
+		glog.V(2).Infof("EmptyFolderCleaner: failed to delete empty folder %s: %v", folder, err)
+		return
+	}
+
+	// Clean up cache entry
+	efc.mu.Lock()
+	delete(efc.folderCounts, folder)
+	efc.mu.Unlock()
+
+	// Note: No need to recursively check parent folder here.
+	// The deletion of this folder will generate a metadata event,
+	// which will trigger OnDeleteEvent for the parent folder.
+}
+
+// countItems counts items in a folder (up to maxCountCheck)
+func (efc *EmptyFolderCleaner) countItems(ctx context.Context, folder string) (int, error) {
+	return efc.filer.CountDirectoryEntries(ctx, util.FullPath(folder), efc.maxCountCheck)
+}
+
+// deleteFolder deletes an empty folder
+func (efc *EmptyFolderCleaner) deleteFolder(ctx context.Context, folder string) error {
+	return efc.filer.DeleteEntryMetaAndData(ctx, util.FullPath(folder), false, false, false, false, nil, 0)
+}
+
+// isUnderPath checks if child is under parent path
+func isUnderPath(child, parent string) bool {
+	if parent == "" || parent == "/" {
+		return true
+	}
+	// Ensure parent ends without slash for proper prefix matching
+	if len(parent) > 0 && parent[len(parent)-1] == '/' {
+		parent = parent[:len(parent)-1]
+	}
+	// Child must start with parent and then have a / or be exactly parent
+	if len(child) < len(parent) {
+		return false
+	}
+	if child[:len(parent)] != parent {
+		return false
+	}
+	if len(child) == len(parent) {
+		return true
+	}
+	return child[len(parent)] == '/'
+}
+
+// isUnderBucketPath checks if directory is inside a bucket (under /buckets/<bucket>/...)
+// This ensures we only clean up folders inside buckets, not the buckets themselves
+func isUnderBucketPath(directory, bucketPath string) bool {
+	if bucketPath == "" {
+		return true
+	}
+	// Ensure bucketPath ends without slash
+	if len(bucketPath) > 0 && bucketPath[len(bucketPath)-1] == '/' {
+		bucketPath = bucketPath[:len(bucketPath)-1]
+	}
+	// Directory must be under bucketPath
+	if !isUnderPath(directory, bucketPath) {
+		return false
+	}
+	// Directory must be at least /buckets/<bucket>/<something>
+	// i.e., depth must be at least bucketPath depth + 2
+	// For /buckets (depth 1), we need at least /buckets/mybucket/folder (depth 3)
+	bucketPathDepth := strings.Count(bucketPath, "/")
+	directoryDepth := strings.Count(directory, "/")
+	return directoryDepth >= bucketPathDepth+2
+}
+
+// cacheEvictionLoop periodically removes stale entries from folderCounts
+func (efc *EmptyFolderCleaner) cacheEvictionLoop() {
+	ticker := time.NewTicker(efc.cacheExpiry)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-efc.stopCh:
+			return
+		case <-ticker.C:
+			efc.evictStaleCacheEntries()
+		}
+	}
+}
+
+// evictStaleCacheEntries removes cache entries that haven't been accessed recently
+func (efc *EmptyFolderCleaner) evictStaleCacheEntries() {
+	efc.mu.Lock()
+	defer efc.mu.Unlock()
+
+	now := time.Now()
+	expiredCount := 0
+	for folder, state := range efc.folderCounts {
+		// Skip if folder is in cleanup queue
+		if efc.cleanupQueue.Contains(folder) {
+			continue
+		}
+
+		// Find the most recent activity time for this folder
+		lastActivity := state.lastCheck
+		if state.lastAddTime.After(lastActivity) {
+			lastActivity = state.lastAddTime
+		}
+		if state.lastDelTime.After(lastActivity) {
+			lastActivity = state.lastDelTime
+		}
+
+		// Evict if no activity within cache expiry period
+		if now.Sub(lastActivity) > efc.cacheExpiry {
+			delete(efc.folderCounts, folder)
+			expiredCount++
+		}
+	}
+
+	if expiredCount > 0 {
+		glog.V(3).Infof("EmptyFolderCleaner: evicted %d stale cache entries", expiredCount)
+	}
+}
+
+// Stop stops the cleaner and cancels all pending tasks
+func (efc *EmptyFolderCleaner) Stop() {
+	close(efc.stopCh)
+
+	efc.mu.Lock()
+	defer efc.mu.Unlock()
+
+	efc.enabled = false
+	efc.cleanupQueue.Clear()
+	efc.folderCounts = make(map[string]*folderState) // Clear cache on stop
+}
+
+// GetPendingCleanupCount returns the number of pending cleanup tasks (for testing)
+func (efc *EmptyFolderCleaner) GetPendingCleanupCount() int {
+	return efc.cleanupQueue.Len()
+}
+
+// GetCachedFolderCount returns the cached count for a folder (for testing)
+func (efc *EmptyFolderCleaner) GetCachedFolderCount(folder string) (int, bool) {
+	efc.mu.RLock()
+	defer efc.mu.RUnlock()
+	if state, exists := efc.folderCounts[folder]; exists {
+		return state.roughCount, true
+	}
+	return 0, false
+}
+

weed/filer/empty_folder_cleanup/empty_folder_cleaner_test.go

@@ -0,0 +1,569 @@
+package empty_folder_cleanup
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+func Test_isUnderPath(t *testing.T) {
+	tests := []struct {
+		name     string
+		child    string
+		parent   string
+		expected bool
+	}{
+		{"child under parent", "/buckets/mybucket/folder/file.txt", "/buckets", true},
+		{"child is parent", "/buckets", "/buckets", true},
+		{"child not under parent", "/other/path", "/buckets", false},
+		{"empty parent", "/any/path", "", true},
+		{"root parent", "/any/path", "/", true},
+		{"parent with trailing slash", "/buckets/mybucket", "/buckets/", true},
+		{"similar prefix but not under", "/buckets-other/file", "/buckets", false},
+		{"deeply nested", "/buckets/a/b/c/d/e/f", "/buckets/a/b", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := isUnderPath(tt.child, tt.parent)
+			if result != tt.expected {
+				t.Errorf("isUnderPath(%q, %q) = %v, want %v", tt.child, tt.parent, result, tt.expected)
+			}
+		})
+	}
+}
+
+func Test_isUnderBucketPath(t *testing.T) {
+	tests := []struct {
+		name       string
+		directory  string
+		bucketPath string
+		expected   bool
+	}{
+		// Should NOT process - bucket path itself
+		{"bucket path itself", "/buckets", "/buckets", false},
+		// Should NOT process - bucket directory (immediate child)
+		{"bucket directory", "/buckets/mybucket", "/buckets", false},
+		// Should process - folder inside bucket
+		{"folder in bucket", "/buckets/mybucket/folder", "/buckets", true},
+		// Should process - nested folder
+		{"nested folder", "/buckets/mybucket/a/b/c", "/buckets", true},
+		// Should NOT process - outside buckets
+		{"outside buckets", "/other/path", "/buckets", false},
+		// Empty bucket path allows all
+		{"empty bucket path", "/any/path", "", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := isUnderBucketPath(tt.directory, tt.bucketPath)
+			if result != tt.expected {
+				t.Errorf("isUnderBucketPath(%q, %q) = %v, want %v", tt.directory, tt.bucketPath, result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestEmptyFolderCleaner_ownsFolder(t *testing.T) {
+	// Create a LockRing with multiple servers
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+
+	servers := []pb.ServerAddress{
+		"filer1:8888",
+		"filer2:8888",
+		"filer3:8888",
+	}
+	lockRing.SetSnapshot(servers)
+
+	// Create cleaner for filer1
+	cleaner1 := &EmptyFolderCleaner{
+		lockRing: lockRing,
+		host:     "filer1:8888",
+	}
+
+	// Create cleaner for filer2
+	cleaner2 := &EmptyFolderCleaner{
+		lockRing: lockRing,
+		host:     "filer2:8888",
+	}
+
+	// Create cleaner for filer3
+	cleaner3 := &EmptyFolderCleaner{
+		lockRing: lockRing,
+		host:     "filer3:8888",
+	}
+
+	// Test that exactly one filer owns each folder
+	testFolders := []string{
+		"/buckets/mybucket/folder1",
+		"/buckets/mybucket/folder2",
+		"/buckets/mybucket/folder3",
+		"/buckets/mybucket/a/b/c",
+		"/buckets/otherbucket/x",
+	}
+
+	for _, folder := range testFolders {
+		ownCount := 0
+		if cleaner1.ownsFolder(folder) {
+			ownCount++
+		}
+		if cleaner2.ownsFolder(folder) {
+			ownCount++
+		}
+		if cleaner3.ownsFolder(folder) {
+			ownCount++
+		}
+
+		if ownCount != 1 {
+			t.Errorf("folder %q owned by %d filers, expected exactly 1", folder, ownCount)
+		}
+	}
+}
+
+func TestEmptyFolderCleaner_ownsFolder_singleServer(t *testing.T) {
+	// Create a LockRing with a single server
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+	cleaner := &EmptyFolderCleaner{
+		lockRing: lockRing,
+		host:     "filer1:8888",
+	}
+
+	// Single filer should own all folders
+	testFolders := []string{
+		"/buckets/mybucket/folder1",
+		"/buckets/mybucket/folder2",
+		"/buckets/otherbucket/x",
+	}
+
+	for _, folder := range testFolders {
+		if !cleaner.ownsFolder(folder) {
+			t.Errorf("single filer should own folder %q", folder)
+		}
+	}
+}
+
+func TestEmptyFolderCleaner_ownsFolder_emptyRing(t *testing.T) {
+	// Create an empty LockRing
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+
+	cleaner := &EmptyFolderCleaner{
+		lockRing: lockRing,
+		host:     "filer1:8888",
+	}
+
+	// With empty ring, should own all folders
+	if !cleaner.ownsFolder("/buckets/mybucket/folder") {
+		t.Error("should own folder with empty ring")
+	}
+}
+
+func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanup(t *testing.T) {
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+	cleaner := &EmptyFolderCleaner{
+		lockRing:     lockRing,
+		host:         "filer1:8888",
+		bucketPath:   "/buckets",
+		enabled:      true,
+		folderCounts: make(map[string]*folderState),
+		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+		stopCh:       make(chan struct{}),
+	}
+
+	folder := "/buckets/mybucket/testfolder"
+	now := time.Now()
+
+	// Simulate delete event
+	cleaner.OnDeleteEvent(folder, "file.txt", false, now)
+
+	// Check that cleanup is queued
+	if cleaner.GetPendingCleanupCount() != 1 {
+		t.Errorf("expected 1 pending cleanup, got %d", cleaner.GetPendingCleanupCount())
+	}
+
+	// Simulate create event
+	cleaner.OnCreateEvent(folder, "newfile.txt", false)
+
+	// Check that cleanup is cancelled
+	if cleaner.GetPendingCleanupCount() != 0 {
+		t.Errorf("expected 0 pending cleanups after create, got %d", cleaner.GetPendingCleanupCount())
+	}
+
+	cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_OnDeleteEvent_deduplication(t *testing.T) {
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+	cleaner := &EmptyFolderCleaner{
+		lockRing:     lockRing,
+		host:         "filer1:8888",
+		bucketPath:   "/buckets",
+		enabled:      true,
+		folderCounts: make(map[string]*folderState),
+		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+		stopCh:       make(chan struct{}),
+	}
+
+	folder := "/buckets/mybucket/testfolder"
+	now := time.Now()
+
+	// Simulate multiple delete events for same folder
+	for i := 0; i < 5; i++ {
+		cleaner.OnDeleteEvent(folder, "file"+string(rune('0'+i))+".txt", false, now.Add(time.Duration(i)*time.Second))
+	}
+
+	// Check that only 1 cleanup is queued (deduplicated)
+	if cleaner.GetPendingCleanupCount() != 1 {
+		t.Errorf("expected 1 pending cleanup after deduplication, got %d", cleaner.GetPendingCleanupCount())
+	}
+
+	cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_OnDeleteEvent_multipleFolders(t *testing.T) {
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+	cleaner := &EmptyFolderCleaner{
+		lockRing:     lockRing,
+		host:         "filer1:8888",
+		bucketPath:   "/buckets",
+		enabled:      true,
+		folderCounts: make(map[string]*folderState),
+		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+		stopCh:       make(chan struct{}),
+	}
+
+	now := time.Now()
+
+	// Delete files in different folders
+	cleaner.OnDeleteEvent("/buckets/mybucket/folder1", "file.txt", false, now)
+	cleaner.OnDeleteEvent("/buckets/mybucket/folder2", "file.txt", false, now.Add(1*time.Second))
+	cleaner.OnDeleteEvent("/buckets/mybucket/folder3", "file.txt", false, now.Add(2*time.Second))
+
+	// Each folder should be queued
+	if cleaner.GetPendingCleanupCount() != 3 {
+		t.Errorf("expected 3 pending cleanups, got %d", cleaner.GetPendingCleanupCount())
+	}
+
+	cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_OnDeleteEvent_notOwner(t *testing.T) {
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888", "filer2:8888"})
+
+	// Create cleaner for filer that doesn't own the folder
+	cleaner := &EmptyFolderCleaner{
+		lockRing:     lockRing,
+		host:         "filer1:8888",
+		bucketPath:   "/buckets",
+		enabled:      true,
+		folderCounts: make(map[string]*folderState),
+		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+		stopCh:       make(chan struct{}),
+	}
+
+	now := time.Now()
+
+	// Try many folders, looking for one that filer1 doesn't own
+	foundNonOwned := false
+	for i := 0; i < 100; i++ {
+		folder := "/buckets/mybucket/folder" + string(rune('0'+i%10)) + string(rune('0'+i/10))
+		if !cleaner.ownsFolder(folder) {
+			// This folder is not owned by filer1
+			cleaner.OnDeleteEvent(folder, "file.txt", false, now)
+			if cleaner.GetPendingCleanupCount() != 0 {
+				t.Errorf("non-owner should not queue cleanup for folder %s", folder)
+			}
+			foundNonOwned = true
+			break
+		}
+	}
+
+	if !foundNonOwned {
+		t.Skip("could not find a folder not owned by filer1")
+	}
+
+	cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_OnDeleteEvent_disabled(t *testing.T) {
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+	cleaner := &EmptyFolderCleaner{
+		lockRing:     lockRing,
+		host:         "filer1:8888",
+		bucketPath:   "/buckets",
+		enabled:      false, // Disabled
+		folderCounts: make(map[string]*folderState),
+		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+		stopCh:       make(chan struct{}),
+	}
+
+	folder := "/buckets/mybucket/testfolder"
+	now := time.Now()
+
+	// Simulate delete event
+	cleaner.OnDeleteEvent(folder, "file.txt", false, now)
+
+	// Check that no cleanup is queued when disabled
+	if cleaner.GetPendingCleanupCount() != 0 {
+		t.Errorf("disabled cleaner should not queue cleanup, got %d", cleaner.GetPendingCleanupCount())
+	}
+
+	cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_OnDeleteEvent_directoryDeletion(t *testing.T) {
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+	cleaner := &EmptyFolderCleaner{
+		lockRing:     lockRing,
+		host:         "filer1:8888",
+		bucketPath:   "/buckets",
+		enabled:      true,
+		folderCounts: make(map[string]*folderState),
+		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+		stopCh:       make(chan struct{}),
+	}
+
+	folder := "/buckets/mybucket/testfolder"
+	now := time.Now()
+
+	// Simulate directory delete event - should trigger cleanup
+	// because subdirectory deletion also makes parent potentially empty
+	cleaner.OnDeleteEvent(folder, "subdir", true, now)
+
+	// Check that cleanup IS queued for directory deletion
+	if cleaner.GetPendingCleanupCount() != 1 {
+		t.Errorf("directory deletion should trigger cleanup, got %d", cleaner.GetPendingCleanupCount())
+	}
+
+	cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_cachedCounts(t *testing.T) {
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+	cleaner := &EmptyFolderCleaner{
+		lockRing:     lockRing,
+		host:         "filer1:8888",
+		bucketPath:   "/buckets",
+		enabled:      true,
+		folderCounts: make(map[string]*folderState),
+		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+		stopCh:       make(chan struct{}),
+	}
+
+	folder := "/buckets/mybucket/testfolder"
+
+	// Initialize cached count
+	cleaner.folderCounts[folder] = &folderState{roughCount: 5}
+
+	// Simulate create events
+	cleaner.OnCreateEvent(folder, "newfile1.txt", false)
+	cleaner.OnCreateEvent(folder, "newfile2.txt", false)
+
+	// Check cached count increased
+	count, exists := cleaner.GetCachedFolderCount(folder)
+	if !exists {
+		t.Error("cached folder count should exist")
+	}
+	if count != 7 {
+		t.Errorf("expected cached count 7, got %d", count)
+	}
+
+	// Simulate delete events
+	now := time.Now()
+	cleaner.OnDeleteEvent(folder, "file1.txt", false, now)
+	cleaner.OnDeleteEvent(folder, "file2.txt", false, now.Add(1*time.Second))
+
+	// Check cached count decreased
+	count, exists = cleaner.GetCachedFolderCount(folder)
+	if !exists {
+		t.Error("cached folder count should exist")
+	}
+	if count != 5 {
+		t.Errorf("expected cached count 5, got %d", count)
+	}
+
+	cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_Stop(t *testing.T) {
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+	cleaner := &EmptyFolderCleaner{
+		lockRing:     lockRing,
+		host:         "filer1:8888",
+		bucketPath:   "/buckets",
+		enabled:      true,
+		folderCounts: make(map[string]*folderState),
+		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+		stopCh:       make(chan struct{}),
+	}
+
+	now := time.Now()
+
+	// Queue some cleanups
+	cleaner.OnDeleteEvent("/buckets/mybucket/folder1", "file1.txt", false, now)
+	cleaner.OnDeleteEvent("/buckets/mybucket/folder2", "file2.txt", false, now.Add(1*time.Second))
+	cleaner.OnDeleteEvent("/buckets/mybucket/folder3", "file3.txt", false, now.Add(2*time.Second))
+
+	// Verify cleanups are queued
+	if cleaner.GetPendingCleanupCount() < 1 {
+		t.Error("expected at least 1 pending cleanup before stop")
+	}
+
+	// Stop the cleaner
+	cleaner.Stop()
+
+	// Verify all cleanups are cancelled
+	if cleaner.GetPendingCleanupCount() != 0 {
+		t.Errorf("expected 0 pending cleanups after stop, got %d", cleaner.GetPendingCleanupCount())
+	}
+}
+
+func TestEmptyFolderCleaner_cacheEviction(t *testing.T) {
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+	cleaner := &EmptyFolderCleaner{
+		lockRing:     lockRing,
+		host:         "filer1:8888",
+		bucketPath:   "/buckets",
+		enabled:      true,
+		folderCounts: make(map[string]*folderState),
+		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+		cacheExpiry:  100 * time.Millisecond, // Short expiry for testing
+		stopCh:       make(chan struct{}),
+	}
+
+	folder1 := "/buckets/mybucket/folder1"
+	folder2 := "/buckets/mybucket/folder2"
+	folder3 := "/buckets/mybucket/folder3"
+
+	// Add some cache entries with old timestamps
+	oldTime := time.Now().Add(-1 * time.Hour)
+	cleaner.folderCounts[folder1] = &folderState{roughCount: 5, lastCheck: oldTime}
+	cleaner.folderCounts[folder2] = &folderState{roughCount: 3, lastCheck: oldTime}
+	// folder3 has recent activity
+	cleaner.folderCounts[folder3] = &folderState{roughCount: 2, lastCheck: time.Now()}
+
+	// Verify all entries exist
+	if len(cleaner.folderCounts) != 3 {
+		t.Errorf("expected 3 cache entries, got %d", len(cleaner.folderCounts))
+	}
+
+	// Run eviction
+	cleaner.evictStaleCacheEntries()
+
+	// Verify stale entries are evicted
+	if len(cleaner.folderCounts) != 1 {
+		t.Errorf("expected 1 cache entry after eviction, got %d", len(cleaner.folderCounts))
+	}
+
+	// Verify the recent entry still exists
+	if _, exists := cleaner.folderCounts[folder3]; !exists {
+		t.Error("expected folder3 to still exist in cache")
+	}
+
+	// Verify stale entries are removed
+	if _, exists := cleaner.folderCounts[folder1]; exists {
+		t.Error("expected folder1 to be evicted")
+	}
+	if _, exists := cleaner.folderCounts[folder2]; exists {
+		t.Error("expected folder2 to be evicted")
+	}
+
+	cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_cacheEviction_skipsEntriesInQueue(t *testing.T) {
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+	cleaner := &EmptyFolderCleaner{
+		lockRing:     lockRing,
+		host:         "filer1:8888",
+		bucketPath:   "/buckets",
+		enabled:      true,
+		folderCounts: make(map[string]*folderState),
+		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+		cacheExpiry:  100 * time.Millisecond,
+		stopCh:       make(chan struct{}),
+	}
+
+	folder := "/buckets/mybucket/folder"
+	oldTime := time.Now().Add(-1 * time.Hour)
+
+	// Add a stale cache entry
+	cleaner.folderCounts[folder] = &folderState{roughCount: 0, lastCheck: oldTime}
+	// Also add to cleanup queue
+	cleaner.cleanupQueue.Add(folder, time.Now())
+
+	// Run eviction
+	cleaner.evictStaleCacheEntries()
+
+	// Verify entry is NOT evicted because it's in cleanup queue
+	if _, exists := cleaner.folderCounts[folder]; !exists {
+		t.Error("expected folder to still exist in cache (is in cleanup queue)")
+	}
+
+	cleaner.Stop()
+}
+
+func TestEmptyFolderCleaner_queueFIFOOrder(t *testing.T) {
+	lockRing := lock_manager.NewLockRing(5 * time.Second)
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+
+	cleaner := &EmptyFolderCleaner{
+		lockRing:     lockRing,
+		host:         "filer1:8888",
+		bucketPath:   "/buckets",
+		enabled:      true,
+		folderCounts: make(map[string]*folderState),
+		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
+		stopCh:       make(chan struct{}),
+	}
+
+	now := time.Now()
+
+	// Add folders in order
+	folders := []string{
+		"/buckets/mybucket/folder1",
+		"/buckets/mybucket/folder2",
+		"/buckets/mybucket/folder3",
+	}
+	for i, folder := range folders {
+		cleaner.OnDeleteEvent(folder, "file.txt", false, now.Add(time.Duration(i)*time.Second))
+	}
+
+	// Verify queue length
+	if cleaner.GetPendingCleanupCount() != 3 {
+		t.Errorf("expected 3 queued folders, got %d", cleaner.GetPendingCleanupCount())
+	}
+
+	// Verify time-sorted order by popping
+	for i, expected := range folders {
+		folder, ok := cleaner.cleanupQueue.Pop()
+		if !ok || folder != expected {
+			t.Errorf("expected folder %s at index %d, got %s", expected, i, folder)
+		}
+	}
+
+	cleaner.Stop()
+}
+

weed/filer/filer.go

@@ -11,6 +11,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/filer/empty_folder_cleanup"
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -56,6 +57,7 @@ type Filer struct {
 	MaxFilenameLength   uint32
 	deletionQuit        chan struct{}
 	DeletionRetryQueue  *DeletionRetryQueue
+	EmptyFolderCleaner  *empty_folder_cleanup.EmptyFolderCleaner
 }
 
 func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer {
@@ -116,6 +118,9 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
 	f.Dlm.LockRing.SetSnapshot(snapshot)
 	glog.V(0).Infof("%s aggregate from peers %+v", self, snapshot)
 
+	// Initialize the empty folder cleaner using the same LockRing as Dlm for consistent hashing
+	f.EmptyFolderCleaner = empty_folder_cleanup.NewEmptyFolderCleaner(f, f.Dlm.LockRing, self, f.DirBucketsPath)
+
 	f.MetaAggregator = NewMetaAggregator(f, self, f.GrpcDialOption)
 	f.MasterClient.SetOnPeerUpdateFn(func(update *master_pb.ClusterNodeUpdate, startFrom time.Time) {
 		if update.NodeType != cluster.FilerType {
@@ -506,6 +511,9 @@ func (f *Filer) IsDirectoryEmpty(ctx context.Context, dirPath util.FullPath) (bo
 
 func (f *Filer) Shutdown() {
 	close(f.deletionQuit)
+	if f.EmptyFolderCleaner != nil {
+		f.EmptyFolderCleaner.Stop()
+	}
 	f.LocalMetaLogBuffer.ShutdownLogBuffer()
 	f.Store.Shutdown()
 }

weed/filer/filer_notify.go

@@ -66,6 +66,10 @@ func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry
 
 	f.logMetaEvent(ctx, fullpath, eventNotification)
 
+	// Trigger empty folder cleanup for local events
+	// Remote events are handled via MetaAggregator.onMetadataChangeEvent
+	f.triggerLocalEmptyFolderCleanup(oldEntry, newEntry)
+
 }
 
 func (f *Filer) logMetaEvent(ctx context.Context, fullpath string, eventNotification *filer_pb.EventNotification) {
@@ -89,6 +93,41 @@ func (f *Filer) logMetaEvent(ctx context.Context, fullpath string, eventNotifica
 
 }
 
+// triggerLocalEmptyFolderCleanup triggers empty folder cleanup for local events
+// This is needed because onMetadataChangeEvent is only called for remote peer events
+func (f *Filer) triggerLocalEmptyFolderCleanup(oldEntry, newEntry *Entry) {
+	if f.EmptyFolderCleaner == nil || !f.EmptyFolderCleaner.IsEnabled() {
+		return
+	}
+
+	eventTime := time.Now()
+
+	// Handle delete events (oldEntry exists, newEntry is nil)
+	if oldEntry != nil && newEntry == nil {
+		dir, name := oldEntry.FullPath.DirAndName()
+		f.EmptyFolderCleaner.OnDeleteEvent(dir, name, oldEntry.IsDirectory(), eventTime)
+	}
+
+	// Handle create events (oldEntry is nil, newEntry exists)
+	if oldEntry == nil && newEntry != nil {
+		dir, name := newEntry.FullPath.DirAndName()
+		f.EmptyFolderCleaner.OnCreateEvent(dir, name, newEntry.IsDirectory())
+	}
+
+	// Handle rename/move events (both exist but paths differ)
+	if oldEntry != nil && newEntry != nil {
+		oldDir, oldName := oldEntry.FullPath.DirAndName()
+		newDir, newName := newEntry.FullPath.DirAndName()
+
+		if oldDir != newDir || oldName != newName {
+			// Treat old location as delete
+			f.EmptyFolderCleaner.OnDeleteEvent(oldDir, oldName, oldEntry.IsDirectory(), eventTime)
+			// Treat new location as create
+			f.EmptyFolderCleaner.OnCreateEvent(newDir, newName, newEntry.IsDirectory())
+		}
+	}
+}
+
 func (f *Filer) logFlushFunc(logBuffer *log_buffer.LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
 
 	if len(buf) == 0 {

weed/filer/filer_on_meta_event.go

@@ -2,6 +2,7 @@ package filer
 
 import (
 	"bytes"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -13,6 +14,7 @@ func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse)
 	f.maybeReloadFilerConfiguration(event)
 	f.maybeReloadRemoteStorageConfigurationAndMapping(event)
 	f.onBucketEvents(event)
+	f.onEmptyFolderCleanupEvents(event)
 }
 
 func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
@@ -32,6 +34,43 @@ func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
 	}
 }
 
+// onEmptyFolderCleanupEvents handles create/delete events for empty folder cleanup
+func (f *Filer) onEmptyFolderCleanupEvents(event *filer_pb.SubscribeMetadataResponse) {
+	if f.EmptyFolderCleaner == nil || !f.EmptyFolderCleaner.IsEnabled() {
+		return
+	}
+
+	message := event.EventNotification
+	directory := event.Directory
+	eventTime := time.Unix(0, event.TsNs)
+
+	// Handle delete events - trigger folder cleanup check
+	if filer_pb.IsDelete(event) && message.OldEntry != nil {
+		f.EmptyFolderCleaner.OnDeleteEvent(directory, message.OldEntry.Name, message.OldEntry.IsDirectory, eventTime)
+	}
+
+	// Handle create events - cancel pending cleanup for the folder
+	if filer_pb.IsCreate(event) && message.NewEntry != nil {
+		f.EmptyFolderCleaner.OnCreateEvent(directory, message.NewEntry.Name, message.NewEntry.IsDirectory)
+	}
+
+	// Handle rename/move events
+	if filer_pb.IsRename(event) {
+		// Treat the old location as a delete
+		if message.OldEntry != nil {
+			f.EmptyFolderCleaner.OnDeleteEvent(directory, message.OldEntry.Name, message.OldEntry.IsDirectory, eventTime)
+		}
+		// Treat the new location as a create
+		if message.NewEntry != nil {
+			newDir := message.NewParentPath
+			if newDir == "" {
+				newDir = directory
+			}
+			f.EmptyFolderCleaner.OnCreateEvent(newDir, message.NewEntry.Name, message.NewEntry.IsDirectory)
+		}
+	}
+}
+
 func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataResponse) {
 	if DirectoryEtcSeaweedFS != event.Directory {
 		if DirectoryEtcSeaweedFS != event.EventNotification.NewParentPath {

weed/filer/filer_search.go

@@ -41,6 +41,19 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start
 	return entries, hasMore, err
 }
 
+// CountDirectoryEntries counts entries in a directory up to limit
+func (f *Filer) CountDirectoryEntries(ctx context.Context, p util.FullPath, limit int) (count int, err error) {
+	entries, hasMore, err := f.ListDirectoryEntries(ctx, p, "", false, int64(limit), "", "", "")
+	if err != nil {
+		return 0, err
+	}
+	count = len(entries)
+	if hasMore {
+		count = limit // At least this many
+	}
+	return count, nil
+}
+
 // For now, prefix and namePattern are mutually exclusive
 func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
 	if strings.HasSuffix(string(p), "/") && len(p) > 1 {

weed/filer/reader_at.go

@@ -7,6 +7,8 @@ import (
 	"math/rand"
 	"sync"
 
+	"golang.org/x/sync/errgroup"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -19,6 +21,11 @@ import (
 // the prefetch count is derived from the -concurrentReaders option.
 const DefaultPrefetchCount = 4
 
+// minReadConcurrency is the minimum number of parallel chunk fetches.
+// This ensures at least some parallelism even when prefetchCount is low,
+// improving throughput for reads spanning multiple chunks.
+const minReadConcurrency = 4
+
 type ChunkReadAt struct {
 	masterClient  *wdclient.MasterClient
 	chunkViews    *IntervalList[*ChunkView]
@@ -175,67 +182,139 @@ func (c *ChunkReadAt) ReadAtWithTime(ctx context.Context, p []byte, offset int64
 	return c.doReadAt(ctx, p, offset)
 }
 
+// chunkReadTask represents a single chunk read operation for parallel processing
+type chunkReadTask struct {
+	chunk        *ChunkView
+	bufferStart  int64  // start position in the output buffer
+	bufferEnd    int64  // end position in the output buffer
+	chunkOffset  uint64 // offset within the chunk to read from
+	bytesRead    int
+	modifiedTsNs int64
+}
+
 func (c *ChunkReadAt) doReadAt(ctx context.Context, p []byte, offset int64) (n int, ts int64, err error) {
 
+	// Collect all chunk read tasks
+	var tasks []*chunkReadTask
+	var gaps []struct{ start, length int64 } // gaps that need zero-filling
+
 	startOffset, remaining := offset, int64(len(p))
-	var nextChunks *Interval[*ChunkView]
+	var lastChunk *Interval[*ChunkView]
+
 	for x := c.chunkViews.Front(); x != nil; x = x.Next {
 		chunk := x.Value
 		if remaining <= 0 {
 			break
 		}
-		if x.Next != nil {
-			nextChunks = x.Next
-		}
+		lastChunk = x
+
+		// Handle gap before this chunk
 		if startOffset < chunk.ViewOffset {
 			gap := chunk.ViewOffset - startOffset
-			glog.V(4).Infof("zero [%d,%d)", startOffset, chunk.ViewOffset)
-			n += zero(p, startOffset-offset, gap)
+			gaps = append(gaps, struct{ start, length int64 }{startOffset - offset, gap})
 			startOffset, remaining = chunk.ViewOffset, remaining-gap
 			if remaining <= 0 {
 				break
 			}
 		}
-		// fmt.Printf(">>> doReadAt [%d,%d), chunk[%d,%d)\n", offset, offset+int64(len(p)), chunk.ViewOffset, chunk.ViewOffset+int64(chunk.ViewSize))
+
 		chunkStart, chunkStop := max(chunk.ViewOffset, startOffset), min(chunk.ViewOffset+int64(chunk.ViewSize), startOffset+remaining)
 		if chunkStart >= chunkStop {
 			continue
 		}
-		// glog.V(4).Infof("read [%d,%d), %d/%d chunk %s [%d,%d)", chunkStart, chunkStop, i, len(c.chunkViews), chunk.FileId, chunk.ViewOffset-chunk.Offset, chunk.ViewOffset-chunk.Offset+int64(chunk.ViewSize))
-		bufferOffset := chunkStart - chunk.ViewOffset + chunk.OffsetInChunk
-		ts = chunk.ModifiedTsNs
-		copied, err := c.readChunkSliceAt(ctx, p[startOffset-offset:chunkStop-chunkStart+startOffset-offset], chunk, nextChunks, uint64(bufferOffset))
-		if err != nil {
-			glog.Errorf("fetching chunk %+v: %v\n", chunk, err)
-			return copied, ts, err
-		}
 
-		n += copied
-		startOffset, remaining = startOffset+int64(copied), remaining-int64(copied)
+		bufferOffset := chunkStart - chunk.ViewOffset + chunk.OffsetInChunk
+		tasks = append(tasks, &chunkReadTask{
+			chunk:       chunk,
+			bufferStart: startOffset - offset,
+			bufferEnd:   chunkStop - chunkStart + startOffset - offset,
+			chunkOffset: uint64(bufferOffset),
+		})
+
+		startOffset, remaining = chunkStop, remaining-(chunkStop-chunkStart)
 	}
 
-	// glog.V(4).Infof("doReadAt [%d,%d), n:%v, err:%v", offset, offset+int64(len(p)), n, err)
+	// Zero-fill gaps
+	for _, gap := range gaps {
+		glog.V(4).Infof("zero [%d,%d)", offset+gap.start, offset+gap.start+gap.length)
+		n += zero(p, gap.start, gap.length)
+	}
 
-	// zero the remaining bytes if a gap exists at the end of the last chunk (or a fully sparse file)
-	if err == nil && remaining > 0 {
+	// If only one chunk or random access mode, use sequential reading
+	if len(tasks) <= 1 || c.readerPattern.IsRandomMode() {
+		for _, task := range tasks {
+			copied, readErr := c.readChunkSliceAt(ctx, p[task.bufferStart:task.bufferEnd], task.chunk, nil, task.chunkOffset)
+			ts = max(ts, task.chunk.ModifiedTsNs)
+			if readErr != nil {
+				glog.Errorf("fetching chunk %+v: %v\n", task.chunk, readErr)
+				return n + copied, ts, readErr
+			}
+			n += copied
+		}
+	} else {
+		// Parallel chunk fetching for multiple chunks
+		// This significantly improves throughput when chunks are on different volume servers
+		g, gCtx := errgroup.WithContext(ctx)
+
+		// Limit concurrency to avoid overwhelming the system
+		concurrency := c.prefetchCount
+		if concurrency < minReadConcurrency {
+			concurrency = minReadConcurrency
+		}
+		if concurrency > len(tasks) {
+			concurrency = len(tasks)
+		}
+		g.SetLimit(concurrency)
+
+		for _, task := range tasks {
+			g.Go(func() error {
+				// Read directly into the correct position in the output buffer
+				copied, readErr := c.readChunkSliceAtForParallel(gCtx, p[task.bufferStart:task.bufferEnd], task.chunk, task.chunkOffset)
+				task.bytesRead = copied
+				task.modifiedTsNs = task.chunk.ModifiedTsNs
+				return readErr
+			})
+		}
+
+		// Wait for all chunk reads to complete
+		if waitErr := g.Wait(); waitErr != nil {
+			err = waitErr
+		}
+
+		// Aggregate results (order is preserved since we read directly into buffer positions)
+		for _, task := range tasks {
+			n += task.bytesRead
+			ts = max(ts, task.modifiedTsNs)
+		}
+
+		if err != nil {
+			return n, ts, err
+		}
+	}
+
+	// Trigger prefetch for sequential reads
+	if lastChunk != nil && lastChunk.Next != nil && c.prefetchCount > 0 && !c.readerPattern.IsRandomMode() {
+		c.readerCache.MaybeCache(lastChunk.Next, c.prefetchCount)
+	}
+
+	// Zero the remaining bytes if a gap exists at the end
+	if remaining > 0 {
 		var delta int64
 		if c.fileSize >= startOffset {
 			delta = min(remaining, c.fileSize-startOffset)
-			startOffset -= offset
-		}
-		if delta > 0 {
-			glog.V(4).Infof("zero2 [%d,%d) of file size %d bytes", startOffset, startOffset+delta, c.fileSize)
-			n += zero(p, startOffset, delta)
+			bufStart := startOffset - offset
+			if delta > 0 {
+				glog.V(4).Infof("zero2 [%d,%d) of file size %d bytes", startOffset, startOffset+delta, c.fileSize)
+				n += zero(p, bufStart, delta)
+			}
 		}
 	}
 
 	if err == nil && offset+int64(len(p)) >= c.fileSize {
 		err = io.EOF
 	}
-	// fmt.Printf("~~~ filled %d, err: %v\n\n", n, err)
 
 	return
-
 }
 
 func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunkView *ChunkView, nextChunkViews *Interval[*ChunkView], offset uint64) (n int, err error) {
@@ -249,7 +328,7 @@ func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunk
 	}
 
 	shouldCache := (uint64(chunkView.ViewOffset) + chunkView.ChunkSize) <= c.readerCache.chunkCache.GetMaxFilePartSizeInCache()
-	n, err = c.readerCache.ReadChunkAt(buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), shouldCache)
+	n, err = c.readerCache.ReadChunkAt(ctx, buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), shouldCache)
 	if c.lastChunkFid != chunkView.FileId {
 		if chunkView.OffsetInChunk == 0 { // start of a new chunk
 			if c.lastChunkFid != "" {
@@ -266,6 +345,13 @@ func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunk
 	return
 }
 
+// readChunkSliceAtForParallel is a simplified version for parallel chunk fetching
+// It doesn't update lastChunkFid or trigger prefetch (handled by the caller)
+func (c *ChunkReadAt) readChunkSliceAtForParallel(ctx context.Context, buffer []byte, chunkView *ChunkView, offset uint64) (n int, err error) {
+	shouldCache := (uint64(chunkView.ViewOffset) + chunkView.ChunkSize) <= c.readerCache.chunkCache.GetMaxFilePartSizeInCache()
+	return c.readerCache.ReadChunkAt(ctx, buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), shouldCache)
+}
+
 func zero(buffer []byte, start, length int64) int {
 	if length <= 0 {
 		return 0

weed/filer/reader_cache.go

@@ -35,6 +35,7 @@ type SingleChunkCacher struct {
 	shouldCache    bool
 	wg             sync.WaitGroup
 	cacheStartedCh chan struct{}
+	done           chan struct{} // signals when download is complete
 }
 
 func NewReaderCache(limit int, chunkCache chunk_cache.ChunkCache, lookupFileIdFn wdclient.LookupFileIdFunctionType) *ReaderCache {
@@ -93,14 +94,18 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView], count int) {
 	return
 }
 
-func (rc *ReaderCache) ReadChunkAt(buffer []byte, fileId string, cipherKey []byte, isGzipped bool, offset int64, chunkSize int, shouldCache bool) (int, error) {
+func (rc *ReaderCache) ReadChunkAt(ctx context.Context, buffer []byte, fileId string, cipherKey []byte, isGzipped bool, offset int64, chunkSize int, shouldCache bool) (int, error) {
 	rc.Lock()
 
 	if cacher, found := rc.downloaders[fileId]; found {
-		if n, err := cacher.readChunkAt(buffer, offset); n != 0 && err == nil {
-			rc.Unlock()
+		rc.Unlock()
+		n, err := cacher.readChunkAt(ctx, buffer, offset)
+		if n > 0 || err != nil {
 			return n, err
 		}
+		// If n=0 and err=nil, the cacher couldn't provide data for this offset.
+		// Fall through to try chunkCache.
+		rc.Lock()
 	}
 	if shouldCache || rc.lookupFileIdFn == nil {
 		n, err := rc.chunkCache.ReadChunkAt(buffer, fileId, uint64(offset))
@@ -134,7 +139,7 @@ func (rc *ReaderCache) ReadChunkAt(buffer []byte, fileId string, cipherKey []byt
 	rc.downloaders[fileId] = cacher
 	rc.Unlock()
 
-	return cacher.readChunkAt(buffer, offset)
+	return cacher.readChunkAt(ctx, buffer, offset)
 }
 
 func (rc *ReaderCache) UnCache(fileId string) {
@@ -166,38 +171,53 @@ func newSingleChunkCacher(parent *ReaderCache, fileId string, cipherKey []byte,
 		chunkSize:      chunkSize,
 		shouldCache:    shouldCache,
 		cacheStartedCh: make(chan struct{}),
+		done:           make(chan struct{}),
 	}
 }
 
+// startCaching downloads the chunk data in the background.
+// It does NOT hold the lock during the HTTP download to allow concurrent readers
+// to wait efficiently using the done channel.
 func (s *SingleChunkCacher) startCaching() {
 	s.wg.Add(1)
 	defer s.wg.Done()
-	s.Lock()
-	defer s.Unlock()
+	defer close(s.done) // guarantee completion signal even on panic
 
-	s.cacheStartedCh <- struct{}{} // means this has been started
+	s.cacheStartedCh <- struct{}{} // signal that we've started
 
+	// Note: We intentionally use context.Background() here, NOT a request-specific context.
+	// The downloaded chunk is a shared resource - multiple concurrent readers may be waiting
+	// for this same download to complete. If we used a request context and that request was
+	// cancelled, it would abort the download and cause errors for all other waiting readers.
+	// The download should always complete once started to serve all potential consumers.
+
+	// Lookup file ID without holding the lock
 	urlStrings, err := s.parent.lookupFileIdFn(context.Background(), s.chunkFileId)
 	if err != nil {
+		s.Lock()
 		s.err = fmt.Errorf("operation LookupFileId %s failed, err: %v", s.chunkFileId, err)
+		s.Unlock()
 		return
 	}
 
-	s.data = mem.Allocate(s.chunkSize)
+	// Allocate buffer and download without holding the lock
+	// This allows multiple downloads to proceed in parallel
+	data := mem.Allocate(s.chunkSize)
+	_, fetchErr := util_http.RetriedFetchChunkData(context.Background(), data, urlStrings, s.cipherKey, s.isGzipped, true, 0, s.chunkFileId)
 
-	_, s.err = util_http.RetriedFetchChunkData(context.Background(), s.data, urlStrings, s.cipherKey, s.isGzipped, true, 0, s.chunkFileId)
-	if s.err != nil {
-		mem.Free(s.data)
-		s.data = nil
-		return
+	// Now acquire lock to update state
+	s.Lock()
+	if fetchErr != nil {
+		mem.Free(data)
+		s.err = fetchErr
+	} else {
+		s.data = data
+		if s.shouldCache {
+			s.parent.chunkCache.SetChunk(s.chunkFileId, s.data)
+		}
+		atomic.StoreInt64(&s.completedTimeNew, time.Now().UnixNano())
 	}
-
-	if s.shouldCache {
-		s.parent.chunkCache.SetChunk(s.chunkFileId, s.data)
-	}
-	atomic.StoreInt64(&s.completedTimeNew, time.Now().UnixNano())
-
-	return
+	s.Unlock()
 }
 
 func (s *SingleChunkCacher) destroy() {
@@ -209,13 +229,34 @@ func (s *SingleChunkCacher) destroy() {
 	if s.data != nil {
 		mem.Free(s.data)
 		s.data = nil
-		close(s.cacheStartedCh)
 	}
 }
 
-func (s *SingleChunkCacher) readChunkAt(buf []byte, offset int64) (int, error) {
+// readChunkAt reads data from the cached chunk.
+// It waits for the download to complete if it's still in progress.
+// The ctx parameter allows the reader to cancel its wait (but the download continues
+// for other readers - see comment in startCaching about shared resource semantics).
+func (s *SingleChunkCacher) readChunkAt(ctx context.Context, buf []byte, offset int64) (int, error) {
 	s.wg.Add(1)
 	defer s.wg.Done()
+
+	// Wait for download to complete, but allow reader cancellation.
+	// Prioritize checking done first - if data is already available,
+	// return it even if context is also cancelled.
+	select {
+	case <-s.done:
+		// Download already completed, proceed immediately
+	default:
+		// Download not complete, wait for it or context cancellation
+		select {
+		case <-s.done:
+			// Download completed
+		case <-ctx.Done():
+			// Reader cancelled while waiting - download continues for other readers
+			return 0, ctx.Err()
+		}
+	}
+
 	s.Lock()
 	defer s.Unlock()
 
@@ -228,5 +269,4 @@ func (s *SingleChunkCacher) readChunkAt(buf []byte, offset int64) (int, error) {
 	}
 
 	return copy(buf, s.data[offset:]), nil
-
 }

weed/filer/reader_cache_test.go

@@ -0,0 +1,505 @@
+package filer
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// mockChunkCacheForReaderCache implements chunk cache for testing
+type mockChunkCacheForReaderCache struct {
+	data     map[string][]byte
+	hitCount int32
+	mu       sync.Mutex
+}
+
+func newMockChunkCacheForReaderCache() *mockChunkCacheForReaderCache {
+	return &mockChunkCacheForReaderCache{
+		data: make(map[string][]byte),
+	}
+}
+
+func (m *mockChunkCacheForReaderCache) GetChunk(fileId string, minSize uint64) []byte {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if d, ok := m.data[fileId]; ok {
+		atomic.AddInt32(&m.hitCount, 1)
+		return d
+	}
+	return nil
+}
+
+func (m *mockChunkCacheForReaderCache) ReadChunkAt(data []byte, fileId string, offset uint64) (int, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if d, ok := m.data[fileId]; ok && int(offset) < len(d) {
+		atomic.AddInt32(&m.hitCount, 1)
+		n := copy(data, d[offset:])
+		return n, nil
+	}
+	return 0, nil
+}
+
+func (m *mockChunkCacheForReaderCache) SetChunk(fileId string, data []byte) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.data[fileId] = data
+}
+
+func (m *mockChunkCacheForReaderCache) GetMaxFilePartSizeInCache() uint64 {
+	return 1024 * 1024 // 1MB
+}
+
+func (m *mockChunkCacheForReaderCache) IsInCache(fileId string, lockNeeded bool) bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	_, ok := m.data[fileId]
+	return ok
+}
+
+// TestReaderCacheContextCancellation tests that a reader can cancel its wait
+// while the download continues for other readers
+func TestReaderCacheContextCancellation(t *testing.T) {
+	cache := newMockChunkCacheForReaderCache()
+
+	// Create a ReaderCache - we can't easily test the full flow without mocking HTTP,
+	// but we can test the context cancellation in readChunkAt
+	rc := NewReaderCache(10, cache, nil)
+	defer rc.destroy()
+
+	// Pre-populate cache to avoid HTTP calls
+	testData := []byte("test data for context cancellation")
+	cache.SetChunk("test-file-1", testData)
+
+	// Test that context cancellation works
+	ctx, cancel := context.WithCancel(context.Background())
+
+	buffer := make([]byte, len(testData))
+	n, err := rc.ReadChunkAt(ctx, buffer, "test-file-1", nil, false, 0, len(testData), true)
+	if err != nil {
+		t.Errorf("Expected no error, got: %v", err)
+	}
+	if n != len(testData) {
+		t.Errorf("Expected %d bytes, got %d", len(testData), n)
+	}
+
+	// Cancel context and verify it doesn't affect already completed reads
+	cancel()
+
+	// Subsequent read with cancelled context should still work from cache
+	buffer2 := make([]byte, len(testData))
+	n2, err2 := rc.ReadChunkAt(ctx, buffer2, "test-file-1", nil, false, 0, len(testData), true)
+	// Note: This may or may not error depending on whether it hits cache
+	_ = n2
+	_ = err2
+}
+
+// TestReaderCacheFallbackToChunkCache tests that when a cacher returns n=0, err=nil,
+// we fall back to the chunkCache
+func TestReaderCacheFallbackToChunkCache(t *testing.T) {
+	cache := newMockChunkCacheForReaderCache()
+
+	// Pre-populate the chunk cache with data
+	testData := []byte("fallback test data that should be found in chunk cache")
+	cache.SetChunk("fallback-file", testData)
+
+	rc := NewReaderCache(10, cache, nil)
+	defer rc.destroy()
+
+	// Read should hit the chunk cache
+	buffer := make([]byte, len(testData))
+	n, err := rc.ReadChunkAt(context.Background(), buffer, "fallback-file", nil, false, 0, len(testData), true)
+
+	if err != nil {
+		t.Errorf("Expected no error, got: %v", err)
+	}
+	if n != len(testData) {
+		t.Errorf("Expected %d bytes, got %d", len(testData), n)
+	}
+
+	// Verify cache was hit
+	if cache.hitCount == 0 {
+		t.Error("Expected chunk cache to be hit")
+	}
+}
+
+// TestReaderCacheMultipleReadersWaitForSameChunk tests that multiple readers
+// can wait for the same chunk download to complete
+func TestReaderCacheMultipleReadersWaitForSameChunk(t *testing.T) {
+	cache := newMockChunkCacheForReaderCache()
+
+	// Pre-populate cache so we don't need HTTP
+	testData := make([]byte, 1024)
+	for i := range testData {
+		testData[i] = byte(i % 256)
+	}
+	cache.SetChunk("shared-chunk", testData)
+
+	rc := NewReaderCache(10, cache, nil)
+	defer rc.destroy()
+
+	// Launch multiple concurrent readers for the same chunk
+	numReaders := 10
+	var wg sync.WaitGroup
+	errors := make(chan error, numReaders)
+	bytesRead := make(chan int, numReaders)
+
+	for i := 0; i < numReaders; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			buffer := make([]byte, len(testData))
+			n, err := rc.ReadChunkAt(context.Background(), buffer, "shared-chunk", nil, false, 0, len(testData), true)
+			if err != nil {
+				errors <- err
+			}
+			bytesRead <- n
+		}()
+	}
+
+	wg.Wait()
+	close(errors)
+	close(bytesRead)
+
+	// Check for errors
+	for err := range errors {
+		t.Errorf("Reader got error: %v", err)
+	}
+
+	// Verify all readers got the expected data
+	for n := range bytesRead {
+		if n != len(testData) {
+			t.Errorf("Expected %d bytes, got %d", len(testData), n)
+		}
+	}
+}
+
+// TestReaderCachePartialRead tests reading at different offsets
+func TestReaderCachePartialRead(t *testing.T) {
+	cache := newMockChunkCacheForReaderCache()
+
+	testData := []byte("0123456789ABCDEFGHIJ")
+	cache.SetChunk("partial-read-file", testData)
+
+	rc := NewReaderCache(10, cache, nil)
+	defer rc.destroy()
+
+	tests := []struct {
+		name     string
+		offset   int64
+		size     int
+		expected []byte
+	}{
+		{"read from start", 0, 5, []byte("01234")},
+		{"read from middle", 5, 5, []byte("56789")},
+		{"read to end", 15, 5, []byte("FGHIJ")},
+		{"read single byte", 10, 1, []byte("A")},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			buffer := make([]byte, tt.size)
+			n, err := rc.ReadChunkAt(context.Background(), buffer, "partial-read-file", nil, false, tt.offset, len(testData), true)
+
+			if err != nil {
+				t.Errorf("Expected no error, got: %v", err)
+			}
+			if n != tt.size {
+				t.Errorf("Expected %d bytes, got %d", tt.size, n)
+			}
+			if string(buffer[:n]) != string(tt.expected) {
+				t.Errorf("Expected %q, got %q", tt.expected, buffer[:n])
+			}
+		})
+	}
+}
+
+// TestReaderCacheCleanup tests that old downloaders are cleaned up
+func TestReaderCacheCleanup(t *testing.T) {
+	cache := newMockChunkCacheForReaderCache()
+
+	// Create cache with limit of 3
+	rc := NewReaderCache(3, cache, nil)
+	defer rc.destroy()
+
+	// Add data for multiple files
+	for i := 0; i < 5; i++ {
+		fileId := string(rune('A' + i))
+		data := []byte("data for file " + fileId)
+		cache.SetChunk(fileId, data)
+	}
+
+	// Read from multiple files - should trigger cleanup when exceeding limit
+	for i := 0; i < 5; i++ {
+		fileId := string(rune('A' + i))
+		buffer := make([]byte, 20)
+		_, err := rc.ReadChunkAt(context.Background(), buffer, fileId, nil, false, 0, 20, true)
+		if err != nil {
+			t.Errorf("Read error for file %s: %v", fileId, err)
+		}
+	}
+
+	// Cache should still work - reads should succeed
+	for i := 0; i < 5; i++ {
+		fileId := string(rune('A' + i))
+		buffer := make([]byte, 20)
+		n, err := rc.ReadChunkAt(context.Background(), buffer, fileId, nil, false, 0, 20, true)
+		if err != nil {
+			t.Errorf("Second read error for file %s: %v", fileId, err)
+		}
+		if n == 0 {
+			t.Errorf("Expected data for file %s, got 0 bytes", fileId)
+		}
+	}
+}
+
+// TestSingleChunkCacherDoneSignal tests that done channel is always closed
+func TestSingleChunkCacherDoneSignal(t *testing.T) {
+	cache := newMockChunkCacheForReaderCache()
+	rc := NewReaderCache(10, cache, nil)
+	defer rc.destroy()
+
+	// Test that we can read even when data is in cache (done channel should work)
+	testData := []byte("done signal test")
+	cache.SetChunk("done-signal-test", testData)
+
+	// Multiple goroutines reading same chunk
+	var wg sync.WaitGroup
+	for i := 0; i < 5; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			buffer := make([]byte, len(testData))
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+
+			n, err := rc.ReadChunkAt(ctx, buffer, "done-signal-test", nil, false, 0, len(testData), true)
+			if err != nil && err != context.DeadlineExceeded {
+				t.Errorf("Unexpected error: %v", err)
+			}
+			if n == 0 && err == nil {
+				t.Error("Got 0 bytes with no error")
+			}
+		}()
+	}
+
+	// Should complete without hanging
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		// Success
+	case <-time.After(10 * time.Second):
+		t.Fatal("Test timed out - done channel may not be signaled correctly")
+	}
+}
+
+// ============================================================================
+// Tests that exercise SingleChunkCacher concurrency logic
+// ============================================================================
+//
+// These tests use blocking lookupFileIdFn to exercise the wait/cancellation
+// logic in SingleChunkCacher without requiring HTTP calls.
+
+// TestSingleChunkCacherLookupError tests handling of lookup errors
+func TestSingleChunkCacherLookupError(t *testing.T) {
+	cache := newMockChunkCacheForReaderCache()
+	
+	// Lookup function that returns an error
+	lookupFn := func(ctx context.Context, fileId string) ([]string, error) {
+		return nil, fmt.Errorf("lookup failed for %s", fileId)
+	}
+
+	rc := NewReaderCache(10, cache, lookupFn)
+	defer rc.destroy()
+
+	buffer := make([]byte, 100)
+	_, err := rc.ReadChunkAt(context.Background(), buffer, "error-test", nil, false, 0, 100, true)
+	
+	if err == nil {
+		t.Error("Expected an error, got nil")
+	}
+}
+
+// TestSingleChunkCacherContextCancellationDuringLookup tests that a reader can
+// cancel its wait while the lookup is in progress. This exercises the actual
+// SingleChunkCacher wait/cancel logic.
+func TestSingleChunkCacherContextCancellationDuringLookup(t *testing.T) {
+	cache := newMockChunkCacheForReaderCache()
+	lookupStarted := make(chan struct{})
+	lookupCanFinish := make(chan struct{})
+
+	// Lookup function that blocks to simulate slow operation
+	lookupFn := func(ctx context.Context, fileId string) ([]string, error) {
+		close(lookupStarted)
+		<-lookupCanFinish // Block until test allows completion
+		return nil, fmt.Errorf("lookup completed but reader should have cancelled")
+	}
+
+	rc := NewReaderCache(10, cache, lookupFn)
+	defer rc.destroy()
+	defer close(lookupCanFinish) // Ensure cleanup
+
+	ctx, cancel := context.WithCancel(context.Background())
+	readResult := make(chan error, 1)
+
+	go func() {
+		buffer := make([]byte, 100)
+		_, err := rc.ReadChunkAt(ctx, buffer, "cancel-during-lookup", nil, false, 0, 100, true)
+		readResult <- err
+	}()
+
+	// Wait for lookup to start, then cancel the reader's context
+	select {
+	case <-lookupStarted:
+		cancel() // Cancel the reader while lookup is blocked
+	case <-time.After(5 * time.Second):
+		t.Fatal("Lookup never started")
+	}
+
+	// Read should return with context.Canceled
+	select {
+	case err := <-readResult:
+		if err != context.Canceled {
+			t.Errorf("Expected context.Canceled, got: %v", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Read did not complete after context cancellation")
+	}
+}
+
+// TestSingleChunkCacherMultipleReadersWaitForDownload tests that multiple readers
+// can wait for the same SingleChunkCacher download to complete. When lookup fails,
+// all readers should receive the same error.
+func TestSingleChunkCacherMultipleReadersWaitForDownload(t *testing.T) {
+	cache := newMockChunkCacheForReaderCache()
+	lookupStarted := make(chan struct{})
+	lookupCanFinish := make(chan struct{})
+	var lookupStartedOnce sync.Once
+
+	// Lookup function that blocks to simulate slow operation
+	lookupFn := func(ctx context.Context, fileId string) ([]string, error) {
+		lookupStartedOnce.Do(func() { close(lookupStarted) })
+		<-lookupCanFinish
+		return nil, fmt.Errorf("simulated lookup error")
+	}
+
+	rc := NewReaderCache(10, cache, lookupFn)
+	defer rc.destroy()
+
+	numReaders := 5
+	var wg sync.WaitGroup
+	errors := make(chan error, numReaders)
+
+	// Start multiple readers for the same chunk
+	for i := 0; i < numReaders; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			buffer := make([]byte, 100)
+			_, err := rc.ReadChunkAt(context.Background(), buffer, "shared-chunk", nil, false, 0, 100, true)
+			errors <- err
+		}()
+	}
+
+	// Wait for lookup to start, then allow completion
+	select {
+	case <-lookupStarted:
+		close(lookupCanFinish)
+	case <-time.After(5 * time.Second):
+		close(lookupCanFinish)
+		t.Fatal("Lookup never started")
+	}
+
+	wg.Wait()
+	close(errors)
+
+	// All readers should receive an error
+	errorCount := 0
+	for err := range errors {
+		if err != nil {
+			errorCount++
+		}
+	}
+	if errorCount != numReaders {
+		t.Errorf("Expected %d errors, got %d", numReaders, errorCount)
+	}
+}
+
+// TestSingleChunkCacherOneReaderCancelsOthersContinue tests that when one reader
+// cancels, other readers waiting on the same chunk continue to wait.
+func TestSingleChunkCacherOneReaderCancelsOthersContinue(t *testing.T) {
+	cache := newMockChunkCacheForReaderCache()
+	lookupStarted := make(chan struct{})
+	lookupCanFinish := make(chan struct{})
+	var lookupStartedOnce sync.Once
+
+	lookupFn := func(ctx context.Context, fileId string) ([]string, error) {
+		lookupStartedOnce.Do(func() { close(lookupStarted) })
+		<-lookupCanFinish
+		return nil, fmt.Errorf("simulated error after delay")
+	}
+
+	rc := NewReaderCache(10, cache, lookupFn)
+	defer rc.destroy()
+
+	cancelledReaderDone := make(chan error, 1)
+	otherReaderDone := make(chan error, 1)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// Start reader that will be cancelled
+	go func() {
+		buffer := make([]byte, 100)
+		_, err := rc.ReadChunkAt(ctx, buffer, "shared-chunk-2", nil, false, 0, 100, true)
+		cancelledReaderDone <- err
+	}()
+
+	// Start reader that will NOT be cancelled
+	go func() {
+		buffer := make([]byte, 100)
+		_, err := rc.ReadChunkAt(context.Background(), buffer, "shared-chunk-2", nil, false, 0, 100, true)
+		otherReaderDone <- err
+	}()
+
+	// Wait for lookup to start
+	select {
+	case <-lookupStarted:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Lookup never started")
+	}
+
+	// Cancel the first reader
+	cancel()
+
+	// First reader should complete with context.Canceled quickly
+	select {
+	case err := <-cancelledReaderDone:
+		if err != context.Canceled {
+			t.Errorf("Cancelled reader: expected context.Canceled, got: %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Error("Cancelled reader did not complete quickly")
+	}
+
+	// Allow the download to complete
+	close(lookupCanFinish)
+
+	// Other reader should eventually complete (with error since lookup returns error)
+	select {
+	case err := <-otherReaderDone:
+		if err == nil || err == context.Canceled {
+			t.Errorf("Other reader: expected non-nil non-cancelled error, got: %v", err)
+		}
+		// Expected: "simulated error after delay"
+	case <-time.After(5 * time.Second):
+		t.Error("Other reader did not complete")
+	}
+}

weed/filer/redis2/redis_cluster_store.go

@@ -25,20 +25,24 @@ func (store *RedisCluster2Store) Initialize(configuration util.Configuration, pr
 
 	return store.initialize(
 		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
+		configuration.GetString(prefix+"keyPrefix"),
 		configuration.GetBool(prefix+"useReadOnly"),
 		configuration.GetBool(prefix+"routeByLatency"),
 		configuration.GetStringSlice(prefix+"superLargeDirectories"),
 	)
 }
 
-func (store *RedisCluster2Store) initialize(addresses []string, password string, readOnly, routeByLatency bool, superLargeDirectories []string) (err error) {
+func (store *RedisCluster2Store) initialize(addresses []string, username string, password string, keyPrefix string, readOnly, routeByLatency bool, superLargeDirectories []string) (err error) {
 	store.Client = redis.NewClusterClient(&redis.ClusterOptions{
 		Addrs:          addresses,
+		Username:       username,
 		Password:       password,
 		ReadOnly:       readOnly,
 		RouteByLatency: routeByLatency,
 	})
+	store.keyPrefix = keyPrefix
 	store.loadSuperLargeDirectories(superLargeDirectories)
 	return
 }

weed/filer/redis2/redis_sentinel_store.go

@@ -26,10 +26,11 @@ func (store *Redis2SentinelStore) Initialize(configuration util.Configuration, p
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetInt(prefix+"database"),
+		configuration.GetString(prefix+"keyPrefix"),
 	)
 }
 
-func (store *Redis2SentinelStore) initialize(addresses []string, masterName string, username string, password string, database int) (err error) {
+func (store *Redis2SentinelStore) initialize(addresses []string, masterName string, username string, password string, database int, keyPrefix string) (err error) {
 	store.Client = redis.NewFailoverClient(&redis.FailoverOptions{
 		MasterName:      masterName,
 		SentinelAddrs:   addresses,
@@ -41,5 +42,6 @@ func (store *Redis2SentinelStore) initialize(addresses []string, masterName stri
 		ReadTimeout:     time.Second * 30,
 		WriteTimeout:    time.Second * 5,
 	})
+	store.keyPrefix = keyPrefix
 	return
 }

weed/filer/redis2/redis_store.go

@@ -27,8 +27,10 @@ func (store *Redis2Store) GetName() string {
 func (store *Redis2Store) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
 		configuration.GetString(prefix+"address"),
+		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetInt(prefix+"database"),
+		configuration.GetString(prefix+"keyPrefix"),
 		configuration.GetStringSlice(prefix+"superLargeDirectories"),
 		configuration.GetBool(prefix+"enable_mtls"),
 		configuration.GetString(prefix+"ca_cert_path"),
@@ -37,7 +39,13 @@ func (store *Redis2Store) Initialize(configuration util.Configuration, prefix st
 	)
 }
 
-func (store *Redis2Store) initialize(hostPort string, password string, database int, superLargeDirectories []string, enableMtls bool, caCertPath string, clientCertPath string, clientKeyPath string) (err error) {
+func (store *Redis2Store) initialize(hostPort string, username string, password string, database int, keyPrefix string, superLargeDirectories []string, enableMtls bool, caCertPath string, clientCertPath string, clientKeyPath string) (err error) {
+	opt := &redis.Options{
+		Addr:     hostPort,
+		Username: username,
+		Password: password,
+		DB:       database,
+	}
 	if enableMtls {
 		clientCert, err := tls.LoadX509KeyPair(clientCertPath, clientKeyPath)
 		if err != nil {
@@ -59,25 +67,15 @@ func (store *Redis2Store) initialize(hostPort string, password string, database
 			glog.Fatalf("Error parsing redis host and port from %s: %v", hostPort, err)
 		}
 
-		tlsConfig := &tls.Config{
+		opt.TLSConfig = &tls.Config{
 			Certificates: []tls.Certificate{clientCert},
 			RootCAs:      caCertPool,
 			ServerName:   redisHost,
 			MinVersion:   tls.VersionTLS12,
 		}
-		store.Client = redis.NewClient(&redis.Options{
-			Addr:      hostPort,
-			Password:  password,
-			DB:        database,
-			TLSConfig: tlsConfig,
-		})
-	} else {
-		store.Client = redis.NewClient(&redis.Options{
-			Addr:     hostPort,
-			Password: password,
-			DB:       database,
-		})
 	}
+	store.Client = redis.NewClient(opt)
+	store.keyPrefix = keyPrefix
 	store.loadSuperLargeDirectories(superLargeDirectories)
 	return
 }

weed/filer/redis2/universal_redis_store.go

@@ -19,6 +19,7 @@ const (
 
 type UniversalRedis2Store struct {
 	Client                  redis.UniversalClient
+	keyPrefix               string
 	superLargeDirectoryHash map[string]bool
 }
 
@@ -35,6 +36,13 @@ func (store *UniversalRedis2Store) loadSuperLargeDirectories(superLargeDirectori
 	}
 }
 
+func (store *UniversalRedis2Store) getKey(key string) string {
+	if store.keyPrefix == "" {
+		return key
+	}
+	return store.keyPrefix + key
+}
+
 func (store *UniversalRedis2Store) BeginTransaction(ctx context.Context) (context.Context, error) {
 	return ctx, nil
 }
@@ -57,7 +65,7 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer
 	}
 
 	if name != "" {
-		if err = store.Client.ZAddNX(ctx, genDirectoryListKey(dir), redis.Z{Score: 0, Member: name}).Err(); err != nil {
+		if err = store.Client.ZAddNX(ctx, store.getKey(genDirectoryListKey(dir)), redis.Z{Score: 0, Member: name}).Err(); err != nil {
 			return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err)
 		}
 	}
@@ -75,7 +83,7 @@ func (store *UniversalRedis2Store) doInsertEntry(ctx context.Context, entry *fil
 		value = util.MaybeGzipData(value)
 	}
 
-	if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
+	if err = store.Client.Set(ctx, store.getKey(string(entry.FullPath)), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
 		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
 	}
 	return nil
@@ -88,7 +96,7 @@ func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer
 
 func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
 
-	data, err := store.Client.Get(ctx, string(fullpath)).Result()
+	data, err := store.Client.Get(ctx, store.getKey(string(fullpath))).Result()
 	if err == redis.Nil {
 		return nil, filer_pb.ErrNotFound
 	}
@@ -110,12 +118,12 @@ func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.
 
 func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
 
-	_, err = store.Client.Del(ctx, genDirectoryListKey(string(fullpath))).Result()
+	_, err = store.Client.Del(ctx, store.getKey(genDirectoryListKey(string(fullpath)))).Result()
 	if err != nil {
 		return fmt.Errorf("delete dir list %s : %v", fullpath, err)
 	}
 
-	_, err = store.Client.Del(ctx, string(fullpath)).Result()
+	_, err = store.Client.Del(ctx, store.getKey(string(fullpath))).Result()
 	if err != nil {
 		return fmt.Errorf("delete %s : %v", fullpath, err)
 	}
@@ -125,7 +133,7 @@ func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath uti
 		return nil
 	}
 	if name != "" {
-		_, err = store.Client.ZRem(ctx, genDirectoryListKey(dir), name).Result()
+		_, err = store.Client.ZRem(ctx, store.getKey(genDirectoryListKey(dir)), name).Result()
 		if err != nil {
 			return fmt.Errorf("DeleteEntry %s in parent dir: %v", fullpath, err)
 		}
@@ -140,7 +148,7 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful
 		return nil
 	}
 
-	members, err := store.Client.ZRangeByLex(ctx, genDirectoryListKey(string(fullpath)), &redis.ZRangeBy{
+	members, err := store.Client.ZRangeByLex(ctx, store.getKey(genDirectoryListKey(string(fullpath))), &redis.ZRangeBy{
 		Min: "-",
 		Max: "+",
 	}).Result()
@@ -150,12 +158,12 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful
 
 	for _, fileName := range members {
 		path := util.NewFullPath(string(fullpath), fileName)
-		_, err = store.Client.Del(ctx, string(path)).Result()
+		_, err = store.Client.Del(ctx, store.getKey(string(path))).Result()
 		if err != nil {
 			return fmt.Errorf("DeleteFolderChildren %s in parent dir: %v", fullpath, err)
 		}
 		// not efficient, but need to remove if it is a directory
-		store.Client.Del(ctx, genDirectoryListKey(string(path)))
+		store.Client.Del(ctx, store.getKey(genDirectoryListKey(string(path))))
 	}
 
 	return nil
@@ -167,7 +175,7 @@ func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Cont
 
 func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
 
-	dirListKey := genDirectoryListKey(string(dirPath))
+	dirListKey := store.getKey(genDirectoryListKey(string(dirPath)))
 
 	min := "-"
 	if startFileName != "" {
@@ -201,7 +209,7 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dir
 		} else {
 			if entry.TtlSec > 0 {
 				if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
-					store.Client.Del(ctx, string(path)).Result()
+					store.Client.Del(ctx, store.getKey(string(path))).Result()
 					store.Client.ZRem(ctx, dirListKey, fileName).Result()
 					continue
 				}

weed/filer/redis_lua/redis_cluster_store.go

@@ -25,20 +25,24 @@ func (store *RedisLuaClusterStore) Initialize(configuration util.Configuration,
 
 	return store.initialize(
 		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
+		configuration.GetString(prefix+"keyPrefix"),
 		configuration.GetBool(prefix+"useReadOnly"),
 		configuration.GetBool(prefix+"routeByLatency"),
 		configuration.GetStringSlice(prefix+"superLargeDirectories"),
 	)
 }
 
-func (store *RedisLuaClusterStore) initialize(addresses []string, password string, readOnly, routeByLatency bool, superLargeDirectories []string) (err error) {
+func (store *RedisLuaClusterStore) initialize(addresses []string, username string, password string, keyPrefix string, readOnly, routeByLatency bool, superLargeDirectories []string) (err error) {
 	store.Client = redis.NewClusterClient(&redis.ClusterOptions{
 		Addrs:          addresses,
+		Username:       username,
 		Password:       password,
 		ReadOnly:       readOnly,
 		RouteByLatency: routeByLatency,
 	})
+	store.keyPrefix = keyPrefix
 	store.loadSuperLargeDirectories(superLargeDirectories)
 	return
 }

weed/filer/redis_lua/redis_sentinel_store.go

@@ -26,10 +26,11 @@ func (store *RedisLuaSentinelStore) Initialize(configuration util.Configuration,
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetInt(prefix+"database"),
+		configuration.GetString(prefix+"keyPrefix"),
 	)
 }
 
-func (store *RedisLuaSentinelStore) initialize(addresses []string, masterName string, username string, password string, database int) (err error) {
+func (store *RedisLuaSentinelStore) initialize(addresses []string, masterName string, username string, password string, database int, keyPrefix string) (err error) {
 	store.Client = redis.NewFailoverClient(&redis.FailoverOptions{
 		MasterName:      masterName,
 		SentinelAddrs:   addresses,
@@ -41,5 +42,6 @@ func (store *RedisLuaSentinelStore) initialize(addresses []string, masterName st
 		ReadTimeout:     time.Second * 30,
 		WriteTimeout:    time.Second * 5,
 	})
+	store.keyPrefix = keyPrefix
 	return
 }

weed/filer/redis_lua/redis_store.go

@@ -21,18 +21,22 @@ func (store *RedisLuaStore) GetName() string {
 func (store *RedisLuaStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
 		configuration.GetString(prefix+"address"),
+		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetInt(prefix+"database"),
+		configuration.GetString(prefix+"keyPrefix"),
 		configuration.GetStringSlice(prefix+"superLargeDirectories"),
 	)
 }
 
-func (store *RedisLuaStore) initialize(hostPort string, password string, database int, superLargeDirectories []string) (err error) {
+func (store *RedisLuaStore) initialize(hostPort string, username string, password string, database int, keyPrefix string, superLargeDirectories []string) (err error) {
 	store.Client = redis.NewClient(&redis.Options{
 		Addr:     hostPort,
+		Username: username,
 		Password: password,
 		DB:       database,
 	})
+	store.keyPrefix = keyPrefix
 	store.loadSuperLargeDirectories(superLargeDirectories)
 	return
 }

weed/filer/redis_lua/universal_redis_store.go

@@ -20,6 +20,7 @@ const (
 
 type UniversalRedisLuaStore struct {
 	Client                  redis.UniversalClient
+	keyPrefix               string
 	superLargeDirectoryHash map[string]bool
 }
 
@@ -36,6 +37,13 @@ func (store *UniversalRedisLuaStore) loadSuperLargeDirectories(superLargeDirecto
 	}
 }
 
+func (store *UniversalRedisLuaStore) getKey(key string) string {
+	if store.keyPrefix == "" {
+		return key
+	}
+	return store.keyPrefix + key
+}
+
 func (store *UniversalRedisLuaStore) BeginTransaction(ctx context.Context) (context.Context, error) {
 	return ctx, nil
 }
@@ -60,7 +68,7 @@ func (store *UniversalRedisLuaStore) InsertEntry(ctx context.Context, entry *fil
 	dir, name := entry.FullPath.DirAndName()
 
 	err = stored_procedure.InsertEntryScript.Run(ctx, store.Client,
-		[]string{string(entry.FullPath), genDirectoryListKey(dir)},
+		[]string{store.getKey(string(entry.FullPath)), store.getKey(genDirectoryListKey(dir))},
 		value, entry.TtlSec,
 		store.isSuperLargeDirectory(dir), 0, name).Err()
 
@@ -78,7 +86,7 @@ func (store *UniversalRedisLuaStore) UpdateEntry(ctx context.Context, entry *fil
 
 func (store *UniversalRedisLuaStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
 
-	data, err := store.Client.Get(ctx, string(fullpath)).Result()
+	data, err := store.Client.Get(ctx, store.getKey(string(fullpath))).Result()
 	if err == redis.Nil {
 		return nil, filer_pb.ErrNotFound
 	}
@@ -103,7 +111,7 @@ func (store *UniversalRedisLuaStore) DeleteEntry(ctx context.Context, fullpath u
 	dir, name := fullpath.DirAndName()
 
 	err = stored_procedure.DeleteEntryScript.Run(ctx, store.Client,
-		[]string{string(fullpath), genDirectoryListKey(string(fullpath)), genDirectoryListKey(dir)},
+		[]string{store.getKey(string(fullpath)), store.getKey(genDirectoryListKey(string(fullpath))), store.getKey(genDirectoryListKey(dir))},
 		store.isSuperLargeDirectory(dir), name).Err()
 
 	if err != nil {
@@ -120,7 +128,7 @@ func (store *UniversalRedisLuaStore) DeleteFolderChildren(ctx context.Context, f
 	}
 
 	err = stored_procedure.DeleteFolderChildrenScript.Run(ctx, store.Client,
-		[]string{string(fullpath)}).Err()
+		[]string{store.getKey(string(fullpath))}).Err()
 
 	if err != nil {
 		return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err)
@@ -135,7 +143,7 @@ func (store *UniversalRedisLuaStore) ListDirectoryPrefixedEntries(ctx context.Co
 
 func (store *UniversalRedisLuaStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
 
-	dirListKey := genDirectoryListKey(string(dirPath))
+	dirListKey := store.getKey(genDirectoryListKey(string(dirPath)))
 
 	min := "-"
 	if startFileName != "" {

weed/operation/upload_content.go

@@ -90,10 +90,9 @@ func (uploadResult *UploadResult) ToPbFileChunkWithSSE(fileId string, offset int
 }
 
 var (
-	fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", "")
-	uploader        *Uploader
-	uploaderErr     error
-	once            sync.Once
+	uploader    *Uploader
+	uploaderErr error
+	once        sync.Once
 )
 
 // HTTPClient interface for testing
@@ -336,8 +335,9 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction
 		body_writer = multipart.NewWriter(option.BytesBuffer)
 	}
 	h := make(textproto.MIMEHeader)
-	filename := fileNameEscaper.Replace(option.Filename)
-	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename))
+	// Use mime.FormatMediaType for RFC 6266 compliant Content-Disposition,
+	// properly handling non-ASCII characters and special characters
+	h.Set("Content-Disposition", mime.FormatMediaType("form-data", map[string]string{"name": "file", "filename": option.Filename}))
 	h.Set("Idempotency-Key", option.UploadUrl)
 	if option.MimeType == "" {
 		option.MimeType = mime.TypeByExtension(strings.ToLower(filepath.Ext(option.Filename)))

weed/pb/master.proto

@@ -81,6 +81,7 @@ message Heartbeat {
   map<string, uint32> max_volume_counts = 4;
   uint32 grpc_port = 20;
   repeated string location_uuids = 21;
+  string id = 22; // volume server id, independent of ip:port for stable identification
 }
 
 message HeartbeatResponse {
@@ -289,6 +290,7 @@ message DataNodeInfo {
   string id = 1;
   map<string, DiskInfo> diskInfos = 2;
   uint32 grpc_port = 3;
+  string address = 4; // ip:port for connecting to the volume server
 }
 message RackInfo {
   string id = 1;

weed/pb/master_pb/master.pb.go

@@ -44,6 +44,7 @@ type Heartbeat struct {
 	MaxVolumeCounts map[string]uint32                  `protobuf:"bytes,4,rep,name=max_volume_counts,json=maxVolumeCounts,proto3" json:"max_volume_counts,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
 	GrpcPort        uint32                             `protobuf:"varint,20,opt,name=grpc_port,json=grpcPort,proto3" json:"grpc_port,omitempty"`
 	LocationUuids   []string                           `protobuf:"bytes,21,rep,name=location_uuids,json=locationUuids,proto3" json:"location_uuids,omitempty"`
+	Id              string                             `protobuf:"bytes,22,opt,name=id,proto3" json:"id,omitempty"` // volume server id, independent of ip:port for stable identification
 	unknownFields   protoimpl.UnknownFields
 	sizeCache       protoimpl.SizeCache
 }
@@ -204,6 +205,13 @@ func (x *Heartbeat) GetLocationUuids() []string {
 	return nil
 }
 
+func (x *Heartbeat) GetId() string {
+	if x != nil {
+		return x.Id
+	}
+	return ""
+}
+
 type HeartbeatResponse struct {
 	state                  protoimpl.MessageState `protogen:"open.v1"`
 	VolumeSizeLimit        uint64                 `protobuf:"varint,1,opt,name=volume_size_limit,json=volumeSizeLimit,proto3" json:"volume_size_limit,omitempty"`
@@ -2039,6 +2047,7 @@ type DataNodeInfo struct {
 	Id            string                 `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
 	DiskInfos     map[string]*DiskInfo   `protobuf:"bytes,2,rep,name=diskInfos,proto3" json:"diskInfos,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
 	GrpcPort      uint32                 `protobuf:"varint,3,opt,name=grpc_port,json=grpcPort,proto3" json:"grpc_port,omitempty"`
+	Address       string                 `protobuf:"bytes,4,opt,name=address,proto3" json:"address,omitempty"` // ip:port for connecting to the volume server
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -2094,6 +2103,13 @@ func (x *DataNodeInfo) GetGrpcPort() uint32 {
 	return 0
 }
 
+func (x *DataNodeInfo) GetAddress() string {
+	if x != nil {
+		return x.Address
+	}
+	return ""
+}
+
 type RackInfo struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	Id            string                 `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
@@ -4038,7 +4054,7 @@ var File_master_proto protoreflect.FileDescriptor
 
 const file_master_proto_rawDesc = "" +
 	"\n" +
-	"\fmaster.proto\x12\tmaster_pb\"\xc0\a\n" +
+	"\fmaster.proto\x12\tmaster_pb\"\xd0\a\n" +
 	"\tHeartbeat\x12\x0e\n" +
 	"\x02ip\x18\x01 \x01(\tR\x02ip\x12\x12\n" +
 	"\x04port\x18\x02 \x01(\rR\x04port\x12\x1d\n" +
@@ -4063,7 +4079,8 @@ const file_master_proto_rawDesc = "" +
 	"\x10has_no_ec_shards\x18\x13 \x01(\bR\rhasNoEcShards\x12U\n" +
 	"\x11max_volume_counts\x18\x04 \x03(\v2).master_pb.Heartbeat.MaxVolumeCountsEntryR\x0fmaxVolumeCounts\x12\x1b\n" +
 	"\tgrpc_port\x18\x14 \x01(\rR\bgrpcPort\x12%\n" +
-	"\x0elocation_uuids\x18\x15 \x03(\tR\rlocationUuids\x1aB\n" +
+	"\x0elocation_uuids\x18\x15 \x03(\tR\rlocationUuids\x12\x0e\n" +
+	"\x02id\x18\x16 \x01(\tR\x02id\x1aB\n" +
 	"\x14MaxVolumeCountsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
 	"\x05value\x18\x02 \x01(\rR\x05value:\x028\x01\"\xcd\x02\n" +
@@ -4254,11 +4271,12 @@ const file_master_proto_rawDesc = "" +
 	"\fvolume_infos\x18\x06 \x03(\v2#.master_pb.VolumeInformationMessageR\vvolumeInfos\x12P\n" +
 	"\x0eec_shard_infos\x18\a \x03(\v2*.master_pb.VolumeEcShardInformationMessageR\fecShardInfos\x12.\n" +
 	"\x13remote_volume_count\x18\b \x01(\x03R\x11remoteVolumeCount\x12\x17\n" +
-	"\adisk_id\x18\t \x01(\rR\x06diskId\"\xd4\x01\n" +
+	"\adisk_id\x18\t \x01(\rR\x06diskId\"\xee\x01\n" +
 	"\fDataNodeInfo\x12\x0e\n" +
 	"\x02id\x18\x01 \x01(\tR\x02id\x12D\n" +
 	"\tdiskInfos\x18\x02 \x03(\v2&.master_pb.DataNodeInfo.DiskInfosEntryR\tdiskInfos\x12\x1b\n" +
-	"\tgrpc_port\x18\x03 \x01(\rR\bgrpcPort\x1aQ\n" +
+	"\tgrpc_port\x18\x03 \x01(\rR\bgrpcPort\x12\x18\n" +
+	"\aaddress\x18\x04 \x01(\tR\aaddress\x1aQ\n" +
 	"\x0eDiskInfosEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
 	"\x05value\x18\x02 \x01(\v2\x13.master_pb.DiskInfoR\x05value:\x028\x01\"\xf0\x01\n" +

weed/pb/server_address.go

@@ -2,11 +2,12 @@ package pb
 
 import (
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"net"
 	"strconv"
 	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 type ServerAddress string
@@ -32,7 +33,12 @@ func NewServerAddressWithGrpcPort(address string, grpcPort int) ServerAddress {
 }
 
 func NewServerAddressFromDataNode(dn *master_pb.DataNodeInfo) ServerAddress {
-	return NewServerAddressWithGrpcPort(dn.Id, int(dn.GrpcPort))
+	// Use Address field if available (new behavior), fall back to Id for backward compatibility
+	addr := dn.Address
+	if addr == "" {
+		addr = dn.Id // backward compatibility: old nodes use ip:port as id
+	}
+	return NewServerAddressWithGrpcPort(addr, int(dn.GrpcPort))
 }
 
 func NewServerAddressFromLocation(dn *master_pb.Location) ServerAddress {

weed/s3api/auth_signature_v4.go

@@ -53,10 +53,11 @@ func (iam *IdentityAccessManagement) reqSignatureV4Verify(r *http.Request) (*Ide
 
 // Constants specific to this file
 const (
-	emptySHA256              = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-	streamingContentSHA256   = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
-	streamingUnsignedPayload = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
-	unsignedPayload          = "UNSIGNED-PAYLOAD"
+	emptySHA256                   = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	streamingContentSHA256        = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+	streamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
+	streamingUnsignedPayload      = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
+	unsignedPayload               = "UNSIGNED-PAYLOAD"
 	// Limit for IAM/STS request body size to prevent DoS attacks
 	iamRequestBodyLimit = 10 * (1 << 20) // 10 MiB
 )
@@ -214,14 +215,14 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh
 			availableKeys = append(availableKeys, key)
 		}
 		iam.m.RUnlock()
-		
+
 		glog.Warningf("InvalidAccessKeyId: attempted key '%s' not found. Available keys: %d, Auth enabled: %v",
 			authInfo.AccessKey, len(availableKeys), iam.isAuthEnabled)
-		
+
 		if glog.V(2) && len(availableKeys) > 0 {
 			glog.V(2).Infof("Available access keys: %v", availableKeys)
 		}
-		
+
 		return nil, nil, "", nil, s3err.ErrInvalidAccessKeyID
 	}
 
@@ -562,10 +563,10 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.
 		iam.m.RLock()
 		availableKeyCount := len(iam.accessKeyIdent)
 		iam.m.RUnlock()
-		
+
 		glog.Warningf("InvalidAccessKeyId (POST policy): attempted key '%s' not found. Available keys: %d, Auth enabled: %v",
 			credHeader.accessKey, availableKeyCount, iam.isAuthEnabled)
-		
+
 		return s3err.ErrInvalidAccessKeyID
 	}
 

weed/s3api/chunked_reader_v4.go

@@ -53,8 +53,8 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
 
 	// This check ensures we only proceed for streaming uploads.
 	switch authInfo.HashedPayload {
-	case streamingContentSHA256:
-		glog.V(3).Infof("streaming content sha256")
+	case streamingContentSHA256, streamingContentSHA256Trailer:
+		glog.V(3).Infof("streaming content sha256 (with trailer: %v)", authInfo.HashedPayload == streamingContentSHA256Trailer)
 	case streamingUnsignedPayload:
 		glog.V(3).Infof("streaming unsigned payload")
 	default:
@@ -87,9 +87,9 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea
 	var errCode s3err.ErrorCode
 
 	switch contentSha256Header {
-	// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
-	case streamingContentSHA256:
-		glog.V(3).Infof("streaming content sha256")
+	// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' or 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER'
+	case streamingContentSHA256, streamingContentSHA256Trailer:
+		glog.V(3).Infof("streaming content sha256 (with trailer: %v)", contentSha256Header == streamingContentSHA256Trailer)
 		credential, seedSignature, region, service, seedDate, errCode = iam.calculateSeedSignature(req)
 		if errCode != s3err.ErrNone {
 			return nil, errCode
@@ -116,6 +116,7 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea
 	}
 
 	checkSumWriter := getCheckSumWriter(checksumAlgorithm)
+	hasTrailer := amzTrailerHeader != ""
 
 	return &s3ChunkedReader{
 		cred:              credential,
@@ -129,6 +130,7 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea
 		checkSumWriter:    checkSumWriter,
 		state:             readChunkHeader,
 		iam:               iam,
+		hasTrailer:        hasTrailer,
 	}, s3err.ErrNone
 }
 
@@ -170,6 +172,7 @@ type s3ChunkedReader struct {
 	n                 uint64    // Unread bytes in chunk
 	err               error
 	iam               *IdentityAccessManagement
+	hasTrailer        bool
 }
 
 // Read chunk reads the chunk token signature portion.
@@ -281,10 +284,10 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
 			}
 
 			// If we're using unsigned streaming upload, there is no signature to verify at each chunk.
-			if cr.chunkSignature != "" {
-				cr.state = verifyChunk
-			} else if cr.lastChunk {
+			if cr.lastChunk && cr.hasTrailer {
 				cr.state = readTrailerChunk
+			} else if cr.chunkSignature != "" {
+				cr.state = verifyChunk
 			} else {
 				cr.state = readChunkHeader
 			}
@@ -304,7 +307,11 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
 			// This implementation currently only supports the first case.
 			// TODO: Implement the second case (signed upload with additional checksum computation for each chunk)
 
-			extractedCheckSumAlgorithm, extractedChecksum := parseChunkChecksum(cr.reader)
+			extractedCheckSumAlgorithm, extractedChecksum, err := parseChunkChecksum(cr.reader)
+			if err != nil {
+				cr.err = err
+				return 0, err
+			}
 
 			if extractedCheckSumAlgorithm.String() != cr.checkSumAlgorithm {
 				errorMessage := fmt.Sprintf("checksum algorithm in trailer '%s' does not match the one advertised in the header '%s'", extractedCheckSumAlgorithm.String(), cr.checkSumAlgorithm)
@@ -313,6 +320,7 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
 				return 0, cr.err
 			}
 
+			// Validate checksum for data integrity (required for both signed and unsigned streaming with trailers)
 			computedChecksum := cr.checkSumWriter.Sum(nil)
 			base64Checksum := base64.StdEncoding.EncodeToString(computedChecksum)
 			if string(extractedChecksum) != base64Checksum {
@@ -324,11 +332,6 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
 			// TODO: Extract signature from trailer chunk and verify it.
 			// For now, we just read the trailer chunk and discard it.
 
-			// Reading remaining CRLF.
-			for i := 0; i < 2; i++ {
-				cr.err = readCRLF(cr.reader)
-			}
-
 			cr.state = eofChunk
 
 		case readChunk:
@@ -506,41 +509,37 @@ func parseS3ChunkExtension(buf []byte) ([]byte, []byte) {
 	return buf[:semi], parseChunkSignature(buf[semi:])
 }
 
-func parseChunkChecksum(b *bufio.Reader) (ChecksumAlgorithm, []byte) {
-	// When using unsigned upload, this would be the raw contents of  the trailer chunk:
-	//
-	// x-amz-checksum-crc32:YABb/g==\n\r\n\r\n      // Trailer chunk (note optional \n character)
-	// \r\n                                         // CRLF
-	//
-	// When using signed upload with an additional checksum algorithm, this would be the raw contents of the trailer chunk:
-	//
-	// x-amz-checksum-crc32:YABb/g==\n\r\n            // Trailer chunk (note optional \n character)
-	// trailer-signature\r\n
-	// \r\n                                           // CRLF
-	//
+func parseChunkChecksum(b *bufio.Reader) (ChecksumAlgorithm, []byte, error) {
+	// Read trailer lines until empty line
+	var checksumAlgorithm ChecksumAlgorithm
+	var checksum []byte
 
-	// x-amz-checksum-crc32:YABb/g==\n
-	bytesRead, err := readChunkLine(b)
-	if err != nil {
-		return ChecksumAlgorithmNone, nil
+	for {
+		bytesRead, err := readChunkLine(b)
+		if err != nil {
+			return ChecksumAlgorithmNone, nil, err
+		}
+
+		if len(bytesRead) == 0 {
+			break
+		}
+
+		parts := bytes.SplitN(bytesRead, []byte(":"), 2)
+		if len(parts) == 2 {
+			key := string(bytes.TrimSpace(parts[0]))
+			value := bytes.TrimSpace(parts[1])
+			if alg, err := extractChecksumAlgorithm(key); err == nil {
+				if checksumAlgorithm != ChecksumAlgorithmNone {
+					glog.V(3).Infof("multiple checksum headers found in trailer, using last: %s", key)
+				}
+				checksumAlgorithm = alg
+				checksum = value
+			}
+			// Ignore other trailer headers like x-amz-trailer-signature
+		}
 	}
 
-	// Split on ':'
-	parts := bytes.SplitN(bytesRead, []byte(":"), 2)
-	checksumKey := string(parts[0])
-	checksumValue := parts[1]
-
-	// Discard all trailing whitespace characters
-	checksumValue = trimTrailingWhitespace(checksumValue)
-
-	// If the checksum key is not a supported checksum algorithm, return an error.
-	// TODO: Bubble that error up to the caller
-	extractedAlgorithm, err := extractChecksumAlgorithm(checksumKey)
-	if err != nil {
-		return ChecksumAlgorithmNone, nil
-	}
-
-	return extractedAlgorithm, checksumValue
+	return checksumAlgorithm, checksum, nil
 }
 
 func parseChunkSignature(chunk []byte) []byte {

weed/s3api/chunked_reader_v4_test.go

@@ -234,6 +234,150 @@ func TestSignedStreamingUpload(t *testing.T) {
 	assert.Equal(t, chunk1Data+chunk2Data, string(data))
 }
 
+// createTrailerStreamingRequest creates a streaming upload request with trailer for testing.
+// If useValidTrailerSignature is true, uses a correctly calculated trailer signature;
+// otherwise uses an intentionally wrong signature for negative testing.
+func createTrailerStreamingRequest(t *testing.T, useValidTrailerSignature bool) (*http.Request, string) {
+	chunk1Data := "hello world\n"
+	chunk1DataLen := len(chunk1Data)
+	chunk1DataLenHex := fmt.Sprintf("%x", chunk1DataLen)
+
+	// Use current time for signatures
+	now := time.Now().UTC()
+	amzDate := now.Format(iso8601Format)
+	dateStamp := now.Format(yyyymmdd)
+
+	// Calculate seed signature
+	scope := dateStamp + "/" + defaultRegion + "/s3/aws4_request"
+
+	// Build canonical request for seed signature
+	hashedPayload := "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
+	canonicalHeaders := "content-encoding:aws-chunked\n" +
+		"host:s3.amazonaws.com\n" +
+		"x-amz-content-sha256:" + hashedPayload + "\n" +
+		"x-amz-date:" + amzDate + "\n" +
+		fmt.Sprintf("x-amz-decoded-content-length:%d\n", chunk1DataLen) +
+		"x-amz-trailer:x-amz-checksum-crc32\n"
+	signedHeaders := "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-trailer"
+
+	canonicalRequest := "PUT\n" +
+		"/test-bucket/test-object\n" +
+		"\n" +
+		canonicalHeaders + "\n" +
+		signedHeaders + "\n" +
+		hashedPayload
+
+	canonicalRequestHash := getSHA256Hash([]byte(canonicalRequest))
+	stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + canonicalRequestHash
+
+	signingKey := getSigningKey(defaultSecretAccessKey, dateStamp, defaultRegion, "s3")
+	seedSignature := getSignature(signingKey, stringToSign)
+
+	// Calculate chunk signatures
+	chunk1Hash := getSHA256Hash([]byte(chunk1Data))
+	chunk1StringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
+		seedSignature + "\n" + emptySHA256 + "\n" + chunk1Hash
+	chunk1Signature := getSignature(signingKey, chunk1StringToSign)
+
+	// Final chunk (0 bytes)
+	finalStringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
+		chunk1Signature + "\n" + emptySHA256 + "\n" + emptySHA256
+	finalSignature := getSignature(signingKey, finalStringToSign)
+
+	// Calculate CRC32 checksum for trailer
+	crcWriter := crc32.NewIEEE()
+	_, crcErr := crcWriter.Write([]byte(chunk1Data))
+	assert.NoError(t, crcErr)
+	checksum := crcWriter.Sum(nil)
+	base64EncodedChecksum := base64.StdEncoding.EncodeToString(checksum)
+
+	// The on-wire trailer format uses \r\n (HTTP/aws-chunked convention)
+	trailerOnWire := "x-amz-checksum-crc32:" + base64EncodedChecksum + "\r\n"
+
+	// Calculate or use wrong trailer signature
+	var trailerSignature string
+	if useValidTrailerSignature {
+		// The canonical trailer content uses \n for signing (per AWS SigV4 spec)
+		trailerCanonical := "x-amz-checksum-crc32:" + base64EncodedChecksum + "\n"
+		trailerHash := getSHA256Hash([]byte(trailerCanonical))
+		trailerStringToSign := "AWS4-HMAC-SHA256-TRAILER\n" + amzDate + "\n" + scope + "\n" +
+			finalSignature + "\n" + trailerHash
+		trailerSignature = getSignature(signingKey, trailerStringToSign)
+	} else {
+		// Intentionally wrong signature for negative testing
+		trailerSignature = "0000000000000000000000000000000000000000000000000000000000000000"
+	}
+
+	// Build the chunked payload with trailer and trailer signature
+	payload := fmt.Sprintf("%s;chunk-signature=%s\r\n%s\r\n", chunk1DataLenHex, chunk1Signature, chunk1Data) +
+		fmt.Sprintf("0;chunk-signature=%s\r\n", finalSignature) +
+		trailerOnWire +
+		"x-amz-trailer-signature:" + trailerSignature + "\r\n" +
+		"\r\n"
+
+	// Create the request
+	req, err := http.NewRequest("PUT", "http://s3.amazonaws.com/test-bucket/test-object",
+		bytes.NewReader([]byte(payload)))
+	assert.NoError(t, err)
+
+	req.Header.Set("Host", "s3.amazonaws.com")
+	req.Header.Set("x-amz-date", amzDate)
+	req.Header.Set("x-amz-content-sha256", hashedPayload)
+	req.Header.Set("Content-Encoding", "aws-chunked")
+	req.Header.Set("x-amz-decoded-content-length", fmt.Sprintf("%d", chunk1DataLen))
+	req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32")
+
+	authHeader := fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s",
+		defaultAccessKeyId, scope, signedHeaders, seedSignature)
+	req.Header.Set("Authorization", authHeader)
+
+	return req, chunk1Data
+}
+
+// TestSignedStreamingUploadWithTrailer tests streaming uploads with signed chunks and trailers
+// This tests the STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER content-sha256 header value
+// which is used by AWS SDK v2 when checksum validation is enabled
+func TestSignedStreamingUploadWithTrailer(t *testing.T) {
+	iam := setupIam()
+	req, expectedData := createTrailerStreamingRequest(t, true)
+
+	// Test the chunked reader
+	reader, errCode := iam.newChunkedReader(req)
+	assert.Equal(t, s3err.ErrNone, errCode)
+	assert.NotNil(t, reader)
+
+	// Read and verify the payload
+	data, err := io.ReadAll(reader)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedData, string(data))
+}
+
+// TestSignedStreamingUploadWithTrailerInvalidSignature tests behavior with invalid trailer signatures.
+// This is a negative test case for trailer signature validation. It currently verifies that an invalid
+// signature doesn't break content reading, and is prepared for when validation is implemented.
+func TestSignedStreamingUploadWithTrailerInvalidSignature(t *testing.T) {
+	iam := setupIam()
+	req, expectedData := createTrailerStreamingRequest(t, false)
+
+	// Test the chunked reader - it should be created successfully
+	reader, errCode := iam.newChunkedReader(req)
+	assert.Equal(t, s3err.ErrNone, errCode)
+	assert.NotNil(t, reader)
+
+	// Read the payload - currently trailer signature validation may not be implemented,
+	// but this test documents the expected behavior and will catch regressions
+	// if trailer signature validation is added in the future
+	data, err := io.ReadAll(reader)
+	// Note: If trailer signature validation is implemented, this should fail with an error
+	// For now, we just verify the content is correctly extracted
+	if err != nil {
+		assert.Contains(t, err.Error(), "signature", "Error should indicate signature mismatch")
+	} else {
+		// If no error, content should still be correct (trailer sig validation not yet implemented)
+		assert.Equal(t, expectedData, string(data))
+	}
+}
+
 // TestSignedStreamingUploadInvalidSignature tests that invalid chunk signatures are rejected
 // This is a negative test case to ensure signature validation is actually working
 func TestSignedStreamingUploadInvalidSignature(t *testing.T) {

weed/s3api/s3api_auth.go

@@ -48,14 +48,22 @@ func isRequestPostPolicySignatureV4(r *http.Request) bool {
 }
 
 // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
+// Supports both with and without trailer variants:
+// - STREAMING-AWS4-HMAC-SHA256-PAYLOAD (original)
+// - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER (with trailing checksums)
 func isRequestSignStreamingV4(r *http.Request) bool {
-	return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 &&
-		r.Method == http.MethodPut
+	if r.Method != http.MethodPut {
+		return false
+	}
+	contentSha256 := r.Header.Get("x-amz-content-sha256")
+	return contentSha256 == streamingContentSHA256 || contentSha256 == streamingContentSHA256Trailer
 }
 
 func isRequestUnsignedStreaming(r *http.Request) bool {
-	return r.Header.Get("x-amz-content-sha256") == streamingUnsignedPayload &&
-		r.Method == http.MethodPut
+	if r.Method != http.MethodPut {
+		return false
+	}
+	return r.Header.Get("x-amz-content-sha256") == streamingUnsignedPayload
 }
 
 // Authorization type.

weed/s3api/s3api_bucket_config.go

@@ -519,7 +519,9 @@ func (s3a *S3ApiServer) getVersioningState(bucket string) (string, error) {
 	config, errCode := s3a.getBucketConfig(bucket)
 	if errCode != s3err.ErrNone {
 		if errCode == s3err.ErrNoSuchBucket {
-			return "", nil
+			// Signal to callers that the bucket does not exist so they can
+			// decide whether to auto-create it (e.g., in PUT handlers).
+			return "", filer_pb.ErrNotFound
 		}
 		glog.Errorf("getVersioningState: failed to get bucket config for %s: %v", bucket, errCode)
 		return "", fmt.Errorf("failed to get bucket config: %v", errCode)
@@ -546,6 +548,15 @@ func (s3a *S3ApiServer) getBucketVersioningStatus(bucket string) (string, s3err.
 	return config.Versioning, s3err.ErrNone
 }
 
+// isObjectLockEnabled checks if object lock is enabled for a bucket (cached check)
+func (s3a *S3ApiServer) isObjectLockEnabled(bucket string) bool {
+	config, errCode := s3a.getBucketConfig(bucket)
+	if errCode != s3err.ErrNone {
+		return false
+	}
+	return config.ObjectLockConfig != nil
+}
+
 // setBucketVersioningStatus sets the versioning status for a bucket
 func (s3a *S3ApiServer) setBucketVersioningStatus(bucket, status string) s3err.ErrorCode {
 	errCode := s3a.updateBucketConfig(bucket, func(config *BucketConfig) error {

weed/s3api/s3api_object_handlers.go

@@ -659,16 +659,14 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 			glog.V(3).Infof("GetObject: Set PartsCount=%d for multipart GET with PartNumber=%d", partsCount, partNumber)
 
 			// Calculate the byte range for this part
+			// Note: ETag is NOT overridden - AWS S3 returns the complete object's ETag
+			// even when requesting a specific part via PartNumber
 			var startOffset, endOffset int64
 			if partInfo != nil {
 				// Use part boundaries from metadata (accurate for multi-chunk parts)
 				startOffset = objectEntryForSSE.Chunks[partInfo.StartChunk].Offset
 				lastChunk := objectEntryForSSE.Chunks[partInfo.EndChunk-1]
 				endOffset = lastChunk.Offset + int64(lastChunk.Size) - 1
-
-				// Override ETag with the part's ETag from metadata
-				w.Header().Set("ETag", "\""+partInfo.ETag+"\"")
-				glog.V(3).Infof("GetObject: Override ETag with part %d ETag: %s (from metadata)", partNumber, partInfo.ETag)
 			} else {
 				// Fallback: assume 1:1 part-to-chunk mapping (backward compatibility)
 				chunkIndex := partNumber - 1
@@ -680,15 +678,6 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 				partChunk := objectEntryForSSE.Chunks[chunkIndex]
 				startOffset = partChunk.Offset
 				endOffset = partChunk.Offset + int64(partChunk.Size) - 1
-
-				// Override ETag with chunk's ETag (fallback)
-				if partChunk.ETag != "" {
-					if md5Bytes, decodeErr := base64.StdEncoding.DecodeString(partChunk.ETag); decodeErr == nil {
-						partETag := fmt.Sprintf("%x", md5Bytes)
-						w.Header().Set("ETag", "\""+partETag+"\"")
-						glog.V(3).Infof("GetObject: Override ETag with part %d ETag: %s (fallback from chunk)", partNumber, partETag)
-					}
-				}
 			}
 
 			// Check if client supplied a Range header - if so, apply it within the part's boundaries
@@ -2266,7 +2255,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 	if partNumberStr != "" {
 		if partNumber, parseErr := strconv.Atoi(partNumberStr); parseErr == nil && partNumber > 0 {
 			// Get actual parts count from metadata (not chunk count)
-			partsCount, partInfo := s3a.getMultipartInfo(objectEntryForSSE, partNumber)
+			partsCount, _ := s3a.getMultipartInfo(objectEntryForSSE, partNumber)
 
 			// Validate part number
 			if partNumber > partsCount {
@@ -2276,31 +2265,10 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 			}
 
 			// Set parts count header
+			// Note: ETag is NOT overridden - AWS S3 returns the complete object's ETag
+			// even when requesting a specific part via PartNumber
 			w.Header().Set(s3_constants.AmzMpPartsCount, strconv.Itoa(partsCount))
 			glog.V(3).Infof("HeadObject: Set PartsCount=%d for part %d", partsCount, partNumber)
-
-			// Override ETag with the part's ETag
-			if partInfo != nil {
-				// Use part ETag from metadata (accurate for multi-chunk parts)
-				w.Header().Set("ETag", "\""+partInfo.ETag+"\"")
-				glog.V(3).Infof("HeadObject: Override ETag with part %d ETag: %s (from metadata)", partNumber, partInfo.ETag)
-			} else {
-				// Fallback: use chunk's ETag (backward compatibility)
-				chunkIndex := partNumber - 1
-				if chunkIndex >= len(objectEntryForSSE.Chunks) {
-					glog.Warningf("HeadObject: Part %d chunk index %d out of range (chunks: %d)", partNumber, chunkIndex, len(objectEntryForSSE.Chunks))
-					s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPart)
-					return
-				}
-				partChunk := objectEntryForSSE.Chunks[chunkIndex]
-				if partChunk.ETag != "" {
-					if md5Bytes, decodeErr := base64.StdEncoding.DecodeString(partChunk.ETag); decodeErr == nil {
-						partETag := fmt.Sprintf("%x", md5Bytes)
-						w.Header().Set("ETag", "\""+partETag+"\"")
-						glog.V(3).Infof("HeadObject: Override ETag with part %d ETag: %s (fallback from chunk)", partNumber, partETag)
-					}
-				}
-			}
 		}
 	}
 

weed/s3api/s3api_object_handlers_copy.go

@@ -199,7 +199,9 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 	}
 
 	// Process metadata and tags and apply to destination
-	processedMetadata, tagErr := processMetadataBytes(r.Header, entry.Extended, replaceMeta, replaceTagging)
+	// Use dstEntry.Extended (already filtered) as the source, not entry.Extended,
+	// to preserve the encryption header filtering. Fixes GitHub #7562.
+	processedMetadata, tagErr := processMetadataBytes(r.Header, dstEntry.Extended, replaceMeta, replaceTagging)
 	if tagErr != nil {
 		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 		return
@@ -1522,7 +1524,7 @@ func (s3a *S3ApiServer) copyMultipartSSECChunk(chunk *filer_pb.FileChunk, copySo
 }
 
 // copyMultipartCrossEncryption handles all cross-encryption and decrypt-only copy scenarios
-// This unified function supports: SSE-C↔SSE-KMS, SSE-C→Plain, SSE-KMS→Plain
+// This unified function supports: SSE-C↔SSE-KMS↔SSE-S3, and any→Plain
 func (s3a *S3ApiServer) copyMultipartCrossEncryption(entry *filer_pb.Entry, r *http.Request, state *EncryptionState, dstBucket, dstPath string) ([]*filer_pb.FileChunk, map[string][]byte, error) {
 	var dstChunks []*filer_pb.FileChunk
 
@@ -1531,6 +1533,7 @@ func (s3a *S3ApiServer) copyMultipartCrossEncryption(entry *filer_pb.Entry, r *h
 	var destKMSKeyID string
 	var destKMSEncryptionContext map[string]string
 	var destKMSBucketKeyEnabled bool
+	var destSSES3Key *SSES3Key
 
 	if state.DstSSEC {
 		var err error
@@ -1544,7 +1547,13 @@ func (s3a *S3ApiServer) copyMultipartCrossEncryption(entry *filer_pb.Entry, r *h
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to parse destination SSE-KMS headers: %w", err)
 		}
-	} else {
+	} else if state.DstSSES3 {
+		// Generate SSE-S3 key for destination
+		var err error
+		destSSES3Key, err = GenerateSSES3Key()
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to generate SSE-S3 key: %w", err)
+		}
 	}
 
 	// Parse source encryption parameters
@@ -1563,12 +1572,18 @@ func (s3a *S3ApiServer) copyMultipartCrossEncryption(entry *filer_pb.Entry, r *h
 		var err error
 
 		if chunk.GetSseType() == filer_pb.SSEType_SSE_C {
-			copiedChunk, err = s3a.copyCrossEncryptionChunk(chunk, sourceSSECKey, destSSECKey, destKMSKeyID, destKMSEncryptionContext, destKMSBucketKeyEnabled, dstPath, dstBucket, state)
+			copiedChunk, err = s3a.copyCrossEncryptionChunk(chunk, sourceSSECKey, destSSECKey, destKMSKeyID, destKMSEncryptionContext, destKMSBucketKeyEnabled, destSSES3Key, dstPath, dstBucket, state)
 		} else if chunk.GetSseType() == filer_pb.SSEType_SSE_KMS {
-			copiedChunk, err = s3a.copyCrossEncryptionChunk(chunk, nil, destSSECKey, destKMSKeyID, destKMSEncryptionContext, destKMSBucketKeyEnabled, dstPath, dstBucket, state)
+			copiedChunk, err = s3a.copyCrossEncryptionChunk(chunk, nil, destSSECKey, destKMSKeyID, destKMSEncryptionContext, destKMSBucketKeyEnabled, destSSES3Key, dstPath, dstBucket, state)
+		} else if chunk.GetSseType() == filer_pb.SSEType_SSE_S3 {
+			copiedChunk, err = s3a.copyCrossEncryptionChunk(chunk, nil, destSSECKey, destKMSKeyID, destKMSEncryptionContext, destKMSBucketKeyEnabled, destSSES3Key, dstPath, dstBucket, state)
 		} else {
-			// Unencrypted chunk, copy directly
-			copiedChunk, err = s3a.copySingleChunk(chunk, dstPath)
+			// Unencrypted chunk - may need encryption if destination requires it
+			if state.DstSSEC || state.DstSSEKMS || state.DstSSES3 {
+				copiedChunk, err = s3a.copyCrossEncryptionChunk(chunk, nil, destSSECKey, destKMSKeyID, destKMSEncryptionContext, destKMSBucketKeyEnabled, destSSES3Key, dstPath, dstBucket, state)
+			} else {
+				copiedChunk, err = s3a.copySingleChunk(chunk, dstPath)
+			}
 		}
 
 		if err != nil {
@@ -1619,6 +1634,40 @@ func (s3a *S3ApiServer) copyMultipartCrossEncryption(entry *filer_pb.Entry, r *h
 		} else {
 			glog.Errorf("Failed to serialize SSE-KMS metadata: %v", serErr)
 		}
+	} else if state.DstSSES3 && destSSES3Key != nil {
+		// For SSE-S3 destination, create object-level metadata
+		var sses3Metadata *SSES3Key
+		if len(dstChunks) == 0 {
+			// Handle 0-byte files - generate IV for metadata even though there's no content to encrypt
+			if entry.Attributes.FileSize != 0 {
+				return nil, nil, fmt.Errorf("internal error: no chunks created for non-empty SSE-S3 destination object")
+			}
+			// Generate IV for 0-byte object metadata
+			iv := make([]byte, s3_constants.AESBlockSize)
+			if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+				return nil, nil, fmt.Errorf("generate IV for 0-byte object: %w", err)
+			}
+			destSSES3Key.IV = iv
+			sses3Metadata = destSSES3Key
+		} else {
+			// For non-empty objects, use the first chunk's metadata
+			if dstChunks[0].GetSseType() != filer_pb.SSEType_SSE_S3 || len(dstChunks[0].GetSseMetadata()) == 0 {
+				return nil, nil, fmt.Errorf("internal error: first chunk is missing expected SSE-S3 metadata for destination object")
+			}
+			keyManager := GetSSES3KeyManager()
+			var err error
+			sses3Metadata, err = DeserializeSSES3Metadata(dstChunks[0].GetSseMetadata(), keyManager)
+			if err != nil {
+				return nil, nil, fmt.Errorf("failed to deserialize SSE-S3 metadata from first chunk: %w", err)
+			}
+		}
+		// Use the derived key with its IV for object-level metadata
+		keyData, serErr := SerializeSSES3Metadata(sses3Metadata)
+		if serErr != nil {
+			return nil, nil, fmt.Errorf("failed to serialize SSE-S3 metadata: %w", serErr)
+		}
+		dstMetadata[s3_constants.SeaweedFSSSES3Key] = keyData
+		dstMetadata[s3_constants.AmzServerSideEncryption] = []byte("AES256")
 	}
 	// For unencrypted destination, no metadata needed (dstMetadata remains empty)
 
@@ -1626,7 +1675,7 @@ func (s3a *S3ApiServer) copyMultipartCrossEncryption(entry *filer_pb.Entry, r *h
 }
 
 // copyCrossEncryptionChunk handles copying a single chunk with cross-encryption support
-func (s3a *S3ApiServer) copyCrossEncryptionChunk(chunk *filer_pb.FileChunk, sourceSSECKey *SSECustomerKey, destSSECKey *SSECustomerKey, destKMSKeyID string, destKMSEncryptionContext map[string]string, destKMSBucketKeyEnabled bool, dstPath, dstBucket string, state *EncryptionState) (*filer_pb.FileChunk, error) {
+func (s3a *S3ApiServer) copyCrossEncryptionChunk(chunk *filer_pb.FileChunk, sourceSSECKey *SSECustomerKey, destSSECKey *SSECustomerKey, destKMSKeyID string, destKMSEncryptionContext map[string]string, destKMSBucketKeyEnabled bool, destSSES3Key *SSES3Key, dstPath, dstBucket string, state *EncryptionState) (*filer_pb.FileChunk, error) {
 	// Create destination chunk
 	dstChunk := s3a.createDestinationChunk(chunk, chunk.Offset, chunk.Size)
 
@@ -1726,6 +1775,30 @@ func (s3a *S3ApiServer) copyCrossEncryptionChunk(chunk *filer_pb.FileChunk, sour
 			previewLen = len(finalData)
 		}
 
+	} else if chunk.GetSseType() == filer_pb.SSEType_SSE_S3 {
+		// Decrypt SSE-S3 source
+		if len(chunk.GetSseMetadata()) == 0 {
+			return nil, fmt.Errorf("SSE-S3 chunk missing per-chunk metadata")
+		}
+
+		keyManager := GetSSES3KeyManager()
+		sourceSSEKey, err := DeserializeSSES3Metadata(chunk.GetSseMetadata(), keyManager)
+		if err != nil {
+			return nil, fmt.Errorf("failed to deserialize SSE-S3 metadata: %w", err)
+		}
+
+		decryptedReader, decErr := CreateSSES3DecryptedReader(bytes.NewReader(encryptedData), sourceSSEKey, sourceSSEKey.IV)
+		if decErr != nil {
+			return nil, fmt.Errorf("create SSE-S3 decrypted reader: %w", decErr)
+		}
+
+		decryptedData, readErr := io.ReadAll(decryptedReader)
+		if readErr != nil {
+			return nil, fmt.Errorf("decrypt SSE-S3 chunk data: %w", readErr)
+		}
+		finalData = decryptedData
+		glog.V(4).Infof("Decrypted SSE-S3 chunk, size: %d", len(finalData))
+
 	} else {
 		// Source is unencrypted
 		finalData = encryptedData
@@ -1787,6 +1860,36 @@ func (s3a *S3ApiServer) copyCrossEncryptionChunk(chunk *filer_pb.FileChunk, sour
 		dstChunk.SseMetadata = kmsMetadata
 
 		glog.V(4).Infof("Re-encrypted chunk with SSE-KMS")
+
+	} else if state.DstSSES3 && destSSES3Key != nil {
+		// Encrypt with SSE-S3
+		encryptedReader, iv, encErr := CreateSSES3EncryptedReader(bytes.NewReader(finalData), destSSES3Key)
+		if encErr != nil {
+			return nil, fmt.Errorf("create SSE-S3 encrypted reader: %w", encErr)
+		}
+
+		reencryptedData, readErr := io.ReadAll(encryptedReader)
+		if readErr != nil {
+			return nil, fmt.Errorf("re-encrypt with SSE-S3: %w", readErr)
+		}
+		finalData = reencryptedData
+
+		// Create per-chunk SSE-S3 metadata with chunk-specific IV
+		chunkSSEKey := &SSES3Key{
+			Key:       destSSES3Key.Key,
+			KeyID:     destSSES3Key.KeyID,
+			Algorithm: destSSES3Key.Algorithm,
+			IV:        iv,
+		}
+		sses3Metadata, err := SerializeSSES3Metadata(chunkSSEKey)
+		if err != nil {
+			return nil, fmt.Errorf("serialize SSE-S3 metadata: %w", err)
+		}
+
+		dstChunk.SseType = filer_pb.SSEType_SSE_S3
+		dstChunk.SseMetadata = sses3Metadata
+
+		glog.V(4).Infof("Re-encrypted chunk with SSE-S3")
 	}
 	// For unencrypted destination, finalData remains as decrypted plaintext
 

weed/s3api/s3api_object_handlers_copy_unified.go

@@ -1,7 +1,6 @@
 package s3api
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net/http"
@@ -133,9 +132,9 @@ func (s3a *S3ApiServer) executeEncryptCopy(entry *filer_pb.Entry, r *http.Reques
 	}
 
 	if state.DstSSES3 {
-		// Use streaming copy for SSE-S3 encryption
-		chunks, err := s3a.executeStreamingReencryptCopy(entry, r, state, dstPath)
-		return chunks, nil, err
+		// Use chunk-by-chunk copy for SSE-S3 encryption (consistent with SSE-C and SSE-KMS)
+		glog.V(2).Infof("Plain→SSE-S3 copy: using unified multipart encrypt copy")
+		return s3a.copyMultipartCrossEncryption(entry, r, state, dstBucket, dstPath)
 	}
 
 	return nil, nil, fmt.Errorf("unknown target encryption type")
@@ -143,30 +142,18 @@ func (s3a *S3ApiServer) executeEncryptCopy(entry *filer_pb.Entry, r *http.Reques
 
 // executeDecryptCopy handles encrypted → plain copies
 func (s3a *S3ApiServer) executeDecryptCopy(entry *filer_pb.Entry, r *http.Request, state *EncryptionState, dstPath string) ([]*filer_pb.FileChunk, map[string][]byte, error) {
-	// Use unified multipart-aware decrypt copy for all encryption types
-	if state.SrcSSEC || state.SrcSSEKMS {
+	// Use unified multipart-aware decrypt copy for all encryption types (consistent chunk-by-chunk)
+	if state.SrcSSEC || state.SrcSSEKMS || state.SrcSSES3 {
 		glog.V(2).Infof("Encrypted→Plain copy: using unified multipart decrypt copy")
 		return s3a.copyMultipartCrossEncryption(entry, r, state, "", dstPath)
 	}
 
-	if state.SrcSSES3 {
-		// Use streaming copy for SSE-S3 decryption
-		chunks, err := s3a.executeStreamingReencryptCopy(entry, r, state, dstPath)
-		return chunks, nil, err
-	}
-
 	return nil, nil, fmt.Errorf("unknown source encryption type")
 }
 
 // executeReencryptCopy handles encrypted → encrypted copies with different keys/methods
 func (s3a *S3ApiServer) executeReencryptCopy(entry *filer_pb.Entry, r *http.Request, state *EncryptionState, dstBucket, dstPath string) ([]*filer_pb.FileChunk, map[string][]byte, error) {
-	// Check if we should use streaming copy for better performance
-	if s3a.shouldUseStreamingCopy(entry, state) {
-		chunks, err := s3a.executeStreamingReencryptCopy(entry, r, state, dstPath)
-		return chunks, nil, err
-	}
-
-	// Fallback to chunk-by-chunk approach for compatibility
+	// Use chunk-by-chunk approach for all cross-encryption scenarios (consistent behavior)
 	if state.SrcSSEC && state.DstSSEC {
 		return s3a.copyChunksWithSSEC(entry, r)
 	}
@@ -177,83 +164,8 @@ func (s3a *S3ApiServer) executeReencryptCopy(entry *filer_pb.Entry, r *http.Requ
 		return chunks, dstMetadata, err
 	}
 
-	if state.SrcSSEC && state.DstSSEKMS {
-		// SSE-C → SSE-KMS: use unified multipart-aware cross-encryption copy
-		glog.V(2).Infof("SSE-C→SSE-KMS cross-encryption copy: using unified multipart copy")
-		return s3a.copyMultipartCrossEncryption(entry, r, state, dstBucket, dstPath)
-	}
-
-	if state.SrcSSEKMS && state.DstSSEC {
-		// SSE-KMS → SSE-C: use unified multipart-aware cross-encryption copy
-		glog.V(2).Infof("SSE-KMS→SSE-C cross-encryption copy: using unified multipart copy")
-		return s3a.copyMultipartCrossEncryption(entry, r, state, dstBucket, dstPath)
-	}
-
-	// Handle SSE-S3 cross-encryption scenarios
-	if state.SrcSSES3 || state.DstSSES3 {
-		// Any scenario involving SSE-S3 uses streaming copy
-		chunks, err := s3a.executeStreamingReencryptCopy(entry, r, state, dstPath)
-		return chunks, nil, err
-	}
-
-	return nil, nil, fmt.Errorf("unsupported cross-encryption scenario")
-}
-
-// shouldUseStreamingCopy determines if streaming copy should be used
-func (s3a *S3ApiServer) shouldUseStreamingCopy(entry *filer_pb.Entry, state *EncryptionState) bool {
-	// Use streaming copy for large files or when beneficial
-	fileSize := entry.Attributes.FileSize
-
-	// Use streaming for files larger than 10MB
-	if fileSize > 10*1024*1024 {
-		return true
-	}
-
-	// Check if this is a multipart encrypted object
-	isMultipartEncrypted := false
-	if state.IsSourceEncrypted() {
-		encryptedChunks := 0
-		for _, chunk := range entry.GetChunks() {
-			if chunk.GetSseType() != filer_pb.SSEType_NONE {
-				encryptedChunks++
-			}
-		}
-		isMultipartEncrypted = encryptedChunks > 1
-	}
-
-	// For multipart encrypted objects, avoid streaming copy to use per-chunk metadata approach
-	if isMultipartEncrypted {
-		glog.V(3).Infof("Multipart encrypted object detected, using chunk-by-chunk approach")
-		return false
-	}
-
-	// Use streaming for cross-encryption scenarios (for single-part objects only)
-	if state.IsSourceEncrypted() && state.IsTargetEncrypted() {
-		srcType := s3a.getEncryptionTypeString(state.SrcSSEC, state.SrcSSEKMS, state.SrcSSES3)
-		dstType := s3a.getEncryptionTypeString(state.DstSSEC, state.DstSSEKMS, state.DstSSES3)
-		if srcType != dstType {
-			return true
-		}
-	}
-
-	// Use streaming for compressed files
-	if isCompressedEntry(entry) {
-		return true
-	}
-
-	// Use streaming for SSE-S3 scenarios (always)
-	if state.SrcSSES3 || state.DstSSES3 {
-		return true
-	}
-
-	return false
-}
-
-// executeStreamingReencryptCopy performs streaming re-encryption copy
-func (s3a *S3ApiServer) executeStreamingReencryptCopy(entry *filer_pb.Entry, r *http.Request, state *EncryptionState, dstPath string) ([]*filer_pb.FileChunk, error) {
-	// Create streaming copy manager
-	streamingManager := NewStreamingCopyManager(s3a)
-
-	// Execute streaming copy
-	return streamingManager.ExecuteStreamingCopy(context.Background(), entry, r, dstPath, state)
+	// All other cross-encryption scenarios use unified multipart copy
+	// This includes: SSE-C↔SSE-KMS, SSE-C↔SSE-S3, SSE-KMS↔SSE-S3, SSE-S3↔SSE-S3
+	glog.V(2).Infof("Cross-encryption copy: using unified multipart copy")
+	return s3a.copyMultipartCrossEncryption(entry, r, state, dstBucket, dstPath)
 }

weed/s3api/s3api_object_handlers_delete.go

@@ -1,12 +1,10 @@
 package s3api
 
 import (
-	"context"
 	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
-	"slices"
 	"strings"
 
 	"github.com/seaweedfs/seaweedfs/weed/filer"
@@ -55,16 +53,21 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
 		// Handle versioned delete based on specific versioning state
 		if versionId != "" {
 			// Delete specific version (same for both enabled and suspended)
-			// Check object lock permissions before deleting specific version
-			governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
-			if err := s3a.enforceObjectLockProtections(r, bucket, object, versionId, governanceBypassAllowed); err != nil {
-				glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
-				s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
-				return
+			// Only check object lock if enabled for this bucket (avoids expensive entry lookup)
+			var prefetchedEntry *filer_pb.Entry
+			if s3a.isObjectLockEnabled(bucket) {
+				governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
+				var lockErr error
+				prefetchedEntry, lockErr = s3a.enforceObjectLockProtections(r, bucket, object, versionId, governanceBypassAllowed)
+				if lockErr != nil {
+					glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, lockErr)
+					s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
+					return
+				}
 			}
 
-			// Delete specific version
-			err := s3a.deleteSpecificObjectVersion(bucket, object, versionId)
+			// Delete specific version, passing prefetched entry if available
+			err := s3a.deleteSpecificObjectVersion(bucket, object, versionId, prefetchedEntry)
 			if err != nil {
 				glog.Errorf("Failed to delete specific version %s: %v", versionId, err)
 				s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
@@ -93,16 +96,21 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
 				// Suspended versioning: Actually delete the "null" version object
 				glog.V(2).Infof("DeleteObjectHandler: deleting null version for suspended versioning %s/%s", bucket, object)
 
-				// Check object lock permissions before deleting "null" version
-				governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
-				if err := s3a.enforceObjectLockProtections(r, bucket, object, "null", governanceBypassAllowed); err != nil {
-					glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
-					s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
-					return
+				// Only check object lock if enabled for this bucket (avoids expensive entry lookup)
+				var prefetchedEntry *filer_pb.Entry
+				if s3a.isObjectLockEnabled(bucket) {
+					governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
+					var lockErr error
+					prefetchedEntry, lockErr = s3a.enforceObjectLockProtections(r, bucket, object, "null", governanceBypassAllowed)
+					if lockErr != nil {
+						glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, lockErr)
+						s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
+						return
+					}
 				}
 
 				// Delete the "null" version (the regular file)
-				err := s3a.deleteSpecificObjectVersion(bucket, object, "null")
+				err := s3a.deleteSpecificObjectVersion(bucket, object, "null", prefetchedEntry)
 				if err != nil {
 					glog.Errorf("Failed to delete null version: %v", err)
 					s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
@@ -115,34 +123,24 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
 		}
 	} else {
 		// Handle regular delete (non-versioned)
-		// Check object lock permissions before deleting object
-		governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
-		if err := s3a.enforceObjectLockProtections(r, bucket, object, "", governanceBypassAllowed); err != nil {
-			glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
-			s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
-			return
+		// Only check object lock permissions if object lock is enabled for the bucket
+		// This avoids an expensive entry lookup for buckets without object lock
+		if s3a.isObjectLockEnabled(bucket) {
+			governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
+			if _, lockErr := s3a.enforceObjectLockProtections(r, bucket, object, "", governanceBypassAllowed); lockErr != nil {
+				glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, lockErr)
+				s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
+				return
+			}
 		}
 
 		target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object))
 		dir, name := target.DirAndName()
 
 		err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-			// Use operation context that won't be cancelled if request terminates
-			// This ensures deletion completes atomically to avoid inconsistent state
-			opCtx := context.WithoutCancel(r.Context())
-
-			if err := doDeleteEntry(client, dir, name, true, false); err != nil {
-				return err
-			}
-
-			// Cleanup empty directories
-			if !s3a.option.AllowEmptyFolder && strings.LastIndex(object, "/") > 0 {
-				bucketPath := fmt.Sprintf("%s/%s", s3a.option.BucketsPath, bucket)
-				// Recursively delete empty parent directories, stop at bucket path
-				filer_pb.DoDeleteEmptyParentDirectories(opCtx, client, util.FullPath(dir), util.FullPath(bucketPath), nil)
-			}
-
-			return nil
+			return doDeleteEntry(client, dir, name, true, false)
+			// Note: Empty folder cleanup is now handled asynchronously by EmptyFolderCleaner
+			// which listens to metadata events and uses consistent hashing for coordination
 		})
 		if err != nil {
 			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
@@ -222,8 +220,6 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 	var deleteErrors []DeleteError
 	var auditLog *s3err.AccessLog
 
-	directoriesWithDeletion := make(map[string]bool)
-
 	if s3err.Logger != nil {
 		auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
 	}
@@ -245,22 +241,21 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 	versioningConfigured := (versioningState != "")
 
 	s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		// Use operation context that won't be cancelled if request terminates
-		// This ensures batch deletion completes atomically to avoid inconsistent state
-		opCtx := context.WithoutCancel(r.Context())
-
 		// delete file entries
 		for _, object := range deleteObjects.Objects {
 			if object.Key == "" {
 				continue
 			}
 
-			// Check object lock permissions before deletion (only for versioned buckets)
-			if versioningConfigured {
+			// Only check object lock if enabled for this bucket (avoids expensive entry lookup)
+			var prefetchedEntry *filer_pb.Entry
+			if versioningConfigured && s3a.isObjectLockEnabled(bucket) {
 				// Validate governance bypass for this specific object
 				governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object.Key)
-				if err := s3a.enforceObjectLockProtections(r, bucket, object.Key, object.VersionId, governanceBypassAllowed); err != nil {
-					glog.V(2).Infof("DeleteMultipleObjectsHandler: object lock check failed for %s/%s (version: %s): %v", bucket, object.Key, object.VersionId, err)
+				var lockErr error
+				prefetchedEntry, lockErr = s3a.enforceObjectLockProtections(r, bucket, object.Key, object.VersionId, governanceBypassAllowed)
+				if lockErr != nil {
+					glog.V(2).Infof("DeleteMultipleObjectsHandler: object lock check failed for %s/%s (version: %s): %v", bucket, object.Key, object.VersionId, lockErr)
 					deleteErrors = append(deleteErrors, DeleteError{
 						Code:      s3err.GetAPIError(s3err.ErrAccessDenied).Code,
 						Message:   s3err.GetAPIError(s3err.ErrAccessDenied).Description,
@@ -278,7 +273,8 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 				// Handle versioned delete based on specific versioning state
 				if object.VersionId != "" {
 					// Delete specific version (same for both enabled and suspended)
-					err := s3a.deleteSpecificObjectVersion(bucket, object.Key, object.VersionId)
+					// Pass prefetched entry to avoid duplicate lookup
+					err := s3a.deleteSpecificObjectVersion(bucket, object.Key, object.VersionId, prefetchedEntry)
 					if err != nil {
 						deleteErrors = append(deleteErrors, DeleteError{
 							Code:      "",
@@ -309,7 +305,8 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 						// Suspended versioning: Actually delete the "null" version object
 						glog.V(2).Infof("DeleteMultipleObjectsHandler: deleting null version for suspended versioning %s/%s", bucket, object.Key)
 
-						err := s3a.deleteSpecificObjectVersion(bucket, object.Key, "null")
+						// Pass prefetched entry to avoid duplicate lookup
+						err := s3a.deleteSpecificObjectVersion(bucket, object.Key, "null", prefetchedEntry)
 						if err != nil {
 							deleteErrors = append(deleteErrors, DeleteError{
 								Code:      "",
@@ -357,10 +354,6 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 
 				err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
 				if err == nil {
-					// Track directory for empty directory cleanup
-					if !s3a.option.AllowEmptyFolder {
-						directoriesWithDeletion[parentDirectoryPath] = true
-					}
 					deletedObjects = append(deletedObjects, object)
 				} else if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
 					deletedObjects = append(deletedObjects, object)
@@ -380,30 +373,8 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 			}
 		}
 
-		// Cleanup empty directories - optimize by processing deepest first
-		if !s3a.option.AllowEmptyFolder && len(directoriesWithDeletion) > 0 {
-			bucketPath := fmt.Sprintf("%s/%s", s3a.option.BucketsPath, bucket)
-
-			// Collect and sort directories by depth (deepest first) to avoid redundant checks
-			var allDirs []string
-			for dirPath := range directoriesWithDeletion {
-				allDirs = append(allDirs, dirPath)
-			}
-			// Sort by depth (deeper directories first)
-			slices.SortFunc(allDirs, func(a, b string) int {
-				return strings.Count(b, "/") - strings.Count(a, "/")
-			})
-
-			// Track already-checked directories to avoid redundant work
-			checked := make(map[string]bool)
-			for _, dirPath := range allDirs {
-				if !checked[dirPath] {
-					// Recursively delete empty parent directories, stop at bucket path
-					// Mark this directory and all its parents as checked during recursion
-					filer_pb.DoDeleteEmptyParentDirectories(opCtx, client, util.FullPath(dirPath), util.FullPath(bucketPath), checked)
-				}
-			}
-		}
+		// Note: Empty folder cleanup is now handled asynchronously by EmptyFolderCleaner
+		// which listens to metadata events and uses consistent hashing for coordination
 
 		return nil
 	})

weed/s3api/s3api_object_handlers_put.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -165,12 +166,14 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 			return
 		}
 
-		// For non-versioned buckets, check if existing object has object lock protections
+		// For non-versioned buckets with object lock enabled, check if existing object has protections
 		// that would prevent overwrite (PUT operations overwrite existing objects in non-versioned buckets)
-		if !versioningConfigured {
+		// Skip this check for buckets without object lock to avoid expensive entry lookup
+		if !versioningConfigured && s3a.isObjectLockEnabled(bucket) {
 			governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
-			if err := s3a.enforceObjectLockProtections(r, bucket, object, "", governanceBypassAllowed); err != nil {
-				glog.V(2).Infof("PutObjectHandler: object lock permissions check failed for %s/%s: %v", bucket, object, err)
+			// Note: We ignore the returned entry since PUT creates a new entry anyway
+			if _, lockErr := s3a.enforceObjectLockProtections(r, bucket, object, "", governanceBypassAllowed); lockErr != nil {
+				glog.V(2).Infof("PutObjectHandler: object lock permissions check failed for %s/%s: %v", bucket, object, lockErr)
 				s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
 				return
 			}
@@ -548,6 +551,28 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
 		}
 	}
 
+	// Parse and store object tags from X-Amz-Tagging header
+	// Fix for GitHub issue #7589: Tags sent during object upload were not being stored
+	if tagging := r.Header.Get(s3_constants.AmzObjectTagging); tagging != "" {
+		parsedTags, err := url.ParseQuery(tagging)
+		if err != nil {
+			glog.Warningf("putToFiler: Invalid S3 tag format in header '%s': %v", tagging, err)
+			return "", s3err.ErrInvalidTag, SSEResponseMetadata{}
+		}
+		for key, values := range parsedTags {
+			if len(values) > 1 {
+				glog.Warningf("putToFiler: Duplicate tag key '%s' in header", key)
+				return "", s3err.ErrInvalidTag, SSEResponseMetadata{}
+			}
+			value := ""
+			if len(values) > 0 {
+				value = values[0]
+			}
+			entry.Extended[s3_constants.AmzObjectTagging+"-"+key] = []byte(value)
+		}
+		glog.V(3).Infof("putToFiler: stored %d tags from X-Amz-Tagging header", len(parsedTags))
+	}
+
 	// Set SSE-C metadata
 	if customerKey != nil && len(sseIV) > 0 {
 		// Store IV as RAW bytes (matches filer behavior - filer decodes base64 headers and stores raw bytes)
@@ -680,9 +705,9 @@ func filerErrorToS3Error(err error) s3err.ErrorCode {
 	if err == nil {
 		return s3err.ErrNone
 	}
-	
+
 	errString := err.Error()
-	
+
 	switch {
 	case errString == constants.ErrMsgBadDigest:
 		return s3err.ErrBadDigest

weed/s3api/s3api_object_retention.go

@@ -584,8 +584,10 @@ func (s3a *S3ApiServer) evaluateGovernanceBypassRequest(r *http.Request, bucket,
 	return true
 }
 
-// enforceObjectLockProtections enforces object lock protections for operations
-func (s3a *S3ApiServer) enforceObjectLockProtections(request *http.Request, bucket, object, versionId string, governanceBypassAllowed bool) error {
+// enforceObjectLockProtections enforces object lock protections for operations.
+// Returns the fetched entry (if found) to allow callers to reuse it, avoiding duplicate lookups.
+// Returns (nil, nil) if object doesn't exist (not an error for delete operations).
+func (s3a *S3ApiServer) enforceObjectLockProtections(request *http.Request, bucket, object, versionId string, governanceBypassAllowed bool) (*filer_pb.Entry, error) {
 	// Get the object entry to check both retention and legal hold
 	// For delete operations without versionId, we need to check the latest version
 	var entry *filer_pb.Entry
@@ -604,10 +606,10 @@ func (s3a *S3ApiServer) enforceObjectLockProtections(request *http.Request, buck
 		if errors.Is(err, filer_pb.ErrNotFound) || errors.Is(err, ErrObjectNotFound) || errors.Is(err, ErrVersionNotFound) || errors.Is(err, ErrLatestVersionNotFound) {
 			// Object doesn't exist, so it can't be under retention or legal hold - this is normal
 			glog.V(4).Infof("Object %s/%s (versionId: %s) not found during object lock check (expected during delete operations)", bucket, object, versionId)
-			return nil
+			return nil, nil
 		}
 		glog.Warningf("Error retrieving object %s/%s (versionId: %s) for lock check: %v", bucket, object, versionId, err)
-		return err
+		return nil, err
 	}
 
 	// Extract retention information from the entry
@@ -626,25 +628,25 @@ func (s3a *S3ApiServer) enforceObjectLockProtections(request *http.Request, buck
 
 	// If object is under legal hold, it cannot be deleted or modified (including delete marker creation)
 	if legalHoldActive {
-		return ErrObjectUnderLegalHold
+		return entry, ErrObjectUnderLegalHold
 	}
 
 	// If object is under retention, check the mode
 	if retentionActive && retention != nil {
 		if retention.Mode == s3_constants.RetentionModeCompliance {
-			return ErrComplianceModeActive
+			return entry, ErrComplianceModeActive
 		}
 
 		if retention.Mode == s3_constants.RetentionModeGovernance {
 			if !governanceBypassAllowed {
-				return ErrGovernanceModeActive
+				return entry, ErrGovernanceModeActive
 			}
 			// Note: governanceBypassAllowed parameter is already validated by evaluateGovernanceBypassRequest()
 			// which checks both header presence and IAM permissions, so we trust it here
 		}
 	}
 
-	return nil
+	return entry, nil
 }
 
 // ====================================================================

weed/s3api/s3api_object_versioning.go

@@ -637,8 +637,10 @@ func (s3a *S3ApiServer) getSpecificObjectVersion(bucket, object, versionId strin
 	return entry, nil
 }
 
-// deleteSpecificObjectVersion deletes a specific version of an object
-func (s3a *S3ApiServer) deleteSpecificObjectVersion(bucket, object, versionId string) error {
+// deleteSpecificObjectVersion deletes a specific version of an object.
+// The optional prefetchedEntry parameter allows callers to pass an already-fetched entry
+// to avoid duplicate filer lookups (optimization for DELETE operations).
+func (s3a *S3ApiServer) deleteSpecificObjectVersion(bucket, object, versionId string, prefetchedEntry *filer_pb.Entry) error {
 	// Normalize object path to ensure consistency with toFilerPath behavior
 	normalizedObject := removeDuplicateSlashes(object)
 
@@ -651,12 +653,14 @@ func (s3a *S3ApiServer) deleteSpecificObjectVersion(bucket, object, versionId st
 		bucketDir := s3a.option.BucketsPath + "/" + bucket
 		cleanObject := strings.TrimPrefix(normalizedObject, "/")
 
-		// Check if the object exists
-		_, err := s3a.getEntry(bucketDir, cleanObject)
-		if err != nil {
-			// Object doesn't exist - this is OK for delete operations (idempotent)
-			glog.V(2).Infof("deleteSpecificObjectVersion: null version object %s already deleted or doesn't exist", cleanObject)
-			return nil
+		// Check if the object exists (skip if we have a prefetched entry)
+		if prefetchedEntry == nil {
+			_, err := s3a.getEntry(bucketDir, cleanObject)
+			if err != nil {
+				// Object doesn't exist - this is OK for delete operations (idempotent)
+				glog.V(2).Infof("deleteSpecificObjectVersion: null version object %s already deleted or doesn't exist", cleanObject)
+				return nil
+			}
 		}
 
 		// Delete the regular file

weed/s3api/s3api_streaming_copy.go

@@ -59,18 +59,19 @@ func NewStreamingCopyManager(s3a *S3ApiServer) *StreamingCopyManager {
 	}
 }
 
-// ExecuteStreamingCopy performs a streaming copy operation
-func (scm *StreamingCopyManager) ExecuteStreamingCopy(ctx context.Context, entry *filer_pb.Entry, r *http.Request, dstPath string, state *EncryptionState) ([]*filer_pb.FileChunk, error) {
+// ExecuteStreamingCopy performs a streaming copy operation and returns the encryption spec
+// The encryption spec is needed for SSE-S3 to properly set destination metadata (fixes GitHub #7562)
+func (scm *StreamingCopyManager) ExecuteStreamingCopy(ctx context.Context, entry *filer_pb.Entry, r *http.Request, dstPath string, state *EncryptionState) ([]*filer_pb.FileChunk, *EncryptionSpec, error) {
 	// Create streaming copy specification
 	spec, err := scm.createStreamingSpec(entry, r, state)
 	if err != nil {
-		return nil, fmt.Errorf("create streaming spec: %w", err)
+		return nil, nil, fmt.Errorf("create streaming spec: %w", err)
 	}
 
 	// Create source reader from entry
 	sourceReader, err := scm.createSourceReader(entry)
 	if err != nil {
-		return nil, fmt.Errorf("create source reader: %w", err)
+		return nil, nil, fmt.Errorf("create source reader: %w", err)
 	}
 	defer sourceReader.Close()
 
@@ -79,11 +80,16 @@ func (scm *StreamingCopyManager) ExecuteStreamingCopy(ctx context.Context, entry
 	// Create processing pipeline
 	processedReader, err := scm.createProcessingPipeline(spec)
 	if err != nil {
-		return nil, fmt.Errorf("create processing pipeline: %w", err)
+		return nil, nil, fmt.Errorf("create processing pipeline: %w", err)
 	}
 
 	// Stream to destination
-	return scm.streamToDestination(ctx, processedReader, spec, dstPath)
+	chunks, err := scm.streamToDestination(ctx, processedReader, spec, dstPath)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return chunks, spec.EncryptionSpec, nil
 }
 
 // createStreamingSpec creates a streaming specification based on copy parameters
@@ -453,8 +459,8 @@ func (scm *StreamingCopyManager) streamToChunks(ctx context.Context, reader io.R
 	for {
 		n, err := reader.Read(buffer)
 		if n > 0 {
-			// Create chunk for this data
-			chunk, chunkErr := scm.createChunkFromData(buffer[:n], offset, dstPath)
+		// Create chunk for this data, setting SSE type and per-chunk metadata (including chunk-specific IVs for SSE-S3)
+		chunk, chunkErr := scm.createChunkFromData(buffer[:n], offset, dstPath, spec.EncryptionSpec)
 			if chunkErr != nil {
 				return nil, fmt.Errorf("create chunk from data: %w", chunkErr)
 			}
@@ -474,7 +480,7 @@ func (scm *StreamingCopyManager) streamToChunks(ctx context.Context, reader io.R
 }
 
 // createChunkFromData creates a chunk from streaming data
-func (scm *StreamingCopyManager) createChunkFromData(data []byte, offset int64, dstPath string) (*filer_pb.FileChunk, error) {
+func (scm *StreamingCopyManager) createChunkFromData(data []byte, offset int64, dstPath string, encSpec *EncryptionSpec) (*filer_pb.FileChunk, error) {
 	// Assign new volume
 	assignResult, err := scm.s3a.assignNewVolume(dstPath)
 	if err != nil {
@@ -487,6 +493,42 @@ func (scm *StreamingCopyManager) createChunkFromData(data []byte, offset int64,
 		Size:   uint64(len(data)),
 	}
 
+	// Set SSE type and metadata on chunk if destination is encrypted
+	// This is critical for GetObject to know to decrypt the data - fixes GitHub #7562
+	if encSpec != nil && encSpec.NeedsEncryption {
+		switch encSpec.DestinationType {
+		case EncryptionTypeSSEC:
+			chunk.SseType = filer_pb.SSEType_SSE_C
+			// SSE-C metadata is handled at object level, not per-chunk for streaming copy
+		case EncryptionTypeSSEKMS:
+			chunk.SseType = filer_pb.SSEType_SSE_KMS
+			// SSE-KMS metadata is handled at object level, not per-chunk for streaming copy
+		case EncryptionTypeSSES3:
+			chunk.SseType = filer_pb.SSEType_SSE_S3
+			// Create per-chunk SSE-S3 metadata with chunk-specific IV
+			if sseKey, ok := encSpec.DestinationKey.(*SSES3Key); ok {
+				// Calculate chunk-specific IV using base IV and chunk offset
+				baseIV := encSpec.DestinationIV
+				if len(baseIV) == 0 {
+					return nil, fmt.Errorf("SSE-S3 encryption requires DestinationIV to be set for chunk at offset %d", offset)
+				}
+				chunkIV, _ := calculateIVWithOffset(baseIV, offset)
+				// Create chunk key with the chunk-specific IV
+				chunkSSEKey := &SSES3Key{
+					Key:       sseKey.Key,
+					KeyID:     sseKey.KeyID,
+					Algorithm: sseKey.Algorithm,
+					IV:        chunkIV,
+				}
+				chunkMetadata, serErr := SerializeSSES3Metadata(chunkSSEKey)
+				if serErr != nil {
+					return nil, fmt.Errorf("failed to serialize chunk SSE-S3 metadata: %w", serErr)
+				}
+				chunk.SseMetadata = chunkMetadata
+			}
+		}
+	}
+
 	// Set file ID
 	if err := scm.s3a.setChunkFileId(chunk, assignResult); err != nil {
 		return nil, err

weed/server/common.go

@@ -9,9 +9,9 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"mime"
 	"mime/multipart"
 	"net/http"
-	"net/url"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -286,14 +286,15 @@ func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, file
 		return
 	}
 	if filename != "" {
-		filename = url.QueryEscape(filename)
-		contentDisposition := "inline"
+		dispositionType := "inline"
 		if r.FormValue("dl") != "" {
 			if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl {
-				contentDisposition = "attachment"
+				dispositionType = "attachment"
 			}
 		}
-		w.Header().Set("Content-Disposition", contentDisposition+`; filename="`+fileNameEscaper.Replace(filename)+`"`)
+		// Use mime.FormatMediaType for RFC 6266 compliant Content-Disposition,
+		// properly handling non-ASCII characters and special characters
+		w.Header().Set("Content-Disposition", mime.FormatMediaType(dispositionType, map[string]string{"filename": filename}))
 	}
 }
 

weed/server/master_grpc_server.go

@@ -137,8 +137,8 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 			dcName, rackName := ms.Topo.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
 			dc := ms.Topo.GetOrCreateDataCenter(dcName)
 			rack := dc.GetOrCreateRack(rackName)
-			dn = rack.GetOrCreateDataNode(heartbeat.Ip, int(heartbeat.Port), int(heartbeat.GrpcPort), heartbeat.PublicUrl, heartbeat.MaxVolumeCounts)
-			glog.V(0).Infof("added volume server %d: %v:%d %v", dn.Counter, heartbeat.GetIp(), heartbeat.GetPort(), heartbeat.LocationUuids)
+			dn = rack.GetOrCreateDataNode(heartbeat.Ip, int(heartbeat.Port), int(heartbeat.GrpcPort), heartbeat.PublicUrl, heartbeat.Id, heartbeat.MaxVolumeCounts)
+			glog.V(0).Infof("added volume server %d: %v (id=%s, ip=%v:%d) %v", dn.Counter, dn.Id(), heartbeat.Id, heartbeat.GetIp(), heartbeat.GetPort(), heartbeat.LocationUuids)
 			uuidlist, err := ms.RegisterUuids(heartbeat)
 			if err != nil {
 				if stream_err := stream.Send(&master_pb.HeartbeatResponse{

weed/server/master_grpc_server_volume.go

@@ -253,7 +253,7 @@ func (ms *MasterServer) LookupEcVolume(ctx context.Context, req *master_pb.Looku
 		var locations []*master_pb.Location
 		for _, dn := range shardLocations {
 			locations = append(locations, &master_pb.Location{
-				Url:        string(dn.Id()),
+				Url:        dn.Url(),
 				PublicUrl:  dn.PublicUrl,
 				DataCenter: dn.GetDataCenterId(),
 			})

weed/server/volume_grpc_copy.go

@@ -115,7 +115,7 @@ func (vs *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, stre
 		var sendErr error
 		var ioBytePerSecond int64
 		if req.IoBytePerSecond <= 0 {
-			ioBytePerSecond = vs.compactionBytePerSecond
+			ioBytePerSecond = vs.maintenanceBytePerSecond
 		} else {
 			ioBytePerSecond = req.IoBytePerSecond
 		}
@@ -199,7 +199,7 @@ func (vs *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, stre
 }
 
 func (vs *VolumeServer) doCopyFile(client volume_server_pb.VolumeServerClient, isEcVolume bool, collection string, vid, compactRevision uint32, stopOffset uint64, baseFileName, ext string, isAppend, ignoreSourceFileNotFound bool, progressFn storage.ProgressFunc) (modifiedTsNs int64, err error) {
-	return vs.doCopyFileWithThrottler(client, isEcVolume, collection, vid, compactRevision, stopOffset, baseFileName, ext, isAppend, ignoreSourceFileNotFound, progressFn, util.NewWriteThrottler(vs.compactionBytePerSecond))
+	return vs.doCopyFileWithThrottler(client, isEcVolume, collection, vid, compactRevision, stopOffset, baseFileName, ext, isAppend, ignoreSourceFileNotFound, progressFn, util.NewWriteThrottler(vs.maintenanceBytePerSecond))
 }
 
 func (vs *VolumeServer) doCopyFileWithThrottler(client volume_server_pb.VolumeServerClient, isEcVolume bool, collection string, vid, compactRevision uint32, stopOffset uint64, baseFileName, ext string, isAppend, ignoreSourceFileNotFound bool, progressFn storage.ProgressFunc, throttler *util.WriteThrottler) (modifiedTsNs int64, err error) {
@@ -264,7 +264,7 @@ func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName s
 	}
 	dst, err := os.OpenFile(fileName, flags, 0644)
 	if err != nil {
-		return modifiedTsNs, nil
+		return modifiedTsNs, fmt.Errorf("open file %s: %w", fileName, err)
 	}
 	defer dst.Close()
 
@@ -278,9 +278,11 @@ func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName s
 			modifiedTsNs = resp.ModifiedTsNs
 		}
 		if receiveErr != nil {
-			return modifiedTsNs, fmt.Errorf("receiving %s: %v", fileName, receiveErr)
+			return modifiedTsNs, fmt.Errorf("receiving %s: %w", fileName, receiveErr)
+		}
+		if _, writeErr := dst.Write(resp.FileContent); writeErr != nil {
+			return modifiedTsNs, fmt.Errorf("write file %s: %w", fileName, writeErr)
 		}
-		dst.Write(resp.FileContent)
 		progressedBytes += int64(len(resp.FileContent))
 		if progressFn != nil {
 			if !progressFn(progressedBytes) {

weed/server/volume_server.go

@@ -42,20 +42,21 @@ type VolumeServer struct {
 	guard           *security.Guard
 	grpcDialOption  grpc.DialOption
 
-	needleMapKind           storage.NeedleMapKind
-	ldbTimout               int64
-	FixJpgOrientation       bool
-	ReadMode                string
-	compactionBytePerSecond int64
-	metricsAddress          string
-	metricsIntervalSec      int
-	fileSizeLimitBytes      int64
-	isHeartbeating          bool
-	stopChan                chan bool
+	needleMapKind            storage.NeedleMapKind
+	ldbTimout                int64
+	FixJpgOrientation        bool
+	ReadMode                 string
+	compactionBytePerSecond  int64
+	maintenanceBytePerSecond int64
+	metricsAddress           string
+	metricsIntervalSec       int
+	fileSizeLimitBytes       int64
+	isHeartbeating           bool
+	stopChan                 chan bool
 }
 
 func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
-	port int, grpcPort int, publicUrl string,
+	port int, grpcPort int, publicUrl string, id string,
 	folders []string, maxCounts []int32, minFreeSpaces []util.MinFreeSpace, diskTypes []types.DiskType,
 	idxFolder string,
 	needleMapKind storage.NeedleMapKind,
@@ -65,6 +66,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	fixJpgOrientation bool,
 	readMode string,
 	compactionMBPerSecond int,
+	maintenanceMBPerSecond int,
 	fileSizeLimitMB int,
 	concurrentUploadLimit int64,
 	concurrentDownloadLimit int64,
@@ -94,6 +96,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 		ReadMode:                      readMode,
 		grpcDialOption:                security.LoadClientTLS(util.GetViper(), "grpc.volume"),
 		compactionBytePerSecond:       int64(compactionMBPerSecond) * 1024 * 1024,
+		maintenanceBytePerSecond:      int64(maintenanceMBPerSecond) * 1024 * 1024,
 		fileSizeLimitBytes:            int64(fileSizeLimitMB) * 1024 * 1024,
 		isHeartbeating:                true,
 		stopChan:                      make(chan bool),
@@ -114,7 +117,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 
 	vs.checkWithMaster()
 
-	vs.store = storage.NewStore(vs.grpcDialOption, ip, port, grpcPort, publicUrl, folders, maxCounts, minFreeSpaces, idxFolder, vs.needleMapKind, diskTypes, ldbTimeout)
+	vs.store = storage.NewStore(vs.grpcDialOption, ip, port, grpcPort, publicUrl, id, folders, maxCounts, minFreeSpaces, idxFolder, vs.needleMapKind, diskTypes, ldbTimeout)
 	vs.guard = security.NewGuard(whiteList, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec)
 
 	handleStaticResources(adminMux)

weed/server/volume_server_handlers_admin.go

@@ -4,28 +4,33 @@ import (
 	"net/http"
 	"path/filepath"
 
-	"github.com/seaweedfs/seaweedfs/weed/topology"
 	"github.com/seaweedfs/seaweedfs/weed/util/version"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
 )
 
+// healthzHandler checks the local health of the volume server.
+// It only checks local conditions to avoid cascading failures when remote
+// volume servers go down. Previously, this handler checked if all replicated
+// volumes could reach their remote replicas, which caused healthy volume
+// servers to fail health checks when a peer went down.
+// See https://github.com/seaweedfs/seaweedfs/issues/6823
 func (vs *VolumeServer) healthzHandler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Server", "SeaweedFS Volume "+version.VERSION)
-	volumeInfos := vs.store.VolumeInfos()
-	for _, vinfo := range volumeInfos {
-		if len(vinfo.Collection) == 0 {
-			continue
-		}
-		if vinfo.ReplicaPlacement.GetCopyCount() > 1 {
-			_, err := topology.GetWritableRemoteReplications(vs.store, vs.grpcDialOption, vinfo.Id, vs.GetMaster)
-			if err != nil {
-				w.WriteHeader(http.StatusServiceUnavailable)
-				return
-			}
-		}
+
+	// Check if the server is shutting down
+	if vs.store.IsStopping() {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
 	}
+
+	// Check if we can communicate with master
+	if !vs.isHeartbeating {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+
 	w.WriteHeader(http.StatusOK)
 }
 

weed/server/volume_server_handlers_read.go

@@ -34,8 +34,6 @@ import (
 
 const reqIsProxied = "proxied"
 
-var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
-
 func NotFound(w http.ResponseWriter) {
 	stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorGetNotFound).Inc()
 	w.WriteHeader(http.StatusNotFound)

weed/sftpd/sftp_file_writer.go

@@ -72,6 +72,7 @@ func (l listerat) ListAt(ls []os.FileInfo, offset int64) (int, error) {
 type SeaweedSftpFileWriter struct {
 	fs          SftpServer
 	req         *sftp.Request
+	absPath     string // Absolute path after HomeDir translation
 	mu          sync.Mutex
 	tmpFile     *os.File
 	permissions os.FileMode
@@ -105,6 +106,6 @@ func (w *SeaweedSftpFileWriter) Close() error {
 		return err
 	}
 
-	// Stream the file instead of loading it
-	return w.fs.putFile(w.req.Filepath, w.tmpFile, w.fs.user)
+	// Stream the file to the absolute path (after HomeDir translation)
+	return w.fs.putFile(w.absPath, w.tmpFile, w.fs.user)
 }

weed/sftpd/sftp_filer.go

@@ -100,18 +100,26 @@ func (fs *SftpServer) withTimeoutContext(fn func(ctx context.Context) error) err
 // ==================== Command Dispatcher ====================
 
 func (fs *SftpServer) dispatchCmd(r *sftp.Request) error {
-	glog.V(0).Infof("Dispatch: %s %s", r.Method, r.Filepath)
+	absPath, err := fs.toAbsolutePath(r.Filepath)
+	if err != nil {
+		return err
+	}
+	glog.V(1).Infof("Dispatch: %s %s (absolute: %s)", r.Method, r.Filepath, absPath)
 	switch r.Method {
 	case "Remove":
-		return fs.removeEntry(r)
+		return fs.removeEntry(absPath)
 	case "Rename":
-		return fs.renameEntry(r)
+		absTarget, err := fs.toAbsolutePath(r.Target)
+		if err != nil {
+			return err
+		}
+		return fs.renameEntry(absPath, absTarget)
 	case "Mkdir":
-		return fs.makeDir(r)
+		return fs.makeDir(absPath)
 	case "Rmdir":
-		return fs.removeDir(r)
+		return fs.removeDir(absPath)
 	case "Setstat":
-		return fs.setFileStat(r)
+		return fs.setFileStatWithRequest(absPath, r)
 	default:
 		return fmt.Errorf("unsupported: %s", r.Method)
 	}
@@ -120,10 +128,14 @@ func (fs *SftpServer) dispatchCmd(r *sftp.Request) error {
 // ==================== File Operations ====================
 
 func (fs *SftpServer) readFile(r *sftp.Request) (io.ReaderAt, error) {
-	if err := fs.checkFilePermission(r.Filepath, "read"); err != nil {
+	absPath, err := fs.toAbsolutePath(r.Filepath)
+	if err != nil {
 		return nil, err
 	}
-	entry, err := fs.getEntry(r.Filepath)
+	if err := fs.checkFilePermission(absPath, "read"); err != nil {
+		return nil, err
+	}
+	entry, err := fs.getEntry(absPath)
 	if err != nil {
 		return nil, err
 	}
@@ -131,7 +143,11 @@ func (fs *SftpServer) readFile(r *sftp.Request) (io.ReaderAt, error) {
 }
 
 func (fs *SftpServer) newFileWriter(r *sftp.Request) (io.WriterAt, error) {
-	dir, _ := util.FullPath(r.Filepath).DirAndName()
+	absPath, err := fs.toAbsolutePath(r.Filepath)
+	if err != nil {
+		return nil, err
+	}
+	dir, _ := util.FullPath(absPath).DirAndName()
 	if err := fs.checkFilePermission(dir, "write"); err != nil {
 		glog.Errorf("Permission denied for %s", dir)
 		return nil, err
@@ -145,6 +161,7 @@ func (fs *SftpServer) newFileWriter(r *sftp.Request) (io.WriterAt, error) {
 	return &SeaweedSftpFileWriter{
 		fs:          *fs,
 		req:         r,
+		absPath:     absPath,
 		tmpFile:     tmpFile,
 		permissions: 0644,
 		uid:         fs.user.Uid,
@@ -153,16 +170,20 @@ func (fs *SftpServer) newFileWriter(r *sftp.Request) (io.WriterAt, error) {
 	}, nil
 }
 
-func (fs *SftpServer) removeEntry(r *sftp.Request) error {
-	return fs.deleteEntry(r.Filepath, false)
+func (fs *SftpServer) removeEntry(absPath string) error {
+	return fs.deleteEntry(absPath, false)
 }
 
-func (fs *SftpServer) renameEntry(r *sftp.Request) error {
-	if err := fs.checkFilePermission(r.Filepath, "rename"); err != nil {
+func (fs *SftpServer) renameEntry(absPath, absTarget string) error {
+	if err := fs.checkFilePermission(absPath, "rename"); err != nil {
 		return err
 	}
-	oldDir, oldName := util.FullPath(r.Filepath).DirAndName()
-	newDir, newName := util.FullPath(r.Target).DirAndName()
+	targetDir, _ := util.FullPath(absTarget).DirAndName()
+	if err := fs.checkFilePermission(targetDir, "write"); err != nil {
+		return err
+	}
+	oldDir, oldName := util.FullPath(absPath).DirAndName()
+	newDir, newName := util.FullPath(absTarget).DirAndName()
 	return fs.callWithClient(false, func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
 		_, err := client.AtomicRenameEntry(ctx, &filer_pb.AtomicRenameEntryRequest{
 			OldDirectory: oldDir, OldName: oldName,
@@ -172,15 +193,15 @@ func (fs *SftpServer) renameEntry(r *sftp.Request) error {
 	})
 }
 
-func (fs *SftpServer) setFileStat(r *sftp.Request) error {
-	if err := fs.checkFilePermission(r.Filepath, "write"); err != nil {
+func (fs *SftpServer) setFileStatWithRequest(absPath string, r *sftp.Request) error {
+	if err := fs.checkFilePermission(absPath, "write"); err != nil {
 		return err
 	}
-	entry, err := fs.getEntry(r.Filepath)
+	entry, err := fs.getEntry(absPath)
 	if err != nil {
 		return err
 	}
-	dir, _ := util.FullPath(r.Filepath).DirAndName()
+	dir, _ := util.FullPath(absPath).DirAndName()
 	// apply attrs
 	if r.AttrFlags().Permissions {
 		entry.Attributes.FileMode = uint32(r.Attributes().FileMode())
@@ -201,18 +222,22 @@ func (fs *SftpServer) setFileStat(r *sftp.Request) error {
 // ==================== Directory Operations ====================
 
 func (fs *SftpServer) listDir(r *sftp.Request) (sftp.ListerAt, error) {
-	if err := fs.checkFilePermission(r.Filepath, "list"); err != nil {
+	absPath, err := fs.toAbsolutePath(r.Filepath)
+	if err != nil {
+		return nil, err
+	}
+	if err := fs.checkFilePermission(absPath, "list"); err != nil {
 		return nil, err
 	}
 	if r.Method == "Stat" || r.Method == "Lstat" {
-		entry, err := fs.getEntry(r.Filepath)
+		entry, err := fs.getEntry(absPath)
 		if err != nil {
 			return nil, err
 		}
 		fi := &EnhancedFileInfo{FileInfo: FileInfoFromEntry(entry), uid: entry.Attributes.Uid, gid: entry.Attributes.Gid}
 		return listerat([]os.FileInfo{fi}), nil
 	}
-	return fs.listAllPages(r.Filepath)
+	return fs.listAllPages(absPath)
 }
 
 func (fs *SftpServer) listAllPages(dirPath string) (sftp.ListerAt, error) {
@@ -259,18 +284,19 @@ func (fs *SftpServer) fetchDirectoryPage(dirPath, start string) ([]os.FileInfo,
 }
 
 // makeDir creates a new directory with proper permissions.
-func (fs *SftpServer) makeDir(r *sftp.Request) error {
+func (fs *SftpServer) makeDir(absPath string) error {
 	if fs.user == nil {
 		return fmt.Errorf("cannot create directory: no user info")
 	}
-	dir, name := util.FullPath(r.Filepath).DirAndName()
-	if err := fs.checkFilePermission(r.Filepath, "mkdir"); err != nil {
+	dir, name := util.FullPath(absPath).DirAndName()
+	if err := fs.checkFilePermission(dir, "write"); err != nil {
 		return err
 	}
 	// default mode and ownership
 	err := filer_pb.Mkdir(context.Background(), fs, string(dir), name, func(entry *filer_pb.Entry) {
 		mode := uint32(0755 | os.ModeDir)
-		if strings.HasPrefix(r.Filepath, fs.user.HomeDir) {
+		// Defensive check: all paths should be under HomeDir after toAbsolutePath translation
+		if absPath == fs.user.HomeDir || strings.HasPrefix(absPath, fs.user.HomeDir+"/") {
 			mode = uint32(0700 | os.ModeDir)
 		}
 		entry.Attributes.FileMode = mode
@@ -288,8 +314,8 @@ func (fs *SftpServer) makeDir(r *sftp.Request) error {
 }
 
 // removeDir deletes a directory.
-func (fs *SftpServer) removeDir(r *sftp.Request) error {
-	return fs.deleteEntry(r.Filepath, false)
+func (fs *SftpServer) removeDir(absPath string) error {
+	return fs.deleteEntry(absPath, false)
 }
 
 func (fs *SftpServer) putFile(filepath string, reader io.Reader, user *user.User) error {

weed/sftpd/sftp_server.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
+	"strings"
 	"time"
 
 	"github.com/pkg/sftp"
@@ -37,6 +39,28 @@ func NewSftpServer(filerAddr pb.ServerAddress, grpcDialOption grpc.DialOption, d
 	}
 }
 
+// toAbsolutePath translates a user-relative path to an absolute filer path.
+// When a user has HomeDir="/sftp/user", their view of "/" maps to "/sftp/user".
+// This implements chroot-like behavior where the user's home directory
+// becomes their root.
+func (fs *SftpServer) toAbsolutePath(userPath string) (string, error) {
+	// If user has root as home directory, no translation needed
+	if fs.user.HomeDir == "" || fs.user.HomeDir == "/" {
+		return path.Clean(userPath), nil
+	}
+
+	// Concatenate home directory with user path, then clean to resolve any ".." components
+	p := path.Join(fs.user.HomeDir, strings.TrimPrefix(userPath, "/"))
+
+	// Security check: ensure the final path is within the home directory.
+	// This prevents path traversal attacks like `../..` that could escape the chroot jail.
+	if !strings.HasPrefix(p, fs.user.HomeDir+"/") && p != fs.user.HomeDir {
+		return "", fmt.Errorf("path traversal attempt: %s resolves to %s which is outside home dir %s", userPath, p, fs.user.HomeDir)
+	}
+
+	return p, nil
+}
+
 // Fileread is invoked for “get” requests.
 func (fs *SftpServer) Fileread(req *sftp.Request) (io.ReaderAt, error) {
 	return fs.readFile(req)

weed/sftpd/sftp_server_test.go

@@ -0,0 +1,103 @@
+package sftpd
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/sftpd/user"
+	"github.com/stretchr/testify/assert"
+)
+
+func stringPtr(s string) *string {
+	return &s
+}
+
+func TestToAbsolutePath(t *testing.T) {
+	tests := []struct {
+		name        string
+		homeDir     *string // Use pointer to distinguish between unset and empty
+		userPath    string
+		expected    string
+		expectError bool
+	}{
+		{
+			name:     "normal path",
+			userPath: "/foo.txt",
+			expected: "/sftp/testuser/foo.txt",
+		},
+		{
+			name:     "root path",
+			userPath: "/",
+			expected: "/sftp/testuser",
+		},
+		{
+			name:     "path with dot",
+			userPath: "/./foo.txt",
+			expected: "/sftp/testuser/foo.txt",
+		},
+		{
+			name:        "path traversal attempts",
+			userPath:    "/../foo.txt",
+			expectError: true,
+		},
+		{
+			name:        "path traversal attempts 2",
+			userPath:    "../../foo.txt",
+			expectError: true,
+		},
+		{
+			name:        "path traversal attempts 3",
+			userPath:    "/subdir/../../foo.txt",
+			expectError: true,
+		},
+		{
+			name:     "empty path",
+			userPath: "",
+			expected: "/sftp/testuser",
+		},
+		{
+			name:     "multiple slashes",
+			userPath: "//foo.txt",
+			expected: "/sftp/testuser/foo.txt",
+		},
+		{
+			name:     "trailing slash",
+			userPath: "/foo/",
+			expected: "/sftp/testuser/foo",
+		},
+		{
+			name:     "empty HomeDir passthrough",
+			homeDir:  stringPtr(""),
+			userPath: "/foo.txt",
+			expected: "/foo.txt",
+		},
+		{
+			name:     "root HomeDir passthrough",
+			homeDir:  stringPtr("/"),
+			userPath: "/foo.txt",
+			expected: "/foo.txt",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			homeDir := "/sftp/testuser" // default
+			if tt.homeDir != nil {
+				homeDir = *tt.homeDir
+			}
+
+			fs := &SftpServer{
+				user: &user.User{
+					HomeDir: homeDir,
+				},
+			}
+
+			got, err := fs.toAbsolutePath(tt.userPath)
+			if tt.expectError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expected, got)
+			}
+		})
+	}
+}

weed/sftpd/sftp_service.go

@@ -284,8 +284,8 @@ func (s *SFTPService) handleChannel(newChannel ssh.NewChannel, fs *SftpServer) {
 
 // handleSFTP starts the SFTP server on the SSH channel.
 func (s *SFTPService) handleSFTP(channel ssh.Channel, fs *SftpServer) {
-	// Create server options with initial working directory set to user's home
-	serverOptions := sftp.WithStartDirectory(fs.user.HomeDir)
+	// Start at virtual root "/" - toAbsolutePath translates this to the user's HomeDir
+	serverOptions := sftp.WithStartDirectory("/")
 	server := sftp.NewRequestServer(channel, sftp.Handlers{
 		FileGet:  fs,
 		FilePut:  fs,

weed/sftpd/user/filestore.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path"
 	"sync"
 
 	"golang.org/x/crypto/ssh"
@@ -99,6 +100,10 @@ func (s *FileStore) loadUsers() error {
 				user.PublicKeys[i] = string(pubKey.Marshal())
 			}
 		}
+		// Clean HomeDir to handle trailing slashes and normalize path
+		if user.HomeDir != "" {
+			user.HomeDir = path.Clean(user.HomeDir)
+		}
 		s.users[user.Username] = user
 
 	}

weed/shell/command_ec_common.go

@@ -26,12 +26,25 @@ type DataCenterId string
 type EcNodeId string
 type RackId string
 
+// EcDisk represents a single disk on a volume server
+type EcDisk struct {
+	diskId       uint32
+	diskType     string
+	freeEcSlots  int
+	ecShardCount int // Total EC shards on this disk
+	// Map of volumeId -> shardBits for shards on this disk
+	ecShards map[needle.VolumeId]erasure_coding.ShardBits
+}
+
 type EcNode struct {
 	info       *master_pb.DataNodeInfo
 	dc         DataCenterId
 	rack       RackId
 	freeEcSlot int
+	// disks maps diskId -> EcDisk for disk-level balancing
+	disks map[uint32]*EcDisk
 }
+
 type CandidateEcNode struct {
 	ecNode     *EcNode
 	shardCount int
@@ -229,7 +242,7 @@ func collectCollectionsForVolumeIds(t *master_pb.TopologyInfo, vids []needle.Vol
 	return collections
 }
 
-func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, destinationEcNode *EcNode, applyBalancing bool) (err error) {
+func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, destinationEcNode *EcNode, destDiskId uint32, applyBalancing bool) (err error) {
 
 	if !commandEnv.isLocked() {
 		return fmt.Errorf("lock is lost")
@@ -242,7 +255,7 @@ func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode,
 		existingServerAddress := pb.NewServerAddressFromDataNode(existingLocation.info)
 
 		// ask destination node to copy shard and the ecx file from source node, and mount it
-		copiedShardIds, err = oneServerCopyAndMountEcShardsFromSource(commandEnv.option.GrpcDialOption, destinationEcNode, []uint32{uint32(shardId)}, vid, collection, existingServerAddress)
+		copiedShardIds, err = oneServerCopyAndMountEcShardsFromSource(commandEnv.option.GrpcDialOption, destinationEcNode, []uint32{uint32(shardId)}, vid, collection, existingServerAddress, destDiskId)
 		if err != nil {
 			return err
 		}
@@ -259,7 +272,11 @@ func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode,
 			return err
 		}
 
-		fmt.Printf("moved ec shard %d.%d %s => %s\n", vid, shardId, existingLocation.info.Id, destinationEcNode.info.Id)
+		if destDiskId > 0 {
+			fmt.Printf("moved ec shard %d.%d %s => %s (disk %d)\n", vid, shardId, existingLocation.info.Id, destinationEcNode.info.Id, destDiskId)
+		} else {
+			fmt.Printf("moved ec shard %d.%d %s => %s\n", vid, shardId, existingLocation.info.Id, destinationEcNode.info.Id)
+		}
 
 	}
 
@@ -272,7 +289,7 @@ func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode,
 
 func oneServerCopyAndMountEcShardsFromSource(grpcDialOption grpc.DialOption,
 	targetServer *EcNode, shardIdsToCopy []uint32,
-	volumeId needle.VolumeId, collection string, existingLocation pb.ServerAddress) (copiedShardIds []uint32, err error) {
+	volumeId needle.VolumeId, collection string, existingLocation pb.ServerAddress, destDiskId uint32) (copiedShardIds []uint32, err error) {
 
 	fmt.Printf("allocate %d.%v %s => %s\n", volumeId, shardIdsToCopy, existingLocation, targetServer.info.Id)
 
@@ -289,6 +306,7 @@ func oneServerCopyAndMountEcShardsFromSource(grpcDialOption grpc.DialOption,
 				CopyEcjFile:    true,
 				CopyVifFile:    true,
 				SourceDataNode: string(existingLocation),
+				DiskId:         destDiskId,
 			})
 			if copyErr != nil {
 				return fmt.Errorf("copy %d.%v %s => %s : %v\n", volumeId, shardIdsToCopy, existingLocation, targetServer.info.Id, copyErr)
@@ -410,12 +428,74 @@ func collectEcVolumeServersByDc(topo *master_pb.TopologyInfo, selectedDataCenter
 		}
 
 		freeEcSlots := countFreeShardSlots(dn, types.HardDriveType)
-		ecNodes = append(ecNodes, &EcNode{
+		ecNode := &EcNode{
 			info:       dn,
 			dc:         dc,
 			rack:       rack,
 			freeEcSlot: int(freeEcSlots),
-		})
+			disks:      make(map[uint32]*EcDisk),
+		}
+
+		// Build disk-level information from volumes and EC shards
+		// First, discover all unique disk IDs from VolumeInfos (includes empty disks)
+		allDiskIds := make(map[uint32]string) // diskId -> diskType
+		for diskType, diskInfo := range dn.DiskInfos {
+			if diskInfo == nil {
+				continue
+			}
+			// Get all disk IDs from volumes
+			for _, vi := range diskInfo.VolumeInfos {
+				allDiskIds[vi.DiskId] = diskType
+			}
+			// Also get disk IDs from EC shards
+			for _, ecShardInfo := range diskInfo.EcShardInfos {
+				allDiskIds[ecShardInfo.DiskId] = diskType
+			}
+		}
+
+		// Group EC shards by disk_id
+		diskShards := make(map[uint32]map[needle.VolumeId]erasure_coding.ShardBits)
+		for _, diskInfo := range dn.DiskInfos {
+			if diskInfo == nil {
+				continue
+			}
+			for _, ecShardInfo := range diskInfo.EcShardInfos {
+				diskId := ecShardInfo.DiskId
+				if diskShards[diskId] == nil {
+					diskShards[diskId] = make(map[needle.VolumeId]erasure_coding.ShardBits)
+				}
+				vid := needle.VolumeId(ecShardInfo.Id)
+				diskShards[diskId][vid] = erasure_coding.ShardBits(ecShardInfo.EcIndexBits)
+			}
+		}
+
+		// Create EcDisk for each discovered disk
+		diskCount := len(allDiskIds)
+		if diskCount == 0 {
+			diskCount = 1
+		}
+		freePerDisk := int(freeEcSlots) / diskCount
+
+		for diskId, diskType := range allDiskIds {
+			shards := diskShards[diskId]
+			if shards == nil {
+				shards = make(map[needle.VolumeId]erasure_coding.ShardBits)
+			}
+			totalShardCount := 0
+			for _, shardBits := range shards {
+				totalShardCount += shardBits.ShardIdCount()
+			}
+
+			ecNode.disks[diskId] = &EcDisk{
+				diskId:       diskId,
+				diskType:     diskType,
+				freeEcSlots:  freePerDisk,
+				ecShardCount: totalShardCount,
+				ecShards:     shards,
+			}
+		}
+
+		ecNodes = append(ecNodes, ecNode)
 		totalFreeEcSlots += freeEcSlots
 	})
 	return
@@ -884,10 +964,16 @@ func (ecb *ecBalancer) doBalanceEcRack(ecRack *EcRack) error {
 				for _, shards := range fullDiskInfo.EcShardInfos {
 					if _, found := emptyNodeIds[shards.Id]; !found {
 						for _, shardId := range erasure_coding.ShardBits(shards.EcIndexBits).ShardIds() {
+							vid := needle.VolumeId(shards.Id)
+							destDiskId := pickBestDiskOnNode(emptyNode, vid)
 
-							fmt.Printf("%s moves ec shards %d.%d to %s\n", fullNode.info.Id, shards.Id, shardId, emptyNode.info.Id)
+							if destDiskId > 0 {
+								fmt.Printf("%s moves ec shards %d.%d to %s (disk %d)\n", fullNode.info.Id, shards.Id, shardId, emptyNode.info.Id, destDiskId)
+							} else {
+								fmt.Printf("%s moves ec shards %d.%d to %s\n", fullNode.info.Id, shards.Id, shardId, emptyNode.info.Id)
+							}
 
-							err := moveMountedShardToEcNode(ecb.commandEnv, fullNode, shards.Collection, needle.VolumeId(shards.Id), shardId, emptyNode, ecb.applyBalancing)
+							err := moveMountedShardToEcNode(ecb.commandEnv, fullNode, shards.Collection, vid, shardId, emptyNode, destDiskId, ecb.applyBalancing)
 							if err != nil {
 								return err
 							}
@@ -957,18 +1043,98 @@ func (ecb *ecBalancer) pickEcNodeToBalanceShardsInto(vid needle.VolumeId, existi
 	if len(targets) == 0 {
 		return nil, errors.New(details)
 	}
+
+	// When multiple nodes have the same shard count, prefer nodes with better disk distribution
+	// (i.e., nodes with more disks that have fewer shards of this volume)
+	if len(targets) > 1 {
+		slices.SortFunc(targets, func(a, b *EcNode) int {
+			aScore := diskDistributionScore(a, vid)
+			bScore := diskDistributionScore(b, vid)
+			return aScore - bScore // Lower score is better
+		})
+		return targets[0], nil
+	}
+
 	return targets[rand.IntN(len(targets))], nil
 }
 
-func (ecb *ecBalancer) pickOneEcNodeAndMoveOneShard(existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, possibleDestinationEcNodes []*EcNode) error {
-	destNode, err := ecb.pickEcNodeToBalanceShardsInto(vid, existingLocation, possibleDestinationEcNodes)
+// diskDistributionScore calculates a score for how well-distributed shards are on the node's disks
+// Lower score is better (means more room for balanced distribution)
+func diskDistributionScore(ecNode *EcNode, vid needle.VolumeId) int {
+	if len(ecNode.disks) == 0 {
+		return 0
+	}
+
+	// Sum the existing shard count for this volume on each disk
+	// Lower total means more room for new shards
+	score := 0
+	for _, disk := range ecNode.disks {
+		if shardBits, ok := disk.ecShards[vid]; ok {
+			score += shardBits.ShardIdCount() * 10 // Weight shards of this volume heavily
+		}
+		score += disk.ecShardCount // Also consider total shards on disk
+	}
+	return score
+}
+
+// pickBestDiskOnNode selects the best disk on a node for placing a new EC shard
+// It prefers disks with fewer shards and more free slots
+func pickBestDiskOnNode(ecNode *EcNode, vid needle.VolumeId) uint32 {
+	if len(ecNode.disks) == 0 {
+		return 0 // No disk info available, let the server decide
+	}
+
+	var bestDiskId uint32
+	bestScore := -1
+
+	for diskId, disk := range ecNode.disks {
+		if disk.freeEcSlots <= 0 {
+			continue
+		}
+
+		// Check if this volume already has shards on this disk
+		existingShards := 0
+		if shardBits, ok := disk.ecShards[vid]; ok {
+			existingShards = shardBits.ShardIdCount()
+		}
+
+		// Score: prefer disks with fewer total shards and fewer shards of this volume
+		// Lower score is better
+		score := disk.ecShardCount*10 + existingShards*100
+
+		if bestScore == -1 || score < bestScore {
+			bestScore = score
+			bestDiskId = diskId
+		}
+	}
+
+	return bestDiskId
+}
+
+// pickEcNodeAndDiskToBalanceShardsInto picks both a destination node and specific disk
+func (ecb *ecBalancer) pickEcNodeAndDiskToBalanceShardsInto(vid needle.VolumeId, existingLocation *EcNode, possibleDestinations []*EcNode) (*EcNode, uint32, error) {
+	node, err := ecb.pickEcNodeToBalanceShardsInto(vid, existingLocation, possibleDestinations)
 	if err != nil {
-		fmt.Printf("WARNING: Could not find suitable taget node for %d.%d:\n%s", vid, shardId, err.Error())
+		return nil, 0, err
+	}
+
+	diskId := pickBestDiskOnNode(node, vid)
+	return node, diskId, nil
+}
+
+func (ecb *ecBalancer) pickOneEcNodeAndMoveOneShard(existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, possibleDestinationEcNodes []*EcNode) error {
+	destNode, destDiskId, err := ecb.pickEcNodeAndDiskToBalanceShardsInto(vid, existingLocation, possibleDestinationEcNodes)
+	if err != nil {
+		fmt.Printf("WARNING: Could not find suitable target node for %d.%d:\n%s", vid, shardId, err.Error())
 		return nil
 	}
 
-	fmt.Printf("%s moves ec shard %d.%d to %s\n", existingLocation.info.Id, vid, shardId, destNode.info.Id)
-	return moveMountedShardToEcNode(ecb.commandEnv, existingLocation, collection, vid, shardId, destNode, ecb.applyBalancing)
+	if destDiskId > 0 {
+		fmt.Printf("%s moves ec shard %d.%d to %s (disk %d)\n", existingLocation.info.Id, vid, shardId, destNode.info.Id, destDiskId)
+	} else {
+		fmt.Printf("%s moves ec shard %d.%d to %s\n", existingLocation.info.Id, vid, shardId, destNode.info.Id)
+	}
+	return moveMountedShardToEcNode(ecb.commandEnv, existingLocation, collection, vid, shardId, destNode, destDiskId, ecb.applyBalancing)
 }
 
 func pickNEcShardsToMoveFrom(ecNodes []*EcNode, vid needle.VolumeId, n int) map[erasure_coding.ShardId]*EcNode {

weed/shell/command_volume_check_disk.go

@@ -8,7 +8,10 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"math/rand/v2"
 	"net/http"
+	"strings"
+	"sync"
 	"time"
 
 	"slices"
@@ -31,6 +34,7 @@ type commandVolumeCheckDisk struct{}
 type volumeCheckDisk struct {
 	commandEnv *CommandEnv
 	writer     io.Writer
+	writerMu   sync.Mutex
 	now        time.Time
 
 	slowMode           bool
@@ -39,6 +43,8 @@ type volumeCheckDisk struct {
 	syncDeletions      bool
 	fixReadOnly        bool
 	nonRepairThreshold float64
+
+	ewg *ErrorWaitGroup
 }
 
 func (c *commandVolumeCheckDisk) Name() string {
@@ -58,9 +64,9 @@ func (c *commandVolumeCheckDisk) Help() string {
 			append entries in B and not in A to A
 
 	optionally, for each non-writable volume replica A
-		if volume is not full
+		select a writable volume replica B
+		if entries in A don't match B
 			prune late volume entries not matching its index file
-			select a writable volume replica B
 			append missing entries from B into A
 			mark the volume as writable (healthy)
 
@@ -91,6 +97,7 @@ func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, write
 	applyChangesAlias := fsckCommand.Bool("force", false, "apply the fix (alias for -apply)")
 	fixReadOnly := fsckCommand.Bool("fixReadOnly", false, "apply the fix even on readonly volumes (EXPERIMENTAL!)")
 	syncDeletions := fsckCommand.Bool("syncDeleted", false, "sync of deletions the fix")
+	maxParallelization := fsckCommand.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
 	nonRepairThreshold := fsckCommand.Float64("nonRepairThreshold", 0.3, "repair when missing keys is not more than this limit")
 	if err = fsckCommand.Parse(args); err != nil {
 		return nil
@@ -114,6 +121,8 @@ func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, write
 		syncDeletions:      *syncDeletions,
 		fixReadOnly:        *fixReadOnly,
 		nonRepairThreshold: *nonRepairThreshold,
+
+		ewg: NewErrorWaitGroup(*maxParallelization),
 	}
 
 	// collect topology information
@@ -136,23 +145,21 @@ func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, write
 	if err := vcd.checkWritableVolumes(volumeReplicas); err != nil {
 		return err
 	}
-	if err := vcd.checkReadOnlyVolumes(volumeReplicas); err != nil {
-		return err
-	}
+	vcd.checkReadOnlyVolumes(volumeReplicas)
 
-	return nil
+	return vcd.ewg.Wait()
 }
 
 // checkWritableVolumes fixes volume replicas which are not read-only.
 func (vcd *volumeCheckDisk) checkWritableVolumes(volumeReplicas map[uint32][]*VolumeReplica) error {
-	vcd.write("Pass #1 (writable volumes)\n")
+	vcd.write("Pass #1 (writable volumes)")
 
 	for _, replicas := range volumeReplicas {
 		// filter readonly replica
 		var writableReplicas []*VolumeReplica
 		for _, replica := range replicas {
 			if replica.info.ReadOnly {
-				vcd.write("skipping readonly volume %d on %s\n", replica.info.Id, replica.location.dataNode.Id)
+				vcd.write("skipping readonly volume %d on %s", replica.info.Id, replica.location.dataNode.Id)
 			} else {
 				writableReplicas = append(writableReplicas, replica)
 			}
@@ -165,16 +172,23 @@ func (vcd *volumeCheckDisk) checkWritableVolumes(volumeReplicas map[uint32][]*Vo
 			a, b := writableReplicas[0], writableReplicas[1]
 			shouldSkip, err := vcd.shouldSkipVolume(a, b)
 			if err != nil {
-				vcd.write("error checking if volume %d should be skipped: %v\n", a.info.Id, err)
+				vcd.write("error checking if volume %d should be skipped: %v", a.info.Id, err)
 				// Continue with sync despite error to be safe
 			} else if shouldSkip {
 				// always choose the larger volume to be the source
 				writableReplicas = append(writableReplicas[:1], writableReplicas[2:]...)
 				continue
 			}
-			if err := vcd.syncTwoReplicas(a, b, true); err != nil {
-				vcd.write("sync volume %d on %s and %s: %v\n", a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id, err)
+
+			modified, err := vcd.syncTwoReplicas(a, b, true)
+			if err != nil {
+				vcd.write("failed to sync volumes %d on %s and %s: %v", a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id, err)
+			} else {
+				if modified {
+					vcd.write("synced %s and %s for volume %d", a.location.dataNode.Id, b.location.dataNode.Id, a.info.Id)
+				}
 			}
+
 			// always choose the larger volume to be the source
 			if a.info.FileCount > b.info.FileCount {
 				writableReplicas = append(writableReplicas[:1], writableReplicas[2:]...)
@@ -203,7 +217,7 @@ func (vcd *volumeCheckDisk) makeVolumeWritable(vid uint32, vr *VolumeReplica) er
 		return err
 	}
 
-	vcd.write("volume %d on %s is now writable\n", vid, vr.location.dataNode.Id)
+	vcd.write("volume %d on %s is now writable", vid, vr.location.dataNode.Id)
 	return nil
 }
 
@@ -223,71 +237,81 @@ func (vcd *volumeCheckDisk) makeVolumeReadonly(vid uint32, vr *VolumeReplica) er
 		return err
 	}
 
-	vcd.write("volume %d on %s is now read-only\n", vid, vr.location.dataNode.Id)
+	vcd.write("volume %d on %s is now read-only", vid, vr.location.dataNode.Id)
 	return nil
 }
 
-func (vcd *volumeCheckDisk) checkReadOnlyVolumes(volumeReplicas map[uint32][]*VolumeReplica) error {
+func (vcd *volumeCheckDisk) checkReadOnlyVolumes(volumeReplicas map[uint32][]*VolumeReplica) {
 	if !vcd.fixReadOnly {
-		return nil
+		return
 	}
-	vcd.write("Pass #2 (read-only volumes)\n")
+	vcd.write("Pass #2 (read-only volumes)")
 
 	for vid, replicas := range volumeReplicas {
-		var source *VolumeReplica = nil
 		roReplicas := []*VolumeReplica{}
+		rwReplicas := []*VolumeReplica{}
 
 		for _, r := range replicas {
 			if r.info.ReadOnly {
 				roReplicas = append(roReplicas, r)
 			} else {
-				// we assume all writable replicas are identical by this point, after the checkWritableVolumes() pass.
-				source = r
+				rwReplicas = append(rwReplicas, r)
 			}
 		}
 		if len(roReplicas) == 0 {
-			vcd.write("no read-only replicas for volume %d\n", vid)
+			vcd.write("no read-only replicas for volume %d", vid)
 			continue
 		}
-		if source == nil {
-			vcd.write("got %d read-only replicas for volume %d and no writable replicas to fix from\n", len(roReplicas), vid)
+		if len(rwReplicas) == 0 {
+			vcd.write("got %d read-only replicas for volume %d and no writable replicas to fix from", len(roReplicas), vid)
 			continue
 		}
 
-		// attempt to fix read-only replicas from the know good source
+		// attempt to fix read-only replicas from known good sources
 		for _, r := range roReplicas {
-			// TODO: skip full readonly volumes.
+			// select a random writable source replica. we assume these are identical by this point, after the checkWritableVolumes() pass.
+			source := rwReplicas[rand.IntN(len(rwReplicas))]
+
 			skip, err := vcd.shouldSkipVolume(r, source)
 			if err != nil {
-				vcd.write("error checking if volume %d should be skipped: %v\n", r.info.Id, err)
+				vcd.ewg.AddErrorf("failed to check if volume %d should be skipped: %v\n", r.info.Id, err)
 				continue
 			}
 			if skip {
 				continue
 			}
 
-			// make volume writable...
-			if err := vcd.makeVolumeWritable(vid, r); err != nil {
-				return err
-			}
-
-			// ...fix it...
-			// TODO: test whether syncTwoReplicas() is enough to prune garbage entries on broken volumes.
-			if err := vcd.syncTwoReplicas(source, r, false); err != nil {
-				vcd.write("sync read-only volume %d on %s from %s: %v\n", vid, r.location.dataNode.Id, source.location.dataNode.Id, err)
-
-				// ...or revert it back to read-only, if something went wrong.
-				if roErr := vcd.makeVolumeReadonly(vid, r); roErr != nil {
-					return fmt.Errorf("failed to make volume %d on %s readonly after: %v: %v", vid, r.location.dataNode.Id, err, roErr)
+			vcd.ewg.Add(func() error {
+				// make volume writable...
+				if err := vcd.makeVolumeWritable(vid, r); err != nil {
+					return err
 				}
-				vcd.write("volume %d on %s is now read-only\n", vid, r.location.dataNode.Id)
 
-				return err
-			}
+				// ...try to fix it...
+				// TODO: test whether syncTwoReplicas() is enough to prune garbage entries on broken volumes...
+				modified, err := vcd.syncTwoReplicas(source, r, false)
+				if err != nil {
+					vcd.write("sync read-only volume %d on %s from %s: %v", vid, r.location.dataNode.Id, source.location.dataNode.Id, err)
+
+					if roErr := vcd.makeVolumeReadonly(vid, r); roErr != nil {
+						return fmt.Errorf("failed to revert volume %d on %s to readonly after: %v: %v", vid, r.location.dataNode.Id, err, roErr)
+					}
+					return err
+				} else {
+					if modified {
+						vcd.write("volume %d on %s is now synced to %d and writable", vid, r.location.dataNode.Id, source.location.dataNode.Id)
+					} else {
+						// ...or restore back to read-only, if no changes were made.
+						if err := vcd.makeVolumeReadonly(vid, r); err != nil {
+							return fmt.Errorf("failed to revert volume %d on %s to readonly: %v", vid, r.location.dataNode.Id, err)
+						}
+					}
+				}
+
+				return nil
+			})
 		}
 	}
-
-	return nil
 }
 
 func (vcd *volumeCheckDisk) grpcDialOption() grpc.DialOption {
@@ -295,12 +319,15 @@ func (vcd *volumeCheckDisk) grpcDialOption() grpc.DialOption {
 }
 
 func (vcd *volumeCheckDisk) write(format string, a ...any) {
-	fmt.Fprintf(vcd.writer, format, a...)
+	vcd.writerMu.Lock()
+	defer vcd.writerMu.Unlock()
+	fmt.Fprintf(vcd.writer, strings.TrimRight(format, "\r\n "), a...)
+	fmt.Fprint(vcd.writer, "\n")
 }
 
 func (vcd *volumeCheckDisk) writeVerbose(format string, a ...any) {
 	if vcd.verbose {
-		fmt.Fprintf(vcd.writer, format, a...)
+		vcd.write(format, a...)
 	}
 }
 
@@ -386,7 +413,7 @@ func (vcd *volumeCheckDisk) shouldSkipVolume(a, b *VolumeReplica) (bool, error)
 			if doSyncDeletedCount && !eqDeletedFileCount {
 				return false, nil
 			}
-			vcd.writeVerbose("skipping active volumes %d with the same file counts on %s and %s\n",
+			vcd.writeVerbose("skipping active volumes %d with the same file counts on %s and %s",
 				a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id)
 		} else {
 			return false, nil
@@ -397,35 +424,39 @@ func (vcd *volumeCheckDisk) shouldSkipVolume(a, b *VolumeReplica) (bool, error)
 
 // syncTwoReplicas attempts to sync all entries from a source volume replica into a target. If bi-directional mode
 // is enabled, changes from target are also synced back into the source.
-func (vcd *volumeCheckDisk) syncTwoReplicas(source, target *VolumeReplica, bidi bool) (err error) {
+// Returns true if source and/or target were modified, false otherwise.
+func (vcd *volumeCheckDisk) syncTwoReplicas(source, target *VolumeReplica, bidi bool) (modified bool, err error) {
 	sourceHasChanges, targetHasChanges := true, true
 	const maxIterations = 5
 	iteration := 0
 
+	modified = false
+
 	for (sourceHasChanges || targetHasChanges) && iteration < maxIterations {
 		iteration++
-		vcd.writeVerbose("sync iteration %d/%d for volume %d\n", iteration, maxIterations, source.info.Id)
+		vcd.writeVerbose("sync iteration %d/%d for volume %d", iteration, maxIterations, source.info.Id)
 
 		prevSourceHasChanges, prevTargetHasChanges := sourceHasChanges, targetHasChanges
 		if sourceHasChanges, targetHasChanges, err = vcd.checkBoth(source, target, bidi); err != nil {
-			return err
+			return modified, err
 		}
+		modified = modified || sourceHasChanges || targetHasChanges
 
 		// Detect if we're stuck in a loop with no progress
 		if iteration > 1 && prevSourceHasChanges == sourceHasChanges && prevTargetHasChanges == targetHasChanges && (sourceHasChanges || targetHasChanges) {
-			vcd.write("volume %d sync is not making progress between %s and %s after iteration %d, stopping to prevent infinite loop\n",
+			vcd.write("volume %d sync is not making progress between %s and %s after iteration %d, stopping to prevent infinite loop",
 				source.info.Id, source.location.dataNode.Id, target.location.dataNode.Id, iteration)
-			return fmt.Errorf("sync not making progress after %d iterations", iteration)
+			return modified, fmt.Errorf("sync not making progress after %d iterations", iteration)
 		}
 	}
 
 	if iteration >= maxIterations && (sourceHasChanges || targetHasChanges) {
-		vcd.write("volume %d sync reached maximum iterations (%d) between %s and %s, may need manual intervention\n",
+		vcd.write("volume %d sync reached maximum iterations (%d) between %s and %s, may need manual intervention",
 			source.info.Id, maxIterations, source.location.dataNode.Id, target.location.dataNode.Id)
-		return fmt.Errorf("reached maximum sync iterations (%d)", maxIterations)
+		return modified, fmt.Errorf("reached maximum sync iterations (%d)", maxIterations)
 	}
 
-	return nil
+	return modified, nil
 }
 
 // checkBoth performs a sync between source and target volume replicas. If bi-directional mode is enabled, changes from target are also synced back into the source.
@@ -510,7 +541,7 @@ func (vcd *volumeCheckDisk) doVolumeCheckDisk(minuend, subtrahend *needle_map.Me
 		return nil
 	})
 
-	vcd.write("volume %d %s has %d entries, %s missed %d and partially deleted %d entries\n",
+	vcd.write("volume %d %s has %d entries, %s missed %d and partially deleted %d entries",
 		source.info.Id, source.location.dataNode.Id, counter, target.location.dataNode.Id, len(missingNeedles), len(partiallyDeletedNeedles))
 
 	if counter == 0 || (len(missingNeedles) == 0 && len(partiallyDeletedNeedles) == 0) {
@@ -534,7 +565,7 @@ func (vcd *volumeCheckDisk) doVolumeCheckDisk(minuend, subtrahend *needle_map.Me
 			continue
 		}
 
-		vcd.writeVerbose("read %s %s => %s\n", needleValue.Key.FileId(source.info.Id), source.location.dataNode.Id, target.location.dataNode.Id)
+		vcd.writeVerbose("read %s %s => %s", needleValue.Key.FileId(source.info.Id), source.location.dataNode.Id, target.location.dataNode.Id)
 		hasChanges = true
 
 		if err = vcd.writeNeedleBlobToTarget(pb.NewServerAddressFromDataNode(target.location.dataNode), source.info.Id, needleValue, needleBlob); err != nil {
@@ -547,7 +578,7 @@ func (vcd *volumeCheckDisk) doVolumeCheckDisk(minuend, subtrahend *needle_map.Me
 		var fidList []string
 		for _, needleValue := range partiallyDeletedNeedles {
 			fidList = append(fidList, needleValue.Key.FileId(source.info.Id))
-			vcd.writeVerbose("delete %s %s => %s\n", needleValue.Key.FileId(source.info.Id), source.location.dataNode.Id, target.location.dataNode.Id)
+			vcd.writeVerbose("delete %s %s => %s", needleValue.Key.FileId(source.info.Id), source.location.dataNode.Id, target.location.dataNode.Id)
 		}
 		deleteResults := operation.DeleteFileIdsAtOneVolumeServer(
 			pb.NewServerAddressFromDataNode(target.location.dataNode),
@@ -602,7 +633,7 @@ func (vcd *volumeCheckDisk) readIndexDatabase(db *needle_map.MemDb, collection s
 		return err
 	}
 
-	vcd.writeVerbose("load collection %s volume %d index size %d from %s ...\n", collection, volumeId, buf.Len(), volumeServer)
+	vcd.writeVerbose("load collection %s volume %d index size %d from %s ...", collection, volumeId, buf.Len(), volumeServer)
 	return db.LoadFilterFromReaderAt(bytes.NewReader(buf.Bytes()), true, false)
 }
 
@@ -614,7 +645,7 @@ func (vcd *volumeCheckDisk) copyVolumeIndexFile(collection string, volumeId uint
 
 		copyFileClient, err := volumeServerClient.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
 			VolumeId:                 volumeId,
-			Ext:                      ".idx",
+			Ext:                      ext,
 			CompactionRevision:       math.MaxUint32,
 			StopOffset:               math.MaxInt64,
 			Collection:               collection,

weed/shell/command_volume_check_disk_test.go

@@ -278,14 +278,14 @@ func TestVolumeCheckDiskHelperMethods(t *testing.T) {
 	}
 
 	// Test write method
-	vcd.write("test %s\n", "message")
+	vcd.write("test %s", "message")
 	if buf.String() != "test message\n" {
 		t.Errorf("write() output = %q, want %q", buf.String(), "test message\n")
 	}
 
 	// Test writeVerbose with verbose=true
 	buf.Reset()
-	vcd.writeVerbose("verbose %d\n", 123)
+	vcd.writeVerbose("verbose %d", 123)
 	if buf.String() != "verbose 123\n" {
 		t.Errorf("writeVerbose() with verbose=true output = %q, want %q", buf.String(), "verbose 123\n")
 	}
@@ -293,7 +293,7 @@ func TestVolumeCheckDiskHelperMethods(t *testing.T) {
 	// Test writeVerbose with verbose=false
 	buf.Reset()
 	vcd.verbose = false
-	vcd.writeVerbose("should not appear\n")
+	vcd.writeVerbose("should not appear")
 	if buf.String() != "" {
 		t.Errorf("writeVerbose() with verbose=false output = %q, want empty", buf.String())
 	}