3.99

Bumps [github.com/rclone/rclone](https://github.com/rclone/rclone) from 1.71.1 to 1.71.2.
- [Release notes](https://github.com/rclone/rclone/releases)
- [Changelog](https://github.com/rclone/rclone/blob/master/RELEASE.md)
- [Commits](https://github.com/rclone/rclone/compare/v1.71.1...v1.71.2)

---
updated-dependencies:
- dependency-name: github.com/rclone/rclone
  dependency-version: 1.71.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/e2e.yml

@@ -134,7 +134,7 @@ jobs:
 
     - name: Archive logs
       if: always()
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v5
       with:
         name: output-logs
         path: docker/output.log

.github/workflows/fuse-integration.yml

@@ -183,7 +183,7 @@ jobs:
         
     - name: Upload Test Artifacts
       if: always()
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v5
       with:
         name: fuse-integration-test-results
         path: |

.github/workflows/postgres-tests.yml

@@ -62,7 +62,7 @@ jobs:
 
     - name: Archive logs
       if: always()
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v5
       with:
         name: postgres-logs
         path: test/postgres/postgres-output.log

.github/workflows/s3-go-tests.yml

@@ -76,7 +76,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-versioning-test-logs-${{ matrix.test-type }}
           path: test/s3/versioning/weed-test*.log
@@ -124,7 +124,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-versioning-compatibility-logs
           path: test/s3/versioning/weed-test*.log
@@ -172,7 +172,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-cors-compatibility-logs
           path: test/s3/cors/weed-test*.log
@@ -239,7 +239,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-retention-test-logs-${{ matrix.test-type }}
           path: test/s3/retention/weed-test*.log
@@ -306,7 +306,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-cors-test-logs-${{ matrix.test-type }}
           path: test/s3/cors/weed-test*.log
@@ -355,7 +355,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-retention-worm-logs
           path: test/s3/retention/weed-test*.log
@@ -405,7 +405,7 @@ jobs:
 
       - name: Upload stress test logs
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-versioning-stress-logs
           path: test/s3/versioning/weed-test*.log

.github/workflows/s3-iam-tests.yml

@@ -65,7 +65,7 @@ jobs:
 
       - name: Upload test results on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: iam-unit-test-results
           path: |
@@ -162,7 +162,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-iam-integration-logs-${{ matrix.test-type }}
           path: test/s3/iam/weed-*.log
@@ -222,7 +222,7 @@ jobs:
 
       - name: Upload distributed test logs
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-iam-distributed-logs
           path: test/s3/iam/weed-*.log
@@ -274,7 +274,7 @@ jobs:
 
       - name: Upload performance test results
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-iam-performance-results
           path: |

.github/workflows/s3-keycloak-tests.yml

@@ -152,7 +152,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-keycloak-test-logs
           path: |

.github/workflows/s3-sse-tests.yml

@@ -93,7 +93,7 @@ jobs:
 
       - name: Upload test logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-sse-test-logs-${{ matrix.test-type }}
           path: test/s3/sse/weed-test*.log
@@ -141,7 +141,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-sse-compatibility-logs
           path: test/s3/sse/weed-test*.log
@@ -190,7 +190,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-sse-metadata-persistence-logs
           path: test/s3/sse/weed-test*.log
@@ -239,7 +239,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-sse-copy-operations-logs
           path: test/s3/sse/weed-test*.log
@@ -288,7 +288,7 @@ jobs:
 
       - name: Upload server logs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-sse-multipart-logs
           path: test/s3/sse/weed-test*.log
@@ -338,7 +338,7 @@ jobs:
 
       - name: Upload performance test logs
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: s3-sse-performance-logs
           path: test/s3/sse/weed-test*.log

docker/compose/master-cloud.toml

@@ -13,7 +13,7 @@ scripts = """
   ec.rebuild -force
   ec.balance -force
   volume.balance -force
-  volume.fix.replication
+  volume.fix.replication -force
   unlock
 """
 sleep_minutes = 17          # sleep minutes between each script execution

go.mod

@@ -67,7 +67,7 @@ require (
 	github.com/prometheus/client_golang v1.23.2
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect
-	github.com/prometheus/procfs v0.17.0
+	github.com/prometheus/procfs v0.19.1
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
@@ -99,7 +99,7 @@ require (
 	golang.org/x/crypto v0.43.0
 	golang.org/x/exp v0.0.0-20250811191247-51f88131bc50
 	golang.org/x/image v0.32.0
-	golang.org/x/net v0.45.0
+	golang.org/x/net v0.46.0
 	golang.org/x/oauth2 v0.30.0 // indirect
 	golang.org/x/sys v0.37.0
 	golang.org/x/text v0.30.0 // indirect
@@ -125,14 +125,13 @@ require (
 	github.com/a-h/templ v0.3.943
 	github.com/arangodb/go-driver v1.6.7
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.39.2
+	github.com/aws/aws-sdk-go-v2 v1.39.4
 	github.com/aws/aws-sdk-go-v2/config v1.31.3
-	github.com/aws/aws-sdk-go-v2/credentials v1.18.10
+	github.com/aws/aws-sdk-go-v2/credentials v1.18.19
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.88.3
-	github.com/cockroachdb/cockroachdb-parser v0.25.3-0.20250917200107-7bd0408a9e14
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
-	github.com/getsentry/sentry-go v0.35.3
+	github.com/getsentry/sentry-go v0.36.1
 	github.com/gin-contrib/sessions v1.0.4
 	github.com/gin-gonic/gin v1.11.0
 	github.com/golang-jwt/jwt/v5 v5.3.0
@@ -148,14 +147,14 @@ require (
 	github.com/minio/crc64nvme v1.1.1
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/parquet-go/parquet-go v0.25.1
-	github.com/pkg/sftp v1.13.9
+	github.com/pkg/sftp v1.13.10
 	github.com/rabbitmq/amqp091-go v1.10.0
-	github.com/rclone/rclone v1.71.1
+	github.com/rclone/rclone v1.71.2
 	github.com/rdleal/intervalst v1.5.0
 	github.com/redis/go-redis/v9 v9.14.1
 	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/shirou/gopsutil/v4 v4.25.9
-	github.com/tarantool/go-tarantool/v2 v2.4.0
+	github.com/tarantool/go-tarantool/v2 v2.4.1
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
@@ -261,22 +260,22 @@ require (
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 // indirect
-	github.com/aws/smithy-go v1.23.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 // indirect
+	github.com/aws/smithy-go v1.23.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
@@ -401,6 +400,7 @@ require (
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
 	github.com/sagikazarmark/locafero v0.11.0 // indirect
 	github.com/samber/lo v1.51.0 // indirect
+	github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
 	github.com/sony/gobreaker v1.0.0 // indirect
@@ -409,7 +409,7 @@ require (
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 // indirect
+	github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c // indirect
 	github.com/tarantool/go-iproto v1.1.0 // indirect
 	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
 	github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect

go.sum

@@ -664,32 +664,32 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.39.2 h1:EJLg8IdbzgeD7xgvZ+I8M1e0fL0ptn/M47lianzth0I=
-github.com/aws/aws-sdk-go-v2 v1.39.2/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY=
+github.com/aws/aws-sdk-go-v2 v1.39.4 h1:qTsQKcdQPHnfGYBBs+Btl8QwxJeoWcOcPcixK90mRhg=
+github.com/aws/aws-sdk-go-v2 v1.39.4/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 h1:i8p8P4diljCr60PpJp6qZXNlgX4m2yQFpYk+9ZT+J4E=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1/go.mod h1:ddqbooRZYNoJ2dsTwOty16rM+/Aqmk/GOXrK8cg7V00=
 github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
 github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.10 h1:xdJnXCouCx8Y0NncgoptztUocIYLKeQxrCgN6x9sdhg=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.10/go.mod h1:7tQk08ntj914F/5i9jC4+2HQTAuJirq7m1vZVIhEkWs=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 h1:wbjnrrMnKew78/juW7I2BtKQwa1qlf6EjQgS69uYY14=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6/go.mod h1:AtiqqNrDioJXuUgz3+3T0mBWN7Hro2n9wll2zRUc0ww=
+github.com/aws/aws-sdk-go-v2/credentials v1.18.19 h1:Jc1zzwkSY1QbkEcLujwqRTXOdvW8ppND3jRBb/VhBQc=
+github.com/aws/aws-sdk-go-v2/credentials v1.18.19/go.mod h1:DIfQ9fAk5H0pGtnqfqkbSIzky82qYnGvh06ASQXXg6A=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11 h1:X7X4YKb+c0rkI6d4uJ5tEMxXgCZ+jZ/D6mvkno8c8Uw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11/go.mod h1:EqM6vPZQsZHYvC4Cai35UDg/f5NCEU+vp0WfbVqVcZc=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 h1:se2vOWGD3dWQUtfn4wEjRQJb1HK1XsNIt825gskZ970=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9/go.mod h1:hijCGH2VfbZQxqCDN7bwz/4dzxV+hkyhjawAtdPWKZA=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 h1:6RBnKZLkJM4hQ+kN6E7yWFveOTg8NLPHAkqrs4ZPlTU=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9/go.mod h1:V9rQKRmK7AWuEsOMnHzKj8WyrIir1yUJbZxDuZLFvXI=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 h1:7AANQZkF3ihM8fbdftpjhken0TP9sBzFbV/Ze/Y4HXA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11/go.mod h1:NTF4QCGkm6fzVwncpkFQqoquQyOolcyXfbpC98urj+c=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 h1:ShdtWUZT37LCAA4Mw2kJAJtzaszfSHFb5n25sdcv4YE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11/go.mod h1:7bUb2sSr2MZ3M/N+VyETLTQtInemHXb/Fl3s8CLzm0Y=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9 h1:w9LnHqTq8MEdlnyhV4Bwfizd65lfNCNgdlNC6mM5paE=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9/go.mod h1:LGEP6EK4nj+bwWNdrvX/FnDTFowdBNwcSPuZu/ouFys=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.9 h1:by3nYZLR9l8bUH7kgaMU4dJgYFjyRdFEfORlDpPILB4=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.9/go.mod h1:IWjQYlqw4EX9jw2g3qnEPPWvCE6bS8fKzhMed1OK7c8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 h1:5r34CgVOD4WZudeEKZ9/iKpiT6cM1JyEROpXjOcdWv8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9/go.mod h1:dB12CEbNWPbzO2uC6QSWHteqOg4JfBVJOojbAoAUb5I=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 h1:GpMf3z2KJa4RnJ0ew3Hac+hRFYLZ9DDjfgXjuW+pB54=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11/go.mod h1:6MZP3ZI4QQsgUCFTwMZA2V0sEriNQ8k2hmoHF3qjimQ=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9 h1:wuZ5uW2uhJR63zwNlqWH2W4aL4ZjeJP3o92/W+odDY4=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9/go.mod h1:/G58M2fGszCrOzvJUkDdY8O9kycodunH4VdT5oBAqls=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.88.3 h1:P18I4ipbk+b/3dZNq5YYh+Hq6XC0vp5RWkLp1tJldDA=
@@ -698,14 +698,14 @@ github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnY
 github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 h1:8OLZnVJPvjnrxEwHFg9hVUof/P4sibH+Ea4KKuqAGSg=
-github.com/aws/aws-sdk-go-v2/service/sso v1.29.1/go.mod h1:27M3BpVi0C02UiQh1w9nsBEit6pLhlaH3NHna6WUbDE=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 h1:gKWSTnqudpo8dAxqBqZnDoDWCiEh/40FziUjr/mo6uA=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2/go.mod h1:x7+rkNmRoEN1U13A6JE2fXne9EWyJy54o3n6d4mGaXQ=
-github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 h1:YZPjhyaGzhDQEvsffDEcpycq49nl7fiGcfJTIo8BszI=
-github.com/aws/aws-sdk-go-v2/service/sts v1.38.2/go.mod h1:2dIN8qhQfv37BdUYGgEC8Q3tteM3zFxTI1MLO2O3J3c=
-github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE=
-github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 h1:M5nimZmugcZUO9wG7iVtROxPhiqyZX6ejS1lxlDPbTU=
+github.com/aws/aws-sdk-go-v2/service/sso v1.29.8/go.mod h1:mbef/pgKhtKRwrigPPs7SSSKZgytzP8PQ6P6JAAdqyM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 h1:S5GuJZpYxE0lKeMHKn+BRTz6PTFpgThyJ+5mYfux7BM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3/go.mod h1:X4OF+BTd7HIb3L+tc4UlWHVrpgwZZIVENU15pRDVTI0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 h1:Ekml5vGg6sHSZLZJQJagefnVe6PmqC2oiRkBq4F7fU0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.38.9/go.mod h1:/e15V+o1zFHWdH3u7lpI3rVBcxszktIKuHKCY2/py+k=
+github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M=
+github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
 github.com/bazelbuild/rules_go v0.46.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -926,8 +926,8 @@ github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBv
 github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok=
 github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
 github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
-github.com/getsentry/sentry-go v0.35.3 h1:u5IJaEqZyPdWqe/hKlBKBBnMTSxB/HenCqF3QLabeds=
-github.com/getsentry/sentry-go v0.35.3/go.mod h1:mdL49ixwT2yi57k5eh7mpnDyPybixPzlzEJFu0Z76QA=
+github.com/getsentry/sentry-go v0.36.1 h1:kMJt0WWsxWATUxkvFgVBZdIeHSk/Oiv5P0jZ9e5m/Lw=
+github.com/getsentry/sentry-go v0.36.1/go.mod h1:p5Im24mJBeruET8Q4bbcMfCQ+F+Iadc4L48tB1apo2c=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sessions v1.0.4 h1:ha6CNdpYiTOK/hTp05miJLbpTSNfOnFg5Jm2kbcqy8U=
 github.com/gin-contrib/sessions v1.0.4/go.mod h1:ccmkrb2z6iU2osiAHZG3x3J4suJK+OU27oqzlWOqQgs=
@@ -1525,8 +1525,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
-github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw=
-github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA=
+github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
+github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
 github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM=
 github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
@@ -1567,8 +1567,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
-github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
+github.com/prometheus/procfs v0.19.1 h1:QVtROpTkphuXuNlnCv3m1ut3JytkXHtQ3xvck/YmzMM=
+github.com/prometheus/procfs v0.19.1/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
 github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
@@ -1577,8 +1577,8 @@ github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2e
 github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
 github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
 github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
-github.com/rclone/rclone v1.71.1 h1:cpODfWTRz5i/WAzXsyW85tzfIKNsd1aq8CE8lUB+0zg=
-github.com/rclone/rclone v1.71.1/go.mod h1:NLyX57FrnZ9nVLTY5TRdMmGelrGKbIRYGcgRkNdqqlA=
+github.com/rclone/rclone v1.71.2 h1:3Jk5xNPFrZhVABRuN/OPvApuZQddpE2tkhYMuEn1Ud4=
+github.com/rclone/rclone v1.71.2/go.mod h1:dCK9FzPDlpkbQJ9M7MmWsmv3X5nibfWe+ogJXu6gSgM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
@@ -1623,8 +1623,8 @@ github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71e
 github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM=
 github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
 github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
-github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021182748-d0c58c67297e h1:CDKaiOOEslsh0eARMVDJubE1FoI1IQ57NwpuDvfSJ0s=
-github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021182748-d0c58c67297e/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
+github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138 h1:bX1vBF7GQjPeFQsCAZ8gCQGS/nJQnekL7gZ4Qg/pF4E=
+github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
 github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
 github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
 github.com/seaweedfs/raft v1.1.3 h1:5B6hgneQ7IuU4Ceom/f6QUt8pEeqjcsRo+IxlyPZCws=
@@ -1702,13 +1702,13 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
-github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 h1:Sa+sR8aaAMFwxhXWENEnE6ZpqhZ9d7u1RT2722Rw6hc=
-github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5/go.mod h1:UdZiFUFu6e2WjjtjxivwXWcwc1N/8zgbkBR9QNucUOY=
+github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c h1:BLopNCyqewbE8+BtlIp/Juzu8AJGxz0gHdGADnsblVc=
+github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c/go.mod h1:ykucQyiE9Q2qx1wLlEtZkkNn1IURib/2O+Mvd25i1Fo=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
 github.com/tarantool/go-iproto v1.1.0 h1:HULVOIHsiehI+FnHfM7wMDntuzUddO09DKqu2WnFQ5A=
 github.com/tarantool/go-iproto v1.1.0/go.mod h1:LNCtdyZxojUed8SbOiYHoc3v9NvaZTB7p96hUySMlIo=
-github.com/tarantool/go-tarantool/v2 v2.4.0 h1:cfGngxdknpVVbd/vF2LvaoWsKjsLV9i3xC859XgsJlI=
-github.com/tarantool/go-tarantool/v2 v2.4.0/go.mod h1:MTbhdjFc3Jl63Lgi/UJr5D+QbT+QegqOzsNJGmaw7VM=
+github.com/tarantool/go-tarantool/v2 v2.4.1 h1:Bk9mh+gMPVmHTSefHvVBpEkf6P2UZA/8xa5kqgyQtyo=
+github.com/tarantool/go-tarantool/v2 v2.4.1/go.mod h1:MTbhdjFc3Jl63Lgi/UJr5D+QbT+QegqOzsNJGmaw7VM=
 github.com/the42/cartconvert v0.0.0-20131203171324-aae784c392b8 h1:I4DY8wLxJXCrMYzDM6lKCGc3IQwJX0PlTLsd3nQqI3c=
 github.com/the42/cartconvert v0.0.0-20131203171324-aae784c392b8/go.mod h1:fWO/msnJVhHqN1yX6OBoxSyfj7TEj1hHiL8bJSQsK30=
 github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
@@ -2065,8 +2065,8 @@ golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM=
-golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
+golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
+golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.98"
+appVersion: "3.99"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.398
+version: 4.0.399

k8s/charts/seaweedfs/values.yaml

@@ -979,7 +979,7 @@ s3:
   # Custom command line arguments to add to the s3 command
   # Example to fix connection idle seconds:
   extraArgs: ["-idleTimeout=30"]
-  #extraArgs: []
+  # extraArgs: []
 
   # used to configure livenessProbe on s3 containers
   #

test/kafka/go.mod

@@ -168,14 +168,14 @@ require (
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pkg/sftp v1.13.9 // indirect
+	github.com/pkg/sftp v1.13.10 // indirect
 	github.com/pkg/xattr v0.4.12 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect
-	github.com/prometheus/procfs v0.17.0 // indirect
+	github.com/prometheus/procfs v0.19.1 // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
 	github.com/rclone/rclone v1.71.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
@@ -231,7 +231,7 @@ require (
 	golang.org/x/crypto v0.43.0 // indirect
 	golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 // indirect
 	golang.org/x/image v0.32.0 // indirect
-	golang.org/x/net v0.45.0 // indirect
+	golang.org/x/net v0.46.0 // indirect
 	golang.org/x/oauth2 v0.30.0 // indirect
 	golang.org/x/sync v0.17.0 // indirect
 	golang.org/x/sys v0.37.0 // indirect

test/kafka/go.sum

@@ -516,8 +516,8 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjL
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw=
-github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA=
+github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
+github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
 github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM=
 github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -532,8 +532,8 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw
 github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
 github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
 github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
-github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
-github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
+github.com/prometheus/procfs v0.19.1 h1:QVtROpTkphuXuNlnCv3m1ut3JytkXHtQ3xvck/YmzMM=
+github.com/prometheus/procfs v0.19.1/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
 github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
@@ -815,8 +815,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM=
-golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
+golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
+golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

weed/admin/dash/admin_data.go

@@ -3,6 +3,7 @@ package dash
 import (
 	"context"
 	"net/http"
+	"sort"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -108,6 +109,13 @@ func (s *AdminServer) GetAdminData(username string) (AdminData, error) {
 		glog.Errorf("Failed to get cluster volume servers: %v", err)
 		return AdminData{}, err
 	}
+	// Sort the servers so they show up in consistent order after each reload
+	sort.Slice(volumeServersData.VolumeServers, func(i, j int) bool {
+		s1Name := volumeServersData.VolumeServers[i].GetDisplayAddress()
+		s2Name := volumeServersData.VolumeServers[j].GetDisplayAddress()
+
+		return s1Name < s2Name
+	})
 
 	// Get master nodes status
 	masterNodes := s.getMasterNodesStatus()

weed/admin/dash/auth_middleware.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/gin-contrib/sessions"
 	"github.com/gin-gonic/gin"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 )
 
 // ShowLogin displays the login page
@@ -31,9 +32,16 @@ func (s *AdminServer) HandleLogin(username, password string) gin.HandlerFunc {
 
 		if loginUsername == username && loginPassword == password {
 			session := sessions.Default(c)
+			// Clear any existing invalid session data before setting new values
+			session.Clear()
 			session.Set("authenticated", true)
 			session.Set("username", loginUsername)
-			session.Save()
+			if err := session.Save(); err != nil {
+				// Log the detailed error server-side for diagnostics
+				glog.Errorf("Failed to save session for user %s: %v", loginUsername, err)
+				c.Redirect(http.StatusSeeOther, "/login?error=Unable to create session. Please try again or contact administrator.")
+				return
+			}
 
 			c.Redirect(http.StatusSeeOther, "/admin")
 			return
@@ -48,6 +56,8 @@ func (s *AdminServer) HandleLogin(username, password string) gin.HandlerFunc {
 func (s *AdminServer) HandleLogout(c *gin.Context) {
 	session := sessions.Default(c)
 	session.Clear()
-	session.Save()
+	if err := session.Save(); err != nil {
+		glog.Warningf("Failed to save session during logout: %v", err)
+	}
 	c.Redirect(http.StatusSeeOther, "/login")
 }

weed/admin/dash/types.go

@@ -51,6 +51,13 @@ type VolumeServer struct {
 	EcShardDetails []VolumeServerEcInfo `json:"ec_shard_details"` // Detailed EC shard information
 }
 
+func (vs *VolumeServer) GetDisplayAddress() string {
+	if vs.PublicURL != "" {
+		return vs.PublicURL
+	}
+	return vs.Address
+}
+
 // VolumeServerEcInfo represents EC shard information for a specific volume on a server
 type VolumeServerEcInfo struct {
 	VolumeID     uint32        `json:"volume_id"`

weed/admin/handlers/admin_handlers.go

@@ -48,6 +48,11 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
 	// Health check (no auth required)
 	r.GET("/health", h.HealthCheck)
 
+	// Favicon route (no auth required) - redirect to static version
+	r.GET("/favicon.ico", func(c *gin.Context) {
+		c.Redirect(http.StatusMovedPermanently, "/static/favicon.ico")
+	})
+
 	if authRequired {
 		// Authentication routes (no auth required)
 		r.GET("/login", h.authHandlers.ShowLogin)

weed/admin/view/app/admin.templ

@@ -172,7 +172,12 @@ templ Admin(data dash.AdminData) {
                                 <tbody>
                                     for _, master := range data.MasterNodes {
                                         <tr>
-                                            <td>{master.Address}</td>
+                                            <td>
+                                                <a href={templ.SafeURL(fmt.Sprintf("http://%s/ui/index.html", master.Address))} target="_blank">
+                                                    {master.Address}
+                                                    <i class="fas fa-external-link-alt ms-1 text-muted"></i>
+                                                </a>
+                                            </td>
                                             <td>
                                                 if master.IsLeader {
                                                     <span class="badge bg-primary">Leader</span>
@@ -275,8 +280,8 @@ templ Admin(data dash.AdminData) {
                                         <tr>
                                             <td>{vs.ID}</td>
                                             <td>
-                                                <a href={templ.SafeURL(fmt.Sprintf("http://%s/ui/index.html", vs.PublicURL))} target="_blank">
-                                                    {vs.Address}
+                                                <a href={templ.SafeURL(fmt.Sprintf("http://%s/ui/index.html", vs.GetDisplayAddress()))} target="_blank">
+                                                    {vs.GetDisplayAddress()}
                                                     <i class="fas fa-external-link-alt ms-1 text-muted"></i>
                                                 </a>
                                             </td>

weed/admin/view/app/admin_templ.go

@@ -117,323 +117,323 @@ func Admin(data dash.AdminData) templ.Component {
 			return templ_7745c5c3_Err
 		}
 		for _, master := range data.MasterNodes {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "<tr><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "<tr><td><a href=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			var templ_7745c5c3_Var8 string
-			templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(master.Address)
+			var templ_7745c5c3_Var8 templ.SafeURL
+			templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("http://%s/ui/index.html", master.Address)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 175, Col: 63}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 176, Col: 126}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "</td><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "\" target=\"_blank\">")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			var templ_7745c5c3_Var9 string
+			templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(master.Address)
+			if templ_7745c5c3_Err != nil {
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 177, Col: 67}
+			}
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, " <i class=\"fas fa-external-link-alt ms-1 text-muted\"></i></a></td><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			if master.IsLeader {
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "<span class=\"badge bg-primary\">Leader</span>")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "<span class=\"badge bg-primary\">Leader</span>")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 			} else {
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "<span class=\"badge bg-secondary\">Follower</span>")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "<span class=\"badge bg-secondary\">Follower</span>")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "</td></tr>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "</td></tr>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "</tbody></table></div></div></div></div><!-- System Health --><div class=\"col-lg-8\"><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"fas fa-chart-pie me-2\"></i>Cluster</h6></div><div class=\"card-body text-center\"><div class=\"row\"><div class=\"col-3\"><div class=\"card bg-light\"><div class=\"card-body\"><h5>")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var9 string
-		templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.MasterNodes)))
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 205, Col: 85}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "</h5><small class=\"text-muted\">Masters</small></div></div></div><div class=\"col-3\"><div class=\"card bg-light\"><div class=\"card-body\"><h5>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "</tbody></table></div></div></div></div><!-- System Health --><div class=\"col-lg-8\"><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"fas fa-chart-pie me-2\"></i>Cluster</h6></div><div class=\"card-body text-center\"><div class=\"row\"><div class=\"col-3\"><div class=\"card bg-light\"><div class=\"card-body\"><h5>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var10 string
-		templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.VolumeServers)))
+		templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.MasterNodes)))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 213, Col: 87}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 210, Col: 85}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "</h5><small class=\"text-muted\">Volume Servers</small></div></div></div><div class=\"col-3\"><div class=\"card bg-light\"><div class=\"card-body\"><h5>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "</h5><small class=\"text-muted\">Masters</small></div></div></div><div class=\"col-3\"><div class=\"card bg-light\"><div class=\"card-body\"><h5>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var11 string
-		templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.FilerNodes)))
+		templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.VolumeServers)))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 221, Col: 84}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 218, Col: 87}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "</h5><small class=\"text-muted\">Filers</small></div></div></div><div class=\"col-3\"><div class=\"card bg-light\"><div class=\"card-body\"><h5>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "</h5><small class=\"text-muted\">Volume Servers</small></div></div></div><div class=\"col-3\"><div class=\"card bg-light\"><div class=\"card-body\"><h5>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var12 string
-		templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.MessageBrokers)))
+		templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.FilerNodes)))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 229, Col: 88}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 226, Col: 84}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "</h5><small class=\"text-muted\">Message Brokers</small></div></div></div></div></div></div></div></div><!-- Volume Servers --><div class=\"row\"><div class=\"col-12\"><div class=\"card shadow mb-4\"><div class=\"card-header py-3 d-flex flex-row align-items-center justify-content-between\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"fas fa-database me-2\"></i>Volume Servers</h6><div class=\"dropdown no-arrow\"><a class=\"dropdown-toggle\" href=\"#\" role=\"button\" data-bs-toggle=\"dropdown\"><i class=\"fas fa-ellipsis-v fa-sm fa-fw text-gray-400\"></i></a><div class=\"dropdown-menu dropdown-menu-right shadow animated--fade-in\"><div class=\"dropdown-header\">Actions:</div><a class=\"dropdown-item\" href=\"/volumes\">View Details</a> <a class=\"dropdown-item\" href=\"/cluster\">Topology View</a></div></div></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\"><thead><tr><th>ID</th><th>Address</th><th>Data Center</th><th>Rack</th><th>Volumes</th><th>EC Shards</th><th>Capacity</th></tr></thead> <tbody>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "</h5><small class=\"text-muted\">Filers</small></div></div></div><div class=\"col-3\"><div class=\"card bg-light\"><div class=\"card-body\"><h5>")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var13 string
+		templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.MessageBrokers)))
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 234, Col: 88}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "</h5><small class=\"text-muted\">Message Brokers</small></div></div></div></div></div></div></div></div><!-- Volume Servers --><div class=\"row\"><div class=\"col-12\"><div class=\"card shadow mb-4\"><div class=\"card-header py-3 d-flex flex-row align-items-center justify-content-between\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"fas fa-database me-2\"></i>Volume Servers</h6><div class=\"dropdown no-arrow\"><a class=\"dropdown-toggle\" href=\"#\" role=\"button\" data-bs-toggle=\"dropdown\"><i class=\"fas fa-ellipsis-v fa-sm fa-fw text-gray-400\"></i></a><div class=\"dropdown-menu dropdown-menu-right shadow animated--fade-in\"><div class=\"dropdown-header\">Actions:</div><a class=\"dropdown-item\" href=\"/volumes\">View Details</a> <a class=\"dropdown-item\" href=\"/cluster\">Topology View</a></div></div></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\"><thead><tr><th>ID</th><th>Address</th><th>Data Center</th><th>Rack</th><th>Volumes</th><th>EC Shards</th><th>Capacity</th></tr></thead> <tbody>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		for _, vs := range data.VolumeServers {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "<tr><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "<tr><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			var templ_7745c5c3_Var13 string
-			templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(vs.ID)
+			var templ_7745c5c3_Var14 string
+			templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(vs.ID)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 276, Col: 54}
-			}
-			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "</td><td><a href=\"")
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-			var templ_7745c5c3_Var14 templ.SafeURL
-			templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("http://%s/ui/index.html", vs.PublicURL)))
-			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 278, Col: 124}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 281, Col: 54}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "\" target=\"_blank\">")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "</td><td><a href=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			var templ_7745c5c3_Var15 string
-			templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(vs.Address)
+			var templ_7745c5c3_Var15 templ.SafeURL
+			templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("http://%s/ui/index.html", vs.GetDisplayAddress())))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 279, Col: 63}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 283, Col: 134}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, " <i class=\"fas fa-external-link-alt ms-1 text-muted\"></i></a></td><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "\" target=\"_blank\">")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var16 string
-			templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(vs.DataCenter)
+			templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(vs.GetDisplayAddress())
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 283, Col: 62}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 284, Col: 75}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "</td><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, " <i class=\"fas fa-external-link-alt ms-1 text-muted\"></i></a></td><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var17 string
-			templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(vs.Rack)
+			templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(vs.DataCenter)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 284, Col: 56}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 288, Col: 62}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "</td><td><div class=\"progress\" style=\"height: 20px;\"><div class=\"progress-bar\" role=\"progressbar\" style=\"")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "</td><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var18 string
-			templ_7745c5c3_Var18, templ_7745c5c3_Err = templruntime.SanitizeStyleAttributeValues(fmt.Sprintf("width: %d%%", calculatePercent(vs.Volumes, vs.MaxVolumes)))
+			templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(vs.Rack)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 288, Col: 135}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 289, Col: 56}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "\">")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "</td><td><div class=\"progress\" style=\"height: 20px;\"><div class=\"progress-bar\" role=\"progressbar\" style=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var19 string
-			templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d/%d", vs.Volumes, vs.MaxVolumes))
+			templ_7745c5c3_Var19, templ_7745c5c3_Err = templruntime.SanitizeStyleAttributeValues(fmt.Sprintf("width: %d%%", calculatePercent(vs.Volumes, vs.MaxVolumes)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 289, Col: 104}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 293, Col: 135}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "</div></div></td><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "\">")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			var templ_7745c5c3_Var20 string
+			templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d/%d", vs.Volumes, vs.MaxVolumes))
+			if templ_7745c5c3_Err != nil {
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 294, Col: 104}
+			}
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "</div></div></td><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			if vs.EcShards > 0 {
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "<span class=\"badge bg-info text-white me-1\">")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "<span class=\"badge bg-info text-white me-1\">")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				var templ_7745c5c3_Var20 string
-				templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", vs.EcShards))
+				var templ_7745c5c3_Var21 string
+				templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", vs.EcShards))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 295, Col: 127}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 300, Col: 127}
 				}
-				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "</span> ")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "</span> ")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 				if vs.EcVolumes > 0 {
-					templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "<small class=\"text-muted\">(")
+					templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "<small class=\"text-muted\">(")
 					if templ_7745c5c3_Err != nil {
 						return templ_7745c5c3_Err
 					}
-					var templ_7745c5c3_Var21 string
-					templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d vol", vs.EcVolumes))
+					var templ_7745c5c3_Var22 string
+					templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d vol", vs.EcVolumes))
 					if templ_7745c5c3_Err != nil {
-						return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 297, Col: 119}
+						return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 302, Col: 119}
 					}
-					_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21))
+					_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22))
 					if templ_7745c5c3_Err != nil {
 						return templ_7745c5c3_Err
 					}
-					templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, ")</small>")
+					templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, ")</small>")
 					if templ_7745c5c3_Err != nil {
 						return templ_7745c5c3_Err
 					}
 				}
 			} else {
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "<span class=\"text-muted\">-</span>")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "<span class=\"text-muted\">-</span>")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "</td><td>")
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-			var templ_7745c5c3_Var22 string
-			templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(vs.DiskUsage))
-			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 303, Col: 74}
-			}
-			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22))
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, " / ")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "</td><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var23 string
-			templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(vs.DiskCapacity))
+			templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(vs.DiskUsage))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 303, Col: 107}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 308, Col: 74}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "</td></tr>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, " / ")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-		}
-		if len(data.VolumeServers) == 0 {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "<tr><td colspan=\"7\" class=\"text-center text-muted py-4\"><i class=\"fas fa-info-circle me-2\"></i> No volume servers found</td></tr>")
+			var templ_7745c5c3_Var24 string
+			templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(vs.DiskCapacity))
 			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "</tbody></table></div></div></div></div></div><!-- Filer Nodes --><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow mb-4\"><div class=\"card-header py-3 d-flex flex-row align-items-center justify-content-between\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"fas fa-folder me-2\"></i>Filer Nodes</h6><div class=\"dropdown no-arrow\"><a class=\"dropdown-toggle\" href=\"#\" role=\"button\" data-bs-toggle=\"dropdown\"><i class=\"fas fa-ellipsis-v fa-sm fa-fw text-gray-400\"></i></a><div class=\"dropdown-menu dropdown-menu-right shadow animated--fade-in\"><div class=\"dropdown-header\">Actions:</div><a class=\"dropdown-item\" href=\"/filer\">File Browser</a> <a class=\"dropdown-item\" href=\"/cluster\">Topology View</a></div></div></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\"><thead><tr><th>Address</th><th>Data Center</th><th>Rack</th><th>Last Updated</th></tr></thead> <tbody>")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		for _, filer := range data.FilerNodes {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "<tr><td><a href=\"")
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-			var templ_7745c5c3_Var24 templ.SafeURL
-			templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("http://%s", filer.Address)))
-			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 356, Col: 111}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 308, Col: 107}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "\" target=\"_blank\">")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "</td></tr>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			var templ_7745c5c3_Var25 string
-			templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(filer.Address)
+		}
+		if len(data.VolumeServers) == 0 {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "<tr><td colspan=\"7\" class=\"text-center text-muted py-4\"><i class=\"fas fa-info-circle me-2\"></i> No volume servers found</td></tr>")
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 357, Col: 66}
+				return templ_7745c5c3_Err
+			}
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "</tbody></table></div></div></div></div></div><!-- Filer Nodes --><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow mb-4\"><div class=\"card-header py-3 d-flex flex-row align-items-center justify-content-between\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"fas fa-folder me-2\"></i>Filer Nodes</h6><div class=\"dropdown no-arrow\"><a class=\"dropdown-toggle\" href=\"#\" role=\"button\" data-bs-toggle=\"dropdown\"><i class=\"fas fa-ellipsis-v fa-sm fa-fw text-gray-400\"></i></a><div class=\"dropdown-menu dropdown-menu-right shadow animated--fade-in\"><div class=\"dropdown-header\">Actions:</div><a class=\"dropdown-item\" href=\"/filer\">File Browser</a> <a class=\"dropdown-item\" href=\"/cluster\">Topology View</a></div></div></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\"><thead><tr><th>Address</th><th>Data Center</th><th>Rack</th><th>Last Updated</th></tr></thead> <tbody>")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		for _, filer := range data.FilerNodes {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "<tr><td><a href=\"")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			var templ_7745c5c3_Var25 templ.SafeURL
+			templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("http://%s", filer.Address)))
+			if templ_7745c5c3_Err != nil {
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 361, Col: 111}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, " <i class=\"fas fa-external-link-alt ms-1 text-muted\"></i></a></td><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "\" target=\"_blank\">")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var26 string
-			templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(filer.DataCenter)
+			templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(filer.Address)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 361, Col: 65}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 362, Col: 66}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "</td><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, " <i class=\"fas fa-external-link-alt ms-1 text-muted\"></i></a></td><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var27 string
-			templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(filer.Rack)
+			templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(filer.DataCenter)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 362, Col: 59}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 366, Col: 65}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27))
 			if templ_7745c5c3_Err != nil {
@@ -444,39 +444,52 @@ func Admin(data dash.AdminData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var28 string
-			templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(filer.LastUpdated.Format("2006-01-02 15:04:05"))
+			templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(filer.Rack)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 363, Col: 96}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 367, Col: 59}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "</td></tr>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "</td><td>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			var templ_7745c5c3_Var29 string
+			templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(filer.LastUpdated.Format("2006-01-02 15:04:05"))
+			if templ_7745c5c3_Err != nil {
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 368, Col: 96}
+			}
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "</td></tr>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
 		if len(data.FilerNodes) == 0 {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "<tr><td colspan=\"4\" class=\"text-center text-muted py-4\"><i class=\"fas fa-info-circle me-2\"></i> No filer nodes found</td></tr>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "<tr><td colspan=\"4\" class=\"text-center text-muted py-4\"><i class=\"fas fa-info-circle me-2\"></i> No filer nodes found</td></tr>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "</tbody></table></div></div></div></div></div><!-- Last Updated --><div class=\"row\"><div class=\"col-12\"><small class=\"text-muted\"><i class=\"fas fa-clock me-1\"></i> Last updated: ")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "</tbody></table></div></div></div></div></div><!-- Last Updated --><div class=\"row\"><div class=\"col-12\"><small class=\"text-muted\"><i class=\"fas fa-clock me-1\"></i> Last updated: ")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		var templ_7745c5c3_Var29 string
-		templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04:05"))
+		var templ_7745c5c3_Var30 string
+		templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04:05"))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 387, Col: 81}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 392, Col: 81}
 		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "</small></div></div></div>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "</small></div></div></div>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}

weed/command/admin.go

@@ -191,31 +191,7 @@ func startAdminServer(ctx context.Context, options AdminOptions) error {
 	r := gin.New()
 	r.Use(gin.Logger(), gin.Recovery())
 
-	// Session store - always auto-generate session key
-	sessionKeyBytes := make([]byte, 32)
-	_, err := rand.Read(sessionKeyBytes)
-	if err != nil {
-		return fmt.Errorf("failed to generate session key: %w", err)
-	}
-	store := cookie.NewStore(sessionKeyBytes)
-
-	// Configure session options to ensure cookies are properly saved
-	store.Options(sessions.Options{
-		Path:   "/",
-		MaxAge: 3600 * 24, // 24 hours
-	})
-
-	r.Use(sessions.Sessions("admin-session", store))
-
-	// Static files - serve from embedded filesystem
-	staticFS, err := admin.GetStaticFS()
-	if err != nil {
-		log.Printf("Warning: Failed to load embedded static files: %v", err)
-	} else {
-		r.StaticFS("/static", http.FS(staticFS))
-	}
-
-	// Create data directory if specified
+	// Create data directory first if specified (needed for session key storage)
 	var dataDir string
 	if *options.dataDir != "" {
 		// Expand tilde (~) to home directory
@@ -236,6 +212,35 @@ func startAdminServer(ctx context.Context, options AdminOptions) error {
 		fmt.Printf("Data directory created/verified: %s\n", dataDir)
 	}
 
+	// Detect TLS configuration to set Secure cookie flag
+	cookieSecure := viper.GetString("https.admin.key") != ""
+
+	// Session store - load or generate session key
+	sessionKeyBytes, err := loadOrGenerateSessionKey(dataDir)
+	if err != nil {
+		return fmt.Errorf("failed to get session key: %w", err)
+	}
+	store := cookie.NewStore(sessionKeyBytes)
+
+	// Configure session options to ensure cookies are properly saved
+	store.Options(sessions.Options{
+		Path:     "/",
+		MaxAge:   3600 * 24,    // 24 hours
+		HttpOnly: true,         // Prevent JavaScript access
+		Secure:   cookieSecure, // Set based on actual TLS configuration
+		SameSite: http.SameSiteLaxMode,
+	})
+
+	r.Use(sessions.Sessions("admin-session", store))
+
+	// Static files - serve from embedded filesystem
+	staticFS, err := admin.GetStaticFS()
+	if err != nil {
+		log.Printf("Warning: Failed to load embedded static files: %v", err)
+	} else {
+		r.StaticFS("/static", http.FS(staticFS))
+	}
+
 	// Create admin server
 	adminServer := dash.NewAdminServer(*options.masters, nil, dataDir)
 
@@ -331,6 +336,46 @@ func GetAdminOptions() *AdminOptions {
 	return &AdminOptions{}
 }
 
+// loadOrGenerateSessionKey loads an existing session key from dataDir or generates a new one
+func loadOrGenerateSessionKey(dataDir string) ([]byte, error) {
+	const sessionKeyLength = 32
+	if dataDir == "" {
+		// No persistence, generate random key
+		log.Println("No dataDir specified, generating ephemeral session key")
+		key := make([]byte, sessionKeyLength)
+		_, err := rand.Read(key)
+		return key, err
+	}
+
+	sessionKeyPath := filepath.Join(dataDir, ".session_key")
+
+	// Try to load existing key
+	if data, err := os.ReadFile(sessionKeyPath); err == nil {
+		if len(data) == sessionKeyLength {
+			log.Printf("Loaded persisted session key from %s", sessionKeyPath)
+			return data, nil
+		}
+		log.Printf("Warning: Invalid session key file (expected %d bytes, got %d), generating new key", sessionKeyLength, len(data))
+	} else if !os.IsNotExist(err) {
+		log.Printf("Warning: Failed to read session key from %s: %v. A new key will be generated.", sessionKeyPath, err)
+	}
+
+	// Generate new key
+	key := make([]byte, sessionKeyLength)
+	if _, err := rand.Read(key); err != nil {
+		return nil, err
+	}
+
+	// Save key for future use
+	if err := os.WriteFile(sessionKeyPath, key, 0600); err != nil {
+		log.Printf("Warning: Failed to persist session key: %v", err)
+	} else {
+		log.Printf("Generated and persisted new session key to %s", sessionKeyPath)
+	}
+
+	return key, nil
+}
+
 // expandHomeDir expands the tilde (~) in a path to the user's home directory
 func expandHomeDir(path string) (string, error) {
 	if path == "" {

weed/command/autocomplete.go

@@ -2,6 +2,8 @@ package command
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"github.com/posener/complete"
 	completeinstall "github.com/posener/complete/cmd/install"
 	flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
@@ -39,6 +41,40 @@ func AutocompleteMain(commands []*Command) bool {
 	return cmp.Complete()
 }
 
+func printAutocompleteScript(shell string) bool {
+	bin, err := os.Executable()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to get executable path: %s\n", err)
+		return false
+	}
+	binPath, err := filepath.Abs(bin)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to get absolute path: %s\n", err)
+		return false
+	}
+
+ 	switch shell {
+ 	case "bash":
+ 		fmt.Printf("complete -C %q weed\n", binPath)
+ 	case "zsh":
+ 		fmt.Printf("autoload -U +X bashcompinit && bashcompinit\n")
+ 		fmt.Printf("complete -o nospace -C %q weed\n", binPath)
+ 	case "fish":
+ 		fmt.Printf(`function __complete_weed
+    set -lx COMP_LINE (commandline -cp)
+    test -z (commandline -ct)
+    and set COMP_LINE "$COMP_LINE "
+    %q
+end
+complete -f -c weed -a "(__complete_weed)"
+`, binPath)
+ 	default:
+ 		fmt.Fprintf(os.Stderr, "unsupported shell: %s. Supported shells: bash, zsh, fish\n", shell)
+ 		return false
+ 	}
+	return true
+}
+
 func installAutoCompletion() bool {
 	if runtime.GOOS == "windows" {
 		fmt.Println("Windows is not supported")
@@ -71,9 +107,25 @@ func uninstallAutoCompletion() bool {
 
 var cmdAutocomplete = &Command{
 	Run:       runAutocomplete,
-	UsageLine: "autocomplete",
-	Short:     "install autocomplete",
-	Long: `weed autocomplete is installed in the shell.
+	UsageLine: "autocomplete [shell]",
+	Short:     "generate or install shell autocomplete script",
+	Long: `Generate shell autocomplete script or install it to your shell configuration.
+
+Usage:
+    weed autocomplete [bash|zsh|fish]  # print autocomplete script to stdout
+    weed autocomplete install          # install to shell config files
+
+    When a shell name is provided, the autocomplete script is printed to stdout.
+    You can then add it to your shell configuration manually, e.g.:
+
+        # For bash:
+        weed autocomplete bash >> ~/.bashrc
+
+        # Or use eval in your shell config:
+        eval "$(weed autocomplete bash)"
+
+    When 'install' is provided (or no argument), the script is automatically
+    installed to your shell configuration files.
 
     Supported shells are bash, zsh, and fish.
     Windows is not supported.
@@ -82,11 +134,23 @@ var cmdAutocomplete = &Command{
 }
 
 func runAutocomplete(cmd *Command, args []string) bool {
-	if len(args) != 0 {
-		cmd.Usage()
+	if len(args) == 0 {
+		// Default behavior: install
+		return installAutoCompletion()
 	}
 
-	return installAutoCompletion()
+	if len(args) > 1 {
+		cmd.Usage()
+		return false
+	}
+
+	shell := args[0]
+	if shell == "install" {
+		return installAutoCompletion()
+	}
+
+	// Print the autocomplete script for the specified shell
+	return printAutocompleteScript(shell)
 }
 
 var cmdUnautocomplete = &Command{

weed/command/scaffold/master.toml

@@ -13,7 +13,7 @@ scripts = """
   ec.balance -force
   volume.deleteEmpty -quietFor=24h -force
   volume.balance -force
-  volume.fix.replication
+  volume.fix.replication -force
   s3.clean.uploads -timeAgo=24h
   unlock
 """

weed/filer/filechunk_manifest.go

@@ -109,7 +109,8 @@ func fetchWholeChunk(ctx context.Context, bytesBuffer *bytes.Buffer, lookupFileI
 		glog.ErrorfCtx(ctx, "operation LookupFileId %s failed, err: %v", fileId, err)
 		return err
 	}
-	err = retriedStreamFetchChunkData(ctx, bytesBuffer, urlStrings, "", cipherKey, isGzipped, true, 0, 0)
+	jwt := JwtForVolumeServer(fileId)
+	err = retriedStreamFetchChunkData(ctx, bytesBuffer, urlStrings, jwt, cipherKey, isGzipped, true, 0, 0)
 	if err != nil {
 		return err
 	}
@@ -150,7 +151,7 @@ func retriedStreamFetchChunkData(ctx context.Context, writer io.Writer, urlStrin
 			retriedCnt++
 			var localProcessed int
 			var writeErr error
-			shouldRetry, err = util_http.ReadUrlAsStreamAuthenticated(ctx, urlString+"?readDeleted=true", jwt, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
+			shouldRetry, err = util_http.ReadUrlAsStream(ctx, urlString+"?readDeleted=true", jwt, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
 				// Check for context cancellation during data processing
 				select {
 				case <-ctx.Done():

weed/filer/filer_deletion.go

@@ -2,6 +2,7 @@ package filer
 
 import (
 	"context"
+	"fmt"
 	"strings"
 	"time"
 
@@ -56,13 +57,38 @@ func (f *Filer) loopProcessingDeletion() {
 					fileIds = fileIds[:0]
 				}
 				deletionCount = len(toDeleteFileIds)
-				_, err := operation.DeleteFileIdsWithLookupVolumeId(f.GrpcDialOption, toDeleteFileIds, lookupFunc)
-				if err != nil {
-					if !strings.Contains(err.Error(), storage.ErrorDeleted.Error()) {
-						glog.V(0).Infof("deleting fileIds len=%d error: %v", deletionCount, err)
+				results := operation.DeleteFileIdsWithLookupVolumeId(f.GrpcDialOption, toDeleteFileIds, lookupFunc)
+
+				// Process individual results for better error tracking
+				var successCount, notFoundCount, errorCount int
+				var errorDetails []string
+
+				for _, result := range results {
+					if result.Error == "" {
+						successCount++
+					} else if result.Error == "not found" || strings.Contains(result.Error, storage.ErrorDeleted.Error()) {
+						// Already deleted - acceptable
+						notFoundCount++
+					} else {
+						// Actual error
+						errorCount++
+						if errorCount <= 10 {
+							// Only log first 10 errors to avoid flooding logs
+							errorDetails = append(errorDetails, result.FileId+": "+result.Error)
+						}
 					}
-				} else {
-					glog.V(2).Infof("deleting fileIds %+v", toDeleteFileIds)
+				}
+
+				if successCount > 0 || notFoundCount > 0 {
+					glog.V(2).Infof("deleted %d files successfully, %d already deleted (not found)", successCount, notFoundCount)
+				}
+
+				if errorCount > 0 {
+					logMessage := fmt.Sprintf("failed to delete %d/%d files", errorCount, len(toDeleteFileIds))
+					if errorCount > 10 {
+						logMessage += " (showing first 10)"
+					}
+					glog.V(0).Infof("%s: %v", logMessage, strings.Join(errorDetails, "; "))
 				}
 			}
 		})

weed/filer/filer_notify.go

@@ -3,12 +3,13 @@ package filer
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
 	"io"
 	"regexp"
 	"strings"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+
 	"google.golang.org/protobuf/proto"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"

weed/filer/stream.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
@@ -26,6 +27,30 @@ var getLookupFileIdBackoffSchedule = []time.Duration{
 	1800 * time.Millisecond,
 }
 
+var (
+	jwtSigningReadKey        security.SigningKey
+	jwtSigningReadKeyExpires int
+	loadJwtConfigOnce        sync.Once
+)
+
+func loadJwtConfig() {
+	v := util.GetViper()
+	jwtSigningReadKey = security.SigningKey(v.GetString("jwt.signing.read.key"))
+	jwtSigningReadKeyExpires = v.GetInt("jwt.signing.read.expires_after_seconds")
+	if jwtSigningReadKeyExpires == 0 {
+		jwtSigningReadKeyExpires = 60
+	}
+}
+
+// JwtForVolumeServer generates a JWT token for volume server read operations if jwt.signing.read is configured
+func JwtForVolumeServer(fileId string) string {
+	loadJwtConfigOnce.Do(loadJwtConfig)
+	if len(jwtSigningReadKey) == 0 {
+		return ""
+	}
+	return string(security.GenJwtForVolumeServer(jwtSigningReadKey, jwtSigningReadKeyExpires, fileId))
+}
+
 func HasData(entry *filer_pb.Entry) bool {
 
 	if len(entry.Content) > 0 {
@@ -152,7 +177,7 @@ func PrepareStreamContentWithThrottler(ctx context.Context, masterClient wdclien
 }
 
 func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
-	streamFn, err := PrepareStreamContent(masterClient, noJwtFunc, chunks, offset, size)
+	streamFn, err := PrepareStreamContent(masterClient, JwtForVolumeServer, chunks, offset, size)
 	if err != nil {
 		return err
 	}
@@ -351,8 +376,9 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
 	}
 	var buffer bytes.Buffer
 	var shouldRetry bool
+	jwt := JwtForVolumeServer(chunkView.FileId)
 	for _, urlString := range urlStrings {
-		shouldRetry, err = util_http.ReadUrlAsStream(context.Background(), urlString+"?readDeleted=true", chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize), func(data []byte) {
+		shouldRetry, err = util_http.ReadUrlAsStream(context.Background(), urlString+"?readDeleted=true", jwt, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize), func(data []byte) {
 			buffer.Write(data)
 		})
 		if !shouldRetry {

weed/operation/chunked_file.go

@@ -80,11 +80,9 @@ func (cm *ChunkManifest) DeleteChunks(masterFn GetMasterFn, usePublicUrl bool, g
 	for _, ci := range cm.Chunks {
 		fileIds = append(fileIds, ci.Fid)
 	}
-	results, err := DeleteFileIds(masterFn, usePublicUrl, grpcDialOption, fileIds)
-	if err != nil {
-		glog.V(0).Infof("delete %+v: %v", fileIds, err)
-		return fmt.Errorf("chunk delete: %w", err)
-	}
+	results := DeleteFileIds(masterFn, usePublicUrl, grpcDialOption, fileIds)
+
+	// Check for any errors in results
 	for _, result := range results {
 		if result.Error != "" {
 			glog.V(0).Infof("delete file %+v: %v", result.FileId, result.Error)

weed/operation/delete_content.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"google.golang.org/grpc"
 	"net/http"
 	"strings"
 	"sync"
 
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"google.golang.org/grpc"
+
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 )
 
@@ -29,7 +30,8 @@ func ParseFileId(fid string) (vid string, key_cookie string, err error) {
 }
 
 // DeleteFileIds batch deletes a list of fileIds
-func DeleteFileIds(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc.DialOption, fileIds []string) ([]*volume_server_pb.DeleteResult, error) {
+// Returns individual results for each file ID. Check result.Error for per-file failures.
+func DeleteFileIds(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc.DialOption, fileIds []string) []*volume_server_pb.DeleteResult {
 
 	lookupFunc := func(vids []string) (results map[string]*LookupResult, err error) {
 		results, err = LookupVolumeIds(masterFn, grpcDialOption, vids)
@@ -47,7 +49,7 @@ func DeleteFileIds(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc.
 
 }
 
-func DeleteFileIdsWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]*LookupResult, error)) ([]*volume_server_pb.DeleteResult, error) {
+func DeleteFileIdsWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]*LookupResult, error)) []*volume_server_pb.DeleteResult {
 
 	var ret []*volume_server_pb.DeleteResult
 
@@ -72,17 +74,30 @@ func DeleteFileIdsWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []s
 
 	lookupResults, err := lookupFunc(vids)
 	if err != nil {
-		return ret, err
+		// Lookup failed - return error results for all file IDs that passed parsing
+		for _, fids := range vid_to_fileIds {
+			for _, fileId := range fids {
+				ret = append(ret, &volume_server_pb.DeleteResult{
+					FileId: fileId,
+					Status: http.StatusInternalServerError,
+					Error:  fmt.Sprintf("lookup error: %v", err),
+				})
+			}
+		}
+		return ret
 	}
 
 	server_to_fileIds := make(map[pb.ServerAddress][]string)
 	for vid, result := range lookupResults {
 		if result.Error != "" {
-			ret = append(ret, &volume_server_pb.DeleteResult{
-				FileId: vid,
-				Status: http.StatusBadRequest,
-				Error:  result.Error},
-			)
+			// Lookup error for this volume - mark all its files as failed
+			for _, fileId := range vid_to_fileIds[vid] {
+				ret = append(ret, &volume_server_pb.DeleteResult{
+					FileId: fileId,
+					Status: http.StatusBadRequest,
+					Error:  result.Error},
+				)
+			}
 			continue
 		}
 		for _, location := range result.Locations {
@@ -102,11 +117,7 @@ func DeleteFileIdsWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []s
 		go func(server pb.ServerAddress, fidList []string) {
 			defer wg.Done()
 
-			if deleteResults, deleteErr := DeleteFileIdsAtOneVolumeServer(server, grpcDialOption, fidList, false); deleteErr != nil {
-				err = deleteErr
-			} else if deleteResults != nil {
-				resultChan <- deleteResults
-			}
+			resultChan <- DeleteFileIdsAtOneVolumeServer(server, grpcDialOption, fidList, false)
 
 		}(server, fidList)
 	}
@@ -117,13 +128,16 @@ func DeleteFileIdsWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []s
 		ret = append(ret, result...)
 	}
 
-	return ret, err
+	return ret
 }
 
 // DeleteFileIdsAtOneVolumeServer deletes a list of files that is on one volume server via gRpc
-func DeleteFileIdsAtOneVolumeServer(volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) (ret []*volume_server_pb.DeleteResult, err error) {
+// Returns individual results for each file ID. Check result.Error for per-file failures.
+func DeleteFileIdsAtOneVolumeServer(volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) []*volume_server_pb.DeleteResult {
 
-	err = WithVolumeServerClient(false, volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+	var ret []*volume_server_pb.DeleteResult
+
+	err := WithVolumeServerClient(false, volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
 
 		req := &volume_server_pb.BatchDeleteRequest{
 			FileIds:         fileIds,
@@ -144,15 +158,17 @@ func DeleteFileIdsAtOneVolumeServer(volumeServer pb.ServerAddress, grpcDialOptio
 	})
 
 	if err != nil {
-		return
-	}
-
-	for _, result := range ret {
-		if result.Error != "" && result.Error != "not found" {
-			return nil, fmt.Errorf("delete fileId %s: %v", result.FileId, result.Error)
+		// Connection or communication error - return error results for all files
+		ret = make([]*volume_server_pb.DeleteResult, 0, len(fileIds))
+		for _, fileId := range fileIds {
+			ret = append(ret, &volume_server_pb.DeleteResult{
+				FileId: fileId,
+				Status: http.StatusInternalServerError,
+				Error:  err.Error(),
+			})
 		}
 	}
 
-	return
+	return ret
 
 }

weed/query/engine/cockroach_parser.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/cockroachdb/cockroachdb-parser/pkg/sql/parser"
-	"github.com/cockroachdb/cockroachdb-parser/pkg/sql/sem/tree"
+	"github.com/seaweedfs/cockroachdb-parser/pkg/sql/parser"
+	"github.com/seaweedfs/cockroachdb-parser/pkg/sql/sem/tree"
 )
 
 // CockroachSQLParser wraps CockroachDB's PostgreSQL-compatible SQL parser for use in SeaweedFS

weed/replication/repl_util/replication_util.go

@@ -2,6 +2,7 @@ package repl_util
 
 import (
 	"context"
+
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/replication/source"
@@ -20,9 +21,10 @@ func CopyFromChunkViews(chunkViews *filer.IntervalList[*filer.ChunkView], filerS
 
 		var writeErr error
 		var shouldRetry bool
+		jwt := filer.JwtForVolumeServer(chunk.FileId)
 
 		for _, fileUrl := range fileUrls {
-			shouldRetry, err = util_http.ReadUrlAsStream(context.Background(), fileUrl, chunk.CipherKey, chunk.IsGzipped, chunk.IsFullChunk(), chunk.OffsetInChunk, int(chunk.ViewSize), func(data []byte) {
+			shouldRetry, err = util_http.ReadUrlAsStream(context.Background(), fileUrl, jwt, chunk.CipherKey, chunk.IsGzipped, chunk.IsFullChunk(), chunk.OffsetInChunk, int(chunk.ViewSize), func(data []byte) {
 				writeErr = writeFunc(data)
 			})
 			if err != nil {

weed/s3api/auth_signature_v4.go

@@ -25,7 +25,6 @@ import (
 	"encoding/hex"
 	"io"
 	"net/http"
-	"path"
 	"regexp"
 	"sort"
 	"strconv"
@@ -33,17 +32,20 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
 
 func (iam *IdentityAccessManagement) reqSignatureV4Verify(r *http.Request) (*Identity, s3err.ErrorCode) {
-	sha256sum := getContentSha256Cksum(r)
 	switch {
 	case isRequestSignatureV4(r):
-		return iam.doesSignatureMatch(sha256sum, r)
+		identity, _, errCode := iam.doesSignatureMatch(r)
+		return identity, errCode
 	case isRequestPresignedSignatureV4(r):
-		return iam.doesPresignedSignatureMatch(sha256sum, r)
+		identity, _, errCode := iam.doesPresignedSignatureMatch(r)
+		return identity, errCode
 	}
 	return nil, s3err.ErrAccessDenied
 }
@@ -154,236 +156,298 @@ func parseSignV4(v4Auth string) (sv signValues, aec s3err.ErrorCode) {
 	return signV4Values, s3err.ErrNone
 }
 
-// doesSignatureMatch verifies the request signature.
-func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r *http.Request) (*Identity, s3err.ErrorCode) {
+// buildPathWithForwardedPrefix combines forwarded prefix with URL path while preserving S3 key semantics.
+// This function avoids path.Clean which would collapse "//" and dot segments, breaking S3 signatures.
+// It only normalizes the join boundary to avoid double slashes between prefix and path.
+func buildPathWithForwardedPrefix(forwardedPrefix, urlPath string) string {
+	if forwardedPrefix == "" {
+		return urlPath
+	}
+	// Ensure single leading slash on prefix
+	if !strings.HasPrefix(forwardedPrefix, "/") {
+		forwardedPrefix = "/" + forwardedPrefix
+	}
+	// Join without collapsing interior segments; only fix a double slash at the boundary
+	var joined string
+	if strings.HasSuffix(forwardedPrefix, "/") && strings.HasPrefix(urlPath, "/") {
+		joined = forwardedPrefix + urlPath[1:]
+	} else if !strings.HasSuffix(forwardedPrefix, "/") && !strings.HasPrefix(urlPath, "/") {
+		joined = forwardedPrefix + "/" + urlPath
+	} else {
+		joined = forwardedPrefix + urlPath
+	}
+	// Trailing slash semantics inherited from urlPath (already present if needed)
+	return joined
+}
 
-	// Copy request
-	req := *r
+// v4AuthInfo holds the parsed authentication data from a request,
+// whether it's from the Authorization header or presigned URL query parameters.
+type v4AuthInfo struct {
+	Signature     string
+	AccessKey     string
+	SignedHeaders []string
+	Date          time.Time
+	Region        string
+	Service       string
+	Scope         string
+	HashedPayload string
+	IsPresigned   bool
+}
 
-	// Save authorization header.
-	v4Auth := req.Header.Get("Authorization")
+// verifyV4Signature is the single entry point for verifying any AWS Signature V4 request.
+// It handles standard requests, presigned URLs, and the seed signature for streaming uploads.
+func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCheckPermissions bool) (identity *Identity, credential *Credential, calculatedSignature string, authInfo *v4AuthInfo, errCode s3err.ErrorCode) {
+	// 1. Extract authentication information from header or query parameters
+	authInfo, errCode = extractV4AuthInfo(r)
+	if errCode != s3err.ErrNone {
+		return nil, nil, "", nil, errCode
+	}
 
-	// Parse signature version '4' header.
-	signV4Values, errCode := parseSignV4(v4Auth)
+	// 2. Lookup user and credentials
+	identity, cred, found := iam.lookupByAccessKey(authInfo.AccessKey)
+	if !found {
+		return nil, nil, "", nil, s3err.ErrInvalidAccessKeyID
+	}
+
+	// 3. Perform permission check
+	if shouldCheckPermissions {
+		bucket, object := s3_constants.GetBucketAndObject(r)
+		action := s3_constants.ACTION_READ
+		if r.Method != http.MethodGet && r.Method != http.MethodHead {
+			action = s3_constants.ACTION_WRITE
+		}
+		if !identity.canDo(Action(action), bucket, object) {
+			return nil, nil, "", nil, s3err.ErrAccessDenied
+		}
+	}
+
+	// 4. Handle presigned request expiration
+	if authInfo.IsPresigned {
+		if errCode = checkPresignedRequestExpiry(r, authInfo.Date); errCode != s3err.ErrNone {
+			return nil, nil, "", nil, errCode
+		}
+	}
+
+	// 5. Extract headers that were part of the signature
+	extractedSignedHeaders, errCode := extractSignedHeaders(authInfo.SignedHeaders, r)
+	if errCode != s3err.ErrNone {
+		return nil, nil, "", nil, errCode
+	}
+
+	// 6. Get the query string for the canonical request
+	queryStr := getCanonicalQueryString(r, authInfo.IsPresigned)
+
+	// 7. Define a closure for the core verification logic to avoid repetition
+	verify := func(urlPath string) (string, s3err.ErrorCode) {
+		return calculateAndVerifySignature(
+			cred.SecretKey,
+			r.Method,
+			urlPath,
+			queryStr,
+			extractedSignedHeaders,
+			authInfo,
+		)
+	}
+
+	// 8. Verify the signature, trying with X-Forwarded-Prefix first
+	if forwardedPrefix := r.Header.Get("X-Forwarded-Prefix"); forwardedPrefix != "" {
+		cleanedPath := buildPathWithForwardedPrefix(forwardedPrefix, r.URL.Path)
+		calculatedSignature, errCode = verify(cleanedPath)
+		if errCode == s3err.ErrNone {
+			return identity, cred, calculatedSignature, authInfo, s3err.ErrNone
+		}
+	}
+
+	// 9. Verify with the original path
+	calculatedSignature, errCode = verify(r.URL.Path)
+	if errCode != s3err.ErrNone {
+		return nil, nil, "", nil, errCode
+	}
+
+	return identity, cred, calculatedSignature, authInfo, s3err.ErrNone
+}
+
+// calculateAndVerifySignature contains the core logic for creating the canonical request,
+// string-to-sign, and comparing the final signature.
+func calculateAndVerifySignature(secretKey, method, urlPath, queryStr string, extractedSignedHeaders http.Header, authInfo *v4AuthInfo) (string, s3err.ErrorCode) {
+	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, authInfo.HashedPayload, queryStr, urlPath, method)
+	stringToSign := getStringToSign(canonicalRequest, authInfo.Date, authInfo.Scope)
+	signingKey := getSigningKey(secretKey, authInfo.Date.Format(yyyymmdd), authInfo.Region, authInfo.Service)
+	newSignature := getSignature(signingKey, stringToSign)
+
+	if !compareSignatureV4(newSignature, authInfo.Signature) {
+		glog.V(4).Infof("Signature mismatch. Details:\n- CanonicalRequest: %q\n- StringToSign: %q\n- Calculated: %s, Provided: %s",
+			canonicalRequest, stringToSign, newSignature, authInfo.Signature)
+		return "", s3err.ErrSignatureDoesNotMatch
+	}
+
+	return newSignature, s3err.ErrNone
+}
+
+func extractV4AuthInfo(r *http.Request) (*v4AuthInfo, s3err.ErrorCode) {
+	if isRequestPresignedSignatureV4(r) {
+		return extractV4AuthInfoFromQuery(r)
+	}
+	return extractV4AuthInfoFromHeader(r)
+}
+
+func extractV4AuthInfoFromHeader(r *http.Request) (*v4AuthInfo, s3err.ErrorCode) {
+	authHeader := r.Header.Get("Authorization")
+	signV4Values, errCode := parseSignV4(authHeader)
 	if errCode != s3err.ErrNone {
 		return nil, errCode
 	}
 
-	// Compute payload hash for non-S3 services
-	if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil {
-		var err error
-		hashedPayload, err = streamHashRequestBody(r, iamRequestBodyLimit)
+	var t time.Time
+	if xamz := r.Header.Get("x-amz-date"); xamz != "" {
+		parsed, err := time.Parse(iso8601Format, xamz)
 		if err != nil {
+			return nil, s3err.ErrMalformedDate
+		}
+		t = parsed
+	} else {
+		ds := r.Header.Get("Date")
+		if ds == "" {
+			return nil, s3err.ErrMissingDateHeader
+		}
+		parsed, err := http.ParseTime(ds)
+		if err != nil {
+			return nil, s3err.ErrMalformedDate
+		}
+		t = parsed.UTC()
+	}
+
+	// Validate clock skew: requests cannot be older than 15 minutes from server time to prevent replay attacks
+	const maxSkew = 15 * time.Minute
+	now := time.Now().UTC()
+	if now.Sub(t) > maxSkew || t.Sub(now) > maxSkew {
+		return nil, s3err.ErrRequestTimeTooSkewed
+	}
+
+	hashedPayload := getContentSha256Cksum(r)
+	if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil {
+		var hashErr error
+		hashedPayload, hashErr = streamHashRequestBody(r, iamRequestBodyLimit)
+		if hashErr != nil {
 			return nil, s3err.ErrInternalError
 		}
 	}
 
-	// Extract all the signed headers along with its values.
-	extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r)
+	return &v4AuthInfo{
+		Signature:     signV4Values.Signature,
+		AccessKey:     signV4Values.Credential.accessKey,
+		SignedHeaders: signV4Values.SignedHeaders,
+		Date:          t,
+		Region:        signV4Values.Credential.scope.region,
+		Service:       signV4Values.Credential.scope.service,
+		Scope:         signV4Values.Credential.getScope(),
+		HashedPayload: hashedPayload,
+		IsPresigned:   false,
+	}, s3err.ErrNone
+}
+
+func extractV4AuthInfoFromQuery(r *http.Request) (*v4AuthInfo, s3err.ErrorCode) {
+	query := r.URL.Query()
+
+	// Validate all required query parameters upfront for fail-fast behavior
+	if query.Get("X-Amz-Algorithm") != signV4Algorithm {
+		return nil, s3err.ErrSignatureVersionNotSupported
+	}
+	if query.Get("X-Amz-Date") == "" {
+		return nil, s3err.ErrMissingDateHeader
+	}
+	if query.Get("X-Amz-Credential") == "" {
+		return nil, s3err.ErrMissingFields
+	}
+	if query.Get("X-Amz-Signature") == "" {
+		return nil, s3err.ErrMissingFields
+	}
+	if query.Get("X-Amz-SignedHeaders") == "" {
+		return nil, s3err.ErrMissingFields
+	}
+	if query.Get("X-Amz-Expires") == "" {
+		return nil, s3err.ErrInvalidQueryParams
+	}
+
+	// Parse date
+	dateStr := query.Get("X-Amz-Date")
+	t, err := time.Parse(iso8601Format, dateStr)
+	if err != nil {
+		return nil, s3err.ErrMalformedDate
+	}
+
+	// Parse credential header
+	credHeader, errCode := parseCredentialHeader("Credential=" + query.Get("X-Amz-Credential"))
 	if errCode != s3err.ErrNone {
 		return nil, errCode
 	}
 
-	cred := signV4Values.Credential
-	identity, foundCred, found := iam.lookupByAccessKey(cred.accessKey)
-	if !found {
-		return nil, s3err.ErrInvalidAccessKeyID
+	// For presigned URLs, X-Amz-Content-Sha256 must come from the query parameter
+	// (or default to UNSIGNED-PAYLOAD) because that's what was used for signing.
+	// We must NOT check the request header as it wasn't part of the signature calculation.
+	hashedPayload := query.Get("X-Amz-Content-Sha256")
+	if hashedPayload == "" {
+		hashedPayload = unsignedPayload
 	}
 
-	// Extract date, if not present throw error.
-	var dateStr string
-	if dateStr = req.Header.Get("x-amz-date"); dateStr == "" {
-		if dateStr = r.Header.Get("Date"); dateStr == "" {
-			return nil, s3err.ErrMissingDateHeader
-		}
-	}
-	// Parse date header.
-	t, e := time.Parse(iso8601Format, dateStr)
-	if e != nil {
-		return nil, s3err.ErrMalformedDate
-	}
-
-	// Query string.
-	queryStr := req.URL.Query().Encode()
-
-	// Check if reverse proxy is forwarding with prefix
-	if forwardedPrefix := r.Header.Get("X-Forwarded-Prefix"); forwardedPrefix != "" {
-		// Try signature verification with the forwarded prefix first.
-		// This handles cases where reverse proxies strip URL prefixes and add the X-Forwarded-Prefix header.
-		cleanedPath := buildPathWithForwardedPrefix(forwardedPrefix, req.URL.Path)
-		errCode = iam.verifySignatureWithPath(extractedSignedHeaders, hashedPayload, queryStr, cleanedPath, req.Method, foundCred.SecretKey, t, signV4Values)
-		if errCode == s3err.ErrNone {
-			return identity, errCode
-		}
-	}
-
-	// Try normal signature verification (without prefix)
-	errCode = iam.verifySignatureWithPath(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method, foundCred.SecretKey, t, signV4Values)
-	if errCode == s3err.ErrNone {
-		return identity, errCode
-	}
-
-	return nil, errCode
+	return &v4AuthInfo{
+		Signature:     query.Get("X-Amz-Signature"),
+		AccessKey:     credHeader.accessKey,
+		SignedHeaders: strings.Split(query.Get("X-Amz-SignedHeaders"), ";"),
+		Date:          t,
+		Region:        credHeader.scope.region,
+		Service:       credHeader.scope.service,
+		Scope:         credHeader.getScope(),
+		HashedPayload: hashedPayload,
+		IsPresigned:   true,
+	}, s3err.ErrNone
 }
 
-// buildPathWithForwardedPrefix combines forwarded prefix with URL path while preserving trailing slashes.
-// This ensures compatibility with S3 SDK signatures that include trailing slashes for directory operations.
-func buildPathWithForwardedPrefix(forwardedPrefix, urlPath string) string {
-	fullPath := forwardedPrefix + urlPath
-	hasTrailingSlash := strings.HasSuffix(urlPath, "/") && urlPath != "/"
-	cleanedPath := path.Clean(fullPath)
-	if hasTrailingSlash && !strings.HasSuffix(cleanedPath, "/") {
-		cleanedPath += "/"
+func getCanonicalQueryString(r *http.Request, isPresigned bool) string {
+	var queryToEncode string
+	if !isPresigned {
+		queryToEncode = r.URL.Query().Encode()
+	} else {
+		queryForCanonical := r.URL.Query()
+		queryForCanonical.Del("X-Amz-Signature")
+		queryToEncode = queryForCanonical.Encode()
 	}
-	return cleanedPath
+	return queryToEncode
 }
 
-// verifySignatureWithPath verifies signature with a given path (used for both normal and prefixed paths).
-func (iam *IdentityAccessManagement) verifySignatureWithPath(extractedSignedHeaders http.Header, hashedPayload, queryStr, urlPath, method, secretKey string, t time.Time, signV4Values signValues) s3err.ErrorCode {
-	// Get canonical request.
-	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, urlPath, method)
-
-	// Get string to sign from canonical request.
-	stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope())
-
-	// Get hmac signing key.
-	signingKey := getSigningKey(secretKey, signV4Values.Credential.scope.date.Format(yyyymmdd), signV4Values.Credential.scope.region, signV4Values.Credential.scope.service)
-
-	// Calculate signature.
-	newSignature := getSignature(signingKey, stringToSign)
-
-	// Verify if signature match.
-	if !compareSignatureV4(newSignature, signV4Values.Signature) {
-		return s3err.ErrSignatureDoesNotMatch
+func checkPresignedRequestExpiry(r *http.Request, t time.Time) s3err.ErrorCode {
+	expiresStr := r.URL.Query().Get("X-Amz-Expires")
+	// X-Amz-Expires is validated as required in extractV4AuthInfoFromQuery,
+	// so it should never be empty here
+	expires, err := strconv.ParseInt(expiresStr, 10, 64)
+	if err != nil {
+		return s3err.ErrMalformedDate
 	}
 
+	// The maximum value for X-Amz-Expires is 604800 seconds (7 days)
+	// Allow 0 but it will immediately fail expiration check
+	if expires < 0 {
+		return s3err.ErrNegativeExpires
+	}
+	if expires > 604800 {
+		return s3err.ErrMaximumExpires
+	}
+
+	expirationTime := t.Add(time.Duration(expires) * time.Second)
+	if time.Now().UTC().After(expirationTime) {
+		return s3err.ErrExpiredPresignRequest
+	}
 	return s3err.ErrNone
 }
 
-// verifyPresignedSignatureWithPath verifies presigned signature with a given path (used for both normal and prefixed paths).
-func (iam *IdentityAccessManagement) verifyPresignedSignatureWithPath(extractedSignedHeaders http.Header, hashedPayload, queryStr, urlPath, method, secretKey string, t time.Time, credHeader credentialHeader, signature string) s3err.ErrorCode {
-	// Get canonical request.
-	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, urlPath, method)
-
-	// Get string to sign from canonical request.
-	stringToSign := getStringToSign(canonicalRequest, t, credHeader.getScope())
-
-	// Get hmac signing key.
-	signingKey := getSigningKey(secretKey, credHeader.scope.date.Format(yyyymmdd), credHeader.scope.region, credHeader.scope.service)
-
-	// Calculate expected signature.
-	expectedSignature := getSignature(signingKey, stringToSign)
-
-	// Verify if signature match.
-	if !compareSignatureV4(expectedSignature, signature) {
-		return s3err.ErrSignatureDoesNotMatch
-	}
-
-	return s3err.ErrNone
+func (iam *IdentityAccessManagement) doesSignatureMatch(r *http.Request) (*Identity, string, s3err.ErrorCode) {
+	identity, _, calculatedSignature, _, errCode := iam.verifyV4Signature(r, false)
+	return identity, calculatedSignature, errCode
 }
 
-// Simple implementation for presigned signature verification
-func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload string, r *http.Request) (*Identity, s3err.ErrorCode) {
-	// Parse presigned signature values from query parameters
-	query := r.URL.Query()
-
-	// Check required parameters
-	algorithm := query.Get("X-Amz-Algorithm")
-	if algorithm != signV4Algorithm {
-		return nil, s3err.ErrSignatureVersionNotSupported
-	}
-
-	credential := query.Get("X-Amz-Credential")
-	if credential == "" {
-		return nil, s3err.ErrMissingFields
-	}
-
-	signature := query.Get("X-Amz-Signature")
-	if signature == "" {
-		return nil, s3err.ErrMissingFields
-	}
-
-	signedHeadersStr := query.Get("X-Amz-SignedHeaders")
-	if signedHeadersStr == "" {
-		return nil, s3err.ErrMissingFields
-	}
-
-	dateStr := query.Get("X-Amz-Date")
-	if dateStr == "" {
-		return nil, s3err.ErrMissingDateHeader
-	}
-
-	// Parse credential
-	credHeader, err := parseCredentialHeader("Credential=" + credential)
-	if err != s3err.ErrNone {
-		return nil, err
-	}
-
-	// Look up identity by access key
-	identity, foundCred, found := iam.lookupByAccessKey(credHeader.accessKey)
-	if !found {
-		return nil, s3err.ErrInvalidAccessKeyID
-	}
-
-	// Parse date
-	t, e := time.Parse(iso8601Format, dateStr)
-	if e != nil {
-		return nil, s3err.ErrMalformedDate
-	}
-
-	// Check expiration
-	expiresStr := query.Get("X-Amz-Expires")
-	if expiresStr != "" {
-		expires, parseErr := strconv.ParseInt(expiresStr, 10, 64)
-		if parseErr != nil {
-			return nil, s3err.ErrMalformedDate
-		}
-		// Check if current time is after the expiration time
-		expirationTime := t.Add(time.Duration(expires) * time.Second)
-		if time.Now().UTC().After(expirationTime) {
-			return nil, s3err.ErrExpiredPresignRequest
-		}
-	}
-
-	// Parse signed headers
-	signedHeaders := strings.Split(signedHeadersStr, ";")
-
-	// Extract signed headers from request
-	extractedSignedHeaders := make(http.Header)
-	for _, header := range signedHeaders {
-		if header == "host" {
-			extractedSignedHeaders[header] = []string{extractHostHeader(r)}
-			continue
-		}
-		if values := r.Header[http.CanonicalHeaderKey(header)]; len(values) > 0 {
-			extractedSignedHeaders[http.CanonicalHeaderKey(header)] = values
-		}
-	}
-
-	// Remove signature from query for canonical request calculation
-	queryForCanonical := r.URL.Query()
-	queryForCanonical.Del("X-Amz-Signature")
-	queryStr := strings.Replace(queryForCanonical.Encode(), "+", "%20", -1)
-
-	var errCode s3err.ErrorCode
-	// Check if reverse proxy is forwarding with prefix for presigned URLs
-	if forwardedPrefix := r.Header.Get("X-Forwarded-Prefix"); forwardedPrefix != "" {
-		// Try signature verification with the forwarded prefix first.
-		// This handles cases where reverse proxies strip URL prefixes and add the X-Forwarded-Prefix header.
-		cleanedPath := buildPathWithForwardedPrefix(forwardedPrefix, r.URL.Path)
-		errCode = iam.verifyPresignedSignatureWithPath(extractedSignedHeaders, hashedPayload, queryStr, cleanedPath, r.Method, foundCred.SecretKey, t, credHeader, signature)
-		if errCode == s3err.ErrNone {
-			return identity, errCode
-		}
-	}
-
-	// Try normal signature verification (without prefix)
-	errCode = iam.verifyPresignedSignatureWithPath(extractedSignedHeaders, hashedPayload, queryStr, r.URL.Path, r.Method, foundCred.SecretKey, t, credHeader, signature)
-	if errCode == s3err.ErrNone {
-		return identity, errCode
-	}
-
-	return nil, errCode
+func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(r *http.Request) (*Identity, string, s3err.ErrorCode) {
+	identity, _, calculatedSignature, _, errCode := iam.verifyV4Signature(r, false)
+	return identity, calculatedSignature, errCode
 }
 
 // credentialHeader data type represents structured form of Credential
@@ -531,7 +595,7 @@ func extractHostHeader(r *http.Request) string {
 		// Check if reverse proxy also forwarded the port
 		if forwardedPort := r.Header.Get("X-Forwarded-Port"); forwardedPort != "" {
 			// Determine the protocol to check for standard ports
-			proto := r.Header.Get("X-Forwarded-Proto")
+			proto := strings.ToLower(r.Header.Get("X-Forwarded-Proto"))
 			// Only add port if it's not the standard port for the protocol
 			if (proto == "https" && forwardedPort != "443") || (proto != "https" && forwardedPort != "80") {
 				return forwardedHost + ":" + forwardedPort

weed/s3api/auth_signature_v4_test.go

@@ -0,0 +1,91 @@
+package s3api
+
+import (
+	"testing"
+)
+
+func TestBuildPathWithForwardedPrefix(t *testing.T) {
+	tests := []struct {
+		name            string
+		forwardedPrefix string
+		urlPath         string
+		expected        string
+	}{
+		{
+			name:            "empty prefix returns urlPath",
+			forwardedPrefix: "",
+			urlPath:         "/bucket/obj",
+			expected:        "/bucket/obj",
+		},
+		{
+			name:            "prefix without trailing slash",
+			forwardedPrefix: "/storage",
+			urlPath:         "/bucket/obj",
+			expected:        "/storage/bucket/obj",
+		},
+		{
+			name:            "prefix with trailing slash",
+			forwardedPrefix: "/storage/",
+			urlPath:         "/bucket/obj",
+			expected:        "/storage/bucket/obj",
+		},
+		{
+			name:            "prefix without leading slash",
+			forwardedPrefix: "storage",
+			urlPath:         "/bucket/obj",
+			expected:        "/storage/bucket/obj",
+		},
+		{
+			name:            "prefix without leading slash and with trailing slash",
+			forwardedPrefix: "storage/",
+			urlPath:         "/bucket/obj",
+			expected:        "/storage/bucket/obj",
+		},
+		{
+			name:            "preserve double slashes in key",
+			forwardedPrefix: "/storage",
+			urlPath:         "/bucket//obj",
+			expected:        "/storage/bucket//obj",
+		},
+		{
+			name:            "preserve trailing slash in urlPath",
+			forwardedPrefix: "/storage",
+			urlPath:         "/bucket/folder/",
+			expected:        "/storage/bucket/folder/",
+		},
+		{
+			name:            "preserve trailing slash with prefix having trailing slash",
+			forwardedPrefix: "/storage/",
+			urlPath:         "/bucket/folder/",
+			expected:        "/storage/bucket/folder/",
+		},
+		{
+			name:            "root path",
+			forwardedPrefix: "/storage",
+			urlPath:         "/",
+			expected:        "/storage/",
+		},
+		{
+			name:            "complex key with multiple slashes",
+			forwardedPrefix: "/api/v1",
+			urlPath:         "/bucket/path//with///slashes",
+			expected:        "/api/v1/bucket/path//with///slashes",
+		},
+		{
+			name:            "urlPath without leading slash",
+			forwardedPrefix: "/storage",
+			urlPath:         "bucket/obj",
+			expected:        "/storage/bucket/obj",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := buildPathWithForwardedPrefix(tt.forwardedPrefix, tt.urlPath)
+			if result != tt.expected {
+				t.Errorf("buildPathWithForwardedPrefix(%q, %q) = %q, want %q",
+					tt.forwardedPrefix, tt.urlPath, result, tt.expected)
+			}
+		})
+	}
+}

weed/s3api/auto_signature_v4_test.go

@@ -229,8 +229,12 @@ func preSignV4(iam *IdentityAccessManagement, req *http.Request, accessKey, secr
 	// Set the query on the URL (without signature yet)
 	req.URL.RawQuery = query.Encode()
 
-	// Get the payload hash
-	hashedPayload := getContentSha256Cksum(req)
+	// For presigned URLs, the payload hash must be UNSIGNED-PAYLOAD (or from query param if explicitly set)
+	// We should NOT use request headers as they're not part of the presigned URL
+	hashedPayload := query.Get("X-Amz-Content-Sha256")
+	if hashedPayload == "" {
+		hashedPayload = unsignedPayload
+	}
 
 	// Extract signed headers
 	extractedSignedHeaders := make(http.Header)
@@ -314,7 +318,7 @@ func TestSignatureV4WithForwardedPrefix(t *testing.T) {
 			signV4WithPath(r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", tt.expectedPath)
 
 			// Test signature verification
-			_, errCode := iam.doesSignatureMatch(getContentSha256Cksum(r), r)
+			_, _, errCode := iam.doesSignatureMatch(r)
 			if errCode != s3err.ErrNone {
 				t.Errorf("Expected successful signature validation with X-Forwarded-Prefix %q, got error: %v (code: %d)", tt.forwardedPrefix, errCode, int(errCode))
 			}
@@ -380,7 +384,7 @@ func TestSignatureV4WithForwardedPrefixTrailingSlash(t *testing.T) {
 			signV4WithPath(r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", tt.expectedPath)
 
 			// Test signature verification - this should succeed even with trailing slashes
-			_, errCode := iam.doesSignatureMatch(getContentSha256Cksum(r), r)
+			_, _, errCode := iam.doesSignatureMatch(r)
 			if errCode != s3err.ErrNone {
 				t.Errorf("Expected successful signature validation with trailing slash in path %q, got error: %v (code: %d)", tt.urlPath, errCode, int(errCode))
 			}
@@ -475,7 +479,7 @@ func TestSignatureV4WithForwardedPort(t *testing.T) {
 			signV4WithPath(r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", r.URL.Path)
 
 			// Test signature verification
-			_, errCode := iam.doesSignatureMatch(getContentSha256Cksum(r), r)
+			_, _, errCode := iam.doesSignatureMatch(r)
 			if errCode != s3err.ErrNone {
 				t.Errorf("Expected successful signature validation with forwarded port, got error: %v (code: %d)", errCode, int(errCode))
 			}
@@ -508,12 +512,50 @@ func TestPresignedSignatureV4Basic(t *testing.T) {
 	}
 
 	// Test presigned signature verification
-	_, errCode := iam.doesPresignedSignatureMatch(getContentSha256Cksum(r), r)
+	_, _, errCode := iam.doesPresignedSignatureMatch(r)
 	if errCode != s3err.ErrNone {
 		t.Errorf("Expected successful presigned signature validation, got error: %v (code: %d)", errCode, int(errCode))
 	}
 }
 
+// TestPresignedSignatureV4MissingExpires verifies that X-Amz-Expires is required for presigned URLs
+func TestPresignedSignatureV4MissingExpires(t *testing.T) {
+	iam := newTestIAM()
+
+	// Create a presigned request
+	r, err := newTestRequest("GET", "https://example.com/test-bucket/test-object", 0, nil)
+	if err != nil {
+		t.Fatalf("Failed to create test request: %v", err)
+	}
+
+	r = mux.SetURLVars(r, map[string]string{
+		"bucket": "test-bucket",
+		"object": "test-object",
+	})
+	r.Header.Set("Host", "example.com")
+
+	// Manually construct presigned URL query parameters WITHOUT X-Amz-Expires
+	now := time.Now().UTC()
+	dateStr := now.Format(iso8601Format)
+	scope := fmt.Sprintf("%s/%s/%s/%s", now.Format(yyyymmdd), "us-east-1", "s3", "aws4_request")
+	credential := fmt.Sprintf("%s/%s", "AKIAIOSFODNN7EXAMPLE", scope)
+
+	query := r.URL.Query()
+	query.Set("X-Amz-Algorithm", signV4Algorithm)
+	query.Set("X-Amz-Credential", credential)
+	query.Set("X-Amz-Date", dateStr)
+	// Intentionally NOT setting X-Amz-Expires
+	query.Set("X-Amz-SignedHeaders", "host")
+	query.Set("X-Amz-Signature", "dummy-signature") // Signature doesn't matter, should fail earlier
+	r.URL.RawQuery = query.Encode()
+
+	// Test presigned signature verification - should fail with ErrInvalidQueryParams
+	_, _, errCode := iam.doesPresignedSignatureMatch(r)
+	if errCode != s3err.ErrInvalidQueryParams {
+		t.Errorf("Expected ErrInvalidQueryParams for missing X-Amz-Expires, got: %v (code: %d)", errCode, int(errCode))
+	}
+}
+
 // Test X-Forwarded-Prefix support for presigned URLs
 func TestPresignedSignatureV4WithForwardedPrefix(t *testing.T) {
 	tests := []struct {
@@ -573,7 +615,8 @@ func TestPresignedSignatureV4WithForwardedPrefix(t *testing.T) {
 			r.Header.Set("X-Forwarded-Host", "example.com")
 
 			// Test presigned signature verification
-			_, errCode := iam.doesPresignedSignatureMatch(getContentSha256Cksum(r), r)
+			_, _, errCode := iam.doesPresignedSignatureMatch(r)
+
 			if errCode != s3err.ErrNone {
 				t.Errorf("Expected successful presigned signature validation with X-Forwarded-Prefix %q, got error: %v (code: %d)", tt.forwardedPrefix, errCode, int(errCode))
 			}
@@ -640,7 +683,8 @@ func TestPresignedSignatureV4WithForwardedPrefixTrailingSlash(t *testing.T) {
 			r.Header.Set("X-Forwarded-Host", "example.com")
 
 			// Test presigned signature verification - this should succeed with trailing slashes
-			_, errCode := iam.doesPresignedSignatureMatch(getContentSha256Cksum(r), r)
+			_, _, errCode := iam.doesPresignedSignatureMatch(r)
+
 			if errCode != s3err.ErrNone {
 				t.Errorf("Expected successful presigned signature validation with trailing slash in path %q, got error: %v (code: %d)", tt.strippedPath, errCode, int(errCode))
 			}
@@ -669,8 +713,12 @@ func preSignV4WithPath(iam *IdentityAccessManagement, req *http.Request, accessK
 	// Set the query on the URL (without signature yet)
 	req.URL.RawQuery = query.Encode()
 
-	// Get the payload hash
-	hashedPayload := getContentSha256Cksum(req)
+	// For presigned URLs, the payload hash must be UNSIGNED-PAYLOAD (or from query param if explicitly set)
+	// We should NOT use request headers as they're not part of the presigned URL
+	hashedPayload := query.Get("X-Amz-Content-Sha256")
+	if hashedPayload == "" {
+		hashedPayload = unsignedPayload
+	}
 
 	// Extract signed headers
 	extractedSignedHeaders := make(http.Header)
@@ -884,7 +932,7 @@ func signRequestV4(req *http.Request, accessKey, secretKey string) error {
 		return fmt.Errorf("Invalid hashed payload")
 	}
 
-	currTime := time.Now()
+	currTime := time.Now().UTC()
 
 	// Set x-amz-date.
 	req.Header.Set("x-amz-date", currTime.Format(iso8601Format))
@@ -1061,10 +1109,6 @@ func TestIAMPayloadHashComputation(t *testing.T) {
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
 	req.Header.Set("Host", "localhost:8111")
 
-	// Compute expected payload hash
-	expectedHash := sha256.Sum256([]byte(testPayload))
-	expectedHashStr := hex.EncodeToString(expectedHash[:])
-
 	// Create an IAM-style authorization header with "iam" service instead of "s3"
 	now := time.Now().UTC()
 	dateStr := now.Format("20060102T150405Z")
@@ -1079,7 +1123,7 @@ func TestIAMPayloadHashComputation(t *testing.T) {
 
 	// Test the doesSignatureMatch function directly
 	// This should now compute the correct payload hash for IAM requests
-	identity, errCode := iam.doesSignatureMatch(expectedHashStr, req)
+	identity, _, errCode := iam.doesSignatureMatch(req)
 
 	// Even though the signature will fail (dummy signature),
 	// the fact that we get past the credential parsing means the payload hash was computed correctly
@@ -1141,7 +1185,7 @@ func TestS3PayloadHashNoRegression(t *testing.T) {
 	req.Header.Set("Authorization", authHeader)
 
 	// This should use the emptySHA256 hash and not try to read the body
-	identity, errCode := iam.doesSignatureMatch(emptySHA256, req)
+	identity, _, errCode := iam.doesSignatureMatch(req)
 
 	// Should get signature mismatch (because of dummy signature) but not other errors
 	assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode)
@@ -1192,7 +1236,7 @@ func TestIAMEmptyBodyPayloadHash(t *testing.T) {
 	req.Header.Set("Authorization", authHeader)
 
 	// Even with an IAM request, empty body should result in emptySHA256
-	identity, errCode := iam.doesSignatureMatch(emptySHA256, req)
+	identity, _, errCode := iam.doesSignatureMatch(req)
 
 	// Should get signature mismatch (because of dummy signature) but not other errors
 	assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode)
@@ -1235,10 +1279,6 @@ func TestSTSPayloadHashComputation(t *testing.T) {
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
 	req.Header.Set("Host", "localhost:8112")
 
-	// Compute expected payload hash
-	expectedHash := sha256.Sum256([]byte(testPayload))
-	expectedHashStr := hex.EncodeToString(expectedHash[:])
-
 	// Create an STS-style authorization header with "sts" service
 	now := time.Now().UTC()
 	dateStr := now.Format("20060102T150405Z")
@@ -1252,7 +1292,7 @@ func TestSTSPayloadHashComputation(t *testing.T) {
 
 	// Test the doesSignatureMatch function
 	// This should compute the correct payload hash for STS requests (non-S3 service)
-	identity, errCode := iam.doesSignatureMatch(expectedHashStr, req)
+	identity, _, errCode := iam.doesSignatureMatch(req)
 
 	// Should get signature mismatch (dummy signature) but payload hash should be computed correctly
 	assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode)
@@ -1317,7 +1357,7 @@ func TestGitHubIssue7080Scenario(t *testing.T) {
 
 	// Since we're using a dummy signature, we expect signature mismatch, but the important
 	// thing is that it doesn't fail earlier due to payload hash computation issues
-	identity, errCode := iam.doesSignatureMatch(emptySHA256, req)
+	identity, _, errCode := iam.doesSignatureMatch(req)
 
 	// The error should be signature mismatch, not payload related
 	assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode)
@@ -1357,32 +1397,37 @@ func TestIAMSignatureServiceMatching(t *testing.T) {
 	// Use the exact payload and headers from the failing logs
 	testPayload := "Action=CreateAccessKey&UserName=admin&Version=2010-05-08"
 
+	// Use current time to avoid clock skew validation failures
+	now := time.Now().UTC()
+	amzDate := now.Format(iso8601Format)
+	dateStamp := now.Format(yyyymmdd)
+
 	// Create request exactly as shown in logs
 	req, err := http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(testPayload))
 	assert.NoError(t, err)
 
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
 	req.Header.Set("Host", "localhost:8111")
-	req.Header.Set("X-Amz-Date", "20250805T082934Z")
+	req.Header.Set("X-Amz-Date", amzDate)
 
 	// Calculate the expected signature using the correct IAM service
 	// This simulates what botocore/AWS SDK would calculate
-	credentialScope := "20250805/us-east-1/iam/aws4_request"
+	credentialScope := dateStamp + "/us-east-1/iam/aws4_request"
 
 	// Calculate the actual payload hash for our test payload
 	actualPayloadHash := getSHA256Hash([]byte(testPayload))
 
 	// Build the canonical request with the actual payload hash
-	canonicalRequest := "POST\n/\n\ncontent-type:application/x-www-form-urlencoded; charset=utf-8\nhost:localhost:8111\nx-amz-date:20250805T082934Z\n\ncontent-type;host;x-amz-date\n" + actualPayloadHash
+	canonicalRequest := "POST\n/\n\ncontent-type:application/x-www-form-urlencoded; charset=utf-8\nhost:localhost:8111\nx-amz-date:" + amzDate + "\n\ncontent-type;host;x-amz-date\n" + actualPayloadHash
 
 	// Calculate the canonical request hash
 	canonicalRequestHash := getSHA256Hash([]byte(canonicalRequest))
 
 	// Build the string to sign
-	stringToSign := "AWS4-HMAC-SHA256\n20250805T082934Z\n" + credentialScope + "\n" + canonicalRequestHash
+	stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + credentialScope + "\n" + canonicalRequestHash
 
 	// Calculate expected signature using IAM service (what client sends)
-	expectedSigningKey := getSigningKey("power_user_secret", "20250805", "us-east-1", "iam")
+	expectedSigningKey := getSigningKey("power_user_secret", dateStamp, "us-east-1", "iam")
 	expectedSignature := getSignature(expectedSigningKey, stringToSign)
 
 	// Create authorization header with the correct signature
@@ -1391,7 +1436,8 @@ func TestIAMSignatureServiceMatching(t *testing.T) {
 	req.Header.Set("Authorization", authHeader)
 
 	// Now test that SeaweedFS computes the same signature with our fix
-	identity, errCode := iam.doesSignatureMatch(actualPayloadHash, req)
+	identity, computedSignature, errCode := iam.doesSignatureMatch(req)
+	assert.Equal(t, expectedSignature, computedSignature)
 
 	// With the fix, the signatures should match and we should get a successful authentication
 	assert.Equal(t, s3err.ErrNone, errCode)
@@ -1481,7 +1527,7 @@ func TestIAMLargeBodySecurityLimit(t *testing.T) {
 	req.Header.Set("Authorization", authHeader)
 
 	// The function should complete successfully but limit the body to 10 MiB
-	identity, errCode := iam.doesSignatureMatch(emptySHA256, req)
+	identity, _, errCode := iam.doesSignatureMatch(req)
 
 	// Should get signature mismatch (dummy signature) but not internal error
 	assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode)

weed/s3api/chunked_reader_v4.go

@@ -34,7 +34,6 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 
 	"github.com/dustin/go-humanize"
@@ -47,23 +46,13 @@ import (
 // returns signature, error otherwise if the signature mismatches or any other
 // error while parsing and validating.
 func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cred *Credential, signature string, region string, service string, date time.Time, errCode s3err.ErrorCode) {
-
-	// Copy request.
-	req := *r
-
-	// Save authorization header.
-	v4Auth := req.Header.Get("Authorization")
-
-	// Parse signature version '4' header.
-	signV4Values, errCode := parseSignV4(v4Auth)
+	_, credential, calculatedSignature, authInfo, errCode := iam.verifyV4Signature(r, true)
 	if errCode != s3err.ErrNone {
 		return nil, "", "", "", time.Time{}, errCode
 	}
 
-	contentSha256Header := req.Header.Get("X-Amz-Content-Sha256")
-
-	switch contentSha256Header {
-	// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
+	// This check ensures we only proceed for streaming uploads.
+	switch authInfo.HashedPayload {
 	case streamingContentSHA256:
 		glog.V(3).Infof("streaming content sha256")
 	case streamingUnsignedPayload:
@@ -72,64 +61,7 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
 		return nil, "", "", "", time.Time{}, s3err.ErrContentSHA256Mismatch
 	}
 
-	// Payload streaming.
-	payload := contentSha256Header
-
-	// Extract all the signed headers along with its values.
-	extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r)
-	if errCode != s3err.ErrNone {
-		return nil, "", "", "", time.Time{}, errCode
-	}
-	// Verify if the access key id matches.
-	identity, cred, found := iam.lookupByAccessKey(signV4Values.Credential.accessKey)
-	if !found {
-		return nil, "", "", "", time.Time{}, s3err.ErrInvalidAccessKeyID
-	}
-
-	bucket, object := s3_constants.GetBucketAndObject(r)
-	if !identity.canDo(s3_constants.ACTION_WRITE, bucket, object) {
-		errCode = s3err.ErrAccessDenied
-		return
-	}
-
-	// Verify if region is valid.
-	region = signV4Values.Credential.scope.region
-
-	// Extract date, if not present throw error.
-	var dateStr string
-	if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" {
-		if dateStr = r.Header.Get("Date"); dateStr == "" {
-			return nil, "", "", "", time.Time{}, s3err.ErrMissingDateHeader
-		}
-	}
-
-	// Parse date header.
-	date, err := time.Parse(iso8601Format, dateStr)
-	if err != nil {
-		return nil, "", "", "", time.Time{}, s3err.ErrMalformedDate
-	}
-	// Query string.
-	queryStr := req.URL.Query().Encode()
-
-	// Get canonical request.
-	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, payload, queryStr, req.URL.Path, req.Method)
-
-	// Get string to sign from canonical request.
-	stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope())
-
-	// Get hmac signing key.
-	signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date.Format(yyyymmdd), region, signV4Values.Credential.scope.service)
-
-	// Calculate signature.
-	newSignature := getSignature(signingKey, stringToSign)
-
-	// Verify if signature match.
-	if !compareSignatureV4(newSignature, signV4Values.Signature) {
-		return nil, "", "", "", time.Time{}, s3err.ErrSignatureDoesNotMatch
-	}
-
-	// Return calculated signature.
-	return cred, newSignature, region, signV4Values.Credential.scope.service, date, s3err.ErrNone
+	return credential, calculatedSignature, authInfo.Region, authInfo.Service, authInfo.Date, s3err.ErrNone
 }
 
 const maxLineLength = 4 * humanize.KiByte // assumed <= bufio.defaultBufSize 4KiB
@@ -149,7 +81,7 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea
 	contentSha256Header := req.Header.Get("X-Amz-Content-Sha256")
 	authorizationHeader := req.Header.Get("Authorization")
 
-	var ident *Credential
+	var credential *Credential
 	var seedSignature, region, service string
 	var seedDate time.Time
 	var errCode s3err.ErrorCode
@@ -158,7 +90,7 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea
 	// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
 	case streamingContentSHA256:
 		glog.V(3).Infof("streaming content sha256")
-		ident, seedSignature, region, service, seedDate, errCode = iam.calculateSeedSignature(req)
+		credential, seedSignature, region, service, seedDate, errCode = iam.calculateSeedSignature(req)
 		if errCode != s3err.ErrNone {
 			return nil, errCode
 		}
@@ -186,7 +118,7 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea
 	checkSumWriter := getCheckSumWriter(checksumAlgorithm)
 
 	return &s3ChunkedReader{
-		cred:              ident,
+		cred:              credential,
 		reader:            bufio.NewReader(req.Body),
 		seedSignature:     seedSignature,
 		seedDate:          seedDate,

weed/s3api/chunked_reader_v4_test.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"hash/crc32"
 
@@ -16,66 +17,19 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+// getDefaultTimestamp returns a current timestamp for tests
+func getDefaultTimestamp() string {
+	return time.Now().UTC().Format(iso8601Format)
+}
+
 const (
-	defaultTimestamp       = "20130524T000000Z"
+	defaultTimestamp       = "20130524T000000Z" // Legacy constant for reference
 	defaultBucketName      = "examplebucket"
 	defaultAccessKeyId     = "AKIAIOSFODNN7EXAMPLE"
 	defaultSecretAccessKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
 	defaultRegion          = "us-east-1"
 )
 
-func generatestreamingAws4HmacSha256Payload() string {
-	// This test will implement the following scenario:
-	// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
-
-	chunk1 := "10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n" +
-		strings.Repeat("a", 65536) + "\r\n"
-	chunk2 := "400;chunk-signature=0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497\r\n" +
-		strings.Repeat("a", 1024) + "\r\n"
-	chunk3 := "0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n" +
-		"\r\n" // The last chunk is empty
-
-	payload := chunk1 + chunk2 + chunk3
-	return payload
-}
-
-func NewRequeststreamingAws4HmacSha256Payload() (*http.Request, error) {
-	// This test will implement the following scenario:
-	// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
-
-	payload := generatestreamingAws4HmacSha256Payload()
-	req, err := http.NewRequest("PUT", "http://s3.amazonaws.com/examplebucket/chunkObject.txt", bytes.NewReader([]byte(payload)))
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Set("Host", "s3.amazonaws.com")
-	req.Header.Set("x-amz-date", defaultTimestamp)
-	req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
-	req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9")
-	req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
-	req.Header.Set("Content-Encoding", "aws-chunked")
-	req.Header.Set("x-amz-decoded-content-length", "66560")
-	req.Header.Set("Content-Length", "66824")
-
-	return req, nil
-}
-
-func TestNewSignV4ChunkedReaderstreamingAws4HmacSha256Payload(t *testing.T) {
-	// This test will implement the following scenario:
-	// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
-	req, err := NewRequeststreamingAws4HmacSha256Payload()
-	if err != nil {
-		t.Fatalf("Failed to create request: %v", err)
-	}
-	iam := setupIam()
-
-	// The expected payload a long string of 'a's
-	expectedPayload := strings.Repeat("a", 66560)
-
-	runWithRequest(iam, req, t, expectedPayload)
-}
-
 func generateStreamingUnsignedPayloadTrailerPayload(includeFinalCRLF bool) string {
 	// This test will implement the following scenario:
 	// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
@@ -117,7 +71,7 @@ func NewRequestStreamingUnsignedPayloadTrailer(includeFinalCRLF bool) (*http.Req
 	}
 
 	req.Header.Set("Host", "amzn-s3-demo-bucket")
-	req.Header.Set("x-amz-date", defaultTimestamp)
+	req.Header.Set("x-amz-date", getDefaultTimestamp())
 	req.Header.Set("Content-Encoding", "aws-chunked")
 	req.Header.Set("x-amz-decoded-content-length", "17408")
 	req.Header.Set("x-amz-content-sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER")
@@ -194,3 +148,169 @@ func setupIam() IdentityAccessManagement {
 	iam.accessKeyIdent[defaultAccessKeyId] = iam.identities[0]
 	return iam
 }
+
+// TestSignedStreamingUpload tests streaming uploads with signed chunks
+// This replaces the removed AWS example test with a dynamic signature generation approach
+func TestSignedStreamingUpload(t *testing.T) {
+	iam := setupIam()
+
+	// Create a simple streaming upload with 2 chunks
+	chunk1Data := strings.Repeat("a", 1024)
+	chunk2Data := strings.Repeat("b", 512)
+
+	// Use current time for signatures
+	now := time.Now().UTC()
+	amzDate := now.Format(iso8601Format)
+	dateStamp := now.Format(yyyymmdd)
+
+	// Calculate seed signature
+	scope := dateStamp + "/" + defaultRegion + "/s3/aws4_request"
+
+	// Build canonical request for seed signature
+	hashedPayload := "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+	canonicalHeaders := "content-encoding:aws-chunked\n" +
+		"host:s3.amazonaws.com\n" +
+		"x-amz-content-sha256:" + hashedPayload + "\n" +
+		"x-amz-date:" + amzDate + "\n" +
+		"x-amz-decoded-content-length:1536\n"
+	signedHeaders := "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length"
+
+	canonicalRequest := "PUT\n" +
+		"/test-bucket/test-object\n" +
+		"\n" +
+		canonicalHeaders + "\n" +
+		signedHeaders + "\n" +
+		hashedPayload
+
+	canonicalRequestHash := getSHA256Hash([]byte(canonicalRequest))
+	stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + canonicalRequestHash
+
+	signingKey := getSigningKey(defaultSecretAccessKey, dateStamp, defaultRegion, "s3")
+	seedSignature := getSignature(signingKey, stringToSign)
+
+	// Calculate chunk signatures
+	chunk1Hash := getSHA256Hash([]byte(chunk1Data))
+	chunk1StringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
+		seedSignature + "\n" + emptySHA256 + "\n" + chunk1Hash
+	chunk1Signature := getSignature(signingKey, chunk1StringToSign)
+
+	chunk2Hash := getSHA256Hash([]byte(chunk2Data))
+	chunk2StringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
+		chunk1Signature + "\n" + emptySHA256 + "\n" + chunk2Hash
+	chunk2Signature := getSignature(signingKey, chunk2StringToSign)
+
+	finalStringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
+		chunk2Signature + "\n" + emptySHA256 + "\n" + emptySHA256
+	finalSignature := getSignature(signingKey, finalStringToSign)
+
+	// Build the chunked payload
+	payload := fmt.Sprintf("400;chunk-signature=%s\r\n%s\r\n", chunk1Signature, chunk1Data) +
+		fmt.Sprintf("200;chunk-signature=%s\r\n%s\r\n", chunk2Signature, chunk2Data) +
+		fmt.Sprintf("0;chunk-signature=%s\r\n\r\n", finalSignature)
+
+	// Create the request
+	req, err := http.NewRequest("PUT", "http://s3.amazonaws.com/test-bucket/test-object",
+		bytes.NewReader([]byte(payload)))
+	assert.NoError(t, err)
+
+	req.Header.Set("Host", "s3.amazonaws.com")
+	req.Header.Set("x-amz-date", amzDate)
+	req.Header.Set("x-amz-content-sha256", hashedPayload)
+	req.Header.Set("Content-Encoding", "aws-chunked")
+	req.Header.Set("x-amz-decoded-content-length", "1536")
+
+	authHeader := fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s",
+		defaultAccessKeyId, scope, signedHeaders, seedSignature)
+	req.Header.Set("Authorization", authHeader)
+
+	// Test the chunked reader
+	reader, errCode := iam.newChunkedReader(req)
+	assert.Equal(t, s3err.ErrNone, errCode)
+	assert.NotNil(t, reader)
+
+	// Read and verify the payload
+	data, err := io.ReadAll(reader)
+	assert.NoError(t, err)
+	assert.Equal(t, chunk1Data+chunk2Data, string(data))
+}
+
+// TestSignedStreamingUploadInvalidSignature tests that invalid chunk signatures are rejected
+// This is a negative test case to ensure signature validation is actually working
+func TestSignedStreamingUploadInvalidSignature(t *testing.T) {
+	iam := setupIam()
+
+	// Create a simple streaming upload with 1 chunk
+	chunk1Data := strings.Repeat("a", 1024)
+
+	// Use current time for signatures
+	now := time.Now().UTC()
+	amzDate := now.Format(iso8601Format)
+	dateStamp := now.Format(yyyymmdd)
+
+	// Calculate seed signature
+	scope := dateStamp + "/" + defaultRegion + "/s3/aws4_request"
+
+	// Build canonical request for seed signature
+	hashedPayload := "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+	canonicalHeaders := "content-encoding:aws-chunked\n" +
+		"host:s3.amazonaws.com\n" +
+		"x-amz-content-sha256:" + hashedPayload + "\n" +
+		"x-amz-date:" + amzDate + "\n" +
+		"x-amz-decoded-content-length:1024\n"
+	signedHeaders := "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length"
+
+	canonicalRequest := "PUT\n" +
+		"/test-bucket/test-object\n" +
+		"\n" +
+		canonicalHeaders + "\n" +
+		signedHeaders + "\n" +
+		hashedPayload
+
+	canonicalRequestHash := getSHA256Hash([]byte(canonicalRequest))
+	stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + canonicalRequestHash
+
+	signingKey := getSigningKey(defaultSecretAccessKey, dateStamp, defaultRegion, "s3")
+	seedSignature := getSignature(signingKey, stringToSign)
+
+	// Calculate chunk signature (correct)
+	chunk1Hash := getSHA256Hash([]byte(chunk1Data))
+	chunk1StringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
+		seedSignature + "\n" + emptySHA256 + "\n" + chunk1Hash
+	chunk1Signature := getSignature(signingKey, chunk1StringToSign)
+
+	// Calculate final signature (correct)
+	finalStringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
+		chunk1Signature + "\n" + emptySHA256 + "\n" + emptySHA256
+	finalSignature := getSignature(signingKey, finalStringToSign)
+
+	// Build the chunked payload with INTENTIONALLY WRONG chunk signature
+	// We'll use a modified signature to simulate a tampered request
+	wrongChunkSignature := strings.Replace(chunk1Signature, "a", "b", 1)
+	payload := fmt.Sprintf("400;chunk-signature=%s\r\n%s\r\n", wrongChunkSignature, chunk1Data) +
+		fmt.Sprintf("0;chunk-signature=%s\r\n\r\n", finalSignature)
+
+	// Create the request
+	req, err := http.NewRequest("PUT", "http://s3.amazonaws.com/test-bucket/test-object",
+		bytes.NewReader([]byte(payload)))
+	assert.NoError(t, err)
+
+	req.Header.Set("Host", "s3.amazonaws.com")
+	req.Header.Set("x-amz-date", amzDate)
+	req.Header.Set("x-amz-content-sha256", hashedPayload)
+	req.Header.Set("Content-Encoding", "aws-chunked")
+	req.Header.Set("x-amz-decoded-content-length", "1024")
+
+	authHeader := fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s",
+		defaultAccessKeyId, scope, signedHeaders, seedSignature)
+	req.Header.Set("Authorization", authHeader)
+
+	// Test the chunked reader - it should be created successfully
+	reader, errCode := iam.newChunkedReader(req)
+	assert.Equal(t, s3err.ErrNone, errCode)
+	assert.NotNil(t, reader)
+
+	// Try to read the payload - this should fail with signature validation error
+	_, err = io.ReadAll(reader)
+	assert.Error(t, err, "Expected error when reading chunk with invalid signature")
+	assert.Contains(t, err.Error(), "chunk signature does not match", "Error should indicate chunk signature mismatch")
+}

weed/s3api/s3_constants/header.go

@@ -94,6 +94,9 @@ const (
 	AmzEncryptedDataKey      = "x-amz-encrypted-data-key"
 	AmzEncryptionContextMeta = "x-amz-encryption-context"
 
+	// SeaweedFS internal metadata prefix (used to filter internal headers from client responses)
+	SeaweedFSInternalPrefix = "x-seaweedfs-"
+
 	// SeaweedFS internal metadata keys for encryption (prefixed to avoid automatic HTTP header conversion)
 	SeaweedFSSSEKMSKey = "x-seaweedfs-sse-kms-key" // Key for storing serialized SSE-KMS metadata
 	SeaweedFSSSES3Key  = "x-seaweedfs-sse-s3-key"  // Key for storing serialized SSE-S3 metadata
@@ -157,3 +160,10 @@ var PassThroughHeaders = map[string]string{
 	"response-content-type":        "Content-Type",
 	"response-expires":             "Expires",
 }
+
+// IsSeaweedFSInternalHeader checks if a header key is a SeaweedFS internal header
+// that should be filtered from client responses.
+// Header names are case-insensitive in HTTP, so this function normalizes to lowercase.
+func IsSeaweedFSInternalHeader(headerKey string) bool {
+	return strings.HasPrefix(strings.ToLower(headerKey), SeaweedFSInternalPrefix)
+}

weed/s3api/s3_sse_c_range_test.go

@@ -56,7 +56,8 @@ func TestSSECRangeRequestsSupported(t *testing.T) {
 	}
 	rec := httptest.NewRecorder()
 	w := recorderFlusher{rec}
-	statusCode, _ := s3a.handleSSECResponse(req, proxyResponse, w)
+	// Pass nil for entry since this test focuses on Range request handling
+	statusCode, _ := s3a.handleSSECResponse(req, proxyResponse, w, nil)
 
 	// Range requests should now be allowed to proceed (will be handled by filer layer)
 	// The exact status code depends on the object existence and filer response

weed/s3api/s3_sse_copy_test.go

@@ -43,7 +43,7 @@ func TestSSECObjectCopy(t *testing.T) {
 
 	// Test copy strategy determination
 	sourceMetadata := make(map[string][]byte)
-	StoreIVInMetadata(sourceMetadata, iv)
+	StoreSSECIVInMetadata(sourceMetadata, iv)
 	sourceMetadata[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] = []byte("AES256")
 	sourceMetadata[s3_constants.AmzServerSideEncryptionCustomerKeyMD5] = []byte(sourceKey.KeyMD5)
 

weed/s3api/s3_sse_metadata.go

@@ -2,158 +2,28 @@ package s3api
 
 import (
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 )
 
-// SSE metadata keys for storing encryption information in entry metadata
-const (
-	// MetaSSEIV is the initialization vector used for encryption
-	MetaSSEIV = "X-SeaweedFS-Server-Side-Encryption-Iv"
-
-	// MetaSSEAlgorithm is the encryption algorithm used
-	MetaSSEAlgorithm = "X-SeaweedFS-Server-Side-Encryption-Algorithm"
-
-	// MetaSSECKeyMD5 is the MD5 hash of the SSE-C customer key
-	MetaSSECKeyMD5 = "X-SeaweedFS-Server-Side-Encryption-Customer-Key-MD5"
-
-	// MetaSSEKMSKeyID is the KMS key ID used for encryption
-	MetaSSEKMSKeyID = "X-SeaweedFS-Server-Side-Encryption-KMS-Key-Id"
-
-	// MetaSSEKMSEncryptedKey is the encrypted data key from KMS
-	MetaSSEKMSEncryptedKey = "X-SeaweedFS-Server-Side-Encryption-KMS-Encrypted-Key"
-
-	// MetaSSEKMSContext is the encryption context for KMS
-	MetaSSEKMSContext = "X-SeaweedFS-Server-Side-Encryption-KMS-Context"
-
-	// MetaSSES3KeyID is the key ID for SSE-S3 encryption
-	MetaSSES3KeyID = "X-SeaweedFS-Server-Side-Encryption-S3-Key-Id"
-)
-
-// StoreIVInMetadata stores the IV in entry metadata as base64 encoded string
-func StoreIVInMetadata(metadata map[string][]byte, iv []byte) {
+// StoreSSECIVInMetadata stores the SSE-C IV in entry metadata as base64 encoded string
+// Used by SSE-C for storing IV in entry.Extended
+func StoreSSECIVInMetadata(metadata map[string][]byte, iv []byte) {
 	if len(iv) > 0 {
-		metadata[MetaSSEIV] = []byte(base64.StdEncoding.EncodeToString(iv))
+		metadata[s3_constants.SeaweedFSSSEIV] = []byte(base64.StdEncoding.EncodeToString(iv))
 	}
 }
 
-// GetIVFromMetadata retrieves the IV from entry metadata
-func GetIVFromMetadata(metadata map[string][]byte) ([]byte, error) {
-	if ivBase64, exists := metadata[MetaSSEIV]; exists {
+// GetSSECIVFromMetadata retrieves the SSE-C IV from entry metadata
+// Used by SSE-C for retrieving IV from entry.Extended
+func GetSSECIVFromMetadata(metadata map[string][]byte) ([]byte, error) {
+	if ivBase64, exists := metadata[s3_constants.SeaweedFSSSEIV]; exists {
 		iv, err := base64.StdEncoding.DecodeString(string(ivBase64))
 		if err != nil {
-			return nil, fmt.Errorf("failed to decode IV from metadata: %w", err)
+			return nil, fmt.Errorf("failed to decode SSE-C IV from metadata: %w", err)
 		}
 		return iv, nil
 	}
-	return nil, fmt.Errorf("IV not found in metadata")
-}
-
-// StoreSSECMetadata stores SSE-C related metadata
-func StoreSSECMetadata(metadata map[string][]byte, iv []byte, keyMD5 string) {
-	StoreIVInMetadata(metadata, iv)
-	metadata[MetaSSEAlgorithm] = []byte("AES256")
-	if keyMD5 != "" {
-		metadata[MetaSSECKeyMD5] = []byte(keyMD5)
-	}
-}
-
-// StoreSSEKMSMetadata stores SSE-KMS related metadata
-func StoreSSEKMSMetadata(metadata map[string][]byte, iv []byte, keyID string, encryptedKey []byte, context map[string]string) {
-	StoreIVInMetadata(metadata, iv)
-	metadata[MetaSSEAlgorithm] = []byte("aws:kms")
-	if keyID != "" {
-		metadata[MetaSSEKMSKeyID] = []byte(keyID)
-	}
-	if len(encryptedKey) > 0 {
-		metadata[MetaSSEKMSEncryptedKey] = []byte(base64.StdEncoding.EncodeToString(encryptedKey))
-	}
-	if len(context) > 0 {
-		// Marshal context to JSON to handle special characters correctly
-		contextBytes, err := json.Marshal(context)
-		if err == nil {
-			metadata[MetaSSEKMSContext] = contextBytes
-		}
-		// Note: json.Marshal for map[string]string should never fail, but we handle it gracefully
-	}
-}
-
-// StoreSSES3Metadata stores SSE-S3 related metadata
-func StoreSSES3Metadata(metadata map[string][]byte, iv []byte, keyID string) {
-	StoreIVInMetadata(metadata, iv)
-	metadata[MetaSSEAlgorithm] = []byte("AES256")
-	if keyID != "" {
-		metadata[MetaSSES3KeyID] = []byte(keyID)
-	}
-}
-
-// GetSSECMetadata retrieves SSE-C metadata
-func GetSSECMetadata(metadata map[string][]byte) (iv []byte, keyMD5 string, err error) {
-	iv, err = GetIVFromMetadata(metadata)
-	if err != nil {
-		return nil, "", err
-	}
-
-	if keyMD5Bytes, exists := metadata[MetaSSECKeyMD5]; exists {
-		keyMD5 = string(keyMD5Bytes)
-	}
-
-	return iv, keyMD5, nil
-}
-
-// GetSSEKMSMetadata retrieves SSE-KMS metadata
-func GetSSEKMSMetadata(metadata map[string][]byte) (iv []byte, keyID string, encryptedKey []byte, context map[string]string, err error) {
-	iv, err = GetIVFromMetadata(metadata)
-	if err != nil {
-		return nil, "", nil, nil, err
-	}
-
-	if keyIDBytes, exists := metadata[MetaSSEKMSKeyID]; exists {
-		keyID = string(keyIDBytes)
-	}
-
-	if encKeyBase64, exists := metadata[MetaSSEKMSEncryptedKey]; exists {
-		encryptedKey, err = base64.StdEncoding.DecodeString(string(encKeyBase64))
-		if err != nil {
-			return nil, "", nil, nil, fmt.Errorf("failed to decode encrypted key: %w", err)
-		}
-	}
-
-	// Parse context from JSON
-	if contextBytes, exists := metadata[MetaSSEKMSContext]; exists {
-		context = make(map[string]string)
-		if err := json.Unmarshal(contextBytes, &context); err != nil {
-			return nil, "", nil, nil, fmt.Errorf("failed to parse KMS context JSON: %w", err)
-		}
-	}
-
-	return iv, keyID, encryptedKey, context, nil
-}
-
-// GetSSES3Metadata retrieves SSE-S3 metadata
-func GetSSES3Metadata(metadata map[string][]byte) (iv []byte, keyID string, err error) {
-	iv, err = GetIVFromMetadata(metadata)
-	if err != nil {
-		return nil, "", err
-	}
-
-	if keyIDBytes, exists := metadata[MetaSSES3KeyID]; exists {
-		keyID = string(keyIDBytes)
-	}
-
-	return iv, keyID, nil
-}
-
-// IsSSEEncrypted checks if the metadata indicates any form of SSE encryption
-func IsSSEEncrypted(metadata map[string][]byte) bool {
-	_, exists := metadata[MetaSSEIV]
-	return exists
-}
-
-// GetSSEAlgorithm returns the SSE algorithm from metadata
-func GetSSEAlgorithm(metadata map[string][]byte) string {
-	if alg, exists := metadata[MetaSSEAlgorithm]; exists {
-		return string(alg)
-	}
-	return ""
+	return nil, fmt.Errorf("SSE-C IV not found in metadata")
 }

weed/s3api/s3_sse_s3.go

@@ -1,18 +1,26 @@
 package s3api
 
 import (
+	"context"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
 	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	mathrand "math/rand"
 	"net/http"
+	"os"
+	"strings"
+	"sync"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 // SSE-S3 uses AES-256 encryption with server-managed keys
@@ -112,19 +120,24 @@ func GetSSES3Headers() map[string]string {
 	}
 }
 
-// SerializeSSES3Metadata serializes SSE-S3 metadata for storage
+// SerializeSSES3Metadata serializes SSE-S3 metadata for storage using envelope encryption
 func SerializeSSES3Metadata(key *SSES3Key) ([]byte, error) {
 	if err := ValidateSSES3Key(key); err != nil {
 		return nil, err
 	}
 
-	// For SSE-S3, we typically don't store the actual key in metadata
-	// Instead, we store a key ID or reference that can be used to retrieve the key
-	// from a secure key management system
+	// Encrypt the DEK using the global key manager's super key
+	keyManager := GetSSES3KeyManager()
+	encryptedDEK, nonce, err := keyManager.encryptKeyWithSuperKey(key.Key)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encrypt DEK: %w", err)
+	}
 
 	metadata := map[string]string{
-		"algorithm": key.Algorithm,
-		"keyId":     key.KeyID,
+		"algorithm":    key.Algorithm,
+		"keyId":        key.KeyID,
+		"encryptedDEK": base64.StdEncoding.EncodeToString(encryptedDEK),
+		"nonce":        base64.StdEncoding.EncodeToString(nonce),
 	}
 
 	// Include IV if present (needed for chunk-level decryption)
@@ -141,13 +154,13 @@ func SerializeSSES3Metadata(key *SSES3Key) ([]byte, error) {
 	return data, nil
 }
 
-// DeserializeSSES3Metadata deserializes SSE-S3 metadata from storage and retrieves the actual key
+// DeserializeSSES3Metadata deserializes SSE-S3 metadata from storage and decrypts the DEK
 func DeserializeSSES3Metadata(data []byte, keyManager *SSES3KeyManager) (*SSES3Key, error) {
 	if len(data) == 0 {
 		return nil, fmt.Errorf("empty SSE-S3 metadata")
 	}
 
-	// Parse the JSON metadata to extract keyId
+	// Parse the JSON metadata
 	var metadata map[string]string
 	if err := json.Unmarshal(data, &metadata); err != nil {
 		return nil, fmt.Errorf("failed to parse SSE-S3 metadata: %w", err)
@@ -163,19 +176,40 @@ func DeserializeSSES3Metadata(data []byte, keyManager *SSES3KeyManager) (*SSES3K
 		algorithm = s3_constants.SSEAlgorithmAES256 // Default algorithm
 	}
 
-	// Retrieve the actual key using the keyId
+	// Decode the encrypted DEK and nonce
+	encryptedDEKStr, exists := metadata["encryptedDEK"]
+	if !exists {
+		return nil, fmt.Errorf("encryptedDEK not found in SSE-S3 metadata")
+	}
+	encryptedDEK, err := base64.StdEncoding.DecodeString(encryptedDEKStr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode encrypted DEK: %w", err)
+	}
+
+	nonceStr, exists := metadata["nonce"]
+	if !exists {
+		return nil, fmt.Errorf("nonce not found in SSE-S3 metadata")
+	}
+	nonce, err := base64.StdEncoding.DecodeString(nonceStr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode nonce: %w", err)
+	}
+
+	// Decrypt the DEK using the key manager
 	if keyManager == nil {
 		return nil, fmt.Errorf("key manager is required for SSE-S3 key retrieval")
 	}
 
-	key, err := keyManager.GetOrCreateKey(keyID)
+	dekBytes, err := keyManager.decryptKeyWithSuperKey(encryptedDEK, nonce)
 	if err != nil {
-		return nil, fmt.Errorf("failed to retrieve SSE-S3 key with ID %s: %w", keyID, err)
+		return nil, fmt.Errorf("failed to decrypt DEK: %w", err)
 	}
 
-	// Verify the algorithm matches
-	if key.Algorithm != algorithm {
-		return nil, fmt.Errorf("algorithm mismatch: expected %s, got %s", algorithm, key.Algorithm)
+	// Reconstruct the key
+	key := &SSES3Key{
+		Key:       dekBytes,
+		KeyID:     keyID,
+		Algorithm: algorithm,
 	}
 
 	// Restore IV if present in metadata (for chunk-level decryption)
@@ -190,52 +224,211 @@ func DeserializeSSES3Metadata(data []byte, keyManager *SSES3KeyManager) (*SSES3K
 	return key, nil
 }
 
-// SSES3KeyManager manages SSE-S3 encryption keys
+// SSES3KeyManager manages SSE-S3 encryption keys using envelope encryption
+// Instead of storing keys in memory, it uses a super key (KEK) to encrypt/decrypt DEKs
 type SSES3KeyManager struct {
-	// In a production system, this would interface with a secure key management system
-	keys map[string]*SSES3Key
+	mu          sync.RWMutex
+	superKey    []byte               // 256-bit master key (KEK - Key Encryption Key)
+	filerClient filer_pb.FilerClient // Filer client for KEK persistence
+	kekPath     string               // Path in filer where KEK is stored (e.g., /etc/s3/sse_kek)
 }
 
-// NewSSES3KeyManager creates a new SSE-S3 key manager
+const (
+	// KEK storage directory and file name in filer
+	SSES3KEKDirectory = "/etc/s3"
+	SSES3KEKParentDir = "/etc"
+	SSES3KEKDirName   = "s3"
+	SSES3KEKFileName  = "sse_kek"
+
+	// Full KEK path in filer
+	defaultKEKPath = SSES3KEKDirectory + "/" + SSES3KEKFileName
+)
+
+// NewSSES3KeyManager creates a new SSE-S3 key manager with envelope encryption
 func NewSSES3KeyManager() *SSES3KeyManager {
+	// This will be initialized properly when attached to an S3ApiServer
 	return &SSES3KeyManager{
-		keys: make(map[string]*SSES3Key),
+		kekPath: defaultKEKPath,
 	}
 }
 
+// InitializeWithFiler initializes the key manager with a filer client
+func (km *SSES3KeyManager) InitializeWithFiler(filerClient filer_pb.FilerClient) error {
+	km.mu.Lock()
+	defer km.mu.Unlock()
+
+	km.filerClient = filerClient
+
+	// Try to load existing KEK from filer
+	if err := km.loadSuperKeyFromFiler(); err != nil {
+		// Only generate a new key if it does not exist.
+		// For other errors (e.g. connectivity), we should fail fast to prevent creating a new key
+		// and making existing data undecryptable.
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			glog.V(1).Infof("SSE-S3 KeyManager: KEK not found, generating new KEK (load from filer %s: %v)", km.kekPath, err)
+			if genErr := km.generateAndSaveSuperKeyToFiler(); genErr != nil {
+				return fmt.Errorf("failed to generate and save SSE-S3 super key: %w", genErr)
+			}
+		} else {
+			// A different error occurred (e.g., network issue, permission denied).
+			// Return the error to prevent starting with a broken state.
+			return fmt.Errorf("failed to load SSE-S3 super key from %s: %w", km.kekPath, err)
+		}
+	} else {
+		glog.V(1).Infof("SSE-S3 KeyManager: Loaded KEK from filer %s", km.kekPath)
+	}
+
+	return nil
+}
+
+// loadSuperKeyFromFiler loads the KEK from the filer
+func (km *SSES3KeyManager) loadSuperKeyFromFiler() error {
+	if km.filerClient == nil {
+		return fmt.Errorf("filer client not initialized")
+	}
+
+	// Get the entry from filer
+	entry, err := filer_pb.GetEntry(context.Background(), km.filerClient, util.FullPath(km.kekPath))
+	if err != nil {
+		return fmt.Errorf("failed to get KEK entry from filer: %w", err)
+	}
+
+	// Read the content
+	if len(entry.Content) == 0 {
+		return fmt.Errorf("KEK entry is empty")
+	}
+
+	// Decode hex-encoded key
+	key, err := hex.DecodeString(string(entry.Content))
+	if err != nil {
+		return fmt.Errorf("failed to decode KEK: %w", err)
+	}
+
+	if len(key) != SSES3KeySize {
+		return fmt.Errorf("invalid KEK size: expected %d bytes, got %d", SSES3KeySize, len(key))
+	}
+
+	km.superKey = key
+	return nil
+}
+
+// generateAndSaveSuperKeyToFiler generates a new KEK and saves it to the filer
+func (km *SSES3KeyManager) generateAndSaveSuperKeyToFiler() error {
+	if km.filerClient == nil {
+		return fmt.Errorf("filer client not initialized")
+	}
+
+	// Generate a random 256-bit super key (KEK)
+	superKey := make([]byte, SSES3KeySize)
+	if _, err := io.ReadFull(rand.Reader, superKey); err != nil {
+		return fmt.Errorf("failed to generate KEK: %w", err)
+	}
+
+	// Encode as hex for storage
+	encodedKey := []byte(hex.EncodeToString(superKey))
+
+	// Create the entry in filer
+	// First ensure the parent directory exists
+	if err := filer_pb.Mkdir(context.Background(), km.filerClient, SSES3KEKParentDir, SSES3KEKDirName, func(entry *filer_pb.Entry) {
+		// Set appropriate permissions for the directory
+		entry.Attributes.FileMode = uint32(0700 | os.ModeDir)
+	}); err != nil {
+		// Only ignore "file exists" errors.
+		if !strings.Contains(err.Error(), "file exists") {
+			return fmt.Errorf("failed to create KEK directory %s: %w", SSES3KEKDirectory, err)
+		}
+		glog.V(3).Infof("Parent directory %s already exists, continuing.", SSES3KEKDirectory)
+	}
+
+	// Create the KEK file
+	if err := filer_pb.MkFile(context.Background(), km.filerClient, SSES3KEKDirectory, SSES3KEKFileName, nil, func(entry *filer_pb.Entry) {
+		entry.Content = encodedKey
+		entry.Attributes.FileMode = 0600 // Read/write for owner only
+		entry.Attributes.FileSize = uint64(len(encodedKey))
+	}); err != nil {
+		return fmt.Errorf("failed to create KEK file in filer: %w", err)
+	}
+
+	km.superKey = superKey
+	glog.Infof("SSE-S3 KeyManager: Generated and saved new KEK to filer %s", km.kekPath)
+	return nil
+}
+
 // GetOrCreateKey gets an existing key or creates a new one
+// With envelope encryption, we always generate a new DEK since we don't store them
 func (km *SSES3KeyManager) GetOrCreateKey(keyID string) (*SSES3Key, error) {
-	if keyID == "" {
-		// Generate new key
-		return GenerateSSES3Key()
-	}
+	// Always generate a new key - we use envelope encryption so no need to cache DEKs
+	return GenerateSSES3Key()
+}
 
-	// Check if key exists
-	if key, exists := km.keys[keyID]; exists {
-		return key, nil
-	}
+// encryptKeyWithSuperKey encrypts a DEK using the super key (KEK) with AES-GCM
+func (km *SSES3KeyManager) encryptKeyWithSuperKey(dek []byte) ([]byte, []byte, error) {
+	km.mu.RLock()
+	defer km.mu.RUnlock()
 
-	// Create new key
-	key, err := GenerateSSES3Key()
+	block, err := aes.NewCipher(km.superKey)
 	if err != nil {
-		return nil, err
+		return nil, nil, fmt.Errorf("failed to create cipher: %w", err)
 	}
 
-	key.KeyID = keyID
-	km.keys[keyID] = key
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create GCM: %w", err)
+	}
 
-	return key, nil
+	// Generate random nonce
+	nonce := make([]byte, gcm.NonceSize())
+	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+		return nil, nil, fmt.Errorf("failed to generate nonce: %w", err)
+	}
+
+	// Encrypt the DEK
+	encryptedDEK := gcm.Seal(nil, nonce, dek, nil)
+
+	return encryptedDEK, nonce, nil
 }
 
-// StoreKey stores a key in the manager
+// decryptKeyWithSuperKey decrypts a DEK using the super key (KEK) with AES-GCM
+func (km *SSES3KeyManager) decryptKeyWithSuperKey(encryptedDEK, nonce []byte) ([]byte, error) {
+	km.mu.RLock()
+	defer km.mu.RUnlock()
+
+	block, err := aes.NewCipher(km.superKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create cipher: %w", err)
+	}
+
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create GCM: %w", err)
+	}
+
+	if len(nonce) != gcm.NonceSize() {
+		return nil, fmt.Errorf("invalid nonce size: expected %d, got %d", gcm.NonceSize(), len(nonce))
+	}
+
+	// Decrypt the DEK
+	dek, err := gcm.Open(nil, nonce, encryptedDEK, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decrypt DEK: %w", err)
+	}
+
+	return dek, nil
+}
+
+// StoreKey is now a no-op since we use envelope encryption and don't cache DEKs
+// The encrypted DEK is stored in the object metadata, not in the key manager
 func (km *SSES3KeyManager) StoreKey(key *SSES3Key) {
-	km.keys[key.KeyID] = key
+	// No-op: With envelope encryption, we don't need to store keys in memory
+	// The DEK is encrypted with the super key and stored in object metadata
 }
 
-// GetKey retrieves a key by ID
+// GetKey is now a no-op since we don't cache keys
+// Keys are retrieved by decrypting the encrypted DEK from object metadata
 func (km *SSES3KeyManager) GetKey(keyID string) (*SSES3Key, bool) {
-	key, exists := km.keys[keyID]
-	return key, exists
+	// No-op: With envelope encryption, keys are not cached
+	// Each object's metadata contains the encrypted DEK
+	return nil, false
 }
 
 // Global SSE-S3 key manager instance
@@ -246,6 +439,11 @@ func GetSSES3KeyManager() *SSES3KeyManager {
 	return globalSSES3KeyManager
 }
 
+// InitializeGlobalSSES3KeyManager initializes the global key manager with filer access
+func InitializeGlobalSSES3KeyManager(s3ApiServer *S3ApiServer) error {
+	return globalSSES3KeyManager.InitializeWithFiler(s3ApiServer)
+}
+
 // ProcessSSES3Request processes an SSE-S3 request and returns encryption metadata
 func ProcessSSES3Request(r *http.Request) (map[string][]byte, error) {
 	if !IsSSES3RequestInternal(r) {
@@ -287,6 +485,31 @@ func GetSSES3KeyFromMetadata(metadata map[string][]byte, keyManager *SSES3KeyMan
 	return DeserializeSSES3Metadata(keyData, keyManager)
 }
 
+// GetSSES3IV extracts the IV for single-part SSE-S3 objects
+// Priority: 1) object-level metadata (for inline/small files), 2) first chunk metadata
+func GetSSES3IV(entry *filer_pb.Entry, sseS3Key *SSES3Key, keyManager *SSES3KeyManager) ([]byte, error) {
+	// First check if IV is in the object-level key (for small/inline files)
+	if len(sseS3Key.IV) > 0 {
+		return sseS3Key.IV, nil
+	}
+
+	// Fallback: Get IV from first chunk's metadata (for chunked files)
+	if len(entry.GetChunks()) > 0 {
+		chunk := entry.GetChunks()[0]
+		if len(chunk.GetSseMetadata()) > 0 {
+			chunkKey, err := DeserializeSSES3Metadata(chunk.GetSseMetadata(), keyManager)
+			if err != nil {
+				return nil, fmt.Errorf("failed to deserialize chunk SSE-S3 metadata: %w", err)
+			}
+			if len(chunkKey.IV) > 0 {
+				return chunkKey.IV, nil
+			}
+		}
+	}
+
+	return nil, fmt.Errorf("SSE-S3 IV not found in object or chunk metadata")
+}
+
 // CreateSSES3EncryptedReaderWithBaseIV creates an encrypted reader using a base IV for multipart upload consistency.
 // The returned IV is the offset-derived IV, calculated from the input baseIV and offset.
 func CreateSSES3EncryptedReaderWithBaseIV(reader io.Reader, key *SSES3Key, baseIV []byte, offset int64) (io.Reader, []byte /* derivedIV */, error) {

weed/s3api/s3_sse_s3_integration_test.go

@@ -0,0 +1,325 @@
+package s3api
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+)
+
+// NOTE: These are integration tests that test the end-to-end encryption/decryption flow.
+// Full HTTP handler tests (PUT -> GET) would require a complete mock server with filer,
+// which is complex to set up. These tests focus on the critical decrypt path.
+
+// TestSSES3EndToEndSmallFile tests the complete encryption->storage->decryption cycle for small inline files
+// This test would have caught the IV retrieval bug for inline files
+func TestSSES3EndToEndSmallFile(t *testing.T) {
+	// Initialize global SSE-S3 key manager
+	globalSSES3KeyManager = NewSSES3KeyManager()
+	defer func() {
+		globalSSES3KeyManager = NewSSES3KeyManager()
+	}()
+
+	// Set up the key manager with a super key for testing
+	keyManager := GetSSES3KeyManager()
+	keyManager.superKey = make([]byte, 32)
+	for i := range keyManager.superKey {
+		keyManager.superKey[i] = byte(i)
+	}
+
+	testCases := []struct {
+		name string
+		data []byte
+	}{
+		{"tiny file (10 bytes)", []byte("test12345")},
+		{"small file (50 bytes)", []byte("This is a small test file for SSE-S3 encryption")},
+		{"medium file (256 bytes)", bytes.Repeat([]byte("a"), 256)},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Step 1: Encrypt (simulates what happens during PUT)
+			sseS3Key, err := GenerateSSES3Key()
+			if err != nil {
+				t.Fatalf("Failed to generate SSE-S3 key: %v", err)
+			}
+
+			encryptedReader, iv, err := CreateSSES3EncryptedReader(bytes.NewReader(tc.data), sseS3Key)
+			if err != nil {
+				t.Fatalf("Failed to create encrypted reader: %v", err)
+			}
+
+			encryptedData, err := io.ReadAll(encryptedReader)
+			if err != nil {
+				t.Fatalf("Failed to read encrypted data: %v", err)
+			}
+
+			// Store IV in the key (this is critical for inline files!)
+			sseS3Key.IV = iv
+
+			// Serialize the metadata (this is stored in entry.Extended)
+			serializedMetadata, err := SerializeSSES3Metadata(sseS3Key)
+			if err != nil {
+				t.Fatalf("Failed to serialize SSE-S3 metadata: %v", err)
+			}
+
+			// Step 2: Simulate storage (inline file - no chunks)
+			// For inline files, data is in Content, metadata in Extended
+			mockEntry := &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.SeaweedFSSSES3Key:       serializedMetadata,
+					s3_constants.AmzServerSideEncryption: []byte("AES256"),
+				},
+				Content: encryptedData,
+				Chunks:  []*filer_pb.FileChunk{}, // Critical: inline files have NO chunks
+			}
+
+			// Step 3: Decrypt (simulates what happens during GET)
+			// This tests the IV retrieval path for inline files
+			
+			// First, deserialize metadata from storage
+			retrievedKeyData := mockEntry.Extended[s3_constants.SeaweedFSSSES3Key]
+			retrievedKey, err := DeserializeSSES3Metadata(retrievedKeyData, keyManager)
+			if err != nil {
+				t.Fatalf("Failed to deserialize SSE-S3 metadata: %v", err)
+			}
+
+			// CRITICAL TEST: For inline files, IV must be in object-level metadata
+			var retrievedIV []byte
+			if len(retrievedKey.IV) > 0 {
+				// Success path: IV found in object-level key
+				retrievedIV = retrievedKey.IV
+			} else if len(mockEntry.GetChunks()) > 0 {
+				// Fallback path: would check chunks (but inline files have no chunks)
+				t.Fatal("Inline file should have IV in object-level metadata, not chunks")
+			}
+
+			if len(retrievedIV) == 0 {
+				// THIS IS THE BUG WE FIXED: inline files had no way to get IV!
+				t.Fatal("Failed to retrieve IV for inline file - this is the bug we fixed!")
+			}
+
+			// Now decrypt with the retrieved IV
+			decryptedReader, err := CreateSSES3DecryptedReader(bytes.NewReader(encryptedData), retrievedKey, retrievedIV)
+			if err != nil {
+				t.Fatalf("Failed to create decrypted reader: %v", err)
+			}
+
+			decryptedData, err := io.ReadAll(decryptedReader)
+			if err != nil {
+				t.Fatalf("Failed to read decrypted data: %v", err)
+			}
+
+			// Verify decrypted data matches original
+			if !bytes.Equal(decryptedData, tc.data) {
+				t.Errorf("Decrypted data doesn't match original.\nExpected: %q\nGot: %q", tc.data, decryptedData)
+			}
+		})
+	}
+}
+
+// TestSSES3EndToEndChunkedFile tests the complete flow for chunked files
+func TestSSES3EndToEndChunkedFile(t *testing.T) {
+	// Initialize global SSE-S3 key manager
+	globalSSES3KeyManager = NewSSES3KeyManager()
+	defer func() {
+		globalSSES3KeyManager = NewSSES3KeyManager()
+	}()
+
+	keyManager := GetSSES3KeyManager()
+	keyManager.superKey = make([]byte, 32)
+	for i := range keyManager.superKey {
+		keyManager.superKey[i] = byte(i)
+	}
+
+	// Generate SSE-S3 key
+	sseS3Key, err := GenerateSSES3Key()
+	if err != nil {
+		t.Fatalf("Failed to generate SSE-S3 key: %v", err)
+	}
+
+	// Create test data for two chunks
+	chunk1Data := []byte("This is chunk 1 data for SSE-S3 encryption test")
+	chunk2Data := []byte("This is chunk 2 data for SSE-S3 encryption test")
+
+	// Encrypt chunk 1
+	encryptedReader1, iv1, err := CreateSSES3EncryptedReader(bytes.NewReader(chunk1Data), sseS3Key)
+	if err != nil {
+		t.Fatalf("Failed to create encrypted reader for chunk 1: %v", err)
+	}
+	encryptedChunk1, _ := io.ReadAll(encryptedReader1)
+
+	// Encrypt chunk 2
+	encryptedReader2, iv2, err := CreateSSES3EncryptedReader(bytes.NewReader(chunk2Data), sseS3Key)
+	if err != nil {
+		t.Fatalf("Failed to create encrypted reader for chunk 2: %v", err)
+	}
+	encryptedChunk2, _ := io.ReadAll(encryptedReader2)
+
+	// Create metadata for each chunk
+	chunk1Key := &SSES3Key{
+		Key:       sseS3Key.Key,
+		IV:        iv1,
+		Algorithm: sseS3Key.Algorithm,
+		KeyID:     sseS3Key.KeyID,
+	}
+	chunk2Key := &SSES3Key{
+		Key:       sseS3Key.Key,
+		IV:        iv2,
+		Algorithm: sseS3Key.Algorithm,
+		KeyID:     sseS3Key.KeyID,
+	}
+
+	serializedChunk1Meta, _ := SerializeSSES3Metadata(chunk1Key)
+	serializedChunk2Meta, _ := SerializeSSES3Metadata(chunk2Key)
+	serializedObjMeta, _ := SerializeSSES3Metadata(sseS3Key)
+
+	// Create mock entry with chunks
+	mockEntry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSES3Key:       serializedObjMeta,
+			s3_constants.AmzServerSideEncryption: []byte("AES256"),
+		},
+		Chunks: []*filer_pb.FileChunk{
+			{
+				FileId:      "chunk1,123",
+				Offset:      0,
+				Size:        uint64(len(encryptedChunk1)),
+				SseType:     filer_pb.SSEType_SSE_S3,
+				SseMetadata: serializedChunk1Meta,
+			},
+			{
+				FileId:      "chunk2,456",
+				Offset:      int64(len(chunk1Data)),
+				Size:        uint64(len(encryptedChunk2)),
+				SseType:     filer_pb.SSEType_SSE_S3,
+				SseMetadata: serializedChunk2Meta,
+			},
+		},
+	}
+
+	// Verify multipart detection
+	sses3Chunks := 0
+	for _, chunk := range mockEntry.GetChunks() {
+		if chunk.GetSseType() == filer_pb.SSEType_SSE_S3 && len(chunk.GetSseMetadata()) > 0 {
+			sses3Chunks++
+		}
+	}
+
+	isMultipart := sses3Chunks > 1
+	if !isMultipart {
+		t.Error("Expected multipart SSE-S3 object detection")
+	}
+
+	if sses3Chunks != 2 {
+		t.Errorf("Expected 2 SSE-S3 chunks, got %d", sses3Chunks)
+	}
+
+	// Verify each chunk has valid metadata with IV
+	for i, chunk := range mockEntry.GetChunks() {
+		deserializedKey, err := DeserializeSSES3Metadata(chunk.GetSseMetadata(), keyManager)
+		if err != nil {
+			t.Errorf("Failed to deserialize chunk %d metadata: %v", i, err)
+		}
+		if len(deserializedKey.IV) == 0 {
+			t.Errorf("Chunk %d has no IV", i)
+		}
+
+		// Decrypt this chunk to verify it works
+		var chunkData []byte
+		if i == 0 {
+			chunkData = encryptedChunk1
+		} else {
+			chunkData = encryptedChunk2
+		}
+
+		decryptedReader, err := CreateSSES3DecryptedReader(bytes.NewReader(chunkData), deserializedKey, deserializedKey.IV)
+		if err != nil {
+			t.Errorf("Failed to decrypt chunk %d: %v", i, err)
+			continue
+		}
+
+		decrypted, _ := io.ReadAll(decryptedReader)
+		var expectedData []byte
+		if i == 0 {
+			expectedData = chunk1Data
+		} else {
+			expectedData = chunk2Data
+		}
+
+		if !bytes.Equal(decrypted, expectedData) {
+			t.Errorf("Chunk %d decryption failed", i)
+		}
+	}
+}
+
+// TestSSES3EndToEndWithDetectPrimaryType tests that type detection works correctly for different scenarios
+func TestSSES3EndToEndWithDetectPrimaryType(t *testing.T) {
+	s3a := &S3ApiServer{}
+
+	testCases := []struct {
+		name          string
+		entry         *filer_pb.Entry
+		expectedType  string
+		shouldBeSSES3 bool
+	}{
+		{
+			name: "Inline SSE-S3 file (no chunks)",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.AmzServerSideEncryption: []byte("AES256"),
+				},
+				Attributes: &filer_pb.FuseAttributes{},
+				Content:    []byte("encrypted data"),
+				Chunks:     []*filer_pb.FileChunk{},
+			},
+			expectedType:  s3_constants.SSETypeS3,
+			shouldBeSSES3: true,
+		},
+		{
+			name: "Single chunk SSE-S3 file",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.AmzServerSideEncryption: []byte("AES256"),
+				},
+				Attributes: &filer_pb.FuseAttributes{},
+				Chunks: []*filer_pb.FileChunk{
+					{
+						FileId:      "1,123",
+						SseType:     filer_pb.SSEType_SSE_S3,
+						SseMetadata: []byte("metadata"),
+					},
+				},
+			},
+			expectedType:  s3_constants.SSETypeS3,
+			shouldBeSSES3: true,
+		},
+		{
+			name: "SSE-KMS file (has KMS key ID)",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.AmzServerSideEncryption:            []byte("AES256"),
+					s3_constants.AmzServerSideEncryptionAwsKmsKeyId: []byte("kms-key-123"),
+				},
+				Attributes: &filer_pb.FuseAttributes{},
+				Chunks:     []*filer_pb.FileChunk{},
+			},
+			expectedType:  s3_constants.SSETypeKMS,
+			shouldBeSSES3: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			detectedType := s3a.detectPrimarySSEType(tc.entry)
+			if detectedType != tc.expectedType {
+				t.Errorf("Expected type %s, got %s", tc.expectedType, detectedType)
+			}
+			if (detectedType == s3_constants.SSETypeS3) != tc.shouldBeSSES3 {
+				t.Errorf("SSE-S3 detection mismatch: expected %v, got %v", tc.shouldBeSSES3, detectedType == s3_constants.SSETypeS3)
+			}
+		})
+	}
+}

weed/s3api/s3_sse_s3_test.go

@@ -0,0 +1,984 @@
+package s3api
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+)
+
+// TestSSES3EncryptionDecryption tests basic SSE-S3 encryption and decryption
+func TestSSES3EncryptionDecryption(t *testing.T) {
+	// Generate SSE-S3 key
+	sseS3Key, err := GenerateSSES3Key()
+	if err != nil {
+		t.Fatalf("Failed to generate SSE-S3 key: %v", err)
+	}
+
+	// Test data
+	testData := []byte("Hello, World! This is a test of SSE-S3 encryption.")
+
+	// Create encrypted reader
+	dataReader := bytes.NewReader(testData)
+	encryptedReader, iv, err := CreateSSES3EncryptedReader(dataReader, sseS3Key)
+	if err != nil {
+		t.Fatalf("Failed to create encrypted reader: %v", err)
+	}
+
+	// Read encrypted data
+	encryptedData, err := io.ReadAll(encryptedReader)
+	if err != nil {
+		t.Fatalf("Failed to read encrypted data: %v", err)
+	}
+
+	// Verify data is actually encrypted (different from original)
+	if bytes.Equal(encryptedData, testData) {
+		t.Error("Data doesn't appear to be encrypted")
+	}
+
+	// Create decrypted reader
+	encryptedReader2 := bytes.NewReader(encryptedData)
+	decryptedReader, err := CreateSSES3DecryptedReader(encryptedReader2, sseS3Key, iv)
+	if err != nil {
+		t.Fatalf("Failed to create decrypted reader: %v", err)
+	}
+
+	// Read decrypted data
+	decryptedData, err := io.ReadAll(decryptedReader)
+	if err != nil {
+		t.Fatalf("Failed to read decrypted data: %v", err)
+	}
+
+	// Verify decrypted data matches original
+	if !bytes.Equal(decryptedData, testData) {
+		t.Errorf("Decrypted data doesn't match original.\nOriginal: %s\nDecrypted: %s", testData, decryptedData)
+	}
+}
+
+// TestSSES3IsRequestInternal tests detection of SSE-S3 requests
+func TestSSES3IsRequestInternal(t *testing.T) {
+	testCases := []struct {
+		name     string
+		headers  map[string]string
+		expected bool
+	}{
+		{
+			name: "Valid SSE-S3 request",
+			headers: map[string]string{
+				s3_constants.AmzServerSideEncryption: "AES256",
+			},
+			expected: true,
+		},
+		{
+			name:     "No SSE headers",
+			headers:  map[string]string{},
+			expected: false,
+		},
+		{
+			name: "SSE-KMS request",
+			headers: map[string]string{
+				s3_constants.AmzServerSideEncryption: "aws:kms",
+			},
+			expected: false,
+		},
+		{
+			name: "SSE-C request",
+			headers: map[string]string{
+				s3_constants.AmzServerSideEncryptionCustomerAlgorithm: "AES256",
+			},
+			expected: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			req := &http.Request{Header: make(http.Header)}
+			for k, v := range tc.headers {
+				req.Header.Set(k, v)
+			}
+
+			result := IsSSES3RequestInternal(req)
+			if result != tc.expected {
+				t.Errorf("Expected %v, got %v", tc.expected, result)
+			}
+		})
+	}
+}
+
+// TestSSES3MetadataSerialization tests SSE-S3 metadata serialization and deserialization
+func TestSSES3MetadataSerialization(t *testing.T) {
+	// Initialize global key manager
+	globalSSES3KeyManager = NewSSES3KeyManager()
+	defer func() {
+		globalSSES3KeyManager = NewSSES3KeyManager()
+	}()
+
+	// Set up the key manager with a super key for testing
+	keyManager := GetSSES3KeyManager()
+	keyManager.superKey = make([]byte, 32)
+	for i := range keyManager.superKey {
+		keyManager.superKey[i] = byte(i)
+	}
+
+	// Generate SSE-S3 key
+	sseS3Key, err := GenerateSSES3Key()
+	if err != nil {
+		t.Fatalf("Failed to generate SSE-S3 key: %v", err)
+	}
+
+	// Add IV to the key
+	sseS3Key.IV = make([]byte, 16)
+	for i := range sseS3Key.IV {
+		sseS3Key.IV[i] = byte(i * 2)
+	}
+
+	// Serialize metadata
+	serialized, err := SerializeSSES3Metadata(sseS3Key)
+	if err != nil {
+		t.Fatalf("Failed to serialize SSE-S3 metadata: %v", err)
+	}
+
+	if len(serialized) == 0 {
+		t.Error("Serialized metadata is empty")
+	}
+
+	// Deserialize metadata
+	deserializedKey, err := DeserializeSSES3Metadata(serialized, keyManager)
+	if err != nil {
+		t.Fatalf("Failed to deserialize SSE-S3 metadata: %v", err)
+	}
+
+	// Verify key matches
+	if !bytes.Equal(deserializedKey.Key, sseS3Key.Key) {
+		t.Error("Deserialized key doesn't match original key")
+	}
+
+	// Verify IV matches
+	if !bytes.Equal(deserializedKey.IV, sseS3Key.IV) {
+		t.Error("Deserialized IV doesn't match original IV")
+	}
+
+	// Verify algorithm matches
+	if deserializedKey.Algorithm != sseS3Key.Algorithm {
+		t.Errorf("Algorithm mismatch: expected %s, got %s", sseS3Key.Algorithm, deserializedKey.Algorithm)
+	}
+
+	// Verify key ID matches
+	if deserializedKey.KeyID != sseS3Key.KeyID {
+		t.Errorf("Key ID mismatch: expected %s, got %s", sseS3Key.KeyID, deserializedKey.KeyID)
+	}
+}
+
+// TestDetectPrimarySSETypeS3 tests detection of SSE-S3 as primary encryption type
+func TestDetectPrimarySSETypeS3(t *testing.T) {
+	s3a := &S3ApiServer{}
+
+	testCases := []struct {
+		name     string
+		entry    *filer_pb.Entry
+		expected string
+	}{
+		{
+			name: "Single SSE-S3 chunk",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.AmzServerSideEncryption: []byte("AES256"),
+				},
+				Attributes: &filer_pb.FuseAttributes{},
+				Chunks: []*filer_pb.FileChunk{
+					{
+						FileId:      "1,123",
+						Offset:      0,
+						Size:        1024,
+						SseType:     filer_pb.SSEType_SSE_S3,
+						SseMetadata: []byte("metadata"),
+					},
+				},
+			},
+			expected: s3_constants.SSETypeS3,
+		},
+		{
+			name: "Multiple SSE-S3 chunks",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.AmzServerSideEncryption: []byte("AES256"),
+				},
+				Attributes: &filer_pb.FuseAttributes{},
+				Chunks: []*filer_pb.FileChunk{
+					{
+						FileId:      "1,123",
+						Offset:      0,
+						Size:        1024,
+						SseType:     filer_pb.SSEType_SSE_S3,
+						SseMetadata: []byte("metadata1"),
+					},
+					{
+						FileId:      "2,456",
+						Offset:      1024,
+						Size:        1024,
+						SseType:     filer_pb.SSEType_SSE_S3,
+						SseMetadata: []byte("metadata2"),
+					},
+				},
+			},
+			expected: s3_constants.SSETypeS3,
+		},
+		{
+			name: "Mixed SSE-S3 and SSE-KMS chunks (SSE-S3 majority)",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.AmzServerSideEncryption: []byte("AES256"),
+				},
+				Attributes: &filer_pb.FuseAttributes{},
+				Chunks: []*filer_pb.FileChunk{
+					{
+						FileId:      "1,123",
+						Offset:      0,
+						Size:        1024,
+						SseType:     filer_pb.SSEType_SSE_S3,
+						SseMetadata: []byte("metadata1"),
+					},
+					{
+						FileId:      "2,456",
+						Offset:      1024,
+						Size:        1024,
+						SseType:     filer_pb.SSEType_SSE_S3,
+						SseMetadata: []byte("metadata2"),
+					},
+					{
+						FileId:      "3,789",
+						Offset:      2048,
+						Size:        1024,
+						SseType:     filer_pb.SSEType_SSE_KMS,
+						SseMetadata: []byte("metadata3"),
+					},
+				},
+			},
+			expected: s3_constants.SSETypeS3,
+		},
+		{
+			name: "No chunks, SSE-S3 metadata without KMS key ID",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.AmzServerSideEncryption: []byte("AES256"),
+				},
+				Attributes: &filer_pb.FuseAttributes{},
+				Chunks:     []*filer_pb.FileChunk{},
+			},
+			expected: s3_constants.SSETypeS3,
+		},
+		{
+			name: "No chunks, SSE-KMS metadata with KMS key ID",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.AmzServerSideEncryption:            []byte("AES256"),
+					s3_constants.AmzServerSideEncryptionAwsKmsKeyId: []byte("test-key-id"),
+				},
+				Attributes: &filer_pb.FuseAttributes{},
+				Chunks:     []*filer_pb.FileChunk{},
+			},
+			expected: s3_constants.SSETypeKMS,
+		},
+		{
+			name: "SSE-C chunks",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.AmzServerSideEncryptionCustomerAlgorithm: []byte("AES256"),
+				},
+				Attributes: &filer_pb.FuseAttributes{},
+				Chunks: []*filer_pb.FileChunk{
+					{
+						FileId:      "1,123",
+						Offset:      0,
+						Size:        1024,
+						SseType:     filer_pb.SSEType_SSE_C,
+						SseMetadata: []byte("metadata"),
+					},
+				},
+			},
+			expected: s3_constants.SSETypeC,
+		},
+		{
+			name: "Unencrypted",
+			entry: &filer_pb.Entry{
+				Extended:   map[string][]byte{},
+				Attributes: &filer_pb.FuseAttributes{},
+				Chunks: []*filer_pb.FileChunk{
+					{
+						FileId: "1,123",
+						Offset: 0,
+						Size:   1024,
+					},
+				},
+			},
+			expected: "None",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := s3a.detectPrimarySSEType(tc.entry)
+			if result != tc.expected {
+				t.Errorf("Expected %s, got %s", tc.expected, result)
+			}
+		})
+	}
+}
+
+// TestAddSSES3HeadersToResponse tests that SSE-S3 headers are added to responses
+func TestAddSSES3HeadersToResponse(t *testing.T) {
+	s3a := &S3ApiServer{}
+
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.AmzServerSideEncryption: []byte("AES256"),
+		},
+		Attributes: &filer_pb.FuseAttributes{},
+		Chunks: []*filer_pb.FileChunk{
+			{
+				FileId:      "1,123",
+				Offset:      0,
+				Size:        1024,
+				SseType:     filer_pb.SSEType_SSE_S3,
+				SseMetadata: []byte("metadata"),
+			},
+		},
+	}
+
+	proxyResponse := &http.Response{
+		Header: make(http.Header),
+	}
+
+	s3a.addSSEHeadersToResponse(proxyResponse, entry)
+
+	algorithm := proxyResponse.Header.Get(s3_constants.AmzServerSideEncryption)
+	if algorithm != "AES256" {
+		t.Errorf("Expected SSE algorithm AES256, got %s", algorithm)
+	}
+
+	// Should NOT have SSE-C or SSE-KMS specific headers
+	if proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionCustomerAlgorithm) != "" {
+		t.Error("Should not have SSE-C customer algorithm header")
+	}
+
+	if proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionAwsKmsKeyId) != "" {
+		t.Error("Should not have SSE-KMS key ID header")
+	}
+}
+
+// TestSSES3EncryptionWithBaseIV tests multipart encryption with base IV
+func TestSSES3EncryptionWithBaseIV(t *testing.T) {
+	// Generate SSE-S3 key
+	sseS3Key, err := GenerateSSES3Key()
+	if err != nil {
+		t.Fatalf("Failed to generate SSE-S3 key: %v", err)
+	}
+
+	// Generate base IV
+	baseIV := make([]byte, 16)
+	for i := range baseIV {
+		baseIV[i] = byte(i)
+	}
+
+	// Test data for two parts
+	testData1 := []byte("Part 1 of multipart upload test.")
+	testData2 := []byte("Part 2 of multipart upload test.")
+
+	// Encrypt part 1 at offset 0
+	dataReader1 := bytes.NewReader(testData1)
+	encryptedReader1, iv1, err := CreateSSES3EncryptedReaderWithBaseIV(dataReader1, sseS3Key, baseIV, 0)
+	if err != nil {
+		t.Fatalf("Failed to create encrypted reader for part 1: %v", err)
+	}
+
+	encryptedData1, err := io.ReadAll(encryptedReader1)
+	if err != nil {
+		t.Fatalf("Failed to read encrypted data for part 1: %v", err)
+	}
+
+	// Encrypt part 2 at offset (simulating second part)
+	dataReader2 := bytes.NewReader(testData2)
+	offset2 := int64(len(testData1))
+	encryptedReader2, iv2, err := CreateSSES3EncryptedReaderWithBaseIV(dataReader2, sseS3Key, baseIV, offset2)
+	if err != nil {
+		t.Fatalf("Failed to create encrypted reader for part 2: %v", err)
+	}
+
+	encryptedData2, err := io.ReadAll(encryptedReader2)
+	if err != nil {
+		t.Fatalf("Failed to read encrypted data for part 2: %v", err)
+	}
+
+	// IVs should be different (offset-based)
+	if bytes.Equal(iv1, iv2) {
+		t.Error("IVs should be different for different offsets")
+	}
+
+	// Decrypt part 1
+	decryptedReader1, err := CreateSSES3DecryptedReader(bytes.NewReader(encryptedData1), sseS3Key, iv1)
+	if err != nil {
+		t.Fatalf("Failed to create decrypted reader for part 1: %v", err)
+	}
+
+	decryptedData1, err := io.ReadAll(decryptedReader1)
+	if err != nil {
+		t.Fatalf("Failed to read decrypted data for part 1: %v", err)
+	}
+
+	// Decrypt part 2
+	decryptedReader2, err := CreateSSES3DecryptedReader(bytes.NewReader(encryptedData2), sseS3Key, iv2)
+	if err != nil {
+		t.Fatalf("Failed to create decrypted reader for part 2: %v", err)
+	}
+
+	decryptedData2, err := io.ReadAll(decryptedReader2)
+	if err != nil {
+		t.Fatalf("Failed to read decrypted data for part 2: %v", err)
+	}
+
+	// Verify decrypted data matches original
+	if !bytes.Equal(decryptedData1, testData1) {
+		t.Errorf("Decrypted part 1 doesn't match original.\nOriginal: %s\nDecrypted: %s", testData1, decryptedData1)
+	}
+
+	if !bytes.Equal(decryptedData2, testData2) {
+		t.Errorf("Decrypted part 2 doesn't match original.\nOriginal: %s\nDecrypted: %s", testData2, decryptedData2)
+	}
+}
+
+// TestSSES3WrongKeyDecryption tests that wrong key fails decryption
+func TestSSES3WrongKeyDecryption(t *testing.T) {
+	// Generate two different keys
+	sseS3Key1, err := GenerateSSES3Key()
+	if err != nil {
+		t.Fatalf("Failed to generate SSE-S3 key 1: %v", err)
+	}
+
+	sseS3Key2, err := GenerateSSES3Key()
+	if err != nil {
+		t.Fatalf("Failed to generate SSE-S3 key 2: %v", err)
+	}
+
+	// Test data
+	testData := []byte("Secret data encrypted with key 1")
+
+	// Encrypt with key 1
+	dataReader := bytes.NewReader(testData)
+	encryptedReader, iv, err := CreateSSES3EncryptedReader(dataReader, sseS3Key1)
+	if err != nil {
+		t.Fatalf("Failed to create encrypted reader: %v", err)
+	}
+
+	encryptedData, err := io.ReadAll(encryptedReader)
+	if err != nil {
+		t.Fatalf("Failed to read encrypted data: %v", err)
+	}
+
+	// Try to decrypt with key 2 (wrong key)
+	decryptedReader, err := CreateSSES3DecryptedReader(bytes.NewReader(encryptedData), sseS3Key2, iv)
+	if err != nil {
+		t.Fatalf("Failed to create decrypted reader: %v", err)
+	}
+
+	decryptedData, err := io.ReadAll(decryptedReader)
+	if err != nil {
+		t.Fatalf("Failed to read decrypted data: %v", err)
+	}
+
+	// Decrypted data should NOT match original (wrong key produces garbage)
+	if bytes.Equal(decryptedData, testData) {
+		t.Error("Decryption with wrong key should not produce correct plaintext")
+	}
+}
+
+// TestSSES3KeyGeneration tests SSE-S3 key generation
+func TestSSES3KeyGeneration(t *testing.T) {
+	// Generate multiple keys
+	keys := make([]*SSES3Key, 10)
+	for i := range keys {
+		key, err := GenerateSSES3Key()
+		if err != nil {
+			t.Fatalf("Failed to generate SSE-S3 key %d: %v", i, err)
+		}
+		keys[i] = key
+
+		// Verify key properties
+		if len(key.Key) != SSES3KeySize {
+			t.Errorf("Key %d has wrong size: expected %d, got %d", i, SSES3KeySize, len(key.Key))
+		}
+
+		if key.Algorithm != SSES3Algorithm {
+			t.Errorf("Key %d has wrong algorithm: expected %s, got %s", i, SSES3Algorithm, key.Algorithm)
+		}
+
+		if key.KeyID == "" {
+			t.Errorf("Key %d has empty key ID", i)
+		}
+	}
+
+	// Verify keys are unique
+	for i := 0; i < len(keys); i++ {
+		for j := i + 1; j < len(keys); j++ {
+			if bytes.Equal(keys[i].Key, keys[j].Key) {
+				t.Errorf("Keys %d and %d are identical (should be unique)", i, j)
+			}
+			if keys[i].KeyID == keys[j].KeyID {
+				t.Errorf("Key IDs %d and %d are identical (should be unique)", i, j)
+			}
+		}
+	}
+}
+
+// TestSSES3VariousSizes tests SSE-S3 encryption/decryption with various data sizes
+func TestSSES3VariousSizes(t *testing.T) {
+	sizes := []int{1, 15, 16, 17, 100, 1024, 4096, 1048576}
+
+	for _, size := range sizes {
+		t.Run(fmt.Sprintf("size_%d", size), func(t *testing.T) {
+			// Generate test data
+			testData := make([]byte, size)
+			for i := range testData {
+				testData[i] = byte(i % 256)
+			}
+
+			// Generate key
+			sseS3Key, err := GenerateSSES3Key()
+			if err != nil {
+				t.Fatalf("Failed to generate SSE-S3 key: %v", err)
+			}
+
+			// Encrypt
+			dataReader := bytes.NewReader(testData)
+			encryptedReader, iv, err := CreateSSES3EncryptedReader(dataReader, sseS3Key)
+			if err != nil {
+				t.Fatalf("Failed to create encrypted reader: %v", err)
+			}
+
+			encryptedData, err := io.ReadAll(encryptedReader)
+			if err != nil {
+				t.Fatalf("Failed to read encrypted data: %v", err)
+			}
+
+			// Verify encrypted size matches original
+			if len(encryptedData) != size {
+				t.Errorf("Encrypted size mismatch: expected %d, got %d", size, len(encryptedData))
+			}
+
+			// Decrypt
+			decryptedReader, err := CreateSSES3DecryptedReader(bytes.NewReader(encryptedData), sseS3Key, iv)
+			if err != nil {
+				t.Fatalf("Failed to create decrypted reader: %v", err)
+			}
+
+			decryptedData, err := io.ReadAll(decryptedReader)
+			if err != nil {
+				t.Fatalf("Failed to read decrypted data: %v", err)
+			}
+
+			// Verify
+			if !bytes.Equal(decryptedData, testData) {
+				t.Errorf("Decrypted data doesn't match original for size %d", size)
+			}
+		})
+	}
+}
+
+// TestSSES3ResponseHeaders tests that SSE-S3 response headers are set correctly
+func TestSSES3ResponseHeaders(t *testing.T) {
+	w := httptest.NewRecorder()
+
+	// Simulate setting SSE-S3 response headers
+	w.Header().Set(s3_constants.AmzServerSideEncryption, SSES3Algorithm)
+
+	// Verify headers
+	algorithm := w.Header().Get(s3_constants.AmzServerSideEncryption)
+	if algorithm != "AES256" {
+		t.Errorf("Expected algorithm AES256, got %s", algorithm)
+	}
+
+	// Should NOT have customer key headers
+	if w.Header().Get(s3_constants.AmzServerSideEncryptionCustomerAlgorithm) != "" {
+		t.Error("Should not have SSE-C customer algorithm header")
+	}
+
+	if w.Header().Get(s3_constants.AmzServerSideEncryptionCustomerKeyMD5) != "" {
+		t.Error("Should not have SSE-C customer key MD5 header")
+	}
+
+	// Should NOT have KMS key ID
+	if w.Header().Get(s3_constants.AmzServerSideEncryptionAwsKmsKeyId) != "" {
+		t.Error("Should not have SSE-KMS key ID header")
+	}
+}
+
+// TestSSES3IsEncryptedInternal tests detection of SSE-S3 encryption from metadata
+func TestSSES3IsEncryptedInternal(t *testing.T) {
+	testCases := []struct {
+		name     string
+		metadata map[string][]byte
+		expected bool
+	}{
+		{
+			name:     "Empty metadata",
+			metadata: map[string][]byte{},
+			expected: false,
+		},
+		{
+			name: "Valid SSE-S3 metadata",
+			metadata: map[string][]byte{
+				s3_constants.AmzServerSideEncryption: []byte("AES256"),
+			},
+			expected: true,
+		},
+		{
+			name: "SSE-KMS metadata",
+			metadata: map[string][]byte{
+				s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
+			},
+			expected: false,
+		},
+		{
+			name: "SSE-C metadata",
+			metadata: map[string][]byte{
+				s3_constants.AmzServerSideEncryptionCustomerAlgorithm: []byte("AES256"),
+			},
+			expected: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := IsSSES3EncryptedInternal(tc.metadata)
+			if result != tc.expected {
+				t.Errorf("Expected %v, got %v", tc.expected, result)
+			}
+		})
+	}
+}
+
+// TestSSES3InvalidMetadataDeserialization tests error handling for invalid metadata
+func TestSSES3InvalidMetadataDeserialization(t *testing.T) {
+	keyManager := NewSSES3KeyManager()
+	keyManager.superKey = make([]byte, 32)
+
+	testCases := []struct {
+		name        string
+		metadata    []byte
+		shouldError bool
+	}{
+		{
+			name:        "Empty metadata",
+			metadata:    []byte{},
+			shouldError: true,
+		},
+		{
+			name:        "Invalid JSON",
+			metadata:    []byte("not valid json"),
+			shouldError: true,
+		},
+		{
+			name:        "Missing keyId",
+			metadata:    []byte(`{"algorithm":"AES256"}`),
+			shouldError: true,
+		},
+		{
+			name:        "Invalid base64 encrypted DEK",
+			metadata:    []byte(`{"keyId":"test","algorithm":"AES256","encryptedDEK":"not-valid-base64!","nonce":"dGVzdA=="}`),
+			shouldError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := DeserializeSSES3Metadata(tc.metadata, keyManager)
+			if tc.shouldError && err == nil {
+				t.Error("Expected error but got none")
+			}
+			if !tc.shouldError && err != nil {
+				t.Errorf("Unexpected error: %v", err)
+			}
+		})
+	}
+}
+
+// TestGetSSES3Headers tests SSE-S3 header generation
+func TestGetSSES3Headers(t *testing.T) {
+	headers := GetSSES3Headers()
+
+	if len(headers) == 0 {
+		t.Error("Expected headers to be non-empty")
+	}
+
+	algorithm, exists := headers[s3_constants.AmzServerSideEncryption]
+	if !exists {
+		t.Error("Expected AmzServerSideEncryption header to exist")
+	}
+
+	if algorithm != "AES256" {
+		t.Errorf("Expected algorithm AES256, got %s", algorithm)
+	}
+}
+
+// TestProcessSSES3Request tests processing of SSE-S3 requests
+func TestProcessSSES3Request(t *testing.T) {
+	// Initialize global key manager
+	globalSSES3KeyManager = NewSSES3KeyManager()
+	defer func() {
+		globalSSES3KeyManager = NewSSES3KeyManager()
+	}()
+
+	// Set up the key manager with a super key for testing
+	keyManager := GetSSES3KeyManager()
+	keyManager.superKey = make([]byte, 32)
+	for i := range keyManager.superKey {
+		keyManager.superKey[i] = byte(i)
+	}
+
+	// Create SSE-S3 request
+	req := httptest.NewRequest("PUT", "/bucket/object", nil)
+	req.Header.Set(s3_constants.AmzServerSideEncryption, "AES256")
+
+	// Process request
+	metadata, err := ProcessSSES3Request(req)
+	if err != nil {
+		t.Fatalf("Failed to process SSE-S3 request: %v", err)
+	}
+
+	if metadata == nil {
+		t.Fatal("Expected metadata to be non-nil")
+	}
+
+	// Verify metadata contains SSE algorithm
+	if sseAlgo, exists := metadata[s3_constants.AmzServerSideEncryption]; !exists {
+		t.Error("Expected SSE algorithm in metadata")
+	} else if string(sseAlgo) != "AES256" {
+		t.Errorf("Expected AES256, got %s", string(sseAlgo))
+	}
+
+	// Verify metadata contains key data
+	if _, exists := metadata[s3_constants.SeaweedFSSSES3Key]; !exists {
+		t.Error("Expected SSE-S3 key data in metadata")
+	}
+}
+
+// TestGetSSES3KeyFromMetadata tests extraction of SSE-S3 key from metadata
+func TestGetSSES3KeyFromMetadata(t *testing.T) {
+	// Initialize global key manager
+	globalSSES3KeyManager = NewSSES3KeyManager()
+	defer func() {
+		globalSSES3KeyManager = NewSSES3KeyManager()
+	}()
+
+	// Set up the key manager with a super key for testing
+	keyManager := GetSSES3KeyManager()
+	keyManager.superKey = make([]byte, 32)
+	for i := range keyManager.superKey {
+		keyManager.superKey[i] = byte(i)
+	}
+
+	// Generate and serialize key
+	sseS3Key, err := GenerateSSES3Key()
+	if err != nil {
+		t.Fatalf("Failed to generate SSE-S3 key: %v", err)
+	}
+
+	sseS3Key.IV = make([]byte, 16)
+	for i := range sseS3Key.IV {
+		sseS3Key.IV[i] = byte(i)
+	}
+
+	serialized, err := SerializeSSES3Metadata(sseS3Key)
+	if err != nil {
+		t.Fatalf("Failed to serialize SSE-S3 metadata: %v", err)
+	}
+
+	metadata := map[string][]byte{
+		s3_constants.SeaweedFSSSES3Key: serialized,
+	}
+
+	// Extract key
+	extractedKey, err := GetSSES3KeyFromMetadata(metadata, keyManager)
+	if err != nil {
+		t.Fatalf("Failed to get SSE-S3 key from metadata: %v", err)
+	}
+
+	// Verify key matches
+	if !bytes.Equal(extractedKey.Key, sseS3Key.Key) {
+		t.Error("Extracted key doesn't match original key")
+	}
+
+	if !bytes.Equal(extractedKey.IV, sseS3Key.IV) {
+		t.Error("Extracted IV doesn't match original IV")
+	}
+}
+
+// TestSSES3EnvelopeEncryption tests that envelope encryption works correctly
+func TestSSES3EnvelopeEncryption(t *testing.T) {
+	// Initialize key manager with a super key
+	keyManager := NewSSES3KeyManager()
+	keyManager.superKey = make([]byte, 32)
+	for i := range keyManager.superKey {
+		keyManager.superKey[i] = byte(i + 100)
+	}
+
+	// Generate a DEK
+	dek := make([]byte, 32)
+	for i := range dek {
+		dek[i] = byte(i)
+	}
+
+	// Encrypt DEK with super key
+	encryptedDEK, nonce, err := keyManager.encryptKeyWithSuperKey(dek)
+	if err != nil {
+		t.Fatalf("Failed to encrypt DEK: %v", err)
+	}
+
+	if len(encryptedDEK) == 0 {
+		t.Error("Encrypted DEK is empty")
+	}
+
+	if len(nonce) == 0 {
+		t.Error("Nonce is empty")
+	}
+
+	// Decrypt DEK with super key
+	decryptedDEK, err := keyManager.decryptKeyWithSuperKey(encryptedDEK, nonce)
+	if err != nil {
+		t.Fatalf("Failed to decrypt DEK: %v", err)
+	}
+
+	// Verify DEK matches
+	if !bytes.Equal(decryptedDEK, dek) {
+		t.Error("Decrypted DEK doesn't match original DEK")
+	}
+}
+
+// TestValidateSSES3Key tests SSE-S3 key validation
+func TestValidateSSES3Key(t *testing.T) {
+	testCases := []struct {
+		name        string
+		key         *SSES3Key
+		shouldError bool
+		errorMsg    string
+	}{
+		{
+			name:        "Nil key",
+			key:         nil,
+			shouldError: true,
+			errorMsg:    "SSE-S3 key cannot be nil",
+		},
+		{
+			name: "Valid key",
+			key: &SSES3Key{
+				Key:       make([]byte, 32),
+				KeyID:     "test-key",
+				Algorithm: "AES256",
+			},
+			shouldError: false,
+		},
+		{
+			name: "Valid key with IV",
+			key: &SSES3Key{
+				Key:       make([]byte, 32),
+				KeyID:     "test-key",
+				Algorithm: "AES256",
+				IV:        make([]byte, 16),
+			},
+			shouldError: false,
+		},
+		{
+			name: "Invalid key size (too small)",
+			key: &SSES3Key{
+				Key:       make([]byte, 16),
+				KeyID:     "test-key",
+				Algorithm: "AES256",
+			},
+			shouldError: true,
+			errorMsg:    "invalid SSE-S3 key size",
+		},
+		{
+			name: "Invalid key size (too large)",
+			key: &SSES3Key{
+				Key:       make([]byte, 64),
+				KeyID:     "test-key",
+				Algorithm: "AES256",
+			},
+			shouldError: true,
+			errorMsg:    "invalid SSE-S3 key size",
+		},
+		{
+			name: "Nil key bytes",
+			key: &SSES3Key{
+				Key:       nil,
+				KeyID:     "test-key",
+				Algorithm: "AES256",
+			},
+			shouldError: true,
+			errorMsg:    "SSE-S3 key bytes cannot be nil",
+		},
+		{
+			name: "Empty key ID",
+			key: &SSES3Key{
+				Key:       make([]byte, 32),
+				KeyID:     "",
+				Algorithm: "AES256",
+			},
+			shouldError: true,
+			errorMsg:    "SSE-S3 key ID cannot be empty",
+		},
+		{
+			name: "Invalid algorithm",
+			key: &SSES3Key{
+				Key:       make([]byte, 32),
+				KeyID:     "test-key",
+				Algorithm: "INVALID",
+			},
+			shouldError: true,
+			errorMsg:    "invalid SSE-S3 algorithm",
+		},
+		{
+			name: "Invalid IV length",
+			key: &SSES3Key{
+				Key:       make([]byte, 32),
+				KeyID:     "test-key",
+				Algorithm: "AES256",
+				IV:        make([]byte, 8), // Wrong size
+			},
+			shouldError: true,
+			errorMsg:    "invalid SSE-S3 IV length",
+		},
+		{
+			name: "Empty IV is allowed (set during encryption)",
+			key: &SSES3Key{
+				Key:       make([]byte, 32),
+				KeyID:     "test-key",
+				Algorithm: "AES256",
+				IV:        []byte{}, // Empty is OK
+			},
+			shouldError: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := ValidateSSES3Key(tc.key)
+			if tc.shouldError {
+				if err == nil {
+					t.Error("Expected error but got none")
+				} else if tc.errorMsg != "" && !strings.Contains(err.Error(), tc.errorMsg) {
+					t.Errorf("Expected error containing %q, got: %v", tc.errorMsg, err)
+				}
+			} else {
+				if err != nil {
+					t.Errorf("Unexpected error: %v", err)
+				}
+			}
+		})
+	}
+}

weed/s3api/s3_sse_test_utils_test.go

@@ -115,7 +115,7 @@ func CreateTestMetadataWithSSEC(keyPair *TestKeyPair) map[string][]byte {
 	for i := range iv {
 		iv[i] = byte(i)
 	}
-	StoreIVInMetadata(metadata, iv)
+	StoreSSECIVInMetadata(metadata, iv)
 	return metadata
 }
 

weed/s3api/s3_validation_utils.go

@@ -66,10 +66,35 @@ func ValidateSSECKey(customerKey *SSECustomerKey) error {
 	return nil
 }
 
-// ValidateSSES3Key validates that an SSE-S3 key is not nil
+// ValidateSSES3Key validates that an SSE-S3 key has valid structure and contents
 func ValidateSSES3Key(sseKey *SSES3Key) error {
 	if sseKey == nil {
 		return fmt.Errorf("SSE-S3 key cannot be nil")
 	}
+	
+	// Validate key bytes
+	if sseKey.Key == nil {
+		return fmt.Errorf("SSE-S3 key bytes cannot be nil")
+	}
+	if len(sseKey.Key) != SSES3KeySize {
+		return fmt.Errorf("invalid SSE-S3 key size: expected %d bytes, got %d", SSES3KeySize, len(sseKey.Key))
+	}
+	
+	// Validate algorithm
+	if sseKey.Algorithm != SSES3Algorithm {
+		return fmt.Errorf("invalid SSE-S3 algorithm: expected %q, got %q", SSES3Algorithm, sseKey.Algorithm)
+	}
+	
+	// Validate key ID (should not be empty)
+	if sseKey.KeyID == "" {
+		return fmt.Errorf("SSE-S3 key ID cannot be empty")
+	}
+	
+	// IV validation is optional during key creation - it will be set during encryption
+	// If IV is set, validate its length
+	if len(sseKey.IV) > 0 && len(sseKey.IV) != s3_constants.AESBlockSize {
+		return fmt.Errorf("invalid SSE-S3 IV length: expected %d bytes, got %d", s3_constants.AESBlockSize, len(sseKey.IV))
+	}
+	
 	return nil
 }

weed/s3api/s3api_domain_test.go

@@ -0,0 +1,242 @@
+package s3api
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestClassifyDomainNames tests the domain classification logic for mixed virtual-host and path-style S3 access
+// This test validates the fix for issue #7356
+func TestClassifyDomainNames(t *testing.T) {
+	tests := []struct {
+		name                string
+		domainNames         []string
+		expectedPathStyle   []string
+		expectedVirtualHost []string
+		description         string
+	}{
+		{
+			name:                "Mixed path-style and virtual-host with single parent",
+			domainNames:         []string{"s3.mydomain.com", "develop.s3.mydomain.com"},
+			expectedPathStyle:   []string{"develop.s3.mydomain.com"},
+			expectedVirtualHost: []string{"s3.mydomain.com"},
+			description:         "develop.s3.mydomain.com is path-style because s3.mydomain.com is in the list",
+		},
+		{
+			name:                "Multiple subdomains with same parent",
+			domainNames:         []string{"s3.mydomain.com", "develop.s3.mydomain.com", "staging.s3.mydomain.com"},
+			expectedPathStyle:   []string{"develop.s3.mydomain.com", "staging.s3.mydomain.com"},
+			expectedVirtualHost: []string{"s3.mydomain.com"},
+			description:         "Multiple subdomains can be path-style when parent is in the list",
+		},
+		{
+			name:                "Subdomain without parent in list",
+			domainNames:         []string{"develop.s3.mydomain.com"},
+			expectedPathStyle:   []string{},
+			expectedVirtualHost: []string{"develop.s3.mydomain.com"},
+			description:         "Subdomain becomes virtual-host when parent is not in the list",
+		},
+		{
+			name:                "Only top-level domain",
+			domainNames:         []string{"s3.mydomain.com"},
+			expectedPathStyle:   []string{},
+			expectedVirtualHost: []string{"s3.mydomain.com"},
+			description:         "Top-level domain is always virtual-host style",
+		},
+		{
+			name:                "Multiple independent domains",
+			domainNames:         []string{"s3.domain1.com", "s3.domain2.com"},
+			expectedPathStyle:   []string{},
+			expectedVirtualHost: []string{"s3.domain1.com", "s3.domain2.com"},
+			description:         "Independent domains without parent relationships are all virtual-host",
+		},
+		{
+			name:                "Mixed with nested levels",
+			domainNames:         []string{"example.com", "s3.example.com", "api.s3.example.com"},
+			expectedPathStyle:   []string{"s3.example.com", "api.s3.example.com"},
+			expectedVirtualHost: []string{"example.com"},
+			description:         "Both s3.example.com and api.s3.example.com are path-style because their immediate parents are in the list",
+		},
+		{
+			name:                "Domain without dot",
+			domainNames:         []string{"localhost"},
+			expectedPathStyle:   []string{},
+			expectedVirtualHost: []string{"localhost"},
+			description:         "Domain without dot (no subdomain) is virtual-host style",
+		},
+		{
+			name:                "Empty list",
+			domainNames:         []string{},
+			expectedPathStyle:   []string{},
+			expectedVirtualHost: []string{},
+			description:         "Empty domain list returns empty results",
+		},
+		{
+			name:                "Mixed localhost and domain",
+			domainNames:         []string{"localhost", "s3.localhost"},
+			expectedPathStyle:   []string{"s3.localhost"},
+			expectedVirtualHost: []string{"localhost"},
+			description:         "s3.localhost is path-style when localhost is in the list",
+		},
+		{
+			name:                "Three-level subdomain hierarchy",
+			domainNames:         []string{"example.com", "s3.example.com", "dev.s3.example.com", "api.dev.s3.example.com"},
+			expectedPathStyle:   []string{"s3.example.com", "dev.s3.example.com", "api.dev.s3.example.com"},
+			expectedVirtualHost: []string{"example.com"},
+			description:         "Each level that has its parent in the list becomes path-style",
+		},
+		{
+			name:                "Real-world example from issue #7356",
+			domainNames:         []string{"s3.mydomain.com", "develop.s3.mydomain.com", "staging.s3.mydomain.com", "prod.s3.mydomain.com"},
+			expectedPathStyle:   []string{"develop.s3.mydomain.com", "staging.s3.mydomain.com", "prod.s3.mydomain.com"},
+			expectedVirtualHost: []string{"s3.mydomain.com"},
+			description:         "Real-world scenario with multiple environment subdomains",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			pathStyle, virtualHost := classifyDomainNames(tt.domainNames)
+
+			assert.ElementsMatch(t, tt.expectedPathStyle, pathStyle,
+				"Path-style domains mismatch: %s", tt.description)
+			assert.ElementsMatch(t, tt.expectedVirtualHost, virtualHost,
+				"Virtual-host domains mismatch: %s", tt.description)
+		})
+	}
+}
+
+// TestClassifyDomainNamesOrder tests that the function maintains consistent behavior regardless of input order
+func TestClassifyDomainNamesOrder(t *testing.T) {
+	tests := []struct {
+		name        string
+		domainNames []string
+		description string
+	}{
+		{
+			name:        "Parent before child",
+			domainNames: []string{"s3.mydomain.com", "develop.s3.mydomain.com"},
+			description: "Parent domain listed before child",
+		},
+		{
+			name:        "Child before parent",
+			domainNames: []string{"develop.s3.mydomain.com", "s3.mydomain.com"},
+			description: "Child domain listed before parent",
+		},
+		{
+			name:        "Mixed order with multiple children",
+			domainNames: []string{"staging.s3.mydomain.com", "s3.mydomain.com", "develop.s3.mydomain.com"},
+			description: "Children and parent in mixed order",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			pathStyle, virtualHost := classifyDomainNames(tt.domainNames)
+
+			// Regardless of order, the result should be consistent
+			// Parent should be virtual-host
+			assert.Contains(t, virtualHost, "s3.mydomain.com",
+				"Parent should always be virtual-host: %s", tt.description)
+
+			// Children should be path-style
+			if len(tt.domainNames) > 1 {
+				assert.Greater(t, len(pathStyle), 0,
+					"Should have at least one path-style domain: %s", tt.description)
+			}
+		})
+	}
+}
+
+// TestClassifyDomainNamesEdgeCases tests edge cases and special scenarios
+func TestClassifyDomainNamesEdgeCases(t *testing.T) {
+	t.Run("Duplicate domains", func(t *testing.T) {
+		domainNames := []string{"s3.example.com", "s3.example.com", "api.s3.example.com"}
+		pathStyle, virtualHost := classifyDomainNames(domainNames)
+
+		// Even with duplicates, classification should work
+		assert.Contains(t, pathStyle, "api.s3.example.com")
+		assert.Contains(t, virtualHost, "s3.example.com")
+	})
+
+	t.Run("Very long domain name", func(t *testing.T) {
+		domainNames := []string{"very.long.subdomain.hierarchy.example.com", "long.subdomain.hierarchy.example.com"}
+		pathStyle, virtualHost := classifyDomainNames(domainNames)
+
+		// Should handle long domains correctly
+		assert.Contains(t, pathStyle, "very.long.subdomain.hierarchy.example.com")
+		assert.Contains(t, virtualHost, "long.subdomain.hierarchy.example.com")
+	})
+
+	t.Run("Similar but different domains", func(t *testing.T) {
+		domainNames := []string{"s3.example.com", "s3.examples.com", "api.s3.example.com"}
+		pathStyle, virtualHost := classifyDomainNames(domainNames)
+
+		// api.s3.example.com should be path-style (parent s3.example.com is in list)
+		// s3.examples.com should be virtual-host (different domain)
+		assert.Contains(t, pathStyle, "api.s3.example.com")
+		assert.Contains(t, virtualHost, "s3.example.com")
+		assert.Contains(t, virtualHost, "s3.examples.com")
+	})
+
+	t.Run("IP address as domain", func(t *testing.T) {
+		domainNames := []string{"127.0.0.1"}
+		pathStyle, virtualHost := classifyDomainNames(domainNames)
+
+		// IP address should be treated as virtual-host
+		assert.Empty(t, pathStyle)
+		assert.Contains(t, virtualHost, "127.0.0.1")
+	})
+}
+
+// TestClassifyDomainNamesUseCases tests real-world use cases
+func TestClassifyDomainNamesUseCases(t *testing.T) {
+	t.Run("Issue #7356 - Prometheus blackbox exporter scenario", func(t *testing.T) {
+		// From the PR: allow both path-style and virtual-host within same subdomain
+		// curl -H 'Host: develop.s3.mydomain.com' http://127.0.0.1:8000/prometheus-blackbox-exporter/status.html
+		// curl -H 'Host: prometheus-blackbox-exporter.s3.mydomain.com' http://127.0.0.1:8000/status.html
+
+		domainNames := []string{"s3.mydomain.com", "develop.s3.mydomain.com"}
+		pathStyle, virtualHost := classifyDomainNames(domainNames)
+
+		// develop.s3.mydomain.com should be path-style for /bucket/object access
+		assert.Contains(t, pathStyle, "develop.s3.mydomain.com",
+			"develop subdomain should be path-style")
+
+		// s3.mydomain.com should be virtual-host for bucket.s3.mydomain.com access
+		assert.Contains(t, virtualHost, "s3.mydomain.com",
+			"parent domain should be virtual-host")
+	})
+
+	t.Run("Multi-environment setup", func(t *testing.T) {
+		// Common scenario: different environments using different access styles
+		domainNames := []string{
+			"s3.company.com",         // Production - virtual-host style
+			"dev.s3.company.com",     // Development - path-style
+			"test.s3.company.com",    // Testing - path-style
+			"staging.s3.company.com", // Staging - path-style
+		}
+		pathStyle, virtualHost := classifyDomainNames(domainNames)
+
+		assert.Len(t, pathStyle, 3, "Should have 3 path-style domains")
+		assert.Len(t, virtualHost, 1, "Should have 1 virtual-host domain")
+		assert.Contains(t, virtualHost, "s3.company.com")
+	})
+
+	t.Run("Mixed production setup", func(t *testing.T) {
+		// Multiple base domains with their own subdomains
+		domainNames := []string{
+			"s3-us-east.company.com",
+			"api.s3-us-east.company.com",
+			"s3-eu-west.company.com",
+			"api.s3-eu-west.company.com",
+		}
+		pathStyle, virtualHost := classifyDomainNames(domainNames)
+
+		assert.Contains(t, pathStyle, "api.s3-us-east.company.com")
+		assert.Contains(t, pathStyle, "api.s3-eu-west.company.com")
+		assert.Contains(t, virtualHost, "s3-us-east.company.com")
+		assert.Contains(t, virtualHost, "s3-eu-west.company.com")
+	})
+}

weed/s3api/s3api_key_rotation.go

@@ -100,9 +100,9 @@ func (s3a *S3ApiServer) rotateSSEKMSMetadataOnly(entry *filer_pb.Entry, srcKeyID
 // rotateSSECChunks re-encrypts all chunks with new SSE-C key
 func (s3a *S3ApiServer) rotateSSECChunks(entry *filer_pb.Entry, sourceKey, destKey *SSECustomerKey) ([]*filer_pb.FileChunk, error) {
 	// Get IV from entry metadata
-	iv, err := GetIVFromMetadata(entry.Extended)
+	iv, err := GetSSECIVFromMetadata(entry.Extended)
 	if err != nil {
-		return nil, fmt.Errorf("get IV from metadata: %w", err)
+		return nil, fmt.Errorf("get SSE-C IV from metadata: %w", err)
 	}
 
 	var rotatedChunks []*filer_pb.FileChunk
@@ -125,7 +125,7 @@ func (s3a *S3ApiServer) rotateSSECChunks(entry *filer_pb.Entry, sourceKey, destK
 	if entry.Extended == nil {
 		entry.Extended = make(map[string][]byte)
 	}
-	StoreIVInMetadata(entry.Extended, newIV)
+	StoreSSECIVInMetadata(entry.Extended, newIV)
 	entry.Extended[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] = []byte("AES256")
 	entry.Extended[s3_constants.AmzServerSideEncryptionCustomerKeyMD5] = []byte(destKey.KeyMD5)
 
@@ -175,13 +175,14 @@ func (s3a *S3ApiServer) rotateSSECChunk(chunk *filer_pb.FileChunk, sourceKey, de
 	}
 
 	// Get source chunk data
-	srcUrl, err := s3a.lookupVolumeUrl(chunk.GetFileIdString())
+	fileId := chunk.GetFileIdString()
+	srcUrl, err := s3a.lookupVolumeUrl(fileId)
 	if err != nil {
 		return nil, fmt.Errorf("lookup source volume: %w", err)
 	}
 
 	// Download encrypted data
-	encryptedData, err := s3a.downloadChunkData(srcUrl, 0, int64(chunk.Size))
+	encryptedData, err := s3a.downloadChunkData(srcUrl, fileId, 0, int64(chunk.Size))
 	if err != nil {
 		return nil, fmt.Errorf("download chunk data: %w", err)
 	}
@@ -243,13 +244,14 @@ func (s3a *S3ApiServer) rotateSSEKMSChunk(chunk *filer_pb.FileChunk, srcKeyID, d
 	}
 
 	// Get source chunk data
-	srcUrl, err := s3a.lookupVolumeUrl(chunk.GetFileIdString())
+	fileId := chunk.GetFileIdString()
+	srcUrl, err := s3a.lookupVolumeUrl(fileId)
 	if err != nil {
 		return nil, fmt.Errorf("lookup source volume: %w", err)
 	}
 
 	// Download data (this would be encrypted with the old KMS key)
-	chunkData, err := s3a.downloadChunkData(srcUrl, 0, int64(chunk.Size))
+	chunkData, err := s3a.downloadChunkData(srcUrl, fileId, 0, int64(chunk.Size))
 	if err != nil {
 		return nil, fmt.Errorf("download chunk data: %w", err)
 	}

weed/s3api/s3api_object_handlers.go

@@ -278,11 +278,11 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 	glog.V(1).Infof("GetObject: bucket %s, object %s, versioningConfigured=%v, versionId=%s", bucket, object, versioningConfigured, versionId)
 
 	var destUrl string
+	var entry *filer_pb.Entry // Declare entry at function scope for SSE processing
 
 	if versioningConfigured {
 		// Handle versioned GET - all versions are stored in .versions directory
 		var targetVersionId string
-		var entry *filer_pb.Entry
 
 		if versionId != "" {
 			// Request for specific version
@@ -363,6 +363,14 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 		}
 	}
 
+	// Fetch the correct entry for SSE processing (respects versionId)
+	objectEntryForSSE, err := s3a.getObjectEntryForSSE(r, versioningConfigured, entry)
+	if err != nil {
+		glog.Errorf("GetObjectHandler: %v", err)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
 	s3a.proxyToFiler(w, r, destUrl, false, func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) {
 		// Restore the original Range header for SSE processing
 		if sseObject && originalRangeHeader != "" {
@@ -371,14 +379,12 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 		}
 
 		// Add SSE metadata headers based on object metadata before SSE processing
-		bucket, object := s3_constants.GetBucketAndObject(r)
-		objectPath := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)
-		if objectEntry, err := s3a.getEntry("", objectPath); err == nil {
-			s3a.addSSEHeadersToResponse(proxyResponse, objectEntry)
+		if objectEntryForSSE != nil {
+			s3a.addSSEHeadersToResponse(proxyResponse, objectEntryForSSE)
 		}
 
 		// Handle SSE decryption (both SSE-C and SSE-KMS) if needed
-		return s3a.handleSSEResponse(r, proxyResponse, w)
+		return s3a.handleSSEResponse(r, proxyResponse, w, objectEntryForSSE)
 	})
 }
 
@@ -422,11 +428,11 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 	}
 
 	var destUrl string
+	var entry *filer_pb.Entry // Declare entry at function scope for SSE processing
 
 	if versioningConfigured {
 		// Handle versioned HEAD - all versions are stored in .versions directory
 		var targetVersionId string
-		var entry *filer_pb.Entry
 
 		if versionId != "" {
 			// Request for specific version
@@ -488,9 +494,17 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 		destUrl = s3a.toFilerUrl(bucket, object)
 	}
 
+	// Fetch the correct entry for SSE processing (respects versionId)
+	objectEntryForSSE, err := s3a.getObjectEntryForSSE(r, versioningConfigured, entry)
+	if err != nil {
+		glog.Errorf("HeadObjectHandler: %v", err)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
 	s3a.proxyToFiler(w, r, destUrl, false, func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) {
 		// Handle SSE validation (both SSE-C and SSE-KMS) for HEAD requests
-		return s3a.handleSSEResponse(r, proxyResponse, w)
+		return s3a.handleSSEResponse(r, proxyResponse, w, objectEntryForSSE)
 	})
 }
 
@@ -646,20 +660,53 @@ func writeFinalResponse(w http.ResponseWriter, proxyResponse *http.Response, bod
 	return statusCode, bytesTransferred
 }
 
+// getObjectEntryForSSE fetches the correct filer entry for SSE processing
+// For versioned objects, it reuses the already-fetched entry
+// For non-versioned objects, it fetches the entry from the filer
+func (s3a *S3ApiServer) getObjectEntryForSSE(r *http.Request, versioningConfigured bool, versionedEntry *filer_pb.Entry) (*filer_pb.Entry, error) {
+	if versioningConfigured {
+		// For versioned objects, we already have the correct entry
+		return versionedEntry, nil
+	}
+
+	// For non-versioned objects, fetch the entry
+	bucket, object := s3_constants.GetBucketAndObject(r)
+	objectPath := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)
+	fetchedEntry, err := s3a.getEntry("", objectPath)
+	if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return nil, fmt.Errorf("failed to get entry for SSE check %s: %w", objectPath, err)
+	}
+	return fetchedEntry, nil
+}
+
+// copyResponseHeaders copies headers from proxy response to the response writer,
+// excluding internal SeaweedFS headers and optionally excluding body-related headers
+func copyResponseHeaders(w http.ResponseWriter, proxyResponse *http.Response, excludeBodyHeaders bool) {
+	for k, v := range proxyResponse.Header {
+		// Always exclude internal SeaweedFS headers
+		if s3_constants.IsSeaweedFSInternalHeader(k) {
+			continue
+		}
+		// Optionally exclude body-related headers that might change after decryption
+		if excludeBodyHeaders && (k == "Content-Length" || k == "Content-Encoding") {
+			continue
+		}
+		w.Header()[k] = v
+	}
+}
+
 func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) {
 	// Capture existing CORS headers that may have been set by middleware
 	capturedCORSHeaders := captureCORSHeaders(w, corsHeaders)
 
-	// Copy headers from proxy response
-	for k, v := range proxyResponse.Header {
-		w.Header()[k] = v
-	}
+	// Copy headers from proxy response (excluding internal SeaweedFS headers)
+	copyResponseHeaders(w, proxyResponse, false)
 
 	return writeFinalResponse(w, proxyResponse, proxyResponse.Body, capturedCORSHeaders)
 }
 
 // handleSSECResponse handles SSE-C decryption and response processing
-func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) {
+func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter, entry *filer_pb.Entry) (statusCode int, bytesTransferred int64) {
 	// Check if the object has SSE-C metadata
 	sseAlgorithm := proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionCustomerAlgorithm)
 	sseKeyMD5 := proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionCustomerKeyMD5)
@@ -692,9 +739,8 @@ func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http.
 		// Range requests will be handled by the filer layer with proper offset-based decryption
 
 		// Check if this is a chunked or small content SSE-C object
-		bucket, object := s3_constants.GetBucketAndObject(r)
-		objectPath := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)
-		if entry, err := s3a.getEntry("", objectPath); err == nil {
+		// Use the entry parameter passed from the caller (avoids redundant lookup)
+		if entry != nil {
 			// Check for SSE-C chunks
 			sseCChunks := 0
 			for _, chunk := range entry.GetChunks() {
@@ -716,10 +762,8 @@ func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http.
 				// Capture existing CORS headers
 				capturedCORSHeaders := captureCORSHeaders(w, corsHeaders)
 
-				// Copy headers from proxy response
-				for k, v := range proxyResponse.Header {
-					w.Header()[k] = v
-				}
+				// Copy headers from proxy response (excluding internal SeaweedFS headers)
+				copyResponseHeaders(w, proxyResponse, false)
 
 				// Set proper headers for range requests
 				rangeHeader := r.Header.Get("Range")
@@ -785,12 +829,8 @@ func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http.
 		// Capture existing CORS headers that may have been set by middleware
 		capturedCORSHeaders := captureCORSHeaders(w, corsHeaders)
 
-		// Copy headers from proxy response (excluding body-related headers that might change)
-		for k, v := range proxyResponse.Header {
-			if k != "Content-Length" && k != "Content-Encoding" {
-				w.Header()[k] = v
-			}
-		}
+		// Copy headers from proxy response (excluding body-related headers that might change and internal SeaweedFS headers)
+		copyResponseHeaders(w, proxyResponse, true)
 
 		// Set correct Content-Length for SSE-C (only for full object requests)
 		// With IV stored in metadata, the encrypted length equals the original length
@@ -821,29 +861,37 @@ func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http.
 }
 
 // handleSSEResponse handles both SSE-C and SSE-KMS decryption/validation and response processing
-func (s3a *S3ApiServer) handleSSEResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) {
+// The objectEntry parameter should be the correct entry for the requested version (if versioned)
+func (s3a *S3ApiServer) handleSSEResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter, objectEntry *filer_pb.Entry) (statusCode int, bytesTransferred int64) {
 	// Check what the client is expecting based on request headers
 	clientExpectsSSEC := IsSSECRequest(r)
 
 	// Check what the stored object has in headers (may be conflicting after copy)
 	kmsMetadataHeader := proxyResponse.Header.Get(s3_constants.SeaweedFSSSEKMSKeyHeader)
-	sseAlgorithm := proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionCustomerAlgorithm)
 
-	// Get actual object state by examining chunks (most reliable for cross-encryption)
-	bucket, object := s3_constants.GetBucketAndObject(r)
-	objectPath := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)
+	// Detect actual object SSE type from the provided entry (respects versionId)
 	actualObjectType := "Unknown"
-	if objectEntry, err := s3a.getEntry("", objectPath); err == nil {
+	if objectEntry != nil {
 		actualObjectType = s3a.detectPrimarySSEType(objectEntry)
 	}
 
+	// If objectEntry is nil, we cannot determine SSE type from chunks
+	// This should only happen for 404s which will be handled by the proxy
+	if objectEntry == nil {
+		glog.V(4).Infof("Object entry not available for SSE routing, passing through")
+		return passThroughResponse(proxyResponse, w)
+	}
+
 	// Route based on ACTUAL object type (from chunks) rather than conflicting headers
 	if actualObjectType == s3_constants.SSETypeC && clientExpectsSSEC {
 		// Object is SSE-C and client expects SSE-C → SSE-C handler
-		return s3a.handleSSECResponse(r, proxyResponse, w)
+		return s3a.handleSSECResponse(r, proxyResponse, w, objectEntry)
 	} else if actualObjectType == s3_constants.SSETypeKMS && !clientExpectsSSEC {
 		// Object is SSE-KMS and client doesn't expect SSE-C → SSE-KMS handler
-		return s3a.handleSSEKMSResponse(r, proxyResponse, w, kmsMetadataHeader)
+		return s3a.handleSSEKMSResponse(r, proxyResponse, w, objectEntry, kmsMetadataHeader)
+	} else if actualObjectType == s3_constants.SSETypeS3 && !clientExpectsSSEC {
+		// Object is SSE-S3 and client doesn't expect SSE-C → SSE-S3 handler
+		return s3a.handleSSES3Response(r, proxyResponse, w, objectEntry)
 	} else if actualObjectType == "None" && !clientExpectsSSEC {
 		// Object is unencrypted and client doesn't expect SSE-C → pass through
 		return passThroughResponse(proxyResponse, w)
@@ -855,24 +903,23 @@ func (s3a *S3ApiServer) handleSSEResponse(r *http.Request, proxyResponse *http.R
 		// Object is SSE-KMS but client provides SSE-C headers → Error
 		s3err.WriteErrorResponse(w, r, s3err.ErrSSECustomerKeyMissing)
 		return http.StatusBadRequest, 0
+	} else if actualObjectType == s3_constants.SSETypeS3 && clientExpectsSSEC {
+		// Object is SSE-S3 but client provides SSE-C headers → Error (mismatched encryption)
+		s3err.WriteErrorResponse(w, r, s3err.ErrSSEEncryptionTypeMismatch)
+		return http.StatusBadRequest, 0
 	} else if actualObjectType == "None" && clientExpectsSSEC {
 		// Object is unencrypted but client provides SSE-C headers → Error
 		s3err.WriteErrorResponse(w, r, s3err.ErrSSECustomerKeyMissing)
 		return http.StatusBadRequest, 0
 	}
 
-	// Fallback for edge cases - use original logic with header-based detection
-	if clientExpectsSSEC && sseAlgorithm != "" {
-		return s3a.handleSSECResponse(r, proxyResponse, w)
-	} else if !clientExpectsSSEC && kmsMetadataHeader != "" {
-		return s3a.handleSSEKMSResponse(r, proxyResponse, w, kmsMetadataHeader)
-	} else {
-		return passThroughResponse(proxyResponse, w)
-	}
+	// Unknown state - pass through and let proxy handle it
+	glog.V(4).Infof("Unknown SSE state: objectType=%s, clientExpectsSSEC=%v", actualObjectType, clientExpectsSSEC)
+	return passThroughResponse(proxyResponse, w)
 }
 
 // handleSSEKMSResponse handles SSE-KMS decryption and response processing
-func (s3a *S3ApiServer) handleSSEKMSResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter, kmsMetadataHeader string) (statusCode int, bytesTransferred int64) {
+func (s3a *S3ApiServer) handleSSEKMSResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter, entry *filer_pb.Entry, kmsMetadataHeader string) (statusCode int, bytesTransferred int64) {
 	// Deserialize SSE-KMS metadata
 	kmsMetadataBytes, err := base64.StdEncoding.DecodeString(kmsMetadataHeader)
 	if err != nil {
@@ -893,10 +940,8 @@ func (s3a *S3ApiServer) handleSSEKMSResponse(r *http.Request, proxyResponse *htt
 		// Capture existing CORS headers that may have been set by middleware
 		capturedCORSHeaders := captureCORSHeaders(w, corsHeaders)
 
-		// Copy headers from proxy response
-		for k, v := range proxyResponse.Header {
-			w.Header()[k] = v
-		}
+		// Copy headers from proxy response (excluding internal SeaweedFS headers)
+		copyResponseHeaders(w, proxyResponse, false)
 
 		// Add SSE-KMS response headers
 		AddSSEKMSResponseHeaders(w, sseKMSKey)
@@ -908,20 +953,16 @@ func (s3a *S3ApiServer) handleSSEKMSResponse(r *http.Request, proxyResponse *htt
 	// We need to check the object structure to determine if it's multipart encrypted
 	isMultipartSSEKMS := false
 
-	if sseKMSKey != nil {
-		// Get the object entry to check chunk structure
-		bucket, object := s3_constants.GetBucketAndObject(r)
-		objectPath := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)
-		if entry, err := s3a.getEntry("", objectPath); err == nil {
-			// Check for multipart SSE-KMS
-			sseKMSChunks := 0
-			for _, chunk := range entry.GetChunks() {
-				if chunk.GetSseType() == filer_pb.SSEType_SSE_KMS && len(chunk.GetSseMetadata()) > 0 {
-					sseKMSChunks++
-				}
+	if sseKMSKey != nil && entry != nil {
+		// Use the entry parameter passed from the caller (avoids redundant lookup)
+		// Check for multipart SSE-KMS
+		sseKMSChunks := 0
+		for _, chunk := range entry.GetChunks() {
+			if chunk.GetSseType() == filer_pb.SSEType_SSE_KMS && len(chunk.GetSseMetadata()) > 0 {
+				sseKMSChunks++
 			}
-			isMultipartSSEKMS = sseKMSChunks > 1
 		}
+		isMultipartSSEKMS = sseKMSChunks > 1
 	}
 
 	var decryptedReader io.Reader
@@ -950,12 +991,8 @@ func (s3a *S3ApiServer) handleSSEKMSResponse(r *http.Request, proxyResponse *htt
 	// Capture existing CORS headers that may have been set by middleware
 	capturedCORSHeaders := captureCORSHeaders(w, corsHeaders)
 
-	// Copy headers from proxy response (excluding body-related headers that might change)
-	for k, v := range proxyResponse.Header {
-		if k != "Content-Length" && k != "Content-Encoding" {
-			w.Header()[k] = v
-		}
-	}
+	// Copy headers from proxy response (excluding body-related headers that might change and internal SeaweedFS headers)
+	copyResponseHeaders(w, proxyResponse, true)
 
 	// Set correct Content-Length for SSE-KMS
 	if proxyResponse.Header.Get("Content-Range") == "" {
@@ -971,6 +1008,99 @@ func (s3a *S3ApiServer) handleSSEKMSResponse(r *http.Request, proxyResponse *htt
 	return writeFinalResponse(w, proxyResponse, decryptedReader, capturedCORSHeaders)
 }
 
+// handleSSES3Response handles SSE-S3 decryption and response processing
+func (s3a *S3ApiServer) handleSSES3Response(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter, entry *filer_pb.Entry) (statusCode int, bytesTransferred int64) {
+
+	// For HEAD requests, we don't need to decrypt the body, just add response headers
+	if r.Method == "HEAD" {
+		// Capture existing CORS headers that may have been set by middleware
+		capturedCORSHeaders := captureCORSHeaders(w, corsHeaders)
+
+		// Copy headers from proxy response (excluding internal SeaweedFS headers)
+		copyResponseHeaders(w, proxyResponse, false)
+
+		// Add SSE-S3 response headers
+		w.Header().Set(s3_constants.AmzServerSideEncryption, SSES3Algorithm)
+
+		return writeFinalResponse(w, proxyResponse, proxyResponse.Body, capturedCORSHeaders)
+	}
+
+	// For GET requests, check if this is a multipart SSE-S3 object
+	isMultipartSSES3 := false
+	sses3Chunks := 0
+	for _, chunk := range entry.GetChunks() {
+		if chunk.GetSseType() == filer_pb.SSEType_SSE_S3 && len(chunk.GetSseMetadata()) > 0 {
+			sses3Chunks++
+		}
+	}
+	isMultipartSSES3 = sses3Chunks > 1
+
+	var decryptedReader io.Reader
+	if isMultipartSSES3 {
+		// Handle multipart SSE-S3 objects - each chunk needs independent decryption
+		multipartReader, decErr := s3a.createMultipartSSES3DecryptedReader(r, entry)
+		if decErr != nil {
+			glog.Errorf("Failed to create multipart SSE-S3 decrypted reader: %v", decErr)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+			return http.StatusInternalServerError, 0
+		}
+		decryptedReader = multipartReader
+		glog.V(3).Infof("Using multipart SSE-S3 decryption for object")
+	} else {
+		// Handle single-part SSE-S3 objects
+		// Extract SSE-S3 key from metadata
+		keyManager := GetSSES3KeyManager()
+		if keyData, exists := entry.Extended[s3_constants.SeaweedFSSSES3Key]; !exists {
+			glog.Errorf("SSE-S3 key metadata not found in object entry")
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+			return http.StatusInternalServerError, 0
+		} else {
+			sseS3Key, err := DeserializeSSES3Metadata(keyData, keyManager)
+			if err != nil {
+				glog.Errorf("Failed to deserialize SSE-S3 metadata: %v", err)
+				s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+				return http.StatusInternalServerError, 0
+			}
+
+			// Extract IV from metadata using helper function
+			iv, err := GetSSES3IV(entry, sseS3Key, keyManager)
+			if err != nil {
+				glog.Errorf("Failed to get SSE-S3 IV: %v", err)
+				s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+				return http.StatusInternalServerError, 0
+			}
+
+			singlePartReader, decErr := CreateSSES3DecryptedReader(proxyResponse.Body, sseS3Key, iv)
+			if decErr != nil {
+				glog.Errorf("Failed to create SSE-S3 decrypted reader: %v", decErr)
+				s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+				return http.StatusInternalServerError, 0
+			}
+			decryptedReader = singlePartReader
+			glog.V(3).Infof("Using single-part SSE-S3 decryption for object")
+		}
+	}
+
+	// Capture existing CORS headers that may have been set by middleware
+	capturedCORSHeaders := captureCORSHeaders(w, corsHeaders)
+
+	// Copy headers from proxy response (excluding body-related headers that might change and internal SeaweedFS headers)
+	copyResponseHeaders(w, proxyResponse, true)
+
+	// Set correct Content-Length for SSE-S3
+	if proxyResponse.Header.Get("Content-Range") == "" {
+		// For full object requests, encrypted length equals original length
+		if contentLengthStr := proxyResponse.Header.Get("Content-Length"); contentLengthStr != "" {
+			w.Header().Set("Content-Length", contentLengthStr)
+		}
+	}
+
+	// Add SSE-S3 response headers
+	w.Header().Set(s3_constants.AmzServerSideEncryption, SSES3Algorithm)
+
+	return writeFinalResponse(w, proxyResponse, decryptedReader, capturedCORSHeaders)
+}
+
 // addObjectLockHeadersToResponse extracts object lock metadata from entry Extended attributes
 // and adds the appropriate S3 headers to the response
 func (s3a *S3ApiServer) addObjectLockHeadersToResponse(w http.ResponseWriter, entry *filer_pb.Entry) {
@@ -1049,6 +1179,10 @@ func (s3a *S3ApiServer) addSSEHeadersToResponse(proxyResponse *http.Response, en
 			proxyResponse.Header.Set(s3_constants.AmzServerSideEncryptionAwsKmsKeyId, string(kmsKeyID))
 		}
 
+	case s3_constants.SSETypeS3:
+		// Add only SSE-S3 headers
+		proxyResponse.Header.Set(s3_constants.AmzServerSideEncryption, SSES3Algorithm)
+
 	default:
 		// Unencrypted or unknown - don't set any SSE headers
 	}
@@ -1063,10 +1197,26 @@ func (s3a *S3ApiServer) detectPrimarySSEType(entry *filer_pb.Entry) string {
 		hasSSEC := entry.Extended[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] != nil
 		hasSSEKMS := entry.Extended[s3_constants.AmzServerSideEncryption] != nil
 
-		if hasSSEC && !hasSSEKMS {
+		// Check for SSE-S3: algorithm is AES256 but no customer key
+		if hasSSEKMS && !hasSSEC {
+			// Distinguish SSE-S3 from SSE-KMS: check the algorithm value and the presence of a KMS key ID
+			sseAlgo := string(entry.Extended[s3_constants.AmzServerSideEncryption])
+			switch sseAlgo {
+			case s3_constants.SSEAlgorithmAES256:
+				// Could be SSE-S3 or SSE-KMS, check for KMS key ID
+				if _, hasKMSKey := entry.Extended[s3_constants.AmzServerSideEncryptionAwsKmsKeyId]; hasKMSKey {
+					return s3_constants.SSETypeKMS
+				}
+				// No KMS key, this is SSE-S3
+				return s3_constants.SSETypeS3
+			case s3_constants.SSEAlgorithmKMS:
+				return s3_constants.SSETypeKMS
+			default:
+				// Unknown or unsupported algorithm
+				return "None"
+			}
+		} else if hasSSEC && !hasSSEKMS {
 			return s3_constants.SSETypeC
-		} else if hasSSEKMS && !hasSSEC {
-			return s3_constants.SSETypeKMS
 		} else if hasSSEC && hasSSEKMS {
 			// Both present - this should only happen during cross-encryption copies
 			// Use content to determine actual encryption state
@@ -1084,24 +1234,39 @@ func (s3a *S3ApiServer) detectPrimarySSEType(entry *filer_pb.Entry) string {
 	// Count chunk types to determine primary (multipart objects)
 	ssecChunks := 0
 	ssekmsChunks := 0
+	sses3Chunks := 0
 
 	for _, chunk := range entry.GetChunks() {
 		switch chunk.GetSseType() {
 		case filer_pb.SSEType_SSE_C:
 			ssecChunks++
 		case filer_pb.SSEType_SSE_KMS:
-			ssekmsChunks++
+			if len(chunk.GetSseMetadata()) > 0 {
+				ssekmsChunks++
+			}
+		case filer_pb.SSEType_SSE_S3:
+			if len(chunk.GetSseMetadata()) > 0 {
+				sses3Chunks++
+			}
 		}
 	}
 
 	// Primary type is the one with more chunks
-	if ssecChunks > ssekmsChunks {
+	// Note: Tie-breaking follows precedence order SSE-C > SSE-KMS > SSE-S3
+	// Mixed encryption in an object indicates potential corruption and should not occur in normal operation
+	if ssecChunks > ssekmsChunks && ssecChunks > sses3Chunks {
 		return s3_constants.SSETypeC
-	} else if ssekmsChunks > ssecChunks {
+	} else if ssekmsChunks > ssecChunks && ssekmsChunks > sses3Chunks {
 		return s3_constants.SSETypeKMS
+	} else if sses3Chunks > ssecChunks && sses3Chunks > ssekmsChunks {
+		return s3_constants.SSETypeS3
 	} else if ssecChunks > 0 {
-		// Equal number, prefer SSE-C (shouldn't happen in practice)
+		// Equal number or ties - precedence: SSE-C first
 		return s3_constants.SSETypeC
+	} else if ssekmsChunks > 0 {
+		return s3_constants.SSETypeKMS
+	} else if sses3Chunks > 0 {
+		return s3_constants.SSETypeS3
 	}
 
 	return "None"
@@ -1150,21 +1315,9 @@ func (s3a *S3ApiServer) createMultipartSSEKMSDecryptedReader(r *http.Request, pr
 			}
 		}
 
-		// Fallback to object-level metadata (legacy support)
-		if chunkSSEKMSKey == nil {
-			objectMetadataHeader := proxyResponse.Header.Get(s3_constants.SeaweedFSSSEKMSKeyHeader)
-			if objectMetadataHeader != "" {
-				kmsMetadataBytes, decodeErr := base64.StdEncoding.DecodeString(objectMetadataHeader)
-				if decodeErr == nil {
-					kmsKey, _ := DeserializeSSEKMSMetadata(kmsMetadataBytes)
-					if kmsKey != nil {
-						// For object-level metadata (legacy), use absolute file offset as fallback
-						kmsKey.ChunkOffset = chunk.GetOffset()
-						chunkSSEKMSKey = kmsKey
-					}
-				}
-			}
-		}
+		// Note: No fallback to object-level metadata for multipart objects
+		// Each chunk in a multipart SSE-KMS object must have its own unique IV
+		// Falling back to object-level metadata could lead to IV reuse or incorrect decryption
 
 		if chunkSSEKMSKey == nil {
 			return nil, fmt.Errorf("no SSE-KMS metadata found for chunk %s in multipart object", chunk.GetFileIdString())
@@ -1189,6 +1342,86 @@ func (s3a *S3ApiServer) createMultipartSSEKMSDecryptedReader(r *http.Request, pr
 	return multiReader, nil
 }
 
+// createMultipartSSES3DecryptedReader creates a reader for multipart SSE-S3 objects
+func (s3a *S3ApiServer) createMultipartSSES3DecryptedReader(r *http.Request, entry *filer_pb.Entry) (io.Reader, error) {
+	// Sort chunks by offset to ensure correct order
+	chunks := entry.GetChunks()
+	sort.Slice(chunks, func(i, j int) bool {
+		return chunks[i].GetOffset() < chunks[j].GetOffset()
+	})
+
+	// Create readers for each chunk, decrypting them independently
+	var readers []io.Reader
+	keyManager := GetSSES3KeyManager()
+
+	for _, chunk := range chunks {
+		// Get this chunk's encrypted data
+		chunkReader, err := s3a.createEncryptedChunkReader(chunk)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create chunk reader: %v", err)
+		}
+
+		// Handle based on chunk's encryption type
+		if chunk.GetSseType() == filer_pb.SSEType_SSE_S3 {
+			var chunkSSES3Key *SSES3Key
+
+			// Check if this chunk has per-chunk SSE-S3 metadata
+			if len(chunk.GetSseMetadata()) > 0 {
+				// Use the per-chunk SSE-S3 metadata
+				sseKey, err := DeserializeSSES3Metadata(chunk.GetSseMetadata(), keyManager)
+				if err != nil {
+					glog.Errorf("Failed to deserialize per-chunk SSE-S3 metadata for chunk %s: %v", chunk.GetFileIdString(), err)
+					chunkReader.Close()
+					return nil, fmt.Errorf("failed to deserialize SSE-S3 metadata: %v", err)
+				}
+				chunkSSES3Key = sseKey
+			}
+
+			// Note: No fallback to object-level metadata for multipart objects
+			// Each chunk in a multipart SSE-S3 object must have its own unique IV
+			// Falling back to object-level metadata could lead to IV reuse or incorrect decryption
+
+			if chunkSSES3Key == nil {
+				chunkReader.Close()
+				return nil, fmt.Errorf("no SSE-S3 metadata found for chunk %s in multipart object", chunk.GetFileIdString())
+			}
+
+			// Extract IV from chunk metadata
+			if len(chunkSSES3Key.IV) == 0 {
+				chunkReader.Close()
+				return nil, fmt.Errorf("no IV found in SSE-S3 metadata for chunk %s", chunk.GetFileIdString())
+			}
+
+			// Create decrypted reader for this chunk
+			decryptedChunkReader, decErr := CreateSSES3DecryptedReader(chunkReader, chunkSSES3Key, chunkSSES3Key.IV)
+			if decErr != nil {
+				chunkReader.Close()
+				return nil, fmt.Errorf("failed to decrypt chunk: %v", decErr)
+			}
+
+			// Use the streaming decrypted reader directly, ensuring the underlying chunkReader can be closed
+			readers = append(readers, struct {
+				io.Reader
+				io.Closer
+			}{
+				Reader: decryptedChunkReader,
+				Closer: chunkReader,
+			})
+			glog.V(4).Infof("Added streaming decrypted reader for chunk %s in multipart SSE-S3 object", chunk.GetFileIdString())
+		} else {
+			// Non-SSE-S3 chunk (unencrypted or other encryption type), use as-is
+			readers = append(readers, chunkReader)
+			glog.V(4).Infof("Added passthrough reader for non-SSE-S3 chunk %s (type: %v)", chunk.GetFileIdString(), chunk.GetSseType())
+		}
+	}
+
+	// Combine all decrypted chunk readers into a single stream
+	multiReader := NewMultipartSSEReader(readers)
+	glog.V(3).Infof("Created multipart SSE-S3 decrypted reader with %d chunks", len(readers))
+
+	return multiReader, nil
+}
+
 // createEncryptedChunkReader creates a reader for a single encrypted chunk
 func (s3a *S3ApiServer) createEncryptedChunkReader(chunk *filer_pb.FileChunk) (io.ReadCloser, error) {
 	// Get chunk URL

weed/s3api/s3api_object_handlers_copy.go

@@ -734,7 +734,8 @@ func (s3a *S3ApiServer) copySingleChunk(chunk *filer_pb.FileChunk, dstPath strin
 	dstChunk := s3a.createDestinationChunk(chunk, chunk.Offset, chunk.Size)
 
 	// Prepare chunk copy (assign new volume and get source URL)
-	assignResult, srcUrl, err := s3a.prepareChunkCopy(chunk.GetFileIdString(), dstPath)
+	fileId := chunk.GetFileIdString()
+	assignResult, srcUrl, err := s3a.prepareChunkCopy(fileId, dstPath)
 	if err != nil {
 		return nil, err
 	}
@@ -745,7 +746,7 @@ func (s3a *S3ApiServer) copySingleChunk(chunk *filer_pb.FileChunk, dstPath strin
 	}
 
 	// Download and upload the chunk
-	chunkData, err := s3a.downloadChunkData(srcUrl, 0, int64(chunk.Size))
+	chunkData, err := s3a.downloadChunkData(srcUrl, fileId, 0, int64(chunk.Size))
 	if err != nil {
 		return nil, fmt.Errorf("download chunk data: %w", err)
 	}
@@ -763,7 +764,8 @@ func (s3a *S3ApiServer) copySingleChunkForRange(originalChunk, rangeChunk *filer
 	dstChunk := s3a.createDestinationChunk(rangeChunk, rangeChunk.Offset, rangeChunk.Size)
 
 	// Prepare chunk copy (assign new volume and get source URL)
-	assignResult, srcUrl, err := s3a.prepareChunkCopy(originalChunk.GetFileIdString(), dstPath)
+	fileId := originalChunk.GetFileIdString()
+	assignResult, srcUrl, err := s3a.prepareChunkCopy(fileId, dstPath)
 	if err != nil {
 		return nil, err
 	}
@@ -779,7 +781,7 @@ func (s3a *S3ApiServer) copySingleChunkForRange(originalChunk, rangeChunk *filer
 	offsetInChunk := overlapStart - chunkStart
 
 	// Download and upload the chunk portion
-	chunkData, err := s3a.downloadChunkData(srcUrl, offsetInChunk, int64(rangeChunk.Size))
+	chunkData, err := s3a.downloadChunkData(srcUrl, fileId, offsetInChunk, int64(rangeChunk.Size))
 	if err != nil {
 		return nil, fmt.Errorf("download chunk range data: %w", err)
 	}
@@ -1096,9 +1098,10 @@ func (s3a *S3ApiServer) uploadChunkData(chunkData []byte, assignResult *filer_pb
 }
 
 // downloadChunkData downloads chunk data from the source URL
-func (s3a *S3ApiServer) downloadChunkData(srcUrl string, offset, size int64) ([]byte, error) {
+func (s3a *S3ApiServer) downloadChunkData(srcUrl, fileId string, offset, size int64) ([]byte, error) {
+	jwt := filer.JwtForVolumeServer(fileId)
 	var chunkData []byte
-	shouldRetry, err := util_http.ReadUrlAsStream(context.Background(), srcUrl, nil, false, false, offset, int(size), func(data []byte) {
+	shouldRetry, err := util_http.ReadUrlAsStream(context.Background(), srcUrl, jwt, nil, false, false, offset, int(size), func(data []byte) {
 		chunkData = append(chunkData, data...)
 	})
 	if err != nil {
@@ -1152,7 +1155,7 @@ func (s3a *S3ApiServer) copyMultipartSSECChunks(entry *filer_pb.Entry, copySourc
 	dstMetadata := make(map[string][]byte)
 	if destKey != nil && len(destIV) > 0 {
 		// Store the IV and SSE-C headers for single-part compatibility
-		StoreIVInMetadata(dstMetadata, destIV)
+		StoreSSECIVInMetadata(dstMetadata, destIV)
 		dstMetadata[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] = []byte("AES256")
 		dstMetadata[s3_constants.AmzServerSideEncryptionCustomerKeyMD5] = []byte(destKey.KeyMD5)
 		glog.V(2).Infof("Prepared multipart SSE-C destination metadata: %s", dstPath)
@@ -1218,7 +1221,8 @@ func (s3a *S3ApiServer) copyMultipartSSEKMSChunk(chunk *filer_pb.FileChunk, dest
 	dstChunk := s3a.createDestinationChunk(chunk, chunk.Offset, chunk.Size)
 
 	// Prepare chunk copy (assign new volume and get source URL)
-	assignResult, srcUrl, err := s3a.prepareChunkCopy(chunk.GetFileIdString(), dstPath)
+	fileId := chunk.GetFileIdString()
+	assignResult, srcUrl, err := s3a.prepareChunkCopy(fileId, dstPath)
 	if err != nil {
 		return nil, err
 	}
@@ -1229,7 +1233,7 @@ func (s3a *S3ApiServer) copyMultipartSSEKMSChunk(chunk *filer_pb.FileChunk, dest
 	}
 
 	// Download encrypted chunk data
-	encryptedData, err := s3a.downloadChunkData(srcUrl, 0, int64(chunk.Size))
+	encryptedData, err := s3a.downloadChunkData(srcUrl, fileId, 0, int64(chunk.Size))
 	if err != nil {
 		return nil, fmt.Errorf("download encrypted chunk data: %w", err)
 	}
@@ -1315,7 +1319,8 @@ func (s3a *S3ApiServer) copyMultipartSSECChunk(chunk *filer_pb.FileChunk, copySo
 	dstChunk := s3a.createDestinationChunk(chunk, chunk.Offset, chunk.Size)
 
 	// Prepare chunk copy (assign new volume and get source URL)
-	assignResult, srcUrl, err := s3a.prepareChunkCopy(chunk.GetFileIdString(), dstPath)
+	fileId := chunk.GetFileIdString()
+	assignResult, srcUrl, err := s3a.prepareChunkCopy(fileId, dstPath)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -1326,7 +1331,7 @@ func (s3a *S3ApiServer) copyMultipartSSECChunk(chunk *filer_pb.FileChunk, copySo
 	}
 
 	// Download encrypted chunk data
-	encryptedData, err := s3a.downloadChunkData(srcUrl, 0, int64(chunk.Size))
+	encryptedData, err := s3a.downloadChunkData(srcUrl, fileId, 0, int64(chunk.Size))
 	if err != nil {
 		return nil, nil, fmt.Errorf("download encrypted chunk data: %w", err)
 	}
@@ -1504,7 +1509,7 @@ func (s3a *S3ApiServer) copyMultipartCrossEncryption(entry *filer_pb.Entry, r *h
 		if len(dstChunks) > 0 && dstChunks[0].GetSseType() == filer_pb.SSEType_SSE_C && len(dstChunks[0].GetSseMetadata()) > 0 {
 			if ssecMetadata, err := DeserializeSSECMetadata(dstChunks[0].GetSseMetadata()); err == nil {
 				if iv, ivErr := base64.StdEncoding.DecodeString(ssecMetadata.IV); ivErr == nil {
-					StoreIVInMetadata(dstMetadata, iv)
+					StoreSSECIVInMetadata(dstMetadata, iv)
 					dstMetadata[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] = []byte("AES256")
 					dstMetadata[s3_constants.AmzServerSideEncryptionCustomerKeyMD5] = []byte(destSSECKey.KeyMD5)
 				}
@@ -1537,7 +1542,8 @@ func (s3a *S3ApiServer) copyCrossEncryptionChunk(chunk *filer_pb.FileChunk, sour
 	dstChunk := s3a.createDestinationChunk(chunk, chunk.Offset, chunk.Size)
 
 	// Prepare chunk copy (assign new volume and get source URL)
-	assignResult, srcUrl, err := s3a.prepareChunkCopy(chunk.GetFileIdString(), dstPath)
+	fileId := chunk.GetFileIdString()
+	assignResult, srcUrl, err := s3a.prepareChunkCopy(fileId, dstPath)
 	if err != nil {
 		return nil, err
 	}
@@ -1548,7 +1554,7 @@ func (s3a *S3ApiServer) copyCrossEncryptionChunk(chunk *filer_pb.FileChunk, sour
 	}
 
 	// Download encrypted chunk data
-	encryptedData, err := s3a.downloadChunkData(srcUrl, 0, int64(chunk.Size))
+	encryptedData, err := s3a.downloadChunkData(srcUrl, fileId, 0, int64(chunk.Size))
 	if err != nil {
 		return nil, fmt.Errorf("download encrypted chunk data: %w", err)
 	}
@@ -1772,7 +1778,7 @@ func (s3a *S3ApiServer) copyChunksWithSSEC(entry *filer_pb.Entry, r *http.Reques
 		dstMetadata := make(map[string][]byte)
 		if destKey != nil && len(destIV) > 0 {
 			// Store the IV
-			StoreIVInMetadata(dstMetadata, destIV)
+			StoreSSECIVInMetadata(dstMetadata, destIV)
 
 			// Store SSE-C algorithm and key MD5 for proper metadata
 			dstMetadata[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] = []byte("AES256")
@@ -1834,7 +1840,8 @@ func (s3a *S3ApiServer) copyChunkWithReencryption(chunk *filer_pb.FileChunk, cop
 	dstChunk := s3a.createDestinationChunk(chunk, chunk.Offset, chunk.Size)
 
 	// Prepare chunk copy (assign new volume and get source URL)
-	assignResult, srcUrl, err := s3a.prepareChunkCopy(chunk.GetFileIdString(), dstPath)
+	fileId := chunk.GetFileIdString()
+	assignResult, srcUrl, err := s3a.prepareChunkCopy(fileId, dstPath)
 	if err != nil {
 		return nil, err
 	}
@@ -1845,7 +1852,7 @@ func (s3a *S3ApiServer) copyChunkWithReencryption(chunk *filer_pb.FileChunk, cop
 	}
 
 	// Download encrypted chunk data
-	encryptedData, err := s3a.downloadChunkData(srcUrl, 0, int64(chunk.Size))
+	encryptedData, err := s3a.downloadChunkData(srcUrl, fileId, 0, int64(chunk.Size))
 	if err != nil {
 		return nil, fmt.Errorf("download encrypted chunk data: %w", err)
 	}
@@ -1855,7 +1862,7 @@ func (s3a *S3ApiServer) copyChunkWithReencryption(chunk *filer_pb.FileChunk, cop
 	// Decrypt if source is encrypted
 	if copySourceKey != nil {
 		// Get IV from source metadata
-		srcIV, err := GetIVFromMetadata(srcMetadata)
+		srcIV, err := GetSSECIVFromMetadata(srcMetadata)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get IV from metadata: %w", err)
 		}
@@ -2052,7 +2059,8 @@ func (s3a *S3ApiServer) copyChunkWithSSEKMSReencryption(chunk *filer_pb.FileChun
 	dstChunk := s3a.createDestinationChunk(chunk, chunk.Offset, chunk.Size)
 
 	// Prepare chunk copy (assign new volume and get source URL)
-	assignResult, srcUrl, err := s3a.prepareChunkCopy(chunk.GetFileIdString(), dstPath)
+	fileId := chunk.GetFileIdString()
+	assignResult, srcUrl, err := s3a.prepareChunkCopy(fileId, dstPath)
 	if err != nil {
 		return nil, err
 	}
@@ -2063,7 +2071,7 @@ func (s3a *S3ApiServer) copyChunkWithSSEKMSReencryption(chunk *filer_pb.FileChun
 	}
 
 	// Download chunk data
-	chunkData, err := s3a.downloadChunkData(srcUrl, 0, int64(chunk.Size))
+	chunkData, err := s3a.downloadChunkData(srcUrl, fileId, 0, int64(chunk.Size))
 	if err != nil {
 		return nil, fmt.Errorf("download chunk data: %w", err)
 	}

weed/s3api/s3api_server.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"slices"
 	"strings"
 	"time"
 
@@ -147,10 +148,39 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
 
 	s3ApiServer.registerRouter(router)
 
+	// Initialize the global SSE-S3 key manager with filer access
+	if err := InitializeGlobalSSES3KeyManager(s3ApiServer); err != nil {
+		return nil, fmt.Errorf("failed to initialize SSE-S3 key manager: %w", err)
+	}
+
 	go s3ApiServer.subscribeMetaEvents("s3", startTsNs, filer.DirectoryEtcRoot, []string{option.BucketsPath})
 	return s3ApiServer, nil
 }
 
+// classifyDomainNames classifies domains into path-style and virtual-host style domains.
+// A domain is considered path-style if:
+//  1. It contains a dot (has subdomains)
+//  2. Its parent domain is also in the list of configured domains
+//
+// For example, if domains are ["s3.example.com", "develop.s3.example.com"],
+// then "develop.s3.example.com" is path-style (parent "s3.example.com" is in the list),
+// while "s3.example.com" is virtual-host style.
+func classifyDomainNames(domainNames []string) (pathStyleDomains, virtualHostDomains []string) {
+	for _, domainName := range domainNames {
+		parts := strings.SplitN(domainName, ".", 2)
+		if len(parts) == 2 && slices.Contains(domainNames, parts[1]) {
+			// This is a subdomain and its parent is also in the list
+			// Register as path-style: domain.com/bucket/object
+			pathStyleDomains = append(pathStyleDomains, domainName)
+		} else {
+			// This is a top-level domain or its parent is not in the list
+			// Register as virtual-host style: bucket.domain.com/object
+			virtualHostDomains = append(virtualHostDomains, domainName)
+		}
+	}
+	return pathStyleDomains, virtualHostDomains
+}
+
 // handleCORSOriginValidation handles the common CORS origin validation logic
 func (s3a *S3ApiServer) handleCORSOriginValidation(w http.ResponseWriter, r *http.Request) bool {
 	origin := r.Header.Get("Origin")
@@ -191,11 +221,17 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 	var routers []*mux.Router
 	if s3a.option.DomainName != "" {
 		domainNames := strings.Split(s3a.option.DomainName, ",")
-		for _, domainName := range domainNames {
+		pathStyleDomains, virtualHostDomains := classifyDomainNames(domainNames)
+
+		// Register path-style domains
+		for _, domain := range pathStyleDomains {
+			routers = append(routers, apiRouter.Host(domain).PathPrefix("/{bucket}").Subrouter())
+		}
+
+		// Register virtual-host style domains
+		for _, virtualHost := range virtualHostDomains {
 			routers = append(routers, apiRouter.Host(
-				fmt.Sprintf("%s.%s:%d", "{bucket:.+}", domainName, s3a.option.Port)).Subrouter())
-			routers = append(routers, apiRouter.Host(
-				fmt.Sprintf("%s.%s", "{bucket:.+}", domainName)).Subrouter())
+				fmt.Sprintf("%s.%s", "{bucket:.+}", virtualHost)).Subrouter())
 		}
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())

weed/s3api/s3api_streaming_copy.go

@@ -140,10 +140,8 @@ func (scm *StreamingCopyManager) createEncryptionSpec(entry *filer_pb.Entry, r *
 		spec.SourceType = EncryptionTypeSSES3
 		// Extract SSE-S3 key from metadata
 		if keyData, exists := entry.Extended[s3_constants.SeaweedFSSSES3Key]; exists {
-			// TODO: This should use a proper SSE-S3 key manager from S3ApiServer
-			// For now, create a temporary key manager to handle deserialization
-			tempKeyManager := NewSSES3KeyManager()
-			sseKey, err := DeserializeSSES3Metadata(keyData, tempKeyManager)
+			keyManager := GetSSES3KeyManager()
+			sseKey, err := DeserializeSSES3Metadata(keyData, keyManager)
 			if err != nil {
 				return nil, fmt.Errorf("deserialize SSE-S3 metadata: %w", err)
 			}
@@ -258,7 +256,7 @@ func (scm *StreamingCopyManager) createDecryptionReader(reader io.Reader, encSpe
 	case EncryptionTypeSSEC:
 		if sourceKey, ok := encSpec.SourceKey.(*SSECustomerKey); ok {
 			// Get IV from metadata
-			iv, err := GetIVFromMetadata(encSpec.SourceMetadata)
+			iv, err := GetSSECIVFromMetadata(encSpec.SourceMetadata)
 			if err != nil {
 				return nil, fmt.Errorf("get IV from metadata: %w", err)
 			}
@@ -274,10 +272,10 @@ func (scm *StreamingCopyManager) createDecryptionReader(reader io.Reader, encSpe
 
 	case EncryptionTypeSSES3:
 		if sseKey, ok := encSpec.SourceKey.(*SSES3Key); ok {
-			// Get IV from metadata
-			iv, err := GetIVFromMetadata(encSpec.SourceMetadata)
-			if err != nil {
-				return nil, fmt.Errorf("get IV from metadata: %w", err)
+			// For SSE-S3, the IV is stored within the SSES3Key metadata, not as separate metadata
+			iv := sseKey.IV
+			if len(iv) == 0 {
+				return nil, fmt.Errorf("SSE-S3 key is missing IV for streaming copy")
 			}
 			return CreateSSES3DecryptedReader(reader, sseKey, iv)
 		}

weed/s3api/s3err/s3api_errors.go

@@ -102,6 +102,7 @@ const (
 	ErrContentSHA256Mismatch
 	ErrInvalidAccessKeyID
 	ErrRequestNotReadyYet
+	ErrRequestTimeTooSkewed
 	ErrMissingDateHeader
 	ErrInvalidRequest
 	ErrAuthNotSetup
@@ -129,6 +130,7 @@ const (
 	ErrSSECustomerKeyMD5Mismatch
 	ErrSSECustomerKeyMissing
 	ErrSSECustomerKeyNotNeeded
+	ErrSSEEncryptionTypeMismatch
 
 	// SSE-KMS related errors
 	ErrKMSKeyNotFound
@@ -431,6 +433,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		HTTPStatusCode: http.StatusForbidden,
 	},
 
+	ErrRequestTimeTooSkewed: {
+		Code:           "RequestTimeTooSkewed",
+		Description:    "The difference between the request time and the server's time is too large.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+
 	ErrSignatureDoesNotMatch: {
 		Code:           "SignatureDoesNotMatch",
 		Description:    "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
@@ -540,6 +548,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The object was not encrypted with customer provided keys.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrSSEEncryptionTypeMismatch: {
+		Code:           "InvalidRequest",
+		Description:    "The encryption method specified in the request does not match the encryption method used to encrypt the object.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// SSE-KMS error responses
 	ErrKMSKeyNotFound: {

weed/server/filer_grpc_server_sub_meta.go

@@ -69,14 +69,30 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
 		if processedTsNs != 0 {
 			lastReadTime = log_buffer.NewMessagePosition(processedTsNs, -2)
 		} else {
-			nextDayTs := util.GetNextDayTsNano(lastReadTime.Time.UnixNano())
-			position := log_buffer.NewMessagePosition(nextDayTs, -2)
-			found, err := fs.filer.HasPersistedLogFiles(position)
-			if err != nil {
-				return fmt.Errorf("checking persisted log files: %w", err)
-			}
-			if found {
-				lastReadTime = position
+			// No data found on disk
+			// Check if we previously got ResumeFromDiskError from memory, meaning we're in a gap
+			if errors.Is(readInMemoryLogErr, log_buffer.ResumeFromDiskError) {
+				// We have a gap: requested time < earliest memory time, but no data on disk
+				// Skip forward to earliest memory time to avoid infinite loop
+				earliestTime := fs.filer.MetaAggregator.MetaLogBuffer.GetEarliestTime()
+				if !earliestTime.IsZero() && earliestTime.After(lastReadTime.Time) {
+					glog.V(3).Infof("gap detected: skipping from %v to earliest memory time %v for %v",
+						lastReadTime.Time, earliestTime, clientName)
+					// Position at earliest time; time-based reader will include it
+					lastReadTime = log_buffer.NewMessagePosition(earliestTime.UnixNano(), -2)
+					readInMemoryLogErr = nil // Clear the error since we're skipping forward
+				}
+			} else {
+				// First pass or no ResumeFromDiskError yet - check the next day for logs
+				nextDayTs := util.GetNextDayTsNano(lastReadTime.Time.UnixNano())
+				position := log_buffer.NewMessagePosition(nextDayTs, -2)
+				found, err := fs.filer.HasPersistedLogFiles(position)
+				if err != nil {
+					return fmt.Errorf("checking persisted log files: %w", err)
+				}
+				if found {
+					lastReadTime = position
+				}
 			}
 		}
 
@@ -91,12 +107,16 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
 			}
 
 			fs.filer.MetaAggregator.ListenersLock.Lock()
+			atomic.AddInt64(&fs.filer.MetaAggregator.ListenersWaits, 1)
 			fs.filer.MetaAggregator.ListenersCond.Wait()
+			atomic.AddInt64(&fs.filer.MetaAggregator.ListenersWaits, -1)
 			fs.filer.MetaAggregator.ListenersLock.Unlock()
 			return fs.hasClient(req.ClientId, req.ClientEpoch)
 		}, eachLogEntryFn)
 		if readInMemoryLogErr != nil {
 			if errors.Is(readInMemoryLogErr, log_buffer.ResumeFromDiskError) {
+				// Memory says data is too old - will read from disk on next iteration
+				// But if disk also has no data (gap in history), we'll skip forward
 				continue
 			}
 			glog.Errorf("processed to %v: %v", lastReadTime, readInMemoryLogErr)
@@ -150,39 +170,71 @@ func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq
 	var readPersistedLogErr error
 	var readInMemoryLogErr error
 	var isDone bool
+	var lastCheckedFlushTsNs int64 = -1 // Track the last flushed time we checked
+	var lastDiskReadTsNs int64 = -1     // Track the last read position we used for disk read
 
 	for {
-		// println("reading from persisted logs ...")
-		glog.V(0).Infof("read on disk %v local subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime)
-		processedTsNs, isDone, readPersistedLogErr = fs.filer.ReadPersistedLogBuffer(lastReadTime, req.UntilNs, eachLogEntryFn)
-		if readPersistedLogErr != nil {
-			glog.V(0).Infof("read on disk %v local subscribe %s from %+v: %v", clientName, req.PathPrefix, lastReadTime, readPersistedLogErr)
-			return fmt.Errorf("reading from persisted logs: %w", readPersistedLogErr)
-		}
-		if isDone {
-			return nil
-		}
+		// Check if new data has been flushed to disk since last check, or if read position advanced
+		currentFlushTsNs := fs.filer.LocalMetaLogBuffer.GetLastFlushTsNs()
+		currentReadTsNs := lastReadTime.Time.UnixNano()
+		// Read from disk if: first time, new flush observed, or read position advanced (draining backlog)
+		shouldReadFromDisk := lastCheckedFlushTsNs == -1 ||
+			currentFlushTsNs > lastCheckedFlushTsNs ||
+			currentReadTsNs > lastDiskReadTsNs
 
-		if processedTsNs != 0 {
-			lastReadTime = log_buffer.NewMessagePosition(processedTsNs, -2)
-		} else {
-			if readInMemoryLogErr == log_buffer.ResumeFromDiskError {
-				time.Sleep(1127 * time.Millisecond)
-				continue
+		if shouldReadFromDisk {
+			// Record the position we are about to read from
+			lastDiskReadTsNs = currentReadTsNs
+			glog.V(4).Infof("read on disk %v local subscribe %s from %+v (lastFlushed: %v)", clientName, req.PathPrefix, lastReadTime, time.Unix(0, currentFlushTsNs))
+			processedTsNs, isDone, readPersistedLogErr = fs.filer.ReadPersistedLogBuffer(lastReadTime, req.UntilNs, eachLogEntryFn)
+			if readPersistedLogErr != nil {
+				glog.V(0).Infof("read on disk %v local subscribe %s from %+v: %v", clientName, req.PathPrefix, lastReadTime, readPersistedLogErr)
+				return fmt.Errorf("reading from persisted logs: %w", readPersistedLogErr)
 			}
-			// If no persisted entries were read for this day, check the next day for logs
-			nextDayTs := util.GetNextDayTsNano(lastReadTime.Time.UnixNano())
-			position := log_buffer.NewMessagePosition(nextDayTs, -2)
-			found, err := fs.filer.HasPersistedLogFiles(position)
-			if err != nil {
-				return fmt.Errorf("checking persisted log files: %w", err)
+			if isDone {
+				return nil
 			}
-			if found {
-				lastReadTime = position
+
+			// Update the last checked flushed time
+			lastCheckedFlushTsNs = currentFlushTsNs
+
+			if processedTsNs != 0 {
+				lastReadTime = log_buffer.NewMessagePosition(processedTsNs, -2)
+			} else {
+				// No data found on disk
+				// Check if we previously got ResumeFromDiskError from memory, meaning we're in a gap
+				if readInMemoryLogErr == log_buffer.ResumeFromDiskError {
+					// We have a gap: requested time < earliest memory time, but no data on disk
+					// Skip forward to earliest memory time to avoid infinite loop
+					earliestTime := fs.filer.LocalMetaLogBuffer.GetEarliestTime()
+					if !earliestTime.IsZero() && earliestTime.After(lastReadTime.Time) {
+						glog.V(3).Infof("gap detected: skipping from %v to earliest memory time %v for %v",
+							lastReadTime.Time, earliestTime, clientName)
+						// Position at earliest time; time-based reader will include it
+						lastReadTime = log_buffer.NewMessagePosition(earliestTime.UnixNano(), -2)
+						readInMemoryLogErr = nil // Clear the error since we're skipping forward
+					} else {
+						// No memory data yet, just wait
+						time.Sleep(1127 * time.Millisecond)
+						continue
+					}
+				} else {
+					// First pass or no ResumeFromDiskError yet
+					// Check the next day for logs
+					nextDayTs := util.GetNextDayTsNano(lastReadTime.Time.UnixNano())
+					position := log_buffer.NewMessagePosition(nextDayTs, -2)
+					found, err := fs.filer.HasPersistedLogFiles(position)
+					if err != nil {
+						return fmt.Errorf("checking persisted log files: %w", err)
+					}
+					if found {
+						lastReadTime = position
+					}
+				}
 			}
 		}
 
-		glog.V(0).Infof("read in memory %v local subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime)
+		glog.V(3).Infof("read in memory %v local subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime)
 
 		lastReadTime, isDone, readInMemoryLogErr = fs.filer.LocalMetaLogBuffer.LoopProcessLogData("localMeta:"+clientName, lastReadTime, req.UntilNs, func() bool {
 
@@ -205,6 +257,23 @@ func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq
 		}, eachLogEntryFn)
 		if readInMemoryLogErr != nil {
 			if readInMemoryLogErr == log_buffer.ResumeFromDiskError {
+				// Memory buffer says the requested time is too old
+				// Retry disk read if: (a) flush advanced, or (b) read position advanced (draining backlog)
+				currentFlushTsNs := fs.filer.LocalMetaLogBuffer.GetLastFlushTsNs()
+				currentReadTsNs := lastReadTime.Time.UnixNano()
+				if currentFlushTsNs > lastCheckedFlushTsNs || currentReadTsNs > lastDiskReadTsNs {
+					glog.V(0).Infof("retry disk read %v local subscribe %s (lastFlushed: %v -> %v, readTs: %v -> %v)",
+						clientName, req.PathPrefix,
+						time.Unix(0, lastCheckedFlushTsNs), time.Unix(0, currentFlushTsNs),
+						time.Unix(0, lastDiskReadTsNs), time.Unix(0, currentReadTsNs))
+					continue
+				}
+				// No progress possible, wait for new data to arrive (event-driven, not polling)
+				fs.listenersLock.Lock()
+				atomic.AddInt64(&fs.listenersWaits, 1)
+				fs.listenersCond.Wait()
+				atomic.AddInt64(&fs.listenersWaits, -1)
+				fs.listenersLock.Unlock()
 				continue
 			}
 			glog.Errorf("processed to %v: %v", lastReadTime, readInMemoryLogErr)

weed/server/filer_server_handlers_read.go

@@ -192,9 +192,9 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 
 	// print out the header from extended properties
 	for k, v := range entry.Extended {
-		if !strings.HasPrefix(k, "xattr-") && !strings.HasPrefix(k, "x-seaweedfs-") {
+		if !strings.HasPrefix(k, "xattr-") && !s3_constants.IsSeaweedFSInternalHeader(k) {
 			// "xattr-" prefix is set in filesys.XATTR_PREFIX
-			// "x-seaweedfs-" prefix is for internal metadata that should not become HTTP headers
+			// IsSeaweedFSInternalHeader filters internal metadata that should not become HTTP headers
 			w.Header().Set(k, string(v))
 		}
 	}
@@ -241,6 +241,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		w.Header().Set(s3_constants.SeaweedFSSSEKMSKeyHeader, kmsBase64)
 	}
 
+	if _, exists := entry.Extended[s3_constants.SeaweedFSSSES3Key]; exists {
+		// Set standard S3 SSE-S3 response header (not the internal SeaweedFS header)
+		w.Header().Set(s3_constants.AmzServerSideEncryption, s3_constants.SSEAlgorithmAES256)
+	}
+
 	SetEtag(w, etag)
 
 	filename := entry.Name()

weed/server/filer_server_handlers_write_autochunk.go

@@ -377,6 +377,16 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
 		}
 	}
 
+	if sseS3Header := r.Header.Get(s3_constants.SeaweedFSSSES3Key); sseS3Header != "" {
+		// Decode base64-encoded S3 metadata and store
+		if s3Data, err := base64.StdEncoding.DecodeString(sseS3Header); err == nil {
+			entry.Extended[s3_constants.SeaweedFSSSES3Key] = s3Data
+			glog.V(4).Infof("Stored SSE-S3 metadata for %s", entry.FullPath)
+		} else {
+			glog.Errorf("Failed to decode SSE-S3 metadata header for %s: %v", entry.FullPath, err)
+		}
+	}
+
 	dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil, skipCheckParentDirEntry(r), so.MaxFileNameLength)
 	// In test_bucket_listv2_delimiter_basic, the valid object key is the parent folder
 	if dbErr != nil && strings.HasSuffix(dbErr.Error(), " is a file") && isS3Request(r) {

weed/shell/command_fs_cat.go

@@ -34,6 +34,10 @@ func (c *commandFsCat) HasTag(CommandTag) bool {
 
 func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
 
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
 	path, err := commandEnv.parseUrl(findInputDirectory(args))
 	if err != nil {
 		return err

weed/shell/command_fs_cd.go

@@ -34,6 +34,10 @@ func (c *commandFsCd) HasTag(CommandTag) bool {
 
 func (c *commandFsCd) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
 
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
 	path, err := commandEnv.parseUrl(findInputDirectory(args))
 	if err != nil {
 		return err

weed/shell/command_fs_du.go

@@ -36,6 +36,10 @@ func (c *commandFsDu) HasTag(CommandTag) bool {
 
 func (c *commandFsDu) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
 
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
 	path, err := commandEnv.parseUrl(findInputDirectory(args))
 	if err != nil {
 		return err

weed/shell/command_fs_ls.go

@@ -40,6 +40,10 @@ func (c *commandFsLs) HasTag(CommandTag) bool {
 
 func (c *commandFsLs) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
 
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
 	var isLongFormat, showHidden bool
 	for _, arg := range args {
 		if !strings.HasPrefix(arg, "-") {

weed/shell/command_fs_meta_cat.go

@@ -3,11 +3,12 @@ package shell
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/filer"
-	"google.golang.org/protobuf/proto"
 	"io"
 	"sort"
 
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"google.golang.org/protobuf/proto"
+
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
@@ -37,6 +38,10 @@ func (c *commandFsMetaCat) HasTag(CommandTag) bool {
 
 func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
 
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
 	path, err := commandEnv.parseUrl(findInputDirectory(args))
 	if err != nil {
 		return err

weed/shell/command_fs_meta_notify.go

@@ -36,6 +36,10 @@ func (c *commandFsMetaNotify) HasTag(CommandTag) bool {
 
 func (c *commandFsMetaNotify) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
 
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
 	path, err := commandEnv.parseUrl(findInputDirectory(args))
 	if err != nil {
 		return err

weed/shell/command_fs_mkdir.go

@@ -2,11 +2,12 @@ package shell
 
 import (
 	"context"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"io"
 	"os"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 func init() {
@@ -33,6 +34,10 @@ func (c *commandFsMkdir) HasTag(CommandTag) bool {
 
 func (c *commandFsMkdir) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
 
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
 	path, err := commandEnv.parseUrl(findInputDirectory(args))
 	if err != nil {
 		return err

weed/shell/command_fs_mv.go

@@ -40,6 +40,10 @@ func (c *commandFsMv) HasTag(CommandTag) bool {
 
 func (c *commandFsMv) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
 
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
 	if len(args) != 2 {
 		return fmt.Errorf("need to have 2 arguments")
 	}

weed/shell/command_fs_pwd.go

@@ -26,6 +26,10 @@ func (c *commandFsPwd) HasTag(CommandTag) bool {
 
 func (c *commandFsPwd) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
 
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
 	fmt.Fprintf(writer, "%s\n", commandEnv.option.Directory)
 
 	return nil

weed/shell/command_fs_rm.go

@@ -39,6 +39,11 @@ func (c *commandFsRm) HasTag(CommandTag) bool {
 }
 
 func (c *commandFsRm) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
 	isRecursive := false
 	ignoreRecursiveError := false
 	var entries []string

weed/shell/command_fs_tree.go

@@ -35,6 +35,10 @@ func (c *commandFsTree) HasTag(CommandTag) bool {
 
 func (c *commandFsTree) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
 
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
 	path, err := commandEnv.parseUrl(findInputDirectory(args))
 	if err != nil {
 		return err

weed/shell/command_volume_check_disk.go

@@ -11,6 +11,8 @@ import (
 	"sync"
 	"time"
 
+	"slices"
+
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
@@ -18,7 +20,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/server/constants"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
 	"google.golang.org/grpc"
-	"slices"
 )
 
 func init() {
@@ -321,13 +322,15 @@ func doVolumeCheckDisk(minuend, subtrahend *needle_map.MemDb, source, target *Vo
 				fmt.Fprintf(writer, "delete %s %s => %s\n", needleValue.Key.FileId(source.info.Id), source.location.dataNode.Id, target.location.dataNode.Id)
 			}
 		}
-		deleteResults, deleteErr := operation.DeleteFileIdsAtOneVolumeServer(
+		deleteResults := operation.DeleteFileIdsAtOneVolumeServer(
 			pb.NewServerAddressFromDataNode(target.location.dataNode),
 			grpcDialOption, fidList, false)
-		if deleteErr != nil {
-			return hasChanges, deleteErr
-		}
+
+		// Check for errors in results
 		for _, deleteResult := range deleteResults {
+			if deleteResult.Error != "" && deleteResult.Error != "not found" {
+				return hasChanges, fmt.Errorf("delete file %s: %v", deleteResult.FileId, deleteResult.Error)
+			}
 			if deleteResult.Status == http.StatusAccepted && deleteResult.Size > 0 {
 				hasChanges = true
 			}

weed/shell/command_volume_fix_replication.go

@@ -45,8 +45,8 @@ func (c *commandVolumeFixReplication) Help() string {
 	This command also finds all under-replicated volumes, and finds volume servers with free slots.
 	If the free slots satisfy the replication requirement, the volume content is copied over and mounted.
 
-	volume.fix.replication -n                             # do not take action
-	volume.fix.replication                                # actually deleting or copying the volume files and mount the volume
+	volume.fix.replication                                # do not take action
+	volume.fix.replication -force                         # actually deleting or copying the volume files and mount the volume
 	volume.fix.replication -collectionPattern=important*  # fix any collections with prefix "important"
 
 	Note:

weed/shell/command_volume_fsck.go

@@ -384,7 +384,12 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn
 	}
 
 	if !applyPurging {
-		pct := float64(totalOrphanChunkCount*100) / (float64(totalOrphanChunkCount + totalInUseCount))
+		var pct float64
+
+		if totalCount := totalOrphanChunkCount + totalInUseCount; totalCount > 0 {
+			pct = float64(totalOrphanChunkCount) * 100 / (float64(totalCount))
+		}
+
 		fmt.Fprintf(c.writer, "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n",
 			totalOrphanChunkCount+totalInUseCount, totalOrphanChunkCount, pct, totalOrphanDataSize)
 
@@ -697,9 +702,8 @@ func (c *commandVolumeFsck) purgeFileIdsForOneVolume(volumeId uint32, fileIds []
 		go func(server pb.ServerAddress, fidList []string) {
 			defer wg.Done()
 
-			if deleteResults, deleteErr := operation.DeleteFileIdsAtOneVolumeServer(server, c.env.option.GrpcDialOption, fidList, false); deleteErr != nil {
-				err = deleteErr
-			} else if deleteResults != nil {
+			deleteResults := operation.DeleteFileIdsAtOneVolumeServer(server, c.env.option.GrpcDialOption, fidList, false)
+			if deleteResults != nil {
 				resultChan <- deleteResults
 			}
 

weed/shell/commands.go

@@ -3,13 +3,15 @@ package shell
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/operation"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
+	"io"
 	"net/url"
 	"strconv"
 	"strings"
 
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
+
 	"google.golang.org/grpc"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -147,6 +149,37 @@ func findInputDirectory(args []string) (input string) {
 	return input
 }
 
+// isHelpRequest checks if the args contain a help flag (-h, --help, or -help)
+// It also handles combined short flags like -lh or -hl
+func isHelpRequest(args []string) bool {
+	for _, arg := range args {
+		// Check for exact matches
+		if arg == "-h" || arg == "--help" || arg == "-help" {
+			return true
+		}
+		// Check for combined short flags (e.g., -lh, -hl, -rfh)
+		// Limit to reasonable length (2-4 chars total) to avoid matching long options like -verbose
+		if strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "--") && len(arg) > 1 && len(arg) <= 4 {
+			for _, char := range arg[1:] {
+				if char == 'h' {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+// handleHelpRequest checks for help flags and prints the help message if requested.
+// It returns true if the help message was printed, indicating the command should exit.
+func handleHelpRequest(c command, args []string, writer io.Writer) bool {
+	if isHelpRequest(args) {
+		fmt.Fprintln(writer, c.Help())
+		return true
+	}
+	return false
+}
+
 func readNeedleMeta(grpcDialOption grpc.DialOption, volumeServer pb.ServerAddress, volumeId uint32, needleValue needle_map.NeedleValue) (resp *volume_server_pb.ReadNeedleMetaResponse, err error) {
 	err = operation.WithVolumeServerClient(false, volumeServer, grpcDialOption,
 		func(client volume_server_pb.VolumeServerClient) error {

weed/shell/shell_liner.go

@@ -84,6 +84,10 @@ func RunShell(options ShellOptions) {
 			return
 		}
 
+		if strings.TrimSpace(cmd) != "" {
+			line.AppendHistory(cmd)
+		}
+
 		for _, c := range util.StringSplit(cmd, ";") {
 			if processEachCmd(reg, c, commandEnv) {
 				return
@@ -95,8 +99,6 @@ func RunShell(options ShellOptions) {
 func processEachCmd(reg *regexp.Regexp, cmd string, commandEnv *CommandEnv) bool {
 	cmds := reg.FindAllString(cmd, -1)
 
-	line.AppendHistory(cmd)
-
 	if len(cmds) == 0 {
 		return false
 	} else {

weed/storage/disk_location.go

@@ -144,10 +144,26 @@ func (l *DiskLocation) loadExistingVolume(dirEntry os.DirEntry, needleMapKind Ne
 		return false
 	}
 
-	// skip if ec volumes exists
+	// parse out collection, volume id (moved up to use in EC validation)
+	vid, collection, err := volumeIdFromFileName(basename)
+	if err != nil {
+		glog.Warningf("get volume id failed, %s, err : %s", volumeName, err)
+		return false
+	}
+
+	// skip if ec volumes exists, but validate EC files first
 	if skipIfEcVolumesExists {
-		if util.FileExists(l.IdxDirectory + "/" + volumeName + ".ecx") {
-			return false
+		ecxFilePath := filepath.Join(l.IdxDirectory, volumeName+".ecx")
+		if util.FileExists(ecxFilePath) {
+			// Validate EC volume: shard count, size consistency, and expected size vs .dat file
+			if !l.validateEcVolume(collection, vid) {
+				glog.Warningf("EC volume %d validation failed, removing incomplete EC files to allow .dat file loading", vid)
+				l.removeEcVolumeFiles(collection, vid)
+				// Continue to load .dat file
+			} else {
+				// Valid EC volume exists, skip .dat file
+				return false
+			}
 		}
 	}
 
@@ -161,13 +177,6 @@ func (l *DiskLocation) loadExistingVolume(dirEntry os.DirEntry, needleMapKind Ne
 		return false
 	}
 
-	// parse out collection, volume id
-	vid, collection, err := volumeIdFromFileName(basename)
-	if err != nil {
-		glog.Warningf("get volume id failed, %s, err : %s", volumeName, err)
-		return false
-	}
-
 	// avoid loading one volume more than once
 	l.volumesLock.RLock()
 	_, found := l.volumes[vid]

weed/storage/disk_location_ec.go

@@ -10,6 +10,7 @@ import (
 
 	"slices"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 )
@@ -40,6 +41,23 @@ func (l *DiskLocation) DestroyEcVolume(vid needle.VolumeId) {
 	}
 }
 
+// unloadEcVolume removes an EC volume from memory without deleting its files on disk.
+// This is useful for distributed EC volumes where shards may be on other servers.
+func (l *DiskLocation) unloadEcVolume(vid needle.VolumeId) {
+	var toClose *erasure_coding.EcVolume
+	l.ecVolumesLock.Lock()
+	if ecVolume, found := l.ecVolumes[vid]; found {
+		toClose = ecVolume
+		delete(l.ecVolumes, vid)
+	}
+	l.ecVolumesLock.Unlock()
+
+	// Close outside the lock to avoid holding write lock during I/O
+	if toClose != nil {
+		toClose.Close()
+	}
+}
+
 func (l *DiskLocation) CollectEcShards(vid needle.VolumeId, shardFileNames []string) (ecVolume *erasure_coding.EcVolume, found bool) {
 	l.ecVolumesLock.RLock()
 	defer l.ecVolumesLock.RUnlock()
@@ -154,8 +172,18 @@ func (l *DiskLocation) loadAllEcShards() (err error) {
 	slices.SortFunc(dirEntries, func(a, b os.DirEntry) int {
 		return strings.Compare(a.Name(), b.Name())
 	})
+
 	var sameVolumeShards []string
 	var prevVolumeId needle.VolumeId
+	var prevCollection string
+
+	// Helper to reset state between volume processing
+	reset := func() {
+		sameVolumeShards = nil
+		prevVolumeId = 0
+		prevCollection = ""
+	}
+
 	for _, fileInfo := range dirEntries {
 		if fileInfo.IsDir() {
 			continue
@@ -178,24 +206,31 @@ func (l *DiskLocation) loadAllEcShards() (err error) {
 		// 0 byte files should be only appearing erroneously for ec data files
 		// so we ignore them
 		if re.MatchString(ext) && info.Size() > 0 {
-			if prevVolumeId == 0 || volumeId == prevVolumeId {
+			// Group shards by both collection and volumeId to avoid mixing collections
+			if prevVolumeId == 0 || (volumeId == prevVolumeId && collection == prevCollection) {
 				sameVolumeShards = append(sameVolumeShards, fileInfo.Name())
 			} else {
+				// Before starting a new group, check if previous group had orphaned shards
+				l.checkOrphanedShards(sameVolumeShards, prevCollection, prevVolumeId)
 				sameVolumeShards = []string{fileInfo.Name()}
 			}
 			prevVolumeId = volumeId
+			prevCollection = collection
 			continue
 		}
 
-		if ext == ".ecx" && volumeId == prevVolumeId {
-			if err = l.loadEcShards(sameVolumeShards, collection, volumeId); err != nil {
-				return fmt.Errorf("loadEcShards collection:%v volumeId:%d : %v", collection, volumeId, err)
-			}
-			prevVolumeId = volumeId
+		if ext == ".ecx" && volumeId == prevVolumeId && collection == prevCollection {
+			l.handleFoundEcxFile(sameVolumeShards, collection, volumeId)
+			reset()
 			continue
 		}
 
 	}
+
+	// Check for orphaned EC shards without .ecx file at the end of the directory scan
+	// This handles the last group of shards in the directory
+	l.checkOrphanedShards(sameVolumeShards, prevCollection, prevVolumeId)
+
 	return nil
 }
 
@@ -237,3 +272,208 @@ func (l *DiskLocation) EcShardCount() int {
 	}
 	return shardCount
 }
+
+// handleFoundEcxFile processes a complete group of EC shards when their .ecx file is found.
+// This includes validation, loading, and cleanup of incomplete/invalid EC volumes.
+func (l *DiskLocation) handleFoundEcxFile(shards []string, collection string, volumeId needle.VolumeId) {
+	// Check if this is an incomplete EC encoding (not a distributed EC volume)
+	// Key distinction: if .dat file still exists, EC encoding may have failed
+	// If .dat file is gone, this is likely a distributed EC volume with shards on multiple servers
+	baseFileName := erasure_coding.EcShardFileName(collection, l.Directory, int(volumeId))
+	datFileName := baseFileName + ".dat"
+
+	// Determine .dat presence robustly; unexpected errors are treated as "exists"
+	datExists := l.checkDatFileExists(datFileName)
+
+	// Validate EC volume if .dat file exists (incomplete EC encoding scenario)
+	// This checks shard count, shard size consistency, and expected size vs .dat file
+	// If .dat is gone, EC encoding completed and shards are distributed across servers
+	if datExists && !l.validateEcVolume(collection, volumeId) {
+		glog.Warningf("Incomplete or invalid EC volume %d: .dat exists but validation failed, cleaning up EC files...", volumeId)
+		l.removeEcVolumeFiles(collection, volumeId)
+		return
+	}
+
+	// Attempt to load the EC shards
+	if err := l.loadEcShards(shards, collection, volumeId); err != nil {
+		// If EC shards failed to load and .dat still exists, clean up EC files to allow .dat file to be used
+		// If .dat is gone, log error but don't clean up (may be waiting for shards from other servers)
+		if datExists {
+			glog.Warningf("Failed to load EC shards for volume %d and .dat exists: %v, cleaning up EC files to use .dat...", volumeId, err)
+			// Unload first to release FDs, then remove files
+			l.unloadEcVolume(volumeId)
+			l.removeEcVolumeFiles(collection, volumeId)
+		} else {
+			glog.Warningf("Failed to load EC shards for volume %d: %v (this may be normal for distributed EC volumes)", volumeId, err)
+			// Clean up any partially loaded in-memory state. This does not delete files.
+			l.unloadEcVolume(volumeId)
+		}
+		return
+	}
+}
+
+// checkDatFileExists checks if .dat file exists with robust error handling.
+// Unexpected errors (permission, I/O) are treated as "exists" to avoid misclassifying
+// local EC as distributed EC, which is the safer fallback.
+func (l *DiskLocation) checkDatFileExists(datFileName string) bool {
+	if _, err := os.Stat(datFileName); err == nil {
+		return true
+	} else if !os.IsNotExist(err) {
+		glog.Warningf("Failed to stat .dat file %s: %v", datFileName, err)
+		// Safer to assume local .dat exists to avoid misclassifying as distributed EC
+		return true
+	}
+	return false
+}
+
+// checkOrphanedShards checks if the given shards are orphaned (no .ecx file) and cleans them up if needed.
+// Returns true if orphaned shards were found and cleaned up.
+// This handles the case where EC encoding was interrupted before creating the .ecx file.
+func (l *DiskLocation) checkOrphanedShards(shards []string, collection string, volumeId needle.VolumeId) bool {
+	if len(shards) == 0 || volumeId == 0 {
+		return false
+	}
+
+	// Check if .dat file exists (incomplete encoding, not distributed EC)
+	baseFileName := erasure_coding.EcShardFileName(collection, l.Directory, int(volumeId))
+	datFileName := baseFileName + ".dat"
+
+	if l.checkDatFileExists(datFileName) {
+		glog.Warningf("Found %d EC shards without .ecx file for volume %d (incomplete encoding interrupted before .ecx creation), cleaning up...",
+			len(shards), volumeId)
+		l.removeEcVolumeFiles(collection, volumeId)
+		return true
+	}
+	return false
+}
+
+// calculateExpectedShardSize computes the exact expected shard size based on .dat file size
+// The EC encoding process is deterministic:
+// 1. Data is processed in batches of (LargeBlockSize * DataShardsCount) for large blocks
+// 2. Remaining data is processed in batches of (SmallBlockSize * DataShardsCount) for small blocks
+// 3. Each shard gets exactly its portion, with zero-padding applied to incomplete blocks
+func calculateExpectedShardSize(datFileSize int64) int64 {
+	var shardSize int64
+
+	// Process large blocks (1GB * 10 = 10GB batches)
+	largeBatchSize := int64(erasure_coding.ErasureCodingLargeBlockSize) * int64(erasure_coding.DataShardsCount)
+	numLargeBatches := datFileSize / largeBatchSize
+	shardSize = numLargeBatches * int64(erasure_coding.ErasureCodingLargeBlockSize)
+	remainingSize := datFileSize - (numLargeBatches * largeBatchSize)
+
+	// Process remaining data in small blocks (1MB * 10 = 10MB batches)
+	if remainingSize > 0 {
+		smallBatchSize := int64(erasure_coding.ErasureCodingSmallBlockSize) * int64(erasure_coding.DataShardsCount)
+		numSmallBatches := (remainingSize + smallBatchSize - 1) / smallBatchSize // Ceiling division
+		shardSize += numSmallBatches * int64(erasure_coding.ErasureCodingSmallBlockSize)
+	}
+
+	return shardSize
+}
+
+// validateEcVolume checks if EC volume has enough shards to be functional
+// For distributed EC volumes (where .dat is deleted), any number of shards is valid
+// For incomplete EC encoding (where .dat still exists), we need at least DataShardsCount shards
+// Also validates that all shards have the same size (required for Reed-Solomon EC)
+// If .dat exists, it also validates shards match the expected size based on .dat file size
+func (l *DiskLocation) validateEcVolume(collection string, vid needle.VolumeId) bool {
+	baseFileName := erasure_coding.EcShardFileName(collection, l.Directory, int(vid))
+	datFileName := baseFileName + ".dat"
+
+	var expectedShardSize int64 = -1
+	datExists := false
+
+	// If .dat file exists, compute exact expected shard size from it
+	if datFileInfo, err := os.Stat(datFileName); err == nil {
+		datExists = true
+		expectedShardSize = calculateExpectedShardSize(datFileInfo.Size())
+	} else if !os.IsNotExist(err) {
+		// If stat fails with unexpected error (permission, I/O), fail validation
+		// Don't treat this as "distributed EC" - it could be a temporary error
+		glog.Warningf("Failed to stat .dat file %s: %v", datFileName, err)
+		return false
+	}
+
+	shardCount := 0
+	var actualShardSize int64 = -1
+
+	// Count shards and validate they all have the same size (required for Reed-Solomon EC)
+	// Shard files (.ec00 - .ec13) are always in l.Directory, not l.IdxDirectory
+	for i := 0; i < erasure_coding.TotalShardsCount; i++ {
+		shardFileName := baseFileName + erasure_coding.ToExt(i)
+		fi, err := os.Stat(shardFileName)
+
+		if err == nil {
+			// Check if file has non-zero size
+			if fi.Size() > 0 {
+				// Validate all shards are the same size (required for Reed-Solomon EC)
+				if actualShardSize == -1 {
+					actualShardSize = fi.Size()
+				} else if fi.Size() != actualShardSize {
+					glog.Warningf("EC volume %d shard %d has size %d, expected %d (all EC shards must be same size)",
+						vid, i, fi.Size(), actualShardSize)
+					return false
+				}
+				shardCount++
+			}
+		} else if !os.IsNotExist(err) {
+			// If stat fails with unexpected error (permission, I/O), fail validation
+			// This is consistent with .dat file error handling
+			glog.Warningf("Failed to stat shard file %s: %v", shardFileName, err)
+			return false
+		}
+	}
+
+	// If .dat file exists, validate shard size matches expected size
+	if datExists && actualShardSize > 0 && expectedShardSize > 0 {
+		if actualShardSize != expectedShardSize {
+			glog.Warningf("EC volume %d: shard size %d doesn't match expected size %d (based on .dat file size)",
+				vid, actualShardSize, expectedShardSize)
+			return false
+		}
+	}
+
+	// If .dat file is gone, this is a distributed EC volume - any shard count is valid
+	if !datExists {
+		glog.V(1).Infof("EC volume %d: distributed EC (.dat removed) with %d shards", vid, shardCount)
+		return true
+	}
+
+	// If .dat file exists, we need at least DataShardsCount shards locally
+	// Otherwise it's an incomplete EC encoding that should be cleaned up
+	if shardCount < erasure_coding.DataShardsCount {
+		glog.Warningf("EC volume %d has .dat file but only %d shards (need at least %d for local EC)",
+			vid, shardCount, erasure_coding.DataShardsCount)
+		return false
+	}
+
+	return true
+}
+
+// removeEcVolumeFiles removes all EC-related files for a volume
+func (l *DiskLocation) removeEcVolumeFiles(collection string, vid needle.VolumeId) {
+	baseFileName := erasure_coding.EcShardFileName(collection, l.Directory, int(vid))
+	indexBaseFileName := erasure_coding.EcShardFileName(collection, l.IdxDirectory, int(vid))
+
+	// Helper to remove a file with consistent error handling
+	removeFile := func(filePath, description string) {
+		if err := os.Remove(filePath); err != nil {
+			if !os.IsNotExist(err) {
+				glog.Warningf("Failed to remove incomplete %s %s: %v", description, filePath, err)
+			}
+		} else {
+			glog.V(2).Infof("Removed incomplete %s: %s", description, filePath)
+		}
+	}
+
+	// Remove index files first (.ecx, .ecj) before shard files
+	// This ensures that if cleanup is interrupted, the .ecx file won't trigger
+	// EC loading for incomplete/missing shards on next startup
+	removeFile(indexBaseFileName+".ecx", "EC index file")
+	removeFile(indexBaseFileName+".ecj", "EC journal file")
+
+	// Remove all EC shard files (.ec00 ~ .ec13) from data directory
+	for i := 0; i < erasure_coding.TotalShardsCount; i++ {
+		removeFile(baseFileName+erasure_coding.ToExt(i), "EC shard file")
+	}
+}

weed/storage/disk_location_ec_realworld_test.go

@@ -0,0 +1,198 @@
+package storage
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// TestCalculateExpectedShardSizeWithRealEncoding validates our shard size calculation
+// by actually running EC encoding on real files and comparing the results
+func TestCalculateExpectedShardSizeWithRealEncoding(t *testing.T) {
+	tempDir := t.TempDir()
+
+	tests := []struct {
+		name        string
+		datFileSize int64
+		description string
+	}{
+		{
+			name:        "5MB file",
+			datFileSize: 5 * 1024 * 1024,
+			description: "Small file that needs 1 small block per shard",
+		},
+		{
+			name:        "10MB file (exactly 10 small blocks)",
+			datFileSize: 10 * 1024 * 1024,
+			description: "Exactly fits in 1MB small blocks",
+		},
+		{
+			name:        "15MB file",
+			datFileSize: 15 * 1024 * 1024,
+			description: "Requires 2 small blocks per shard",
+		},
+		{
+			name:        "50MB file",
+			datFileSize: 50 * 1024 * 1024,
+			description: "Requires 5 small blocks per shard",
+		},
+		{
+			name:        "100MB file",
+			datFileSize: 100 * 1024 * 1024,
+			description: "Requires 10 small blocks per shard",
+		},
+		{
+			name:        "512MB file",
+			datFileSize: 512 * 1024 * 1024,
+			description: "Requires 52 small blocks per shard (rounded up)",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Create a test .dat file with the specified size
+			baseFileName := filepath.Join(tempDir, "test_volume")
+			datFileName := baseFileName + ".dat"
+
+			// Create .dat file with random data pattern (so it's compressible but realistic)
+			datFile, err := os.Create(datFileName)
+			if err != nil {
+				t.Fatalf("Failed to create .dat file: %v", err)
+			}
+
+			// Write some pattern data (not all zeros, to be more realistic)
+			pattern := make([]byte, 4096)
+			for i := range pattern {
+				pattern[i] = byte(i % 256)
+			}
+
+			written := int64(0)
+			for written < tt.datFileSize {
+				toWrite := tt.datFileSize - written
+				if toWrite > int64(len(pattern)) {
+					toWrite = int64(len(pattern))
+				}
+				n, err := datFile.Write(pattern[:toWrite])
+				if err != nil {
+					t.Fatalf("Failed to write to .dat file: %v", err)
+				}
+				written += int64(n)
+			}
+			datFile.Close()
+
+			// Calculate expected shard size using our function
+			expectedShardSize := calculateExpectedShardSize(tt.datFileSize)
+
+			// Run actual EC encoding
+			err = erasure_coding.WriteEcFiles(baseFileName)
+			if err != nil {
+				t.Fatalf("Failed to encode EC files: %v", err)
+			}
+
+			// Measure actual shard sizes
+			for i := 0; i < erasure_coding.TotalShardsCount; i++ {
+				shardFileName := baseFileName + erasure_coding.ToExt(i)
+				shardInfo, err := os.Stat(shardFileName)
+				if err != nil {
+					t.Fatalf("Failed to stat shard file %s: %v", shardFileName, err)
+				}
+
+				actualShardSize := shardInfo.Size()
+
+				// Verify actual size matches expected size
+				if actualShardSize != expectedShardSize {
+					t.Errorf("Shard %d size mismatch:\n"+
+						"  .dat file size: %d bytes\n"+
+						"  Expected shard size: %d bytes\n"+
+						"  Actual shard size: %d bytes\n"+
+						"  Difference: %d bytes\n"+
+						"  %s",
+						i, tt.datFileSize, expectedShardSize, actualShardSize,
+						actualShardSize-expectedShardSize, tt.description)
+				}
+			}
+
+			// If we got here, all shards match!
+			t.Logf("✓ SUCCESS: .dat size %d → actual shard size %d matches calculated size (%s)",
+				tt.datFileSize, expectedShardSize, tt.description)
+
+			// Cleanup
+			os.Remove(datFileName)
+			for i := 0; i < erasure_coding.TotalShardsCount; i++ {
+				os.Remove(baseFileName + erasure_coding.ToExt(i))
+			}
+		})
+	}
+}
+
+// TestCalculateExpectedShardSizeEdgeCases tests edge cases with real encoding
+func TestCalculateExpectedShardSizeEdgeCases(t *testing.T) {
+	tempDir := t.TempDir()
+
+	tests := []struct {
+		name        string
+		datFileSize int64
+	}{
+		{"1 byte file", 1},
+		{"1KB file", 1024},
+		{"10KB file", 10 * 1024},
+		{"1MB file (1 small block)", 1024 * 1024},
+		{"1MB + 1 byte", 1024*1024 + 1},
+		{"9.9MB (almost 1 small block per shard)", 9*1024*1024 + 900*1024},
+		{"10.1MB (just over 1 small block per shard)", 10*1024*1024 + 100*1024},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			baseFileName := filepath.Join(tempDir, tt.name)
+			datFileName := baseFileName + ".dat"
+
+			// Create .dat file
+			datFile, err := os.Create(datFileName)
+			if err != nil {
+				t.Fatalf("Failed to create .dat file: %v", err)
+			}
+
+			// Write exactly the specified number of bytes
+			data := make([]byte, tt.datFileSize)
+			for i := range data {
+				data[i] = byte(i % 256)
+			}
+			datFile.Write(data)
+			datFile.Close()
+
+			// Calculate expected
+			expectedShardSize := calculateExpectedShardSize(tt.datFileSize)
+
+			// Run actual EC encoding
+			err = erasure_coding.WriteEcFiles(baseFileName)
+			if err != nil {
+				t.Fatalf("Failed to encode EC files: %v", err)
+			}
+
+			// Check first shard (all should be same size)
+			shardFileName := baseFileName + erasure_coding.ToExt(0)
+			shardInfo, err := os.Stat(shardFileName)
+			if err != nil {
+				t.Fatalf("Failed to stat shard file: %v", err)
+			}
+
+			actualShardSize := shardInfo.Size()
+
+			if actualShardSize != expectedShardSize {
+				t.Errorf("File size %d: expected shard %d, got %d (diff: %d)",
+					tt.datFileSize, expectedShardSize, actualShardSize, actualShardSize-expectedShardSize)
+			} else {
+				t.Logf("✓ File size %d → shard size %d (correct)", tt.datFileSize, actualShardSize)
+			}
+
+			// Cleanup
+			os.Remove(datFileName)
+			for i := 0; i < erasure_coding.TotalShardsCount; i++ {
+				os.Remove(baseFileName + erasure_coding.ToExt(i))
+			}
+		})
+	}
+}

weed/storage/disk_location_ec_shard_size_test.go

@@ -0,0 +1,195 @@
+package storage
+
+import (
+	"testing"
+)
+
+func TestCalculateExpectedShardSize(t *testing.T) {
+	const (
+		largeBlock     = 1024 * 1024 * 1024 // 1GB
+		smallBlock     = 1024 * 1024        // 1MB
+		dataShards     = 10
+		largeBatchSize = largeBlock * dataShards // 10GB
+		smallBatchSize = smallBlock * dataShards // 10MB
+	)
+
+	tests := []struct {
+		name              string
+		datFileSize       int64
+		expectedShardSize int64
+		description       string
+	}{
+		// Edge case: empty file
+		{
+			name:              "0 bytes (empty file)",
+			datFileSize:       0,
+			expectedShardSize: 0,
+			description:       "Empty file has 0 shard size",
+		},
+
+		// Boundary tests: exact multiples of large block
+		{
+			name:              "Exact 10GB (1 large batch)",
+			datFileSize:       largeBatchSize, // 10GB = 1 large batch
+			expectedShardSize: largeBlock,     // 1GB per shard
+			description:       "Exactly fits in large blocks",
+		},
+		{
+			name:              "Exact 20GB (2 large batches)",
+			datFileSize:       2 * largeBatchSize, // 20GB
+			expectedShardSize: 2 * largeBlock,     // 2GB per shard
+			description:       "2 complete large batches",
+		},
+		{
+			name:              "Just under large batch (10GB - 1 byte)",
+			datFileSize:       largeBatchSize - 1, // 10,737,418,239 bytes
+			expectedShardSize: 1024 * smallBlock,  // 1024MB = 1GB (needs 1024 small blocks)
+			description:       "Just under 10GB needs 1024 small blocks",
+		},
+		{
+			name:              "Just over large batch (10GB + 1 byte)",
+			datFileSize:       largeBatchSize + 1,      // 10GB + 1 byte
+			expectedShardSize: largeBlock + smallBlock, // 1GB + 1MB
+			description:       "Just over 10GB adds 1 small block",
+		},
+
+		// Boundary tests: exact multiples of small batch
+		{
+			name:              "Exact 10MB (1 small batch)",
+			datFileSize:       smallBatchSize, // 10MB
+			expectedShardSize: smallBlock,     // 1MB per shard
+			description:       "Exactly fits in 1 small batch",
+		},
+		{
+			name:              "Exact 20MB (2 small batches)",
+			datFileSize:       2 * smallBatchSize, // 20MB
+			expectedShardSize: 2 * smallBlock,     // 2MB per shard
+			description:       "2 complete small batches",
+		},
+		{
+			name:              "Just under small batch (10MB - 1 byte)",
+			datFileSize:       smallBatchSize - 1, // 10MB - 1 byte
+			expectedShardSize: smallBlock,         // Still needs 1MB per shard (rounds up)
+			description:       "Just under 10MB rounds up to 1 small block",
+		},
+		{
+			name:              "Just over small batch (10MB + 1 byte)",
+			datFileSize:       smallBatchSize + 1, // 10MB + 1 byte
+			expectedShardSize: 2 * smallBlock,     // 2MB per shard
+			description:       "Just over 10MB needs 2 small blocks",
+		},
+
+		// Mixed: large batch + partial small batch
+		{
+			name:              "10GB + 1MB",
+			datFileSize:       largeBatchSize + 1*1024*1024, // 10GB + 1MB
+			expectedShardSize: largeBlock + smallBlock,      // 1GB + 1MB
+			description:       "1 large batch + 1MB needs 1 small block",
+		},
+		{
+			name:              "10GB + 5MB",
+			datFileSize:       largeBatchSize + 5*1024*1024, // 10GB + 5MB
+			expectedShardSize: largeBlock + smallBlock,      // 1GB + 1MB
+			description:       "1 large batch + 5MB rounds up to 1 small block",
+		},
+		{
+			name:              "10GB + 15MB",
+			datFileSize:       largeBatchSize + 15*1024*1024, // 10GB + 15MB
+			expectedShardSize: largeBlock + 2*smallBlock,     // 1GB + 2MB
+			description:       "1 large batch + 15MB needs 2 small blocks",
+		},
+
+		// Original test cases
+		{
+			name:              "11GB (1 large batch + 103 small blocks)",
+			datFileSize:       11 * 1024 * 1024 * 1024,          // 11GB
+			expectedShardSize: 1*1024*1024*1024 + 103*1024*1024, // 1GB + 103MB (103 small blocks for 1GB remaining)
+			description:       "1GB large + 1GB remaining needs 103 small blocks",
+		},
+		{
+			name:              "5MB (requires 1 small block per shard)",
+			datFileSize:       5 * 1024 * 1024, // 5MB
+			expectedShardSize: 1 * 1024 * 1024, // 1MB per shard (rounded up)
+			description:       "Small file rounds up to 1MB per shard",
+		},
+		{
+			name:              "1KB (minimum size)",
+			datFileSize:       1024,
+			expectedShardSize: 1 * 1024 * 1024, // 1MB per shard (1 small block)
+			description:       "Tiny file needs 1 small block",
+		},
+		{
+			name:              "10.5GB (mixed)",
+			datFileSize:       10*1024*1024*1024 + 512*1024*1024, // 10.5GB
+			expectedShardSize: 1*1024*1024*1024 + 52*1024*1024,   // 1GB + 52MB (52 small blocks for 512MB remaining)
+			description:       "1GB large + 512MB remaining needs 52 small blocks",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actualShardSize := calculateExpectedShardSize(tt.datFileSize)
+
+			if actualShardSize != tt.expectedShardSize {
+				t.Errorf("Expected shard size %d, got %d. %s",
+					tt.expectedShardSize, actualShardSize, tt.description)
+			}
+
+			t.Logf("✓ File size: %d → Shard size: %d (%s)",
+				tt.datFileSize, actualShardSize, tt.description)
+		})
+	}
+}
+
+// TestShardSizeValidationScenarios tests realistic scenarios
+func TestShardSizeValidationScenarios(t *testing.T) {
+	scenarios := []struct {
+		name            string
+		datFileSize     int64
+		actualShardSize int64
+		shouldBeValid   bool
+	}{
+		{
+			name:            "Valid: exact match for 10GB",
+			datFileSize:     10 * 1024 * 1024 * 1024, // 10GB
+			actualShardSize: 1 * 1024 * 1024 * 1024,  // 1GB (exact)
+			shouldBeValid:   true,
+		},
+		{
+			name:            "Invalid: 1 byte too small",
+			datFileSize:     10 * 1024 * 1024 * 1024, // 10GB
+			actualShardSize: 1*1024*1024*1024 - 1,    // 1GB - 1 byte
+			shouldBeValid:   false,
+		},
+		{
+			name:            "Invalid: 1 byte too large",
+			datFileSize:     10 * 1024 * 1024 * 1024, // 10GB
+			actualShardSize: 1*1024*1024*1024 + 1,    // 1GB + 1 byte
+			shouldBeValid:   false,
+		},
+		{
+			name:            "Valid: small file exact match",
+			datFileSize:     5 * 1024 * 1024, // 5MB
+			actualShardSize: 1 * 1024 * 1024, // 1MB (exact)
+			shouldBeValid:   true,
+		},
+		{
+			name:            "Invalid: wrong size for small file",
+			datFileSize:     5 * 1024 * 1024, // 5MB
+			actualShardSize: 500 * 1024,      // 500KB (too small)
+			shouldBeValid:   false,
+		},
+	}
+
+	for _, scenario := range scenarios {
+		t.Run(scenario.name, func(t *testing.T) {
+			expectedSize := calculateExpectedShardSize(scenario.datFileSize)
+			isValid := scenario.actualShardSize == expectedSize
+
+			if isValid != scenario.shouldBeValid {
+				t.Errorf("Expected validation result %v, got %v. Actual shard: %d, Expected: %d",
+					scenario.shouldBeValid, isValid, scenario.actualShardSize, expectedSize)
+			}
+		})
+	}
+}

weed/storage/disk_location_ec_test.go

@@ -0,0 +1,643 @@
+package storage
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// TestIncompleteEcEncodingCleanup tests the cleanup logic for incomplete EC encoding scenarios
+func TestIncompleteEcEncodingCleanup(t *testing.T) {
+	tests := []struct {
+		name              string
+		volumeId          needle.VolumeId
+		collection        string
+		createDatFile     bool
+		createEcxFile     bool
+		createEcjFile     bool
+		numShards         int
+		expectCleanup     bool
+		expectLoadSuccess bool
+	}{
+		{
+			name:              "Incomplete EC: shards without .ecx, .dat exists - should cleanup",
+			volumeId:          100,
+			collection:        "",
+			createDatFile:     true,
+			createEcxFile:     false,
+			createEcjFile:     false,
+			numShards:         14, // All shards but no .ecx
+			expectCleanup:     true,
+			expectLoadSuccess: false,
+		},
+		{
+			name:              "Distributed EC: shards without .ecx, .dat deleted - should NOT cleanup",
+			volumeId:          101,
+			collection:        "",
+			createDatFile:     false,
+			createEcxFile:     false,
+			createEcjFile:     false,
+			numShards:         5, // Partial shards, distributed
+			expectCleanup:     false,
+			expectLoadSuccess: false,
+		},
+		{
+			name:              "Incomplete EC: shards with .ecx but < 10 shards, .dat exists - should cleanup",
+			volumeId:          102,
+			collection:        "",
+			createDatFile:     true,
+			createEcxFile:     true,
+			createEcjFile:     false,
+			numShards:         7, // Less than DataShardsCount (10)
+			expectCleanup:     true,
+			expectLoadSuccess: false,
+		},
+		{
+			name:              "Valid local EC: shards with .ecx, >= 10 shards, .dat exists - should load",
+			volumeId:          103,
+			collection:        "",
+			createDatFile:     true,
+			createEcxFile:     true,
+			createEcjFile:     false,
+			numShards:         14, // All shards
+			expectCleanup:     false,
+			expectLoadSuccess: true, // Would succeed if .ecx was valid
+		},
+		{
+			name:              "Distributed EC: shards with .ecx, .dat deleted - should load",
+			volumeId:          104,
+			collection:        "",
+			createDatFile:     false,
+			createEcxFile:     true,
+			createEcjFile:     false,
+			numShards:         10, // Enough shards
+			expectCleanup:     false,
+			expectLoadSuccess: true, // Would succeed if .ecx was valid
+		},
+		{
+			name:              "Incomplete EC with collection: shards without .ecx, .dat exists - should cleanup",
+			volumeId:          105,
+			collection:        "test_collection",
+			createDatFile:     true,
+			createEcxFile:     false,
+			createEcjFile:     false,
+			numShards:         14,
+			expectCleanup:     true,
+			expectLoadSuccess: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Use per-subtest temp directory for stronger isolation
+			tempDir := t.TempDir()
+
+			// Create DiskLocation
+			minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}
+			diskLocation := &DiskLocation{
+				Directory:              tempDir,
+				DirectoryUuid:          "test-uuid",
+				IdxDirectory:           tempDir,
+				DiskType:               types.HddType,
+				MaxVolumeCount:         100,
+				OriginalMaxVolumeCount: 100,
+				MinFreeSpace:           minFreeSpace,
+			}
+			diskLocation.volumes = make(map[needle.VolumeId]*Volume)
+			diskLocation.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
+
+			// Setup test files
+			baseFileName := erasure_coding.EcShardFileName(tt.collection, tempDir, int(tt.volumeId))
+
+			// Use deterministic but small size: 10MB .dat => 1MB per shard
+			datFileSize := int64(10 * 1024 * 1024) // 10MB
+			expectedShardSize := calculateExpectedShardSize(datFileSize)
+
+			// Create .dat file if needed
+			if tt.createDatFile {
+				datFile, err := os.Create(baseFileName + ".dat")
+				if err != nil {
+					t.Fatalf("Failed to create .dat file: %v", err)
+				}
+				if err := datFile.Truncate(datFileSize); err != nil {
+					t.Fatalf("Failed to truncate .dat file: %v", err)
+				}
+				if err := datFile.Close(); err != nil {
+					t.Fatalf("Failed to close .dat file: %v", err)
+				}
+			}
+
+			// Create EC shard files
+			for i := 0; i < tt.numShards; i++ {
+				shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i))
+				if err != nil {
+					t.Fatalf("Failed to create shard file: %v", err)
+				}
+				if err := shardFile.Truncate(expectedShardSize); err != nil {
+					t.Fatalf("Failed to truncate shard file: %v", err)
+				}
+				if err := shardFile.Close(); err != nil {
+					t.Fatalf("Failed to close shard file: %v", err)
+				}
+			}
+
+			// Create .ecx file if needed
+			if tt.createEcxFile {
+				ecxFile, err := os.Create(baseFileName + ".ecx")
+				if err != nil {
+					t.Fatalf("Failed to create .ecx file: %v", err)
+				}
+				if _, err := ecxFile.WriteString("dummy ecx data"); err != nil {
+					ecxFile.Close()
+					t.Fatalf("Failed to write .ecx file: %v", err)
+				}
+				if err := ecxFile.Close(); err != nil {
+					t.Fatalf("Failed to close .ecx file: %v", err)
+				}
+			}
+
+			// Create .ecj file if needed
+			if tt.createEcjFile {
+				ecjFile, err := os.Create(baseFileName + ".ecj")
+				if err != nil {
+					t.Fatalf("Failed to create .ecj file: %v", err)
+				}
+				if _, err := ecjFile.WriteString("dummy ecj data"); err != nil {
+					ecjFile.Close()
+					t.Fatalf("Failed to write .ecj file: %v", err)
+				}
+				if err := ecjFile.Close(); err != nil {
+					t.Fatalf("Failed to close .ecj file: %v", err)
+				}
+			}
+
+			// Run loadAllEcShards
+			loadErr := diskLocation.loadAllEcShards()
+			if loadErr != nil {
+				t.Logf("loadAllEcShards returned error (expected in some cases): %v", loadErr)
+			}
+
+			// Test idempotency - running again should not cause issues
+			loadErr2 := diskLocation.loadAllEcShards()
+			if loadErr2 != nil {
+				t.Logf("Second loadAllEcShards returned error: %v", loadErr2)
+			}
+
+			// Verify cleanup expectations
+			if tt.expectCleanup {
+				// Check that files were cleaned up
+				if util.FileExists(baseFileName + ".ecx") {
+					t.Errorf("Expected .ecx to be cleaned up but it still exists")
+				}
+				if util.FileExists(baseFileName + ".ecj") {
+					t.Errorf("Expected .ecj to be cleaned up but it still exists")
+				}
+				for i := 0; i < erasure_coding.TotalShardsCount; i++ {
+					shardFile := baseFileName + erasure_coding.ToExt(i)
+					if util.FileExists(shardFile) {
+						t.Errorf("Expected shard %d to be cleaned up but it still exists", i)
+					}
+				}
+				// .dat file should still exist (not cleaned up)
+				if tt.createDatFile && !util.FileExists(baseFileName+".dat") {
+					t.Errorf("Expected .dat file to remain but it was deleted")
+				}
+			} else {
+				// Check that files were NOT cleaned up
+				for i := 0; i < tt.numShards; i++ {
+					shardFile := baseFileName + erasure_coding.ToExt(i)
+					if !util.FileExists(shardFile) {
+						t.Errorf("Expected shard %d to remain but it was cleaned up", i)
+					}
+				}
+				if tt.createEcxFile && !util.FileExists(baseFileName+".ecx") {
+					t.Errorf("Expected .ecx to remain but it was cleaned up")
+				}
+			}
+
+			// Verify load expectations
+			if tt.expectLoadSuccess {
+				if diskLocation.EcShardCount() == 0 {
+					t.Errorf("Expected EC shards to be loaded for volume %d", tt.volumeId)
+				}
+			}
+
+		})
+	}
+}
+
+// TestValidateEcVolume tests the validateEcVolume function
+func TestValidateEcVolume(t *testing.T) {
+	tempDir := t.TempDir()
+
+	minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}
+	diskLocation := &DiskLocation{
+		Directory:     tempDir,
+		DirectoryUuid: "test-uuid",
+		IdxDirectory:  tempDir,
+		DiskType:      types.HddType,
+		MinFreeSpace:  minFreeSpace,
+	}
+
+	tests := []struct {
+		name          string
+		volumeId      needle.VolumeId
+		collection    string
+		createDatFile bool
+		numShards     int
+		expectValid   bool
+	}{
+		{
+			name:          "Valid: .dat exists with 10+ shards",
+			volumeId:      200,
+			collection:    "",
+			createDatFile: true,
+			numShards:     10,
+			expectValid:   true,
+		},
+		{
+			name:          "Invalid: .dat exists with < 10 shards",
+			volumeId:      201,
+			collection:    "",
+			createDatFile: true,
+			numShards:     9,
+			expectValid:   false,
+		},
+		{
+			name:          "Valid: .dat deleted (distributed EC) with any shards",
+			volumeId:      202,
+			collection:    "",
+			createDatFile: false,
+			numShards:     5,
+			expectValid:   true,
+		},
+		{
+			name:          "Valid: .dat deleted (distributed EC) with no shards",
+			volumeId:      203,
+			collection:    "",
+			createDatFile: false,
+			numShards:     0,
+			expectValid:   true,
+		},
+		{
+			name:          "Invalid: zero-byte shard files should not count",
+			volumeId:      204,
+			collection:    "",
+			createDatFile: true,
+			numShards:     0, // Will create 10 zero-byte files below
+			expectValid:   false,
+		},
+		{
+			name:          "Invalid: .dat exists with different size shards",
+			volumeId:      205,
+			collection:    "",
+			createDatFile: true,
+			numShards:     10, // Will create shards with varying sizes
+			expectValid:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			baseFileName := erasure_coding.EcShardFileName(tt.collection, tempDir, int(tt.volumeId))
+
+			// For proper testing, we need to use realistic sizes that match EC encoding
+			// EC uses large blocks (1GB) and small blocks (1MB)
+			// For test purposes, use a small .dat file size that still exercises the logic
+			// 10MB .dat file = 1MB per shard (one small batch, fast and deterministic)
+			datFileSize := int64(10 * 1024 * 1024) // 10MB
+			expectedShardSize := calculateExpectedShardSize(datFileSize)
+
+			// Create .dat file if needed
+			if tt.createDatFile {
+				datFile, err := os.Create(baseFileName + ".dat")
+				if err != nil {
+					t.Fatalf("Failed to create .dat file: %v", err)
+				}
+				// Write minimal data (don't need to fill entire 10GB for tests)
+				datFile.Truncate(datFileSize)
+				datFile.Close()
+			}
+
+			// Create EC shard files with correct size
+			for i := 0; i < tt.numShards; i++ {
+				shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i))
+				if err != nil {
+					t.Fatalf("Failed to create shard file: %v", err)
+				}
+				// Use truncate to create file of correct size without allocating all the space
+				if err := shardFile.Truncate(expectedShardSize); err != nil {
+					shardFile.Close()
+					t.Fatalf("Failed to truncate shard file: %v", err)
+				}
+				if err := shardFile.Close(); err != nil {
+					t.Fatalf("Failed to close shard file: %v", err)
+				}
+			}
+
+			// For zero-byte test case, create empty files for all data shards
+			if tt.volumeId == 204 {
+				for i := 0; i < erasure_coding.DataShardsCount; i++ {
+					shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i))
+					if err != nil {
+						t.Fatalf("Failed to create empty shard file: %v", err)
+					}
+					// Don't write anything - leave as zero-byte
+					shardFile.Close()
+				}
+			}
+
+			// For mismatched shard size test case, create shards with different sizes
+			if tt.volumeId == 205 {
+				for i := 0; i < erasure_coding.DataShardsCount; i++ {
+					shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i))
+					if err != nil {
+						t.Fatalf("Failed to create shard file: %v", err)
+					}
+					// Write different amount of data to each shard
+					data := make([]byte, 100+i*10)
+					shardFile.Write(data)
+					shardFile.Close()
+				}
+			}
+
+			// Test validation
+			isValid := diskLocation.validateEcVolume(tt.collection, tt.volumeId)
+			if isValid != tt.expectValid {
+				t.Errorf("Expected validation result %v but got %v", tt.expectValid, isValid)
+			}
+		})
+	}
+}
+
+// TestRemoveEcVolumeFiles tests the removeEcVolumeFiles function
+func TestRemoveEcVolumeFiles(t *testing.T) {
+	tests := []struct {
+		name           string
+		separateIdxDir bool
+	}{
+		{"Same directory for data and index", false},
+		{"Separate idx directory", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tempDir := t.TempDir()
+
+			var dataDir, idxDir string
+			if tt.separateIdxDir {
+				dataDir = filepath.Join(tempDir, "data")
+				idxDir = filepath.Join(tempDir, "idx")
+				os.MkdirAll(dataDir, 0755)
+				os.MkdirAll(idxDir, 0755)
+			} else {
+				dataDir = tempDir
+				idxDir = tempDir
+			}
+
+			minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}
+			diskLocation := &DiskLocation{
+				Directory:     dataDir,
+				DirectoryUuid: "test-uuid",
+				IdxDirectory:  idxDir,
+				DiskType:      types.HddType,
+				MinFreeSpace:  minFreeSpace,
+			}
+
+			volumeId := needle.VolumeId(300)
+			collection := ""
+			dataBaseFileName := erasure_coding.EcShardFileName(collection, dataDir, int(volumeId))
+			idxBaseFileName := erasure_coding.EcShardFileName(collection, idxDir, int(volumeId))
+
+			// Create all EC shard files in data directory
+			for i := 0; i < erasure_coding.TotalShardsCount; i++ {
+				shardFile, err := os.Create(dataBaseFileName + erasure_coding.ToExt(i))
+				if err != nil {
+					t.Fatalf("Failed to create shard file: %v", err)
+				}
+				if _, err := shardFile.WriteString("dummy shard data"); err != nil {
+					shardFile.Close()
+					t.Fatalf("Failed to write shard file: %v", err)
+				}
+				if err := shardFile.Close(); err != nil {
+					t.Fatalf("Failed to close shard file: %v", err)
+				}
+			}
+
+			// Create .ecx file in idx directory
+			ecxFile, err := os.Create(idxBaseFileName + ".ecx")
+			if err != nil {
+				t.Fatalf("Failed to create .ecx file: %v", err)
+			}
+			if _, err := ecxFile.WriteString("dummy ecx data"); err != nil {
+				ecxFile.Close()
+				t.Fatalf("Failed to write .ecx file: %v", err)
+			}
+			if err := ecxFile.Close(); err != nil {
+				t.Fatalf("Failed to close .ecx file: %v", err)
+			}
+
+			// Create .ecj file in idx directory
+			ecjFile, err := os.Create(idxBaseFileName + ".ecj")
+			if err != nil {
+				t.Fatalf("Failed to create .ecj file: %v", err)
+			}
+			if _, err := ecjFile.WriteString("dummy ecj data"); err != nil {
+				ecjFile.Close()
+				t.Fatalf("Failed to write .ecj file: %v", err)
+			}
+			if err := ecjFile.Close(); err != nil {
+				t.Fatalf("Failed to close .ecj file: %v", err)
+			}
+
+			// Create .dat file in data directory (should NOT be removed)
+			datFile, err := os.Create(dataBaseFileName + ".dat")
+			if err != nil {
+				t.Fatalf("Failed to create .dat file: %v", err)
+			}
+			if _, err := datFile.WriteString("dummy dat data"); err != nil {
+				datFile.Close()
+				t.Fatalf("Failed to write .dat file: %v", err)
+			}
+			if err := datFile.Close(); err != nil {
+				t.Fatalf("Failed to close .dat file: %v", err)
+			}
+
+			// Call removeEcVolumeFiles
+			diskLocation.removeEcVolumeFiles(collection, volumeId)
+
+			// Verify all EC shard files are removed from data directory
+			for i := 0; i < erasure_coding.TotalShardsCount; i++ {
+				shardFile := dataBaseFileName + erasure_coding.ToExt(i)
+				if util.FileExists(shardFile) {
+					t.Errorf("Shard file %d should be removed but still exists", i)
+				}
+			}
+
+			// Verify .ecx file is removed from idx directory
+			if util.FileExists(idxBaseFileName + ".ecx") {
+				t.Errorf(".ecx file should be removed but still exists")
+			}
+
+			// Verify .ecj file is removed from idx directory
+			if util.FileExists(idxBaseFileName + ".ecj") {
+				t.Errorf(".ecj file should be removed but still exists")
+			}
+
+			// Verify .dat file is NOT removed from data directory
+			if !util.FileExists(dataBaseFileName + ".dat") {
+				t.Errorf(".dat file should NOT be removed but was deleted")
+			}
+		})
+	}
+}
+
+// TestEcCleanupWithSeparateIdxDirectory tests EC cleanup when idx directory is different
+func TestEcCleanupWithSeparateIdxDirectory(t *testing.T) {
+	tempDir := t.TempDir()
+
+	idxDir := filepath.Join(tempDir, "idx")
+	dataDir := filepath.Join(tempDir, "data")
+	os.MkdirAll(idxDir, 0755)
+	os.MkdirAll(dataDir, 0755)
+
+	minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}
+	diskLocation := &DiskLocation{
+		Directory:     dataDir,
+		DirectoryUuid: "test-uuid",
+		IdxDirectory:  idxDir,
+		DiskType:      types.HddType,
+		MinFreeSpace:  minFreeSpace,
+	}
+	diskLocation.volumes = make(map[needle.VolumeId]*Volume)
+	diskLocation.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
+
+	volumeId := needle.VolumeId(400)
+	collection := ""
+
+	// Create shards in data directory (shards only go to Directory, not IdxDirectory)
+	dataBaseFileName := erasure_coding.EcShardFileName(collection, dataDir, int(volumeId))
+	for i := 0; i < erasure_coding.TotalShardsCount; i++ {
+		shardFile, err := os.Create(dataBaseFileName + erasure_coding.ToExt(i))
+		if err != nil {
+			t.Fatalf("Failed to create shard file: %v", err)
+		}
+		if _, err := shardFile.WriteString("dummy shard data"); err != nil {
+			t.Fatalf("Failed to write shard file: %v", err)
+		}
+		if err := shardFile.Close(); err != nil {
+			t.Fatalf("Failed to close shard file: %v", err)
+		}
+	}
+
+	// Create .dat in data directory
+	datFile, err := os.Create(dataBaseFileName + ".dat")
+	if err != nil {
+		t.Fatalf("Failed to create .dat file: %v", err)
+	}
+	if _, err := datFile.WriteString("dummy data"); err != nil {
+		t.Fatalf("Failed to write .dat file: %v", err)
+	}
+	if err := datFile.Close(); err != nil {
+		t.Fatalf("Failed to close .dat file: %v", err)
+	}
+
+	// Do not create .ecx: trigger orphaned-shards cleanup when .dat exists
+
+	// Run loadAllEcShards
+	loadErr := diskLocation.loadAllEcShards()
+	if loadErr != nil {
+		t.Logf("loadAllEcShards error: %v", loadErr)
+	}
+
+	// Verify cleanup occurred in data directory (shards)
+	for i := 0; i < erasure_coding.TotalShardsCount; i++ {
+		shardFile := dataBaseFileName + erasure_coding.ToExt(i)
+		if util.FileExists(shardFile) {
+			t.Errorf("Shard file %d should be cleaned up but still exists", i)
+		}
+	}
+
+	// Verify .dat in data directory still exists (only EC files are cleaned up)
+	if !util.FileExists(dataBaseFileName + ".dat") {
+		t.Errorf(".dat file should remain but was deleted")
+	}
+}
+
+// TestDistributedEcVolumeNoFileDeletion verifies that distributed EC volumes
+// (where .dat is deleted) do NOT have their shard files deleted when load fails
+// This tests the critical bug fix where DestroyEcVolume was incorrectly deleting files
+func TestDistributedEcVolumeNoFileDeletion(t *testing.T) {
+	tempDir := t.TempDir()
+
+	minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}
+	diskLocation := &DiskLocation{
+		Directory:     tempDir,
+		DirectoryUuid: "test-uuid",
+		IdxDirectory:  tempDir,
+		DiskType:      types.HddType,
+		MinFreeSpace:  minFreeSpace,
+		ecVolumes:     make(map[needle.VolumeId]*erasure_coding.EcVolume),
+	}
+
+	collection := ""
+	volumeId := needle.VolumeId(500)
+	baseFileName := erasure_coding.EcShardFileName(collection, tempDir, int(volumeId))
+
+	// Create EC shards (only 5 shards - less than DataShardsCount, but OK for distributed EC)
+	numDistributedShards := 5
+	for i := 0; i < numDistributedShards; i++ {
+		shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i))
+		if err != nil {
+			t.Fatalf("Failed to create shard file: %v", err)
+		}
+		if _, err := shardFile.WriteString("dummy shard data"); err != nil {
+			shardFile.Close()
+			t.Fatalf("Failed to write shard file: %v", err)
+		}
+		if err := shardFile.Close(); err != nil {
+			t.Fatalf("Failed to close shard file: %v", err)
+		}
+	}
+
+	// Create .ecx file to trigger EC loading
+	ecxFile, err := os.Create(baseFileName + ".ecx")
+	if err != nil {
+		t.Fatalf("Failed to create .ecx file: %v", err)
+	}
+	if _, err := ecxFile.WriteString("dummy ecx data"); err != nil {
+		ecxFile.Close()
+		t.Fatalf("Failed to write .ecx file: %v", err)
+	}
+	if err := ecxFile.Close(); err != nil {
+		t.Fatalf("Failed to close .ecx file: %v", err)
+	}
+
+	// NO .dat file - this is a distributed EC volume
+
+	// Run loadAllEcShards - this should fail but NOT delete shard files
+	loadErr := diskLocation.loadAllEcShards()
+	if loadErr != nil {
+		t.Logf("loadAllEcShards returned error (expected): %v", loadErr)
+	}
+
+	// CRITICAL CHECK: Verify shard files still exist (should NOT be deleted)
+	for i := 0; i < 5; i++ {
+		shardFile := baseFileName + erasure_coding.ToExt(i)
+		if !util.FileExists(shardFile) {
+			t.Errorf("CRITICAL BUG: Shard file %s was deleted for distributed EC volume!", shardFile)
+		}
+	}
+
+	// Verify .ecx file still exists (should NOT be deleted for distributed EC)
+	if !util.FileExists(baseFileName + ".ecx") {
+		t.Errorf("CRITICAL BUG: .ecx file was deleted for distributed EC volume!")
+	}
+
+	t.Logf("SUCCESS: Distributed EC volume files preserved (not deleted)")
+}

weed/storage/volume_loading.go

@@ -55,6 +55,19 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
 		if err := v.LoadRemoteFile(); err != nil {
 			return fmt.Errorf("load remote file %v: %w", v.volumeInfo, err)
 		}
+		// Set lastModifiedTsSeconds from remote file to prevent premature expiry on startup
+		if len(v.volumeInfo.GetFiles()) > 0 {
+			remoteFileModifiedTime := v.volumeInfo.GetFiles()[0].GetModifiedTime()
+			if remoteFileModifiedTime > 0 {
+				v.lastModifiedTsSeconds = remoteFileModifiedTime
+			} else {
+				// Fallback: use .vif file's modification time
+				if exists, _, _, modifiedTime, _ := util.CheckFile(v.FileName(".vif")); exists {
+					v.lastModifiedTsSeconds = uint64(modifiedTime.Unix())
+				}
+			}
+			glog.V(1).Infof("volume %d remote file lastModifiedTsSeconds set to %d", v.Id, v.lastModifiedTsSeconds)
+		}
 		alreadyHasSuperBlock = true
 	} else if exists, canRead, canWrite, modifiedTime, fileSize := util.CheckFile(v.FileName(".dat")); exists {
 		// open dat file

weed/topology/volume_growth_reservation_test.go

@@ -81,7 +81,11 @@ func TestVolumeGrowth_ReservationBasedAllocation(t *testing.T) {
 		}
 
 		// Simulate successful volume creation
+		// Acquire lock briefly to access children map, then release before updating
+		dn.RLock()
 		disk := dn.children[NodeId(types.HardDriveType.String())].(*Disk)
+		dn.RUnlock()
+
 		deltaDiskUsage := &DiskUsageCounts{
 			volumeCount: 1,
 		}
@@ -135,6 +139,7 @@ func TestVolumeGrowth_ConcurrentAllocationPreventsRaceCondition(t *testing.T) {
 	const concurrentRequests = 10
 	var wg sync.WaitGroup
 	var successCount, failureCount atomic.Int32
+	var commitMutex sync.Mutex // Ensures atomic commit of volume creation + reservation release
 
 	for i := 0; i < concurrentRequests; i++ {
 		wg.Add(1)
@@ -152,8 +157,14 @@ func TestVolumeGrowth_ConcurrentAllocationPreventsRaceCondition(t *testing.T) {
 
 				// Simulate completion: increment volume count BEFORE releasing reservation
 				if reservation != nil {
+					commitMutex.Lock()
+
 					// First, increment the volume count to reflect the created volume
+					// Acquire lock briefly to access children map, then release before updating
+					dn.RLock()
 					disk := dn.children[NodeId(types.HardDriveType.String())].(*Disk)
+					dn.RUnlock()
+
 					deltaDiskUsage := &DiskUsageCounts{
 						volumeCount: 1,
 					}
@@ -161,6 +172,8 @@ func TestVolumeGrowth_ConcurrentAllocationPreventsRaceCondition(t *testing.T) {
 
 					// Then release the reservation
 					reservation.releaseAllReservations()
+
+					commitMutex.Unlock()
 				}
 			}
 		}(i)
@@ -168,23 +181,35 @@ func TestVolumeGrowth_ConcurrentAllocationPreventsRaceCondition(t *testing.T) {
 
 	wg.Wait()
 
-	// With reservation system, only 5 requests should succeed (capacity limit)
-	// The rest should fail due to insufficient capacity
-	if successCount.Load() != 5 {
-		t.Errorf("Expected exactly 5 successful reservations, got %d", successCount.Load())
+	// Collect results
+	successes := successCount.Load()
+	failures := failureCount.Load()
+	total := successes + failures
+
+	if total != concurrentRequests {
+		t.Fatalf("Expected %d total attempts recorded, got %d", concurrentRequests, total)
 	}
 
-	if failureCount.Load() != 5 {
-		t.Errorf("Expected exactly 5 failed reservations, got %d", failureCount.Load())
+	// At most the available capacity should succeed
+	const capacity = 5
+	if successes > capacity {
+		t.Errorf("Expected no more than %d successful reservations, got %d", capacity, successes)
 	}
 
-	// Verify final state
+	// We should see at least the remaining attempts fail
+	minExpectedFailures := concurrentRequests - capacity
+	if failures < int32(minExpectedFailures) {
+		t.Errorf("Expected at least %d failed reservations, got %d", minExpectedFailures, failures)
+	}
+
+	// Verify final state matches the number of successful allocations
 	finalAvailable := dn.AvailableSpaceFor(option)
-	if finalAvailable != 0 {
-		t.Errorf("Expected 0 available space after all allocations, got %d", finalAvailable)
+	expectedAvailable := int64(capacity - successes)
+	if finalAvailable != expectedAvailable {
+		t.Errorf("Expected %d available space after allocations, got %d", expectedAvailable, finalAvailable)
 	}
 
-	t.Logf("Concurrent test completed: %d successes, %d failures", successCount.Load(), failureCount.Load())
+	t.Logf("Concurrent test completed: %d successes, %d failures", successes, failures)
 }
 
 func TestVolumeGrowth_ReservationFailureRollback(t *testing.T) {

weed/util/http/http_global_client_util.go

@@ -305,11 +305,7 @@ func ReadUrl(ctx context.Context, fileUrl string, cipherKey []byte, isContentCom
 	return n, err
 }
 
-func ReadUrlAsStream(ctx context.Context, fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) {
-	return ReadUrlAsStreamAuthenticated(ctx, fileUrl, "", cipherKey, isContentGzipped, isFullChunk, offset, size, fn)
-}
-
-func ReadUrlAsStreamAuthenticated(ctx context.Context, fileUrl, jwt string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) {
+func ReadUrlAsStream(ctx context.Context, fileUrl, jwt string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) {
 	if cipherKey != nil {
 		return readEncryptedUrl(ctx, fileUrl, jwt, cipherKey, isContentGzipped, isFullChunk, offset, size, fn)
 	}
@@ -509,7 +505,7 @@ func RetriedFetchChunkData(ctx context.Context, buffer []byte, urlStrings []stri
 			if strings.Contains(urlString, "%") {
 				urlString = url.PathEscape(urlString)
 			}
-			shouldRetry, err = ReadUrlAsStreamAuthenticated(ctx, urlString+"?readDeleted=true", string(jwt), cipherKey, isGzipped, isFullChunk, offset, len(buffer), func(data []byte) {
+			shouldRetry, err = ReadUrlAsStream(ctx, urlString+"?readDeleted=true", string(jwt), cipherKey, isGzipped, isFullChunk, offset, len(buffer), func(data []byte) {
 				// Check for context cancellation during data processing
 				select {
 				case <-ctx.Done():

weed/util/log_buffer/log_buffer.go

@@ -77,7 +77,7 @@ type LogBuffer struct {
 	maxOffset         int64
 	hasOffsets        bool
 	lastFlushedOffset atomic.Int64 // Highest offset that has been flushed to disk (-1 = nothing flushed yet)
-	lastFlushedTime   atomic.Int64 // Latest timestamp that has been flushed to disk (0 = nothing flushed yet)
+	lastFlushTsNs     atomic.Int64 // Latest timestamp that has been flushed to disk (0 = nothing flushed yet)
 	// Disk chunk cache for historical data reads
 	diskChunkCache *DiskChunkCache
 	sync.RWMutex
@@ -235,14 +235,14 @@ func (logBuffer *LogBuffer) InitializeOffsetFromExistingData(getHighestOffsetFn
 		// Set the next offset to be one after the highest existing offset
 		nextOffset := highestOffset + 1
 		logBuffer.offset = nextOffset
-		// CRITICAL FIX: bufferStartOffset should match offset after initialization
+		// bufferStartOffset should match offset after initialization
 		// This ensures that reads for old offsets (0...highestOffset) will trigger disk reads
 		// New data written after this will start at nextOffset
 		logBuffer.bufferStartOffset = nextOffset
 		// CRITICAL: Track that data [0...highestOffset] is on disk
 		logBuffer.lastFlushedOffset.Store(highestOffset)
 		// Set lastFlushedTime to current time (we know data up to highestOffset is on disk)
-		logBuffer.lastFlushedTime.Store(time.Now().UnixNano())
+		logBuffer.lastFlushTsNs.Store(time.Now().UnixNano())
 		glog.V(0).Infof("Initialized LogBuffer %s offset to %d (highest existing: %d), buffer starts at %d, lastFlushedOffset=%d, lastFlushedTime=%v",
 			logBuffer.name, nextOffset, highestOffset, nextOffset, highestOffset, time.Now())
 	} else {
@@ -299,7 +299,7 @@ func (logBuffer *LogBuffer) AddLogEntryToBuffer(logEntry *filer_pb.LogEntry) {
 	}
 
 	// Track offset ranges for Kafka integration
-	// CRITICAL FIX: Use >= 0 to include offset 0 (first message in a topic)
+	// Use >= 0 to include offset 0 (first message in a topic)
 	if logEntry.Offset >= 0 {
 		if !logBuffer.hasOffsets {
 			logBuffer.minOffset = logEntry.Offset
@@ -385,7 +385,7 @@ func (logBuffer *LogBuffer) AddDataToBuffer(partitionKey, data []byte, processin
 		logBuffer.LastTsNs.Store(processingTsNs)
 	}
 
-	// CRITICAL FIX: Set the offset in the LogEntry before marshaling
+	// Set the offset in the LogEntry before marshaling
 	// This ensures the flushed data contains the correct offset information
 	// Note: This also enables AddToBuffer to work correctly with Kafka-style offset-based reads
 	logEntry.Offset = logBuffer.offset
@@ -414,7 +414,7 @@ func (logBuffer *LogBuffer) AddDataToBuffer(partitionKey, data []byte, processin
 	}
 
 	// Track offset ranges for Kafka integration
-	// CRITICAL FIX: Track the current offset being written
+	// Track the current offset being written
 	if !logBuffer.hasOffsets {
 		logBuffer.minOffset = logBuffer.offset
 		logBuffer.maxOffset = logBuffer.offset
@@ -518,12 +518,12 @@ func (logBuffer *LogBuffer) loopFlush() {
 			logBuffer.lastFlushDataTime = d.stopTime
 
 			// CRITICAL: Track what's been flushed to disk for both offset-based and time-based reads
-			// CRITICAL FIX: Use >= 0 to include offset 0 (first message in a topic)
+			// Use >= 0 to include offset 0 (first message in a topic)
 			if d.maxOffset >= 0 {
 				logBuffer.lastFlushedOffset.Store(d.maxOffset)
 			}
 			if !d.stopTime.IsZero() {
-				logBuffer.lastFlushedTime.Store(d.stopTime.UnixNano())
+				logBuffer.lastFlushTsNs.Store(d.stopTime.UnixNano())
 			}
 
 			// Signal completion if there's a callback channel
@@ -586,8 +586,10 @@ func (logBuffer *LogBuffer) copyToFlushInternal(withCallback bool) *dataToFlush
 		// CRITICAL: logBuffer.offset is the "next offset to assign", so last offset in buffer is offset-1
 		lastOffsetInBuffer := logBuffer.offset - 1
 		logBuffer.buf = logBuffer.prevBuffers.SealBuffer(logBuffer.startTime, logBuffer.stopTime, logBuffer.buf, logBuffer.pos, logBuffer.bufferStartOffset, lastOffsetInBuffer)
-		logBuffer.startTime = time.Unix(0, 0)
-		logBuffer.stopTime = time.Unix(0, 0)
+		// Use zero time (time.Time{}) not epoch time (time.Unix(0,0))
+		// Epoch time (1970) breaks time-based reads after flush
+		logBuffer.startTime = time.Time{}
+		logBuffer.stopTime = time.Time{}
 		logBuffer.pos = 0
 		logBuffer.idx = logBuffer.idx[:0]
 		// DON'T increment offset - it's already pointing to the next offset!
@@ -598,7 +600,7 @@ func (logBuffer *LogBuffer) copyToFlushInternal(withCallback bool) *dataToFlush
 		logBuffer.minOffset = 0
 		logBuffer.maxOffset = 0
 
-		// CRITICAL FIX: Invalidate disk cache chunks after flush
+		// Invalidate disk cache chunks after flush
 		// The cache may contain stale data from before this flush
 		// Invalidating ensures consumers will re-read fresh data from disk after flush
 		logBuffer.invalidateAllDiskCacheChunks()
@@ -629,6 +631,12 @@ func (logBuffer *LogBuffer) GetEarliestPosition() MessagePosition {
 	}
 }
 
+// GetLastFlushTsNs returns the latest flushed timestamp in Unix nanoseconds.
+// Returns 0 if nothing has been flushed yet.
+func (logBuffer *LogBuffer) GetLastFlushTsNs() int64 {
+	return logBuffer.lastFlushTsNs.Load()
+}
+
 func (d *dataToFlush) releaseMemory() {
 	d.data.Reset()
 	bufferPool.Put(d.data)
@@ -639,8 +647,10 @@ func (logBuffer *LogBuffer) ReadFromBuffer(lastReadPosition MessagePosition) (bu
 	defer logBuffer.RUnlock()
 
 	isOffsetBased := lastReadPosition.IsOffsetBased
+	glog.V(2).Infof("[ReadFromBuffer] %s: isOffsetBased=%v, position=%+v, bufferStartOffset=%d, offset=%d, pos=%d",
+		logBuffer.name, isOffsetBased, lastReadPosition, logBuffer.bufferStartOffset, logBuffer.offset, logBuffer.pos)
 
-	// CRITICAL FIX: For offset-based subscriptions, use offset comparisons, not time comparisons!
+	// For offset-based subscriptions, use offset comparisons, not time comparisons!
 	if isOffsetBased {
 		requestedOffset := lastReadPosition.Offset
 
@@ -648,7 +658,7 @@ func (logBuffer *LogBuffer) ReadFromBuffer(lastReadPosition MessagePosition) (bu
 		if requestedOffset >= logBuffer.bufferStartOffset && requestedOffset <= logBuffer.offset {
 			// If current buffer is empty (pos=0), check if data is on disk or not yet written
 			if logBuffer.pos == 0 {
-				// CRITICAL FIX: If buffer is empty but offset range covers the request,
+				// If buffer is empty but offset range covers the request,
 				// it means data was in memory and has been flushed/moved out.
 				// The bufferStartOffset advancing to cover this offset proves data existed.
 				//
@@ -719,15 +729,22 @@ func (logBuffer *LogBuffer) ReadFromBuffer(lastReadPosition MessagePosition) (bu
 	if !logBuffer.startTime.IsZero() {
 		tsMemory = logBuffer.startTime
 	}
-	for _, prevBuf := range logBuffer.prevBuffers.buffers {
-		if !prevBuf.startTime.IsZero() && prevBuf.startTime.Before(tsMemory) {
-			tsMemory = prevBuf.startTime
+	glog.V(2).Infof("[ReadFromBuffer] %s: checking prevBuffers, count=%d, currentStartTime=%v",
+		logBuffer.name, len(logBuffer.prevBuffers.buffers), logBuffer.startTime)
+	for i, prevBuf := range logBuffer.prevBuffers.buffers {
+		glog.V(2).Infof("[ReadFromBuffer] %s: prevBuf[%d]: startTime=%v stopTime=%v size=%d startOffset=%d endOffset=%d",
+			logBuffer.name, i, prevBuf.startTime, prevBuf.stopTime, prevBuf.size, prevBuf.startOffset, prevBuf.offset)
+		if !prevBuf.startTime.IsZero() {
+			// If tsMemory is zero, assign directly; otherwise compare
+			if tsMemory.IsZero() || prevBuf.startTime.Before(tsMemory) {
+				tsMemory = prevBuf.startTime
+			}
 		}
 	}
 	if tsMemory.IsZero() { // case 2.2
 		return nil, -2, nil
 	} else if lastReadPosition.Time.Before(tsMemory) { // case 2.3
-		// CRITICAL FIX: For time-based reads, only check timestamp for disk reads
+		// For time-based reads, only check timestamp for disk reads
 		// Don't use offset comparisons as they're not meaningful for time-based subscriptions
 
 		// Special case: If requested time is zero (Unix epoch), treat as "start from beginning"
@@ -735,45 +752,67 @@ func (logBuffer *LogBuffer) ReadFromBuffer(lastReadPosition MessagePosition) (bu
 		if lastReadPosition.Time.IsZero() || lastReadPosition.Time.Unix() == 0 {
 			// Start from the beginning of memory
 			// Fall through to case 2.1 to read from earliest buffer
-		} else if lastReadPosition.Offset == 0 && lastReadPosition.Time.Before(tsMemory) {
-			// CRITICAL FIX: If this is the first read (offset=0) and time is slightly before memory,
-			// it's likely a race between starting to read and first message being written
-			// Fall through to case 2.1 to read from earliest buffer instead of triggering disk read
-			glog.V(2).Infof("first read at time %v before earliest memory %v, reading from memory",
-				lastReadPosition.Time, tsMemory)
+		} else if lastReadPosition.Offset <= 0 && lastReadPosition.Time.Before(tsMemory) {
+			// Treat first read with sentinel/zero offset as inclusive of earliest in-memory data
+			glog.V(4).Infof("first read (offset=%d) at time %v before earliest memory %v, reading from memory",
+				lastReadPosition.Offset, lastReadPosition.Time, tsMemory)
 		} else {
 			// Data not in memory buffers - read from disk
-			glog.V(0).Infof("resume from disk: requested time %v < earliest memory time %v",
-				lastReadPosition.Time, tsMemory)
+			glog.V(0).Infof("[ReadFromBuffer] %s resume from disk: requested time %v < earliest memory time %v",
+				logBuffer.name, lastReadPosition.Time, tsMemory)
 			return nil, -2, ResumeFromDiskError
 		}
 	}
 
+	glog.V(2).Infof("[ReadFromBuffer] %s: time-based read continuing, tsMemory=%v, lastReadPos=%v",
+		logBuffer.name, tsMemory, lastReadPosition.Time)
+
 	// the following is case 2.1
 
-	if lastReadPosition.Time.Equal(logBuffer.stopTime) {
-		return nil, logBuffer.offset, nil
+	if lastReadPosition.Time.Equal(logBuffer.stopTime) && !logBuffer.stopTime.IsZero() {
+		// For first-read sentinel/zero offset, allow inclusive read at the boundary
+		if lastReadPosition.Offset > 0 {
+			return nil, logBuffer.offset, nil
+		}
 	}
-	if lastReadPosition.Time.After(logBuffer.stopTime) {
+	if lastReadPosition.Time.After(logBuffer.stopTime) && !logBuffer.stopTime.IsZero() {
 		// glog.Fatalf("unexpected last read time %v, older than latest %v", lastReadPosition, m.stopTime)
 		return nil, logBuffer.offset, nil
 	}
-	if lastReadPosition.Time.Before(logBuffer.startTime) {
+	// Also check prevBuffers when current buffer is empty (startTime is zero)
+	if lastReadPosition.Time.Before(logBuffer.startTime) || logBuffer.startTime.IsZero() {
 		for _, buf := range logBuffer.prevBuffers.buffers {
 			if buf.startTime.After(lastReadPosition.Time) {
 				// glog.V(4).Infof("%s return the %d sealed buffer %v", m.name, i, buf.startTime)
 				return copiedBytes(buf.buf[:buf.size]), buf.offset, nil
 			}
 			if !buf.startTime.After(lastReadPosition.Time) && buf.stopTime.After(lastReadPosition.Time) {
-				pos := buf.locateByTs(lastReadPosition.Time)
+				searchTime := lastReadPosition.Time
+				if lastReadPosition.Offset <= 0 {
+					searchTime = searchTime.Add(-time.Nanosecond)
+				}
+				pos := buf.locateByTs(searchTime)
+				glog.V(2).Infof("[ReadFromBuffer] %s: found data in prevBuffer at pos %d, bufSize=%d", logBuffer.name, pos, buf.size)
 				return copiedBytes(buf.buf[pos:buf.size]), buf.offset, nil
 			}
 		}
-		// glog.V(4).Infof("%s return the current buf %v", m.name, lastReadPosition)
-		return copiedBytes(logBuffer.buf[:logBuffer.pos]), logBuffer.offset, nil
+		// If current buffer is not empty, return it
+		if logBuffer.pos > 0 {
+			// glog.V(4).Infof("%s return the current buf %v", m.name, lastReadPosition)
+			return copiedBytes(logBuffer.buf[:logBuffer.pos]), logBuffer.offset, nil
+		}
+		// Buffer is empty and no data in prevBuffers - wait for new data
+		return nil, logBuffer.offset, nil
 	}
 
 	lastTs := lastReadPosition.Time.UnixNano()
+	// Inclusive boundary for first-read sentinel/zero offset
+	searchTs := lastTs
+	if lastReadPosition.Offset <= 0 {
+		if searchTs > math.MinInt64+1 { // prevent underflow
+			searchTs = searchTs - 1
+		}
+	}
 	l, h := 0, len(logBuffer.idx)-1
 
 	/*
@@ -792,14 +831,14 @@ func (logBuffer *LogBuffer) ReadFromBuffer(lastReadPosition MessagePosition) (bu
 		mid := (l + h) / 2
 		pos := logBuffer.idx[mid]
 		_, t := readTs(logBuffer.buf, pos)
-		if t <= lastTs {
+		if t <= searchTs {
 			l = mid + 1
-		} else if lastTs < t {
+		} else if searchTs < t {
 			var prevT int64
 			if mid > 0 {
 				_, prevT = readTs(logBuffer.buf, logBuffer.idx[mid-1])
 			}
-			if prevT <= lastTs {
+			if prevT <= searchTs {
 				return copiedBytes(logBuffer.buf[pos:logBuffer.pos]), logBuffer.offset, nil
 			}
 			h = mid

weed/util/log_buffer/log_buffer_queryability_test.go

@@ -236,3 +236,58 @@ func TestSchemaRegistryScenario(t *testing.T) {
 
 	t.Logf("Schema registry scenario test passed - schema value preserved: %d bytes", len(retrievedEntry.Data))
 }
+
+// TestTimeBasedFirstReadBeforeEarliest ensures starting slightly before earliest memory
+// does not force a disk resume and returns in-memory data (regression test)
+func TestTimeBasedFirstReadBeforeEarliest(t *testing.T) {
+	flushed := false
+	logBuffer := NewLogBuffer("local", 10*time.Minute,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+			// keep in memory; we just want earliest time populated
+			_ = buf
+		},
+		func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+			// disk should not be consulted in this regression path
+			return startPosition, false, nil
+		},
+		func() {})
+
+	// Seed one entry so earliestTime is set
+	baseTs := time.Now().Add(-time.Second)
+	entry := &filer_pb.LogEntry{TsNs: baseTs.UnixNano(), Data: []byte("x"), Key: []byte("k"), Offset: 0}
+	logBuffer.AddLogEntryToBuffer(entry)
+	_ = flushed
+
+	// Start read 1ns before earliest memory, with offset sentinel (-2)
+	startPos := NewMessagePosition(baseTs.Add(-time.Nanosecond).UnixNano(), -2)
+	buf, _, err := logBuffer.ReadFromBuffer(startPos)
+	if err != nil {
+		t.Fatalf("ReadFromBuffer returned err: %v", err)
+	}
+	if buf == nil {
+		t.Fatalf("Expected in-memory data, got nil buffer")
+	}
+}
+
+// TestEarliestTimeExactRead ensures starting exactly at earliest time returns first entry (no skip)
+func TestEarliestTimeExactRead(t *testing.T) {
+	logBuffer := NewLogBuffer("local", 10*time.Minute,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {},
+		func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+			return startPosition, false, nil
+		},
+		func() {})
+
+	ts := time.Now()
+	entry := &filer_pb.LogEntry{TsNs: ts.UnixNano(), Data: []byte("a"), Key: []byte("k"), Offset: 0}
+	logBuffer.AddLogEntryToBuffer(entry)
+
+	startPos := NewMessagePosition(ts.UnixNano(), -2)
+	buf, _, err := logBuffer.ReadFromBuffer(startPos)
+	if err != nil {
+		t.Fatalf("ReadFromBuffer err: %v", err)
+	}
+	if buf == nil || buf.Len() == 0 {
+		t.Fatalf("Expected data at earliest time, got nil/empty")
+	}
+}

weed/util/log_buffer/log_read.go

@@ -270,7 +270,7 @@ func (logBuffer *LogBuffer) LoopProcessLogDataWithOffset(readerName string, star
 				return
 			}
 
-			// CRITICAL FIX: If we're reading offset-based and there's no data in LogBuffer,
+			// If we're reading offset-based and there's no data in LogBuffer,
 			// return ResumeFromDiskError to let Subscribe try reading from disk again.
 			// This prevents infinite blocking when all data is on disk (e.g., after restart).
 			if startPosition.IsOffsetBased {
@@ -355,7 +355,6 @@ func (logBuffer *LogBuffer) LoopProcessLogDataWithOffset(readerName string, star
 				continue
 			}
 
-
 			glog.V(4).Infof("Unmarshaled log entry %d: TsNs=%d, Offset=%d, Key=%s", batchSize+1, logEntry.TsNs, logEntry.Offset, string(logEntry.Key))
 
 			// Handle offset-based filtering for offset-based start positions
@@ -377,7 +376,7 @@ func (logBuffer *LogBuffer) LoopProcessLogDataWithOffset(readerName string, star
 				// println("stopTsNs", stopTsNs, "logEntry.TsNs", logEntry.TsNs)
 				return
 			}
-			// CRITICAL FIX: Use logEntry.Offset + 1 to move PAST the current entry
+			// Use logEntry.Offset + 1 to move PAST the current entry
 			// This prevents infinite loops where we keep requesting the same offset
 			lastReadPosition = NewMessagePosition(logEntry.TsNs, logEntry.Offset+1)
 

weed/util/log_buffer/log_read_stateless.go

@@ -121,7 +121,7 @@ func (logBuffer *LogBuffer) ReadMessagesAtOffset(startOffset int64, maxMessages
 		logBuffer.RUnlock()
 
 		// Data not in memory - attempt disk read if configured
-		// CRITICAL FIX: Don't return error here - data may be on disk!
+		// Don't return error here - data may be on disk!
 		// Fall through to disk read logic below
 		glog.V(2).Infof("[StatelessRead] Data at offset %d not in memory (buffer: %d-%d), attempting disk read",
 			startOffset, bufferStartOffset, currentBufferEnd)

weed/util/version/constants.go

@@ -8,7 +8,7 @@ import (
 
 var (
 	MAJOR_VERSION  = int32(3)
-	MINOR_VERSION  = int32(98)
+	MINOR_VERSION  = int32(99)
 	VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION)
 	VERSION        = util.SizeLimit + " " + VERSION_NUMBER
 	COMMIT         = ""

weed/worker/tasks/balance/balance_task.go

@@ -106,15 +106,8 @@ func (t *BalanceTask) Execute(ctx context.Context, params *worker_pb.TaskParams)
 		glog.Warningf("Tail operation failed (may be normal): %v", err)
 	}
 
-	// Step 5: Unmount from source
-	t.ReportProgress(85.0)
-	t.GetLogger().Info("Unmounting volume from source server")
-	if err := t.unmountVolume(sourceServer, volumeId); err != nil {
-		return fmt.Errorf("failed to unmount volume from source: %v", err)
-	}
-
-	// Step 6: Delete from source
-	t.ReportProgress(95.0)
+	// Step 5: Delete from source
+	t.ReportProgress(90.0)
 	t.GetLogger().Info("Deleting volume from source server")
 	if err := t.deleteVolume(sourceServer, volumeId); err != nil {
 		return fmt.Errorf("failed to delete volume from source: %v", err)

weed/worker/tasks/task_logger.go

@@ -232,6 +232,7 @@ func (l *FileTaskLogger) LogWithFields(level string, message string, fields map[
 
 // Close closes the logger and finalizes metadata
 func (l *FileTaskLogger) Close() error {
+	l.Info("Task logger closed for %s", l.taskID)
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 
@@ -260,7 +261,6 @@ func (l *FileTaskLogger) Close() error {
 	}
 
 	l.closed = true
-	l.Info("Task logger closed for %s", l.taskID)
 
 	return nil
 }

weed/worker/worker.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -23,20 +22,55 @@ import (
 
 // Worker represents a maintenance worker instance
 type Worker struct {
-	id              string
-	config          *types.WorkerConfig
-	registry        *tasks.TaskRegistry
-	currentTasks    map[string]*types.TaskInput
-	adminClient     AdminClient
+	id             string
+	config         *types.WorkerConfig
+	registry       *tasks.TaskRegistry
+	cmds           chan workerCommand
+	state          *workerState
+	taskLogHandler *tasks.TaskLogHandler
+}
+type workerState struct {
 	running         bool
-	stopChan        chan struct{}
-	mutex           sync.RWMutex
+	adminClient     AdminClient
 	startTime       time.Time
-	tasksCompleted  int
-	tasksFailed     int
+	stopChan        chan struct{}
 	heartbeatTicker *time.Ticker
 	requestTicker   *time.Ticker
-	taskLogHandler  *tasks.TaskLogHandler
+	currentTasks    map[string]*types.TaskInput
+	tasksCompleted  int
+	tasksFailed     int
+}
+
+type workerAction string
+
+const (
+	ActionStart             workerAction = "start"
+	ActionStop              workerAction = "stop"
+	ActionGetStatus         workerAction = "getstatus"
+	ActionGetTaskLoad       workerAction = "getload"
+	ActionSetTask           workerAction = "settask"
+	ActionSetAdmin          workerAction = "setadmin"
+	ActionRemoveTask        workerAction = "removetask"
+	ActionGetAdmin          workerAction = "getadmin"
+	ActionIncTaskFail       workerAction = "inctaskfail"
+	ActionIncTaskComplete   workerAction = "inctaskcomplete"
+	ActionGetHbTick         workerAction = "gethbtick"
+	ActionGetReqTick        workerAction = "getreqtick"
+	ActionGetStopChan       workerAction = "getstopchan"
+	ActionSetHbTick         workerAction = "sethbtick"
+	ActionSetReqTick        workerAction = "setreqtick"
+	ActionGetStartTime      workerAction = "getstarttime"
+	ActionGetCompletedTasks workerAction = "getcompletedtasks"
+	ActionGetFailedTasks    workerAction = "getfailedtasks"
+	ActionCancelTask        workerAction = "canceltask"
+	// ... other worker actions like Stop, Status, etc.
+)
+
+type statusResponse chan types.WorkerStatus
+type workerCommand struct {
+	action workerAction
+	data   any
+	resp   chan error // for reporting success/failure
 }
 
 // AdminClient defines the interface for communicating with the admin server
@@ -150,17 +184,223 @@ func NewWorker(config *types.WorkerConfig) (*Worker, error) {
 		id:             workerID,
 		config:         config,
 		registry:       registry,
-		currentTasks:   make(map[string]*types.TaskInput),
-		stopChan:       make(chan struct{}),
-		startTime:      time.Now(),
 		taskLogHandler: taskLogHandler,
+		cmds:           make(chan workerCommand),
 	}
 
 	glog.V(1).Infof("Worker created with %d registered task types", len(registry.GetAll()))
-
+	go worker.managerLoop()
 	return worker, nil
 }
 
+func (w *Worker) managerLoop() {
+	w.state = &workerState{
+		startTime:    time.Now(),
+		stopChan:     make(chan struct{}),
+		currentTasks: make(map[string]*types.TaskInput),
+	}
+out:
+	for cmd := range w.cmds {
+		switch cmd.action {
+		case ActionStart:
+			w.handleStart(cmd)
+		case ActionStop:
+			w.handleStop(cmd)
+			break out
+		case ActionGetStatus:
+			respCh := cmd.data.(statusResponse)
+			var currentTasks []types.TaskInput
+			for _, task := range w.state.currentTasks {
+				currentTasks = append(currentTasks, *task)
+			}
+
+			statusStr := "active"
+			if len(w.state.currentTasks) >= w.config.MaxConcurrent {
+				statusStr = "busy"
+			}
+
+			status := types.WorkerStatus{
+				WorkerID:       w.id,
+				Status:         statusStr,
+				Capabilities:   w.config.Capabilities,
+				MaxConcurrent:  w.config.MaxConcurrent,
+				CurrentLoad:    len(w.state.currentTasks),
+				LastHeartbeat:  time.Now(),
+				CurrentTasks:   currentTasks,
+				Uptime:         time.Since(w.state.startTime),
+				TasksCompleted: w.state.tasksCompleted,
+				TasksFailed:    w.state.tasksFailed,
+			}
+			respCh <- status
+		case ActionGetTaskLoad:
+			respCh := cmd.data.(chan int)
+			respCh <- len(w.state.currentTasks)
+		case ActionSetTask:
+			currentLoad := len(w.state.currentTasks)
+			if currentLoad >= w.config.MaxConcurrent {
+				cmd.resp <- fmt.Errorf("worker is at capacity")
+			}
+			task := cmd.data.(*types.TaskInput)
+			w.state.currentTasks[task.ID] = task
+			cmd.resp <- nil
+		case ActionSetAdmin:
+			admin := cmd.data.(AdminClient)
+			w.state.adminClient = admin
+		case ActionRemoveTask:
+			taskID := cmd.data.(string)
+			delete(w.state.currentTasks, taskID)
+		case ActionGetAdmin:
+			respCh := cmd.data.(chan AdminClient)
+			respCh <- w.state.adminClient
+		case ActionIncTaskFail:
+			w.state.tasksFailed++
+		case ActionIncTaskComplete:
+			w.state.tasksCompleted++
+		case ActionGetHbTick:
+			respCh := cmd.data.(chan *time.Ticker)
+			respCh <- w.state.heartbeatTicker
+		case ActionGetReqTick:
+			respCh := cmd.data.(chan *time.Ticker)
+			respCh <- w.state.requestTicker
+		case ActionSetHbTick:
+			w.state.heartbeatTicker = cmd.data.(*time.Ticker)
+		case ActionSetReqTick:
+			w.state.requestTicker = cmd.data.(*time.Ticker)
+		case ActionGetStopChan:
+			cmd.data.(chan chan struct{}) <- w.state.stopChan
+		case ActionGetStartTime:
+			cmd.data.(chan time.Time) <- w.state.startTime
+		case ActionGetCompletedTasks:
+			cmd.data.(chan int) <- w.state.tasksCompleted
+		case ActionGetFailedTasks:
+			cmd.data.(chan int) <- w.state.tasksFailed
+		case ActionCancelTask:
+			taskID := cmd.data.(string)
+			if task, exists := w.state.currentTasks[taskID]; exists {
+				glog.Infof("Cancelling task %s", task.ID)
+				// TODO: Implement actual task cancellation logic
+			} else {
+				glog.Warningf("Cannot cancel task %s: task not found", taskID)
+			}
+
+		}
+	}
+}
+
+func (w *Worker) getTaskLoad() int {
+	respCh := make(chan int, 1)
+	w.cmds <- workerCommand{
+		action: ActionGetTaskLoad,
+		data:   respCh,
+		resp:   nil,
+	}
+	return <-respCh
+}
+
+func (w *Worker) setTask(task *types.TaskInput) error {
+	resp := make(chan error)
+	w.cmds <- workerCommand{
+		action: ActionSetTask,
+		data:   task,
+		resp:   resp,
+	}
+	if err := <-resp; err != nil {
+		glog.Errorf("TASK REJECTED: Worker %s at capacity (%d/%d) - rejecting task %s",
+			w.id, w.getTaskLoad(), w.config.MaxConcurrent, task.ID)
+		return err
+	}
+	newLoad := w.getTaskLoad()
+
+	glog.Infof("TASK ACCEPTED: Worker %s accepted task %s - current load: %d/%d",
+		w.id, task.ID, newLoad, w.config.MaxConcurrent)
+	return nil
+}
+
+func (w *Worker) removeTask(task *types.TaskInput) int {
+	w.cmds <- workerCommand{
+		action: ActionRemoveTask,
+		data:   task.ID,
+	}
+	return w.getTaskLoad()
+}
+
+func (w *Worker) getAdmin() AdminClient {
+	respCh := make(chan AdminClient, 1)
+	w.cmds <- workerCommand{
+		action: ActionGetAdmin,
+		data:   respCh,
+	}
+	return <-respCh
+}
+
+func (w *Worker) getStopChan() chan struct{} {
+	respCh := make(chan chan struct{}, 1)
+	w.cmds <- workerCommand{
+		action: ActionGetStopChan,
+		data:   respCh,
+	}
+	return <-respCh
+}
+
+func (w *Worker) getHbTick() *time.Ticker {
+	respCh := make(chan *time.Ticker, 1)
+	w.cmds <- workerCommand{
+		action: ActionGetHbTick,
+		data:   respCh,
+	}
+	return <-respCh
+}
+
+func (w *Worker) getReqTick() *time.Ticker {
+	respCh := make(chan *time.Ticker, 1)
+	w.cmds <- workerCommand{
+		action: ActionGetReqTick,
+		data:   respCh,
+	}
+	return <-respCh
+}
+
+func (w *Worker) setHbTick(tick *time.Ticker) *time.Ticker {
+	w.cmds <- workerCommand{
+		action: ActionSetHbTick,
+		data:   tick,
+	}
+	return w.getHbTick()
+}
+
+func (w *Worker) setReqTick(tick *time.Ticker) *time.Ticker {
+	w.cmds <- workerCommand{
+		action: ActionSetReqTick,
+		data:   tick,
+	}
+	return w.getReqTick()
+}
+
+func (w *Worker) getStartTime() time.Time {
+	respCh := make(chan time.Time, 1)
+	w.cmds <- workerCommand{
+		action: ActionGetStartTime,
+		data:   respCh,
+	}
+	return <-respCh
+}
+func (w *Worker) getCompletedTasks() int {
+	respCh := make(chan int, 1)
+	w.cmds <- workerCommand{
+		action: ActionGetCompletedTasks,
+		data:   respCh,
+	}
+	return <-respCh
+}
+func (w *Worker) getFailedTasks() int {
+	respCh := make(chan int, 1)
+	w.cmds <- workerCommand{
+		action: ActionGetFailedTasks,
+		data:   respCh,
+	}
+	return <-respCh
+}
+
 // getTaskLoggerConfig returns the task logger configuration with worker's log directory
 func (w *Worker) getTaskLoggerConfig() tasks.TaskLoggerConfig {
 	config := tasks.DefaultTaskLoggerConfig()
@@ -177,21 +417,29 @@ func (w *Worker) ID() string {
 	return w.id
 }
 
-// Start starts the worker
 func (w *Worker) Start() error {
-	w.mutex.Lock()
-	defer w.mutex.Unlock()
+	resp := make(chan error)
+	w.cmds <- workerCommand{
+		action: ActionStart,
+		resp:   resp,
+	}
+	return <-resp
+}
 
-	if w.running {
-		return fmt.Errorf("worker is already running")
+// Start starts the worker
+func (w *Worker) handleStart(cmd workerCommand) {
+	if w.state.running {
+		cmd.resp <- fmt.Errorf("worker is already running")
+		return
 	}
 
-	if w.adminClient == nil {
-		return fmt.Errorf("admin client is not set")
+	if w.state.adminClient == nil {
+		cmd.resp <- fmt.Errorf("admin client is not set")
+		return
 	}
 
-	w.running = true
-	w.startTime = time.Now()
+	w.state.running = true
+	w.state.startTime = time.Now()
 
 	// Prepare worker info for registration
 	workerInfo := &types.WorkerData{
@@ -204,7 +452,7 @@ func (w *Worker) Start() error {
 	}
 
 	// Register worker info with client first (this stores it for use during connection)
-	if err := w.adminClient.RegisterWorker(workerInfo); err != nil {
+	if err := w.state.adminClient.RegisterWorker(workerInfo); err != nil {
 		glog.V(1).Infof("Worker info stored for registration: %v", err)
 		// This is expected if not connected yet
 	}
@@ -214,7 +462,7 @@ func (w *Worker) Start() error {
 		w.id, w.config.Capabilities, w.config.MaxConcurrent)
 
 	// Try initial connection, but don't fail if it doesn't work immediately
-	if err := w.adminClient.Connect(); err != nil {
+	if err := w.state.adminClient.Connect(); err != nil {
 		glog.Warningf("INITIAL CONNECTION FAILED: Worker %s initial connection to admin server failed, will keep retrying: %v", w.id, err)
 		// Don't return error - let the reconnection loop handle it
 	} else {
@@ -230,54 +478,63 @@ func (w *Worker) Start() error {
 	go w.messageProcessingLoop()
 
 	glog.Infof("WORKER STARTED: Worker %s started successfully (connection attempts will continue in background)", w.id)
-	return nil
+	cmd.resp <- nil
 }
 
-// Stop stops the worker
 func (w *Worker) Stop() error {
-	w.mutex.Lock()
-	defer w.mutex.Unlock()
-
-	if !w.running {
-		return nil
+	resp := make(chan error)
+	w.cmds <- workerCommand{
+		action: ActionStop,
+		resp:   resp,
+	}
+	if err := <-resp; err != nil {
+		return err
 	}
 
-	w.running = false
-	close(w.stopChan)
-
-	// Stop tickers
-	if w.heartbeatTicker != nil {
-		w.heartbeatTicker.Stop()
-	}
-	if w.requestTicker != nil {
-		w.requestTicker.Stop()
-	}
-
-	// Wait for current tasks to complete or timeout
+	// Wait for tasks to finish
 	timeout := time.NewTimer(30 * time.Second)
 	defer timeout.Stop()
-
-	for len(w.currentTasks) > 0 {
+out:
+	for w.getTaskLoad() > 0 {
 		select {
 		case <-timeout.C:
-			glog.Warningf("Worker %s stopping with %d tasks still running", w.id, len(w.currentTasks))
-			break
-		case <-time.After(time.Second):
-			// Check again
+			glog.Warningf("Worker %s stopping with %d tasks still running", w.id, w.getTaskLoad())
+			break out
+		case <-time.After(100 * time.Millisecond):
 		}
 	}
 
 	// Disconnect from admin server
-	if w.adminClient != nil {
-		if err := w.adminClient.Disconnect(); err != nil {
+	if adminClient := w.getAdmin(); adminClient != nil {
+		if err := adminClient.Disconnect(); err != nil {
 			glog.Errorf("Error disconnecting from admin server: %v", err)
 		}
 	}
-
 	glog.Infof("Worker %s stopped", w.id)
 	return nil
 }
 
+// Stop stops the worker
+func (w *Worker) handleStop(cmd workerCommand) {
+	if !w.state.running {
+		cmd.resp <- nil
+		return
+	}
+
+	w.state.running = false
+	close(w.state.stopChan)
+
+	// Stop tickers
+	if w.state.heartbeatTicker != nil {
+		w.state.heartbeatTicker.Stop()
+	}
+	if w.state.requestTicker != nil {
+		w.state.requestTicker.Stop()
+	}
+
+	cmd.resp <- nil
+}
+
 // RegisterTask registers a task factory
 func (w *Worker) RegisterTask(taskType types.TaskType, factory types.TaskFactory) {
 	w.registry.Register(taskType, factory)
@@ -290,31 +547,13 @@ func (w *Worker) GetCapabilities() []types.TaskType {
 
 // GetStatus returns the current worker status
 func (w *Worker) GetStatus() types.WorkerStatus {
-	w.mutex.RLock()
-	defer w.mutex.RUnlock()
-
-	var currentTasks []types.TaskInput
-	for _, task := range w.currentTasks {
-		currentTasks = append(currentTasks, *task)
-	}
-
-	status := "active"
-	if len(w.currentTasks) >= w.config.MaxConcurrent {
-		status = "busy"
-	}
-
-	return types.WorkerStatus{
-		WorkerID:       w.id,
-		Status:         status,
-		Capabilities:   w.config.Capabilities,
-		MaxConcurrent:  w.config.MaxConcurrent,
-		CurrentLoad:    len(w.currentTasks),
-		LastHeartbeat:  time.Now(),
-		CurrentTasks:   currentTasks,
-		Uptime:         time.Since(w.startTime),
-		TasksCompleted: w.tasksCompleted,
-		TasksFailed:    w.tasksFailed,
+	respCh := make(statusResponse, 1)
+	w.cmds <- workerCommand{
+		action: ActionGetStatus,
+		data:   respCh,
+		resp:   nil,
 	}
+	return <-respCh
 }
 
 // HandleTask handles a task execution
@@ -322,22 +561,10 @@ func (w *Worker) HandleTask(task *types.TaskInput) error {
 	glog.V(1).Infof("Worker %s received task %s (type: %s, volume: %d)",
 		w.id, task.ID, task.Type, task.VolumeID)
 
-	w.mutex.Lock()
-	currentLoad := len(w.currentTasks)
-	if currentLoad >= w.config.MaxConcurrent {
-		w.mutex.Unlock()
-		glog.Errorf("TASK REJECTED: Worker %s at capacity (%d/%d) - rejecting task %s",
-			w.id, currentLoad, w.config.MaxConcurrent, task.ID)
-		return fmt.Errorf("worker is at capacity")
+	if err := w.setTask(task); err != nil {
+		return err
 	}
 
-	w.currentTasks[task.ID] = task
-	newLoad := len(w.currentTasks)
-	w.mutex.Unlock()
-
-	glog.Infof("TASK ACCEPTED: Worker %s accepted task %s - current load: %d/%d",
-		w.id, task.ID, newLoad, w.config.MaxConcurrent)
-
 	// Execute task in goroutine
 	go w.executeTask(task)
 
@@ -366,7 +593,10 @@ func (w *Worker) SetTaskRequestInterval(interval time.Duration) {
 
 // SetAdminClient sets the admin client
 func (w *Worker) SetAdminClient(client AdminClient) {
-	w.adminClient = client
+	w.cmds <- workerCommand{
+		action: ActionSetAdmin,
+		data:   client,
+	}
 }
 
 // executeTask executes a task
@@ -374,10 +604,7 @@ func (w *Worker) executeTask(task *types.TaskInput) {
 	startTime := time.Now()
 
 	defer func() {
-		w.mutex.Lock()
-		delete(w.currentTasks, task.ID)
-		currentLoad := len(w.currentTasks)
-		w.mutex.Unlock()
+		currentLoad := w.removeTask(task)
 
 		duration := time.Since(startTime)
 		glog.Infof("TASK EXECUTION FINISHED: Worker %s finished executing task %s after %v - current load: %d/%d",
@@ -388,7 +615,7 @@ func (w *Worker) executeTask(task *types.TaskInput) {
 		w.id, task.ID, task.Type, task.VolumeID, task.Server, task.Collection, startTime.Format(time.RFC3339))
 
 	// Report task start to admin server
-	if err := w.adminClient.UpdateTaskProgress(task.ID, 0.0); err != nil {
+	if err := w.getAdmin().UpdateTaskProgress(task.ID, 0.0); err != nil {
 		glog.V(1).Infof("Failed to report task start to admin: %v", err)
 	}
 
@@ -461,7 +688,7 @@ func (w *Worker) executeTask(task *types.TaskInput) {
 	taskInstance.SetProgressCallback(func(progress float64, stage string) {
 		// Report progress updates to admin server
 		glog.V(2).Infof("Task %s progress: %.1f%% - %s", task.ID, progress, stage)
-		if err := w.adminClient.UpdateTaskProgress(task.ID, progress); err != nil {
+		if err := w.getAdmin().UpdateTaskProgress(task.ID, progress); err != nil {
 			glog.V(1).Infof("Failed to report task progress to admin: %v", err)
 		}
 		if fileLogger != nil {
@@ -481,7 +708,9 @@ func (w *Worker) executeTask(task *types.TaskInput) {
 	// Report completion
 	if err != nil {
 		w.completeTask(task.ID, false, err.Error())
-		w.tasksFailed++
+		w.cmds <- workerCommand{
+			action: ActionIncTaskFail,
+		}
 		glog.Errorf("Worker %s failed to execute task %s: %v", w.id, task.ID, err)
 		if fileLogger != nil {
 			fileLogger.LogStatus("failed", err.Error())
@@ -489,7 +718,9 @@ func (w *Worker) executeTask(task *types.TaskInput) {
 		}
 	} else {
 		w.completeTask(task.ID, true, "")
-		w.tasksCompleted++
+		w.cmds <- workerCommand{
+			action: ActionIncTaskComplete,
+		}
 		glog.Infof("Worker %s completed task %s successfully", w.id, task.ID)
 		if fileLogger != nil {
 			fileLogger.Info("Task %s completed successfully", task.ID)
@@ -499,8 +730,8 @@ func (w *Worker) executeTask(task *types.TaskInput) {
 
 // completeTask reports task completion to admin server
 func (w *Worker) completeTask(taskID string, success bool, errorMsg string) {
-	if w.adminClient != nil {
-		if err := w.adminClient.CompleteTask(taskID, success, errorMsg); err != nil {
+	if w.getAdmin() != nil {
+		if err := w.getAdmin().CompleteTask(taskID, success, errorMsg); err != nil {
 			glog.Errorf("Failed to report task completion: %v", err)
 		}
 	}
@@ -508,14 +739,14 @@ func (w *Worker) completeTask(taskID string, success bool, errorMsg string) {
 
 // heartbeatLoop sends periodic heartbeats to the admin server
 func (w *Worker) heartbeatLoop() {
-	w.heartbeatTicker = time.NewTicker(w.config.HeartbeatInterval)
-	defer w.heartbeatTicker.Stop()
-
+	defer w.setHbTick(time.NewTicker(w.config.HeartbeatInterval)).Stop()
+	ticker := w.getHbTick()
+	stopChan := w.getStopChan()
 	for {
 		select {
-		case <-w.stopChan:
+		case <-stopChan:
 			return
-		case <-w.heartbeatTicker.C:
+		case <-ticker.C:
 			w.sendHeartbeat()
 		}
 	}
@@ -523,14 +754,14 @@ func (w *Worker) heartbeatLoop() {
 
 // taskRequestLoop periodically requests new tasks from the admin server
 func (w *Worker) taskRequestLoop() {
-	w.requestTicker = time.NewTicker(w.config.TaskRequestInterval)
-	defer w.requestTicker.Stop()
-
+	defer w.setReqTick(time.NewTicker(w.config.TaskRequestInterval)).Stop()
+	ticker := w.getReqTick()
+	stopChan := w.getStopChan()
 	for {
 		select {
-		case <-w.stopChan:
+		case <-stopChan:
 			return
-		case <-w.requestTicker.C:
+		case <-ticker.C:
 			w.requestTasks()
 		}
 	}
@@ -538,13 +769,13 @@ func (w *Worker) taskRequestLoop() {
 
 // sendHeartbeat sends heartbeat to admin server
 func (w *Worker) sendHeartbeat() {
-	if w.adminClient != nil {
-		if err := w.adminClient.SendHeartbeat(w.id, &types.WorkerStatus{
+	if w.getAdmin() != nil {
+		if err := w.getAdmin().SendHeartbeat(w.id, &types.WorkerStatus{
 			WorkerID:      w.id,
 			Status:        "active",
 			Capabilities:  w.config.Capabilities,
 			MaxConcurrent: w.config.MaxConcurrent,
-			CurrentLoad:   len(w.currentTasks),
+			CurrentLoad:   w.getTaskLoad(),
 			LastHeartbeat: time.Now(),
 		}); err != nil {
 			glog.Warningf("Failed to send heartbeat: %v", err)
@@ -554,9 +785,7 @@ func (w *Worker) sendHeartbeat() {
 
 // requestTasks requests new tasks from the admin server
 func (w *Worker) requestTasks() {
-	w.mutex.RLock()
-	currentLoad := len(w.currentTasks)
-	w.mutex.RUnlock()
+	currentLoad := w.getTaskLoad()
 
 	if currentLoad >= w.config.MaxConcurrent {
 		glog.V(3).Infof("TASK REQUEST SKIPPED: Worker %s at capacity (%d/%d)",
@@ -564,11 +793,11 @@ func (w *Worker) requestTasks() {
 		return // Already at capacity
 	}
 
-	if w.adminClient != nil {
+	if w.getAdmin() != nil {
 		glog.V(3).Infof("REQUESTING TASK: Worker %s requesting task from admin server (current load: %d/%d, capabilities: %v)",
 			w.id, currentLoad, w.config.MaxConcurrent, w.config.Capabilities)
 
-		task, err := w.adminClient.RequestTask(w.id, w.config.Capabilities)
+		task, err := w.getAdmin().RequestTask(w.id, w.config.Capabilities)
 		if err != nil {
 			glog.V(2).Infof("TASK REQUEST FAILED: Worker %s failed to request task: %v", w.id, err)
 			return
@@ -591,18 +820,6 @@ func (w *Worker) GetTaskRegistry() *tasks.TaskRegistry {
 	return w.registry
 }
 
-// GetCurrentTasks returns the current tasks
-func (w *Worker) GetCurrentTasks() map[string]*types.TaskInput {
-	w.mutex.RLock()
-	defer w.mutex.RUnlock()
-
-	tasks := make(map[string]*types.TaskInput)
-	for id, task := range w.currentTasks {
-		tasks[id] = task
-	}
-	return tasks
-}
-
 // registerWorker registers the worker with the admin server
 func (w *Worker) registerWorker() {
 	workerInfo := &types.WorkerData{
@@ -614,7 +831,7 @@ func (w *Worker) registerWorker() {
 		LastHeartbeat: time.Now(),
 	}
 
-	if err := w.adminClient.RegisterWorker(workerInfo); err != nil {
+	if err := w.getAdmin().RegisterWorker(workerInfo); err != nil {
 		glog.Warningf("Failed to register worker (will retry on next heartbeat): %v", err)
 	} else {
 		glog.Infof("Worker %s registered successfully with admin server", w.id)
@@ -627,15 +844,15 @@ func (w *Worker) connectionMonitorLoop() {
 	defer ticker.Stop()
 
 	lastConnectionStatus := false
-
+	stopChan := w.getStopChan()
 	for {
 		select {
-		case <-w.stopChan:
+		case <-stopChan:
 			glog.V(1).Infof("CONNECTION MONITOR STOPPING: Worker %s connection monitor loop stopping", w.id)
 			return
 		case <-ticker.C:
 			// Monitor connection status and log changes
-			currentConnectionStatus := w.adminClient != nil && w.adminClient.IsConnected()
+			currentConnectionStatus := w.getAdmin() != nil && w.getAdmin().IsConnected()
 
 			if currentConnectionStatus != lastConnectionStatus {
 				if currentConnectionStatus {
@@ -662,19 +879,17 @@ func (w *Worker) GetConfig() *types.WorkerConfig {
 
 // GetPerformanceMetrics returns performance metrics
 func (w *Worker) GetPerformanceMetrics() *types.WorkerPerformance {
-	w.mutex.RLock()
-	defer w.mutex.RUnlock()
 
-	uptime := time.Since(w.startTime)
+	uptime := time.Since(w.getStartTime())
 	var successRate float64
-	totalTasks := w.tasksCompleted + w.tasksFailed
+	totalTasks := w.getCompletedTasks() + w.getFailedTasks()
 	if totalTasks > 0 {
-		successRate = float64(w.tasksCompleted) / float64(totalTasks) * 100
+		successRate = float64(w.getCompletedTasks()) / float64(totalTasks) * 100
 	}
 
 	return &types.WorkerPerformance{
-		TasksCompleted:  w.tasksCompleted,
-		TasksFailed:     w.tasksFailed,
+		TasksCompleted:  w.getCompletedTasks(),
+		TasksFailed:     w.getFailedTasks(),
 		AverageTaskTime: 0, // Would need to track this
 		Uptime:          uptime,
 		SuccessRate:     successRate,
@@ -686,7 +901,7 @@ func (w *Worker) messageProcessingLoop() {
 	glog.Infof("MESSAGE LOOP STARTED: Worker %s message processing loop started", w.id)
 
 	// Get access to the incoming message channel from gRPC client
-	grpcClient, ok := w.adminClient.(*GrpcAdminClient)
+	grpcClient, ok := w.getAdmin().(*GrpcAdminClient)
 	if !ok {
 		glog.Warningf("MESSAGE LOOP UNAVAILABLE: Worker %s admin client is not gRPC client, message processing not available", w.id)
 		return
@@ -694,10 +909,10 @@ func (w *Worker) messageProcessingLoop() {
 
 	incomingChan := grpcClient.GetIncomingChannel()
 	glog.V(1).Infof("MESSAGE CHANNEL READY: Worker %s connected to incoming message channel", w.id)
-
+	stopChan := w.getStopChan()
 	for {
 		select {
-		case <-w.stopChan:
+		case <-stopChan:
 			glog.Infof("MESSAGE LOOP STOPPING: Worker %s message processing loop stopping", w.id)
 			return
 		case message := <-incomingChan:
@@ -773,7 +988,7 @@ func (w *Worker) handleTaskLogRequest(request *worker_pb.TaskLogRequest) {
 		},
 	}
 
-	grpcClient, ok := w.adminClient.(*GrpcAdminClient)
+	grpcClient, ok := w.getAdmin().(*GrpcAdminClient)
 	if !ok {
 		glog.Errorf("Cannot send task log response: admin client is not gRPC client")
 		return
@@ -791,14 +1006,10 @@ func (w *Worker) handleTaskLogRequest(request *worker_pb.TaskLogRequest) {
 func (w *Worker) handleTaskCancellation(cancellation *worker_pb.TaskCancellation) {
 	glog.Infof("Worker %s received task cancellation for task %s", w.id, cancellation.TaskId)
 
-	w.mutex.Lock()
-	defer w.mutex.Unlock()
-
-	if task, exists := w.currentTasks[cancellation.TaskId]; exists {
-		// TODO: Implement task cancellation logic
-		glog.Infof("Cancelling task %s", task.ID)
-	} else {
-		glog.Warningf("Cannot cancel task %s: task not found", cancellation.TaskId)
+	w.cmds <- workerCommand{
+		action: ActionCancelTask,
+		data:   cancellation.TaskId,
+		resp:   nil,
 	}
 }