perf tests run fine

Bumps [github.com/rclone/rclone](https://github.com/rclone/rclone) from 1.69.3 to 1.70.1.
- [Release notes](https://github.com/rclone/rclone/releases)
- [Changelog](https://github.com/rclone/rclone/blob/master/RELEASE.md)
- [Commits](https://github.com/rclone/rclone/compare/v1.69.3...v1.70.1)

---
updated-dependencies:
- dependency-name: github.com/rclone/rclone
  dependency-version: 1.70.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/container_dev.yml

@@ -36,7 +36,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
         with:
           buildkitd-flags: "--debug"
       -

.github/workflows/container_latest.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
         with:
           buildkitd-flags: "--debug"
       -

.github/workflows/container_release1.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'

.github/workflows/container_release2.yml

@@ -38,7 +38,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'

.github/workflows/container_release3.yml

@@ -38,7 +38,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'

.github/workflows/container_release4.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'

.github/workflows/container_release5.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'

go.mod

@@ -69,8 +69,8 @@ require (
 	github.com/posener/complete v1.2.3
 	github.com/pquerna/cachecontrol v0.2.0
 	github.com/prometheus/client_golang v1.22.0
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.62.0 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.64.0 // indirect
 	github.com/prometheus/procfs v0.16.1
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
@@ -94,30 +94,30 @@ require (
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	go.etcd.io/etcd/client/v3 v3.6.0
-	go.mongodb.org/mongo-driver v1.17.3
+	go.etcd.io/etcd/client/v3 v3.6.1
+	go.mongodb.org/mongo-driver v1.17.4
 	go.opencensus.io v0.24.0 // indirect
 	gocloud.dev v0.41.0
 	gocloud.dev/pubsub/natspubsub v0.41.0
 	gocloud.dev/pubsub/rabbitpubsub v0.41.0
-	golang.org/x/crypto v0.38.0
-	golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0
-	golang.org/x/image v0.27.0
-	golang.org/x/net v0.40.0
+	golang.org/x/crypto v0.39.0
+	golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476
+	golang.org/x/image v0.28.0
+	golang.org/x/net v0.41.0
 	golang.org/x/oauth2 v0.30.0 // indirect
 	golang.org/x/sys v0.33.0
-	golang.org/x/text v0.25.0 // indirect
-	golang.org/x/tools v0.33.0
+	golang.org/x/text v0.26.0 // indirect
+	golang.org/x/tools v0.34.0
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
-	google.golang.org/api v0.234.0
+	google.golang.org/api v0.238.0
 	google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect
-	google.golang.org/grpc v1.72.1
+	google.golang.org/grpc v1.73.0
 	google.golang.org/protobuf v1.36.6
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
 	modernc.org/mathutil v1.7.1
 	modernc.org/memory v1.11.0 // indirect
-	modernc.org/sqlite v1.37.1
+	modernc.org/sqlite v1.38.0
 	modernc.org/strutil v1.2.1
 )
 
@@ -125,126 +125,133 @@ require (
 	github.com/Jille/raft-grpc-transport v1.6.1
 	github.com/arangodb/go-driver v1.6.6
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.36.3
-	github.com/aws/aws-sdk-go-v2/config v1.29.14
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.67
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0
+	github.com/aws/aws-sdk-go-v2 v1.36.5
+	github.com/aws/aws-sdk-go-v2/config v1.29.17
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.70
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.80.1
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.0
-	github.com/getsentry/sentry-go v0.31.1
+	github.com/getsentry/sentry-go v0.33.0
 	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
-	github.com/hanwen/go-fuse/v2 v2.7.3-0.20250605191109-50f6569d1a7d
+	github.com/hanwen/go-fuse/v2 v2.8.0
 	github.com/hashicorp/raft v1.7.3
 	github.com/hashicorp/raft-boltdb/v2 v2.3.1
-	github.com/minio/crc64nvme v1.0.1
+	github.com/minio/crc64nvme v1.0.2
 	github.com/orcaman/concurrent-map/v2 v2.0.1
-	github.com/parquet-go/parquet-go v0.24.0
-	github.com/pkg/sftp v1.13.7
+	github.com/parquet-go/parquet-go v0.25.1
+	github.com/pkg/sftp v1.13.9
 	github.com/rabbitmq/amqp091-go v1.10.0
-	github.com/rclone/rclone v1.69.3
-	github.com/rdleal/intervalst v1.4.1
-	github.com/redis/go-redis/v9 v9.8.0
+	github.com/rclone/rclone v1.70.1
+	github.com/rdleal/intervalst v1.5.0
+	github.com/redis/go-redis/v9 v9.10.0
 	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/shirou/gopsutil/v3 v3.24.5
 	github.com/tarantool/go-tarantool/v2 v2.3.2
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.108.3
-	go.etcd.io/etcd/client/pkg/v3 v3.6.0
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.110.1
+	go.etcd.io/etcd/client/pkg/v3 v3.6.1
 	go.uber.org/atomic v1.11.0
-	golang.org/x/sync v0.14.0
+	golang.org/x/sync v0.15.0
 	google.golang.org/grpc/security/advancedtls v1.0.0
 )
 
 require (
-	cel.dev/expr v0.22.1 // indirect
-	cloud.google.com/go/auth v0.16.1 // indirect
+	cel.dev/expr v0.23.0 // indirect
+	cloud.google.com/go/auth v0.16.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.7.0 // indirect
 	cloud.google.com/go/iam v1.5.2 // indirect
 	cloud.google.com/go/monitoring v1.24.2 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.4.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.1 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
-	github.com/Files-com/files-sdk-go/v3 v3.2.97 // indirect
+	github.com/Files-com/files-sdk-go/v3 v3.2.173 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
+	github.com/IBM/go-sdk-core/v5 v5.20.0 // indirect
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
 	github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
-	github.com/ProtonMail/go-crypto v1.1.3 // indirect
+	github.com/ProtonMail/go-crypto v1.3.0 // indirect
 	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
 	github.com/ProtonMail/go-srp v0.0.7 // indirect
-	github.com/ProtonMail/gopenpgp/v2 v2.7.4 // indirect
-	github.com/PuerkitoBio/goquery v1.8.1 // indirect
+	github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
+	github.com/PuerkitoBio/goquery v1.10.3 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
+	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.69 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.77 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.34.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect
-	github.com/aws/smithy-go v1.22.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect
+	github.com/aws/smithy-go v1.22.4 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
-	github.com/bradenaw/juniper v0.15.2 // indirect
+	github.com/bradenaw/juniper v0.15.3 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
 	github.com/buengese/sgzip v0.1.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
 	github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
-	github.com/cloudflare/circl v1.3.7 // indirect
-	github.com/cloudinary/cloudinary-go/v2 v2.9.0 // indirect
-	github.com/cloudsoda/go-smb2 v0.0.0-20241223203758-52b943b88fd6 // indirect
+	github.com/cloudflare/circl v1.6.1 // indirect
+	github.com/cloudinary/cloudinary-go/v2 v2.10.0 // indirect
+	github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
+	github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
 	github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect
 	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
-	github.com/creasty/defaults v1.7.0 // indirect
+	github.com/creasty/defaults v1.8.0 // indirect
 	github.com/cronokirby/saferith v0.33.0 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
-	github.com/ebitengine/purego v0.8.3 // indirect
+	github.com/ebitengine/purego v0.8.4 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
-	github.com/emersion/go-message v0.18.0 // indirect
-	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
-	github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
+	github.com/emersion/go-message v0.18.2 // indirect
+	github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
 	github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/flynn/noise v1.0.1 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.7 // indirect
-	github.com/geoffgarside/ber v1.1.0 // indirect
-	github.com/go-chi/chi/v5 v5.1.0 // indirect
+	github.com/flynn/noise v1.1.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.9 // indirect
+	github.com/geoffgarside/ber v1.2.0 // indirect
+	github.com/go-chi/chi/v5 v5.2.2 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
-	github.com/go-resty/resty/v2 v2.11.0 // indirect
+	github.com/go-openapi/errors v0.22.1 // indirect
+	github.com/go-openapi/strfmt v0.23.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.26.0 // indirect
+	github.com/go-resty/resty/v2 v2.16.5 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
-	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/gofrs/flock v0.12.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
@@ -270,33 +277,35 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/lanrat/extsort v1.0.2 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lpar/date v1.0.0 // indirect
-	github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nats-io/nats.go v1.40.1 // indirect
 	github.com/nats-io/nkeys v0.4.10 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
-	github.com/ncw/swift/v2 v2.0.3 // indirect
+	github.com/ncw/swift/v2 v2.0.4 // indirect
 	github.com/nxadm/tail v1.4.11 // indirect
-	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/onsi/ginkgo/v2 v2.19.0 // indirect
-	github.com/onsi/gomega v1.34.1 // indirect
+	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/onsi/ginkgo/v2 v2.23.3 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.80.0 // indirect
-	github.com/panjf2000/ants/v2 v2.9.1 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.93.0 // indirect
+	github.com/panjf2000/ants/v2 v2.11.3 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
 	github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
@@ -307,31 +316,31 @@ require (
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/xattr v0.4.10 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
-	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
+	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
-	github.com/relvacode/iso8601 v1.3.0 // indirect
+	github.com/relvacode/iso8601 v1.6.0 // indirect
 	github.com/rfjakob/eme v1.1.2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
 	github.com/sagikazarmark/locafero v0.7.0 // indirect
-	github.com/samber/lo v1.47.0 // indirect
-	github.com/shirou/gopsutil/v4 v4.24.12 // indirect
+	github.com/samber/lo v1.50.0 // indirect
+	github.com/shirou/gopsutil/v4 v4.25.5 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
-	github.com/sony/gobreaker v0.5.0 // indirect
+	github.com/sony/gobreaker v1.0.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
-	github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
+	github.com/spacemonkeygo/monkit/v3 v3.0.24 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/t3rm1n4l/go-mega v0.0.0-20241213150454-ec0027fb0002 // indirect
+	github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 // indirect
 	github.com/tarantool/go-iproto v1.1.0 // indirect
 	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
 	github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
 	github.com/tinylib/msgp v1.3.0 // indirect
-	github.com/tklauser/go-sysconf v0.3.13 // indirect
-	github.com/tklauser/numcpus v0.7.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.15 // indirect
+	github.com/tklauser/numcpus v0.10.0 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
 	github.com/unknwon/goconfig v1.0.0 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
@@ -343,36 +352,37 @@ require (
 	github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	github.com/zeebo/blake3 v0.2.3 // indirect
+	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
-	go.etcd.io/bbolt v1.3.10 // indirect
-	go.etcd.io/etcd/api/v3 v3.6.0 // indirect
+	go.etcd.io/bbolt v1.4.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.6.1 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
-	go.opentelemetry.io/otel v1.35.0 // indirect
-	go.opentelemetry.io/otel/metric v1.35.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.35.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
-	go.opentelemetry.io/otel/trace v1.35.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+	go.opentelemetry.io/otel v1.36.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.36.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.36.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/term v0.32.0 // indirect
-	golang.org/x/time v0.11.0 // indirect
+	golang.org/x/time v0.12.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	modernc.org/libc v1.65.7 // indirect
+	modernc.org/libc v1.65.10 // indirect
 	moul.io/http2curl/v2 v2.3.0 // indirect
-	storj.io/common v0.0.0-20240812101423-26b53789c348 // indirect
-	storj.io/drpc v0.0.35-0.20240709171858-0075ac871661 // indirect
-	storj.io/eventkit v0.0.0-20240415002644-1d9596fee086 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+	storj.io/common v0.0.0-20250605163628-70ca83b6228e // indirect
+	storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
+	storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
 	storj.io/infectious v0.0.2 // indirect
-	storj.io/picobuf v0.0.3 // indirect
+	storj.io/picobuf v0.0.4 // indirect
 	storj.io/uplink v1.13.1 // indirect
 )
 

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.90"
+appVersion: "3.92"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.390
+version: 4.0.392

k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@@ -162,6 +162,9 @@ spec:
               {{- if .Values.filer.metricsPort }}
               -metricsPort={{ .Values.filer.metricsPort }} \
               {{- end }}
+              {{- if .Values.filer.metricsIp }}
+              -metricsIp={{ .Values.filer.metricsIp }} \
+              {{- end }}
               {{- if .Values.filer.redirectOnRead }}
               -redirectOnRead \
               {{- end }}
@@ -187,6 +190,7 @@ spec:
               -encryptVolumeData \
               {{- end }}
               -ip=${POD_IP} \
+              -ip.bind={{ .Values.filer.ipBind }} \
               {{- if .Values.filer.filerGroup}}
               -filerGroup={{ .Values.filer.filerGroup}} \
               {{- end }}
@@ -219,7 +223,10 @@ spec:
               -s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
               {{- end }}
               {{- end }}
-              -master={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }}
+              -master={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }} \
+              {{- range .Values.filer.extraArgs }}
+              {{ . }} \
+              {{- end }}
           volumeMounts:
             {{- if (or (eq .Values.filer.logs.type "hostPath") (eq .Values.filer.logs.type "persistentVolumeClaim") (eq .Values.filer.logs.type "emptyDir")) }}
             - name: seaweedfs-filer-log-volume

k8s/charts/seaweedfs/templates/master-statefulset.yaml

@@ -157,18 +157,36 @@ spec:
               {{- if .Values.master.metricsPort }}
               -metricsPort={{ .Values.master.metricsPort }} \
               {{- end }}
+              {{- if .Values.master.metricsIp }}
+              -metricsIp={{ .Values.master.metricsIp }} \
+              {{- end }}
               -volumeSizeLimitMB={{ .Values.master.volumeSizeLimitMB }} \
               {{- if .Values.master.disableHttp }}
               -disableHttp \
               {{- end }}
-              {{- if .Values.master.pulseSeconds }}
-              -pulseSeconds={{ .Values.master.pulseSeconds }} \
+              {{- if .Values.master.resumeState }}
+              -resumeState \
+              {{- end }}
+              {{- if .Values.master.raftHashicorp }}
+              -raftHashicorp \
+              {{- end }}
+              {{- if .Values.master.raftBootstrap }}
+              -raftBootstrap \
+              {{- end }}
+              {{- if .Values.master.electionTimeout }}
+              -electionTimeout={{ .Values.master.electionTimeout }} \
+              {{- end }}
+              {{- if .Values.master.heartbeatInterval }}
+              -heartbeatInterval={{ .Values.master.heartbeatInterval }} \
               {{- end }}
               {{- if .Values.master.garbageThreshold }}
               -garbageThreshold={{ .Values.master.garbageThreshold }} \
               {{- end }}
               -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master.{{ .Release.Namespace }} \
-              -peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
+              -peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} \
+              {{- range .Values.master.extraArgs }}
+              {{ . }} \
+              {{- end }}
           volumeMounts:
             - name : data-{{ .Release.Namespace }}
               mountPath: /data

k8s/charts/seaweedfs/templates/volume-statefulset.yaml

@@ -150,6 +150,9 @@ spec:
                 {{- if .Values.volume.metricsPort }}
                 -metricsPort={{ .Values.volume.metricsPort }} \
                 {{- end }}
+                {{- if .Values.volume.metricsIp }}
+                -metricsIp={{ .Values.volume.metricsIp }} \
+                {{- end }}
                 -dir {{range $index, $dir :=  .Values.volume.dataDirs }}{{if ne $index 0}},{{end}}/{{$dir.name}}{{end}} \
                 {{- if .Values.volume.idx }}
                 -dir.idx=/idx \
@@ -183,7 +186,10 @@ spec:
                 -minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \
                 -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume.{{ .Release.Namespace }} \
                 -compactionMBps={{ .Values.volume.compactionMBps }} \
-                -mserver={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }}
+                -mserver={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }} \
+                {{- range .Values.volume.extraArgs }}
+                {{ . }} \
+                {{- end }}
           volumeMounts:
             {{- range $dir := .Values.volume.dataDirs }}
             {{- if not ( eq $dir.type "custom" ) }}

k8s/charts/seaweedfs/values.yaml

@@ -56,12 +56,11 @@ master:
   port: 9333
   grpcPort: 19333
   metricsPort: 9327
+  metricsIp: ""  # Metrics listen IP. If empty, defaults to ipBind
   ipBind: "0.0.0.0"
   volumePreallocate: false
   volumeSizeLimitMB: 1000
   loggingOverrideLevel: null
-  # number of seconds between heartbeats, default 5
-  pulseSeconds: null
   # threshold to vacuum and reclaim spaces, default 0.3 (30%)
   garbageThreshold: null
   # Prometheus push interval in seconds, default 15
@@ -75,6 +74,25 @@ master:
   # Disable http request, only gRpc operations are allowed
   disableHttp: false
 
+  # Resume previous state on start master server
+  resumeState: false
+  # Use Hashicorp Raft
+  raftHashicorp: false
+  # Whether to bootstrap the Raft cluster. Only use it when use Hashicorp Raft
+  raftBootstrap: false
+
+  # election timeout of master servers
+  electionTimeout: "10s"
+  # heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)
+  heartbeatInterval: "300ms"
+
+  # Custom command line arguments to add to the master command
+  # Example to fix IPv6 metrics connectivity issues:
+  # extraArgs: ["-metricsIp", "0.0.0.0"]
+  # Example with multiple args:
+  # extraArgs: ["-customFlag", "value", "-anotherFlag"]
+  extraArgs: []
+
   config: |-
     # Enter any extra configuration for master.toml here.
     # It may be a multi-line string.
@@ -277,6 +295,7 @@ volume:
   port: 8080
   grpcPort: 18080
   metricsPort: 9327
+  metricsIp: ""  # Metrics listen IP. If empty, defaults to ipBind
   ipBind: "0.0.0.0"
   replicas: 1
   loggingOverrideLevel: null
@@ -289,6 +308,13 @@ volume:
   # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly
   minFreeSpacePercent: 7
 
+  # Custom command line arguments to add to the volume command
+  # Example to fix IPv6 metrics connectivity issues:
+  # extraArgs: ["-metricsIp", "0.0.0.0"]
+  # Example with multiple args:
+  # extraArgs: ["-customFlag", "value", "-anotherFlag"]
+  extraArgs: []
+
   # For each data disk you may use ANY storage-class, example with local-path-provisioner
   # Annotations are optional.
   #  dataDirs:
@@ -520,6 +546,8 @@ filer:
   port: 8888
   grpcPort: 18888
   metricsPort: 9327
+  metricsIp: ""  # Metrics listen IP. If empty, defaults to ipBind
+  ipBind: "0.0.0.0"  # IP address to bind to. Set to 0.0.0.0 to allow external traffic
   loggingOverrideLevel: null
   filerGroup: ""
   # prefer to read and write to volumes in this data center (not set by default)
@@ -547,6 +575,13 @@ filer:
   # Disable http request, only gRpc operations are allowed
   disableHttp: false
 
+  # Custom command line arguments to add to the filer command
+  # Example to fix IPv6 metrics connectivity issues:
+  # extraArgs: ["-metricsIp", "0.0.0.0"]
+  # Example with multiple args:
+  # extraArgs: ["-customFlag", "value", "-anotherFlag"]
+  extraArgs: []
+
   # Add a custom notification.toml to configure filer notifications
   # Example:
   # notificationConfig: |-

test/mq/Dockerfile.test

@@ -0,0 +1,38 @@
+FROM golang:1.24-alpine
+
+# Install necessary tools
+RUN apk add --no-cache \
+    curl \
+    netcat-openbsd \
+    bash \
+    git \
+    build-base \
+    ca-certificates
+
+# Set working directory
+WORKDIR /workspace
+
+# Copy go mod files first for better caching
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy only the necessary source code for building and testing
+COPY weed/ ./weed/
+COPY test/mq/integration/ ./test/mq/integration/
+
+# Build the weed binary for testing (optional - tests can use external cluster)
+RUN cd weed && CGO_ENABLED=1 GOOS=linux go build -o ../weed .
+
+# Create test results directory
+RUN mkdir -p /test-results
+
+# Set up environment
+ENV CGO_ENABLED=1
+ENV GOOS=linux
+ENV GO111MODULE=on
+
+# Default working directory for tests
+WORKDIR /workspace
+
+# Entry point for running tests - use explicit bash
+ENTRYPOINT ["/bin/bash", "-c"] 

test/mq/Makefile

@@ -0,0 +1,223 @@
+.PHONY: help build test test-basic test-performance test-failover test-agent clean up down logs
+
+# Detect architecture and Docker platform compatibility
+ARCH := $(shell uname -m)
+OS := $(shell uname -s)
+ifeq ($(ARCH),arm64)
+    ifeq ($(OS),Darwin)
+        # On Apple Silicon macOS, use native arm64 for better performance
+        DOCKER_PLATFORM := linux/arm64
+    else
+        DOCKER_PLATFORM := linux/arm64
+    endif
+else
+    DOCKER_PLATFORM := linux/amd64
+endif
+
+# Default target
+help:
+	@echo "SeaweedMQ Integration Test Suite"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  build          - Build SeaweedFS Docker images"
+	@echo "  test           - Run all integration tests (in Docker)"
+	@echo "  test-basic     - Run basic pub/sub tests (in Docker)"
+	@echo "  test-native    - Run all tests natively (no Docker test container)"
+	@echo "  test-basic-native - Run basic tests natively (recommended for Apple Silicon)"
+	@echo "  test-performance - Run performance tests"
+	@echo "  test-failover  - Run failover tests"
+	@echo "  test-agent     - Run agent tests"
+	@echo "  up             - Start test environment (local build)"
+	@echo "  up-prod        - Start test environment (production images)"
+	@echo "  up-cluster     - Start cluster only (no test runner)"
+	@echo "  down           - Stop test environment"
+	@echo "  clean          - Clean up test environment and results"
+	@echo "  logs           - Show container logs"
+
+# Build SeaweedFS Docker images
+build:
+	@echo "Building SeaweedFS Docker image for $(DOCKER_PLATFORM)..."
+	cd ../.. && docker build --platform $(DOCKER_PLATFORM) -f docker/Dockerfile.go_build -t chrislusf/seaweedfs:local .
+	@echo "Building test runner image for $(DOCKER_PLATFORM)..."
+	cd ../.. && docker build --platform $(DOCKER_PLATFORM) -f test/mq/Dockerfile.test -t seaweedfs-test-runner .
+
+# Start the test environment
+up: build
+	@echo "Starting SeaweedMQ test environment..."
+	docker-compose -f docker-compose.test.yml up -d master0 master1 master2
+	@echo "Waiting for masters to be ready..."
+	sleep 10
+	docker-compose -f docker-compose.test.yml up -d volume1 volume2 volume3
+	@echo "Waiting for volumes to be ready..."
+	sleep 10
+	docker-compose -f docker-compose.test.yml up -d filer1 filer2
+	@echo "Waiting for filers to be ready..."
+	sleep 15
+	docker-compose -f docker-compose.test.yml up -d broker1 broker2 broker3
+	@echo "Waiting for brokers to be ready..."
+	sleep 20
+	@echo "Test environment is ready!"
+
+# Start the test environment with production images (no build required)
+up-prod: build-test-runner
+	@echo "Starting SeaweedMQ test environment with production images..."
+	docker-compose -f docker-compose.production.yml up -d master0 master1 master2
+	@echo "Waiting for masters to be ready..."
+	sleep 10
+	docker-compose -f docker-compose.production.yml up -d volume1 volume2 volume3
+	@echo "Waiting for volumes to be ready..."
+	sleep 10
+	docker-compose -f docker-compose.production.yml up -d filer1 filer2
+	@echo "Waiting for filers to be ready..."
+	sleep 15
+	docker-compose -f docker-compose.production.yml up -d broker1 broker2 broker3
+	@echo "Waiting for brokers to be ready..."
+	sleep 20
+	@echo "Test environment is ready!"
+
+# Build only the test runner image (for production setup)
+build-test-runner:
+	@echo "Building test runner image for $(DOCKER_PLATFORM)..."
+	cd ../.. && docker build --platform $(DOCKER_PLATFORM) -f test/mq/Dockerfile.test -t seaweedfs-test-runner .
+
+# Start cluster only (no test runner, no build required)
+up-cluster:
+	@echo "Starting SeaweedMQ cluster only..."
+	docker-compose -f docker-compose.cluster.yml up -d master0 master1 master2
+	@echo "Waiting for masters to be ready..."
+	sleep 10
+	docker-compose -f docker-compose.cluster.yml up -d volume1 volume2 volume3
+	@echo "Waiting for volumes to be ready..."
+	sleep 10
+	docker-compose -f docker-compose.cluster.yml up -d filer1 filer2
+	@echo "Waiting for filers to be ready..."
+	sleep 15
+	docker-compose -f docker-compose.cluster.yml up -d broker1 broker2 broker3
+	@echo "Waiting for brokers to be ready..."
+	sleep 20
+	@echo "SeaweedMQ cluster is ready!"
+	@echo "Masters: http://localhost:19333, http://localhost:19334, http://localhost:19335"
+	@echo "Filers: http://localhost:18888, http://localhost:18889"
+	@echo "Brokers: localhost:17777, localhost:17778, localhost:17779"
+
+# Stop the test environment
+down:
+	@echo "Stopping SeaweedMQ test environment..."
+	docker-compose -f docker-compose.test.yml down
+	docker-compose -f docker-compose.production.yml down
+	docker-compose -f docker-compose.cluster.yml down
+
+# Clean up everything
+clean:
+	@echo "Cleaning up test environment..."
+	docker-compose -f docker-compose.test.yml down -v
+	docker system prune -f
+	sudo rm -rf /tmp/test-results/*
+
+# Show container logs
+logs:
+	docker-compose -f docker-compose.test.yml logs -f
+
+# Run all integration tests
+test:
+	@echo "Running all integration tests..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=30m ./test/mq/integration/... -args -test.parallel=4"
+
+# Run basic pub/sub tests
+test-basic:
+	@echo "Running basic pub/sub tests natively (no container restart)..."
+	cd ../.. && SEAWEED_MASTERS="localhost:19333,localhost:19334,localhost:19335" \
+	SEAWEED_BROKERS="localhost:17777,localhost:17778,localhost:17779" \
+	SEAWEED_FILERS="localhost:18888,localhost:18889" \
+	go test -v -timeout=10m ./test/mq/integration/ -run TestBasic
+
+# Run performance tests
+test-performance:
+	@echo "Running performance tests..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=20m ./test/mq/integration/ -run TestPerformance"
+
+# Run failover tests
+test-failover:
+	@echo "Running failover tests..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=15m ./test/mq/integration/ -run TestFailover"
+
+# Run agent tests
+test-agent:
+	@echo "Running agent tests..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=10m ./test/mq/integration/ -run TestAgent"
+
+# Development targets (run tests natively without Docker container)
+test-dev:
+	@echo "Running tests in development mode (using local binaries)..."
+	SEAWEED_MASTERS="localhost:19333,localhost:19334,localhost:19335" \
+	SEAWEED_BROKERS="localhost:17777,localhost:17778,localhost:17779" \
+	SEAWEED_FILERS="localhost:18888,localhost:18889" \
+	go test -v -timeout=10m ./integration/...
+
+# Native test running (no Docker container for tests)
+test-native:
+	@echo "Running tests natively (without Docker container for tests)..."
+	cd ../.. && SEAWEED_MASTERS="localhost:19333,localhost:19334,localhost:19335" \
+	SEAWEED_BROKERS="localhost:17777,localhost:17778,localhost:17779" \
+	SEAWEED_FILERS="localhost:18888,localhost:18889" \
+	go test -v -timeout=10m ./test/mq/integration/...
+
+# Basic native tests
+test-basic-native:
+	@echo "Running basic tests natively..."
+	cd ../.. && SEAWEED_MASTERS="localhost:19333,localhost:19334,localhost:19335" \
+	SEAWEED_BROKERS="localhost:17777,localhost:17778,localhost:17779" \
+	SEAWEED_FILERS="localhost:18888,localhost:18889" \
+	go test -v -timeout=10m ./test/mq/integration/ -run TestBasic
+
+# Quick smoke test
+smoke-test:
+	@echo "Running smoke test..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=5m ./test/mq/integration/ -run TestBasicPublishSubscribe"
+
+# Performance benchmarks
+benchmark:
+	@echo "Running performance benchmarks..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=30m -bench=. ./test/mq/integration/..."
+
+# Check test environment health
+health:
+	@echo "Checking test environment health..."
+	@echo "Masters:"
+	@curl -s http://localhost:19333/cluster/status || echo "Master 0 not accessible"
+	@curl -s http://localhost:19334/cluster/status || echo "Master 1 not accessible"
+	@curl -s http://localhost:19335/cluster/status || echo "Master 2 not accessible"
+	@echo ""
+	@echo "Filers:"
+	@curl -s http://localhost:18888/ || echo "Filer 1 not accessible"
+	@curl -s http://localhost:18889/ || echo "Filer 2 not accessible"
+	@echo ""
+	@echo "Brokers:"
+	@nc -z localhost 17777 && echo "Broker 1 accessible" || echo "Broker 1 not accessible"
+	@nc -z localhost 17778 && echo "Broker 2 accessible" || echo "Broker 2 not accessible"  
+	@nc -z localhost 17779 && echo "Broker 3 accessible" || echo "Broker 3 not accessible"
+
+# Generate test reports
+report:
+	@echo "Generating test reports..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=30m ./test/mq/integration/... -json > /test-results/test-report.json"
+
+# Load testing
+load-test:
+	@echo "Running load tests..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=45m ./test/mq/integration/ -run TestLoad"
+
+# View monitoring dashboards
+monitoring:
+	@echo "Starting monitoring stack..."
+	docker-compose -f docker-compose.test.yml up -d prometheus grafana
+	@echo "Prometheus: http://localhost:19090"
+	@echo "Grafana: http://localhost:13000 (admin/admin)" 

test/mq/README.md

@@ -0,0 +1,414 @@
+# SeaweedMQ Integration Test Suite
+
+This directory contains a comprehensive integration test suite for SeaweedMQ, designed to validate all critical functionalities from basic pub/sub operations to advanced features like auto-scaling, failover, and performance testing.
+
+## Overview
+
+The integration test suite provides:
+
+- **Automated Environment Setup**: Docker Compose based test clusters
+- **Comprehensive Test Coverage**: Basic pub/sub, scaling, failover, performance
+- **Monitoring & Metrics**: Prometheus and Grafana integration
+- **CI/CD Ready**: Configurable for continuous integration pipelines
+- **Load Testing**: Performance benchmarks and stress tests
+
+## Architecture
+
+The test environment consists of:
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│  Master Cluster │    │ Volume Servers  │    │  Filer Cluster  │
+│   (3 nodes)     │    │   (3 nodes)     │    │   (2 nodes)     │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+         │                       │                       │
+         └───────────────────────┼───────────────────────┘
+                                 │
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│ Broker Cluster  │    │ Test Framework  │    │   Monitoring    │
+│   (3 nodes)     │    │   (Go Tests)    │    │ (Prometheus +   │
+└─────────────────┘    └─────────────────┘    │   Grafana)      │
+                                              └─────────────────┘
+```
+
+## Quick Start
+
+### Prerequisites
+
+- Docker and Docker Compose
+- Go 1.21+
+- Make
+- 8GB+ RAM recommended
+- 20GB+ disk space for test data
+
+### Basic Usage
+
+Choose one of three approaches:
+
+#### Option 1: Quick Cluster Setup (Fastest)
+Just starts a SeaweedMQ cluster - no test runner, no build required.
+
+1. **Start Cluster**:
+   ```bash
+   cd test/mq
+   make up-cluster
+   ```
+
+2. **Test manually** or with your own code against:
+   - Masters: http://localhost:19333, http://localhost:19334, http://localhost:19335
+   - Brokers: localhost:17777, localhost:17778, localhost:17779
+   - Filers: http://localhost:18888, http://localhost:18889
+
+#### Option 2: Use Production Images (Recommended for testing)
+Uses official SeaweedFS images from Docker Hub with test runner.
+
+1. **Start Test Environment**:
+   ```bash
+   cd test/mq
+   make up-prod
+   ```
+
+2. **Run Tests**:
+   ```bash
+   make test-basic      # Basic pub/sub tests
+   ```
+
+#### Option 3: Build from Source (Development)
+Builds SeaweedFS from your local source code to test latest changes.
+
+1. **Start Test Environment**:
+   ```bash
+   cd test/mq
+   make up  # This will build and start
+   ```
+
+2. **Run Tests**:
+   ```bash
+   make test-basic      # Basic pub/sub tests
+   ```
+
+#### Common Commands
+
+3. **Run All Tests**:
+   ```bash
+   make test
+   ```
+
+4. **Run Specific Test Categories**:
+   ```bash
+   make test-performance # Performance tests
+   make test-failover   # Failover tests
+   make test-agent      # Agent tests
+   ```
+
+5. **Quick Smoke Test**:
+   ```bash
+   make smoke-test
+   ```
+
+6. **Check Health**:
+   ```bash
+   make health
+   ```
+
+7. **Clean Up**:
+   ```bash
+   make down
+   ```
+
+## Test Categories
+
+### 1. Basic Functionality Tests
+
+**File**: `integration/basic_pubsub_test.go`
+
+- **TestBasicPublishSubscribe**: Basic message publishing and consumption
+- **TestMultipleConsumers**: Load balancing across multiple consumers
+- **TestMessageOrdering**: FIFO ordering within partitions
+- **TestSchemaValidation**: Schema validation and complex nested structures
+
+### 2. Partitioning and Scaling Tests
+
+**File**: `integration/scaling_test.go` (to be implemented)
+
+- **TestPartitionDistribution**: Message distribution across partitions
+- **TestAutoSplitMerge**: Automatic partition split/merge based on load
+- **TestBrokerScaling**: Adding/removing brokers during operation
+- **TestLoadBalancing**: Even load distribution verification
+
+### 3. Failover and Reliability Tests
+
+**File**: `integration/failover_test.go` (to be implemented)
+
+- **TestBrokerFailover**: Leader failover scenarios
+- **TestBrokerRecovery**: Recovery from broker failures
+- **TestMessagePersistence**: Data durability across restarts
+- **TestFollowerReplication**: Leader-follower consistency
+
+### 4. Performance Tests
+
+**File**: `integration/performance_test.go` (to be implemented)
+
+- **TestHighThroughputPublish**: High-volume message publishing
+- **TestHighThroughputSubscribe**: High-volume message consumption
+- **TestLatencyMeasurement**: End-to-end latency analysis
+- **TestResourceUtilization**: CPU, memory, and disk usage
+
+### 5. Agent Tests
+
+**File**: `integration/agent_test.go` (to be implemented)
+
+- **TestAgentPublishSessions**: Session management for publishers
+- **TestAgentSubscribeSessions**: Session management for subscribers
+- **TestAgentFailover**: Agent reconnection and failover
+- **TestAgentConcurrency**: Concurrent session handling
+
+## Configuration
+
+### Environment Variables
+
+The test framework supports configuration via environment variables:
+
+```bash
+# Cluster endpoints
+SEAWEED_MASTERS="master0:9333,master1:9334,master2:9335"
+SEAWEED_BROKERS="broker1:17777,broker2:17778,broker3:17779"
+SEAWEED_FILERS="filer1:8888,filer2:8889"
+
+# Test configuration
+GO_TEST_TIMEOUT="30m"
+TEST_RESULTS_DIR="/test-results"
+```
+
+### Docker Compose Override
+
+Create `docker-compose.override.yml` to customize the test environment:
+
+```yaml
+version: '3.9'
+services:
+  broker1:
+    environment:
+      - CUSTOM_ENV_VAR=value
+  test-runner:
+    volumes:
+      - ./custom-config:/config
+```
+
+## Monitoring and Metrics
+
+### Prometheus Metrics
+
+Access Prometheus at: http://localhost:19090
+
+Key metrics to monitor:
+- Message throughput: `seaweedmq_messages_published_total`
+- Consumer lag: `seaweedmq_consumer_lag_seconds`
+- Broker health: `seaweedmq_broker_health`
+- Resource usage: `seaweedfs_disk_usage_bytes`
+
+### Grafana Dashboards
+
+Access Grafana at: http://localhost:13000 (admin/admin)
+
+Pre-configured dashboards:
+- **SeaweedMQ Overview**: System health and throughput
+- **Performance Metrics**: Latency and resource usage
+- **Error Analysis**: Error rates and failure patterns
+
+## Development
+
+### Writing New Tests
+
+1. **Create Test File**:
+   ```bash
+   touch integration/my_new_test.go
+   ```
+
+2. **Use Test Framework**:
+   ```go
+   func TestMyFeature(t *testing.T) {
+       suite := NewIntegrationTestSuite(t)
+       require.NoError(t, suite.Setup())
+       
+       // Your test logic here
+   }
+   ```
+
+3. **Run Specific Test**:
+   ```bash
+   go test -v ./integration/ -run TestMyFeature
+   ```
+
+### Test Framework Components
+
+**IntegrationTestSuite**: Base test framework with cluster management
+**MessageCollector**: Utility for collecting and verifying received messages
+**TestMessage**: Standard message structure for testing
+**Schema Builders**: Helpers for creating test schemas
+
+### Local Development
+
+Run tests against a local SeaweedMQ cluster:
+
+```bash
+make test-dev
+```
+
+This uses local binaries instead of Docker containers.
+
+## Continuous Integration
+
+### GitHub Actions Example
+
+```yaml
+name: Integration Tests
+on: [push, pull_request]
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.21
+      - name: Run Integration Tests
+        run: |
+          cd test/mq
+          make test
+```
+
+### Jenkins Pipeline
+
+```groovy
+pipeline {
+    agent any
+    stages {
+        stage('Setup') {
+            steps {
+                sh 'cd test/mq && make up'
+            }
+        }
+        stage('Test') {
+            steps {
+                sh 'cd test/mq && make test'
+            }
+            post {
+                always {
+                    sh 'cd test/mq && make down'
+                }
+            }
+        }
+    }
+}
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Port Conflicts**:
+   ```bash
+   # Check port usage
+   netstat -tulpn | grep :19333
+   
+   # Kill conflicting processes
+   sudo kill -9 $(lsof -t -i:19333)
+   ```
+
+2. **Docker Resource Issues**:
+   ```bash
+   # Increase Docker memory (8GB+)
+   # Clean up Docker resources
+   docker system prune -a
+   ```
+
+3. **Test Timeouts**:
+   ```bash
+   # Increase timeout
+   GO_TEST_TIMEOUT=60m make test
+   ```
+
+### Debug Mode
+
+Run tests with verbose logging:
+
+```bash
+docker-compose -f docker-compose.test.yml run --rm test-runner \
+  sh -c "go test -v -race ./test/mq/integration/... -args -test.v"
+```
+
+### Container Logs
+
+View real-time logs:
+
+```bash
+make logs
+
+# Or specific service
+docker-compose -f docker-compose.test.yml logs -f broker1
+```
+
+## Performance Benchmarks
+
+### Throughput Benchmarks
+
+```bash
+make benchmark
+```
+
+Expected performance (on 8-core, 16GB RAM):
+- **Publish Throughput**: 50K+ messages/second/broker
+- **Subscribe Throughput**: 100K+ messages/second/broker
+- **End-to-End Latency**: P95 < 100ms
+- **Storage Efficiency**: < 20% overhead
+
+### Load Testing
+
+```bash
+make load-test
+```
+
+Stress tests with:
+- 1M+ messages
+- 100+ concurrent producers
+- 50+ concurrent consumers
+- Multiple topic scenarios
+
+## Contributing
+
+### Test Guidelines
+
+1. **Test Isolation**: Each test should be independent
+2. **Resource Cleanup**: Always clean up resources in test teardown
+3. **Timeouts**: Set appropriate timeouts for operations
+4. **Error Handling**: Test both success and failure scenarios
+5. **Documentation**: Document test purpose and expected behavior
+
+### Code Style
+
+- Follow Go testing conventions
+- Use testify for assertions
+- Include setup/teardown in test functions
+- Use descriptive test names
+
+## Future Enhancements
+
+- [ ] Chaos engineering tests (network partitions, node failures)
+- [ ] Multi-datacenter deployment testing
+- [ ] Schema evolution compatibility tests
+- [ ] Security and authentication tests
+- [ ] Performance regression detection
+- [ ] Automated load pattern generation
+
+## Support
+
+For issues and questions:
+- Check existing GitHub issues
+- Review SeaweedMQ documentation
+- Join SeaweedFS community discussions
+
+---
+
+*This integration test suite ensures SeaweedMQ's reliability, performance, and functionality across all critical use cases and failure scenarios.* 

test/mq/docker-compose.cluster.yml

@@ -0,0 +1,165 @@
+services:
+  # Masters
+  master0:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "19333:9333"
+    volumes:
+      - /tmp/seaweedfs-test/master0:/data
+    command: "master -port=9333 -mdir=/data -peers=master0:9333,master1:9334,master2:9335 -ip=master0 -defaultReplication=001"
+    networks:
+      - seaweedfs-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master0:9333/cluster/status"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+
+  master1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "19334:9334"
+    volumes:
+      - /tmp/seaweedfs-test/master1:/data
+    command: "master -port=9334 -mdir=/data -peers=master0:9333,master1:9334,master2:9335 -ip=master1 -defaultReplication=001"
+    networks:
+      - seaweedfs-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master1:9334/cluster/status"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+
+  master2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "19335:9335"
+    volumes:
+      - /tmp/seaweedfs-test/master2:/data
+    command: "master -port=9335 -mdir=/data -peers=master0:9333,master1:9334,master2:9335 -ip=master2 -defaultReplication=001"
+    networks:
+      - seaweedfs-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master2:9335/cluster/status"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+
+  # Volume Servers
+  volume1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18080:8080"
+    volumes:
+      - /tmp/seaweedfs-test/volume1:/data
+    command: "volume -port=8080 -mserver=master0:9333,master1:9334,master2:9335 -dir=/data"
+    depends_on:
+      - master0
+      - master1
+      - master2
+    networks:
+      - seaweedfs-test
+
+  volume2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18081:8081"
+    volumes:
+      - /tmp/seaweedfs-test/volume2:/data
+    command: "volume -port=8081 -mserver=master0:9333,master1:9334,master2:9335 -dir=/data"
+    depends_on:
+      - master0
+      - master1
+      - master2
+    networks:
+      - seaweedfs-test
+
+  volume3:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18082:8082"
+    volumes:
+      - /tmp/seaweedfs-test/volume3:/data
+    command: "volume -port=8082 -mserver=master0:9333,master1:9334,master2:9335 -dir=/data"
+    depends_on:
+      - master0
+      - master1
+      - master2
+    networks:
+      - seaweedfs-test
+
+  # Filers
+  filer1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18888:8888"
+    volumes:
+      - /tmp/seaweedfs-test/filer1:/data
+    command: "filer -port=8888 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - volume1
+      - volume2
+      - volume3
+    networks:
+      - seaweedfs-test
+
+  filer2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18889:8889"
+    volumes:
+      - /tmp/seaweedfs-test/filer2:/data
+    command: "filer -port=8889 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - volume1
+      - volume2
+      - volume3
+    networks:
+      - seaweedfs-test
+
+  # Message Queue Brokers
+  broker1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "17777:17777"
+    volumes:
+      - /tmp/seaweedfs-test/broker1:/data
+    command: "mq.broker -port=17777 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - filer1
+      - filer2
+    networks:
+      - seaweedfs-test
+
+  broker2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "17778:17778"
+    volumes:
+      - /tmp/seaweedfs-test/broker2:/data
+    command: "mq.broker -port=17778 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - filer1
+      - filer2
+    networks:
+      - seaweedfs-test
+
+  broker3:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "17779:17779"
+    volumes:
+      - /tmp/seaweedfs-test/broker3:/data
+    command: "mq.broker -port=17779 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - filer1
+      - filer2
+    networks:
+      - seaweedfs-test
+
+networks:
+  seaweedfs-test:
+    driver: bridge 

test/mq/docker-compose.production.yml

@@ -0,0 +1,260 @@
+services:
+  # Masters
+  master0:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "19333:19333"
+      - "9333:9333"
+    volumes:
+      - /tmp/seaweedfs-test/master0:/data
+    command: "master -port=9333 -mdir=/data -peers=master0:9333,master1:9334,master2:9335 -defaultReplication=001"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 6
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 3
+    networks:
+      - seaweedfs-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master0:9333/cluster/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  master1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "19334:9334"
+      - "9334:9334"
+    volumes:
+      - /tmp/seaweedfs-test/master1:/data
+    command: "master -port=9334 -mdir=/data -peers=master0:9333,master1:9334,master2:9335 -defaultReplication=001"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 6
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 3
+    networks:
+      - seaweedfs-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master1:9334/cluster/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  master2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "19335:9335"
+      - "9335:9335"
+    volumes:
+      - /tmp/seaweedfs-test/master2:/data
+    command: "master -port=9335 -mdir=/data -peers=master0:9333,master1:9334,master2:9335 -defaultReplication=001"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 6
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 3
+    networks:
+      - seaweedfs-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master2:9335/cluster/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  # Volume Servers
+  volume1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18080:8080"
+      - "8080:8080"
+    volumes:
+      - /tmp/seaweedfs-test/volume1:/data
+    command: "volume -port=8080 -mserver=master0:9333,master1:9334,master2:9335 -dir=/data"
+    depends_on:
+      master0:
+        condition: service_healthy
+      master1:
+        condition: service_healthy
+      master2:
+        condition: service_healthy
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_VOLUME_MAX: 100
+
+  volume2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18081:8081"
+      - "8081:8081"
+    volumes:
+      - /tmp/seaweedfs-test/volume2:/data
+    command: "volume -port=8081 -mserver=master0:9333,master1:9334,master2:9335 -dir=/data"
+    depends_on:
+      master0:
+        condition: service_healthy
+      master1:
+        condition: service_healthy
+      master2:
+        condition: service_healthy
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_VOLUME_MAX: 100
+
+  volume3:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18082:8082"
+      - "8082:8082"
+    volumes:
+      - /tmp/seaweedfs-test/volume3:/data
+    command: "volume -port=8082 -mserver=master0:9333,master1:9334,master2:9335 -dir=/data"
+    depends_on:
+      master0:
+        condition: service_healthy
+      master1:
+        condition: service_healthy
+      master2:
+        condition: service_healthy
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_VOLUME_MAX: 100
+
+  # Filers
+  filer1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18888:8888"
+      - "8888:8888"
+    volumes:
+      - /tmp/seaweedfs-test/filer1:/data
+    command: "filer -port=8888 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - volume1
+      - volume2
+      - volume3
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_LEVELDB2_DIR: "/data/filerldb2"
+
+  filer2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18889:8889"
+      - "8889:8889"
+    volumes:
+      - /tmp/seaweedfs-test/filer2:/data
+    command: "filer -port=8889 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - volume1
+      - volume2
+      - volume3
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_LEVELDB2_DIR: "/data/filerldb2"
+
+  # Message Queue Brokers
+  broker1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "17777:17777"
+    volumes:
+      - /tmp/seaweedfs-test/broker1:/data
+    command: "mq.broker -port=17777 -filer=filer1:8888,filer2:8889"
+    depends_on:
+      - filer1
+      - filer2
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_MQ_BROKER_CPU_PERCENT: 80
+
+  broker2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "17778:17778"
+    volumes:
+      - /tmp/seaweedfs-test/broker2:/data
+    command: "mq.broker -port=17778 -filer=filer1:8888,filer2:8889"
+    depends_on:
+      - filer1
+      - filer2
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_MQ_BROKER_CPU_PERCENT: 80
+
+  broker3:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "17779:17779"
+    volumes:
+      - /tmp/seaweedfs-test/broker3:/data
+    command: "mq.broker -port=17779 -filer=filer1:8888,filer2:8889"
+    depends_on:
+      - filer1
+      - filer2
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_MQ_BROKER_CPU_PERCENT: 80
+
+  # Test Runner
+  test-runner:
+    image: seaweedfs-test-runner
+    volumes:
+      - ../..:/workspace
+      - /tmp/test-results:/test-results
+    working_dir: /workspace
+    networks:
+      - seaweedfs-test
+    environment:
+      - SEAWEED_MASTERS=master0:9333,master1:9334,master2:9335
+      - SEAWEED_BROKERS=broker1:17777,broker2:17778,broker3:17779
+      - SEAWEED_FILERS=filer1:8888,filer2:8889
+      - GO111MODULE=on
+      - CGO_ENABLED=0
+    depends_on:
+      - broker1
+      - broker2
+      - broker3
+
+  # Monitoring
+  prometheus:
+    image: prom/prometheus:latest
+    ports:
+      - "19090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    networks:
+      - seaweedfs-test
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume1
+      - volume2
+      - volume3
+      - filer1
+      - filer2
+      - broker1
+      - broker2
+      - broker3
+
+  grafana:
+    image: grafana/grafana:latest
+    ports:
+      - "13000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    networks:
+      - seaweedfs-test
+    depends_on:
+      - prometheus
+
+networks:
+  seaweedfs-test:
+    driver: bridge 

test/mq/docker-compose.test.yml

@@ -0,0 +1,341 @@
+services:
+  # Master cluster for coordination and metadata
+  master0:
+    image: chrislusf/seaweedfs:local
+    container_name: test-master0
+    ports:
+      - "19333:9333"
+      - "29333:19333"
+    command: >
+      -v=1 
+      master 
+      -volumeSizeLimitMB=100 
+      -resumeState=false 
+      -ip=master0 
+      -port=9333 
+      -peers=master0:9333,master1:9334,master2:9335 
+      -mdir=/tmp/master0
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+    networks:
+      - seaweedmq-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master0:9333/cluster/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  master1:
+    image: chrislusf/seaweedfs:local
+    container_name: test-master1
+    ports:
+      - "19334:9334"
+      - "29334:19334"
+    command: >
+      -v=1 
+      master 
+      -volumeSizeLimitMB=100 
+      -resumeState=false 
+      -ip=master1 
+      -port=9334 
+      -peers=master0:9333,master1:9334,master2:9335 
+      -mdir=/tmp/master1
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+    networks:
+      - seaweedmq-test
+    depends_on:
+      - master0
+
+  master2:
+    image: chrislusf/seaweedfs:local
+    container_name: test-master2
+    ports:
+      - "19335:9335"
+      - "29335:19335"
+    command: >
+      -v=1 
+      master 
+      -volumeSizeLimitMB=100 
+      -resumeState=false 
+      -ip=master2 
+      -port=9335 
+      -peers=master0:9333,master1:9334,master2:9335 
+      -mdir=/tmp/master2
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+    networks:
+      - seaweedmq-test
+    depends_on:
+      - master0
+
+  # Volume servers for data storage
+  volume1:
+    image: chrislusf/seaweedfs:local
+    container_name: test-volume1
+    ports:
+      - "18080:8080"
+      - "28080:18080"
+    volumes:
+      - volume1-data:/data/volume1
+    entrypoint: ["/bin/sh", "-c"]
+    command: >
+      "mkdir -p /data/volume1 && exec weed -v=1
+      volume 
+      -dataCenter=dc1 
+      -rack=rack1 
+      -mserver=master0:9333,master1:9334,master2:9335 
+      -port=8080 
+      -ip=volume1 
+      -publicUrl=localhost:18080 
+      -preStopSeconds=1
+      -dir=/data/volume1"
+    networks:
+      - seaweedmq-test
+    depends_on:
+      master0:
+        condition: service_healthy
+
+  volume2:
+    image: chrislusf/seaweedfs:local
+    container_name: test-volume2
+    ports:
+      - "18081:8081"
+      - "28081:18081"
+    volumes:
+      - volume2-data:/data/volume2
+    entrypoint: ["/bin/sh", "-c"]
+    command: >
+      "mkdir -p /data/volume2 && exec weed -v=1
+      volume 
+      -dataCenter=dc1 
+      -rack=rack2 
+      -mserver=master0:9333,master1:9334,master2:9335 
+      -port=8081 
+      -ip=volume2 
+      -publicUrl=localhost:18081 
+      -preStopSeconds=1
+      -dir=/data/volume2"
+    networks:
+      - seaweedmq-test
+    depends_on:
+      master0:
+        condition: service_healthy
+
+  volume3:
+    image: chrislusf/seaweedfs:local
+    container_name: test-volume3
+    ports:
+      - "18082:8082"
+      - "28082:18082"
+    volumes:
+      - volume3-data:/data/volume3
+    entrypoint: ["/bin/sh", "-c"]
+    command: >
+      "mkdir -p /data/volume3 && exec weed -v=1
+      volume 
+      -dataCenter=dc2 
+      -rack=rack1 
+      -mserver=master0:9333,master1:9334,master2:9335 
+      -port=8082 
+      -ip=volume3 
+      -publicUrl=localhost:18082 
+      -preStopSeconds=1
+      -dir=/data/volume3"
+    networks:
+      - seaweedmq-test
+    depends_on:
+      master0:
+        condition: service_healthy
+
+  # Filer servers for metadata
+  filer1:
+    image: chrislusf/seaweedfs:local
+    container_name: test-filer1
+    ports:
+      - "18888:8888"
+      - "28888:18888"
+    command: >
+      -v=1
+      filer 
+      -defaultReplicaPlacement=100 
+      -master=master0:9333,master1:9334,master2:9335 
+      -port=8888 
+      -ip=filer1
+      -dataCenter=dc1
+    networks:
+      - seaweedmq-test
+    depends_on:
+      master0:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://filer1:8888/"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  filer2:
+    image: chrislusf/seaweedfs:local
+    container_name: test-filer2
+    ports:
+      - "18889:8889"
+      - "28889:18889"
+    command: >
+      -v=1
+      filer 
+      -defaultReplicaPlacement=100 
+      -master=master0:9333,master1:9334,master2:9335 
+      -port=8889 
+      -ip=filer2
+      -dataCenter=dc2
+    networks:
+      - seaweedmq-test
+    depends_on:
+      filer1:
+        condition: service_healthy
+
+  # Message Queue Brokers
+  broker1:
+    image: chrislusf/seaweedfs:local
+    container_name: test-broker1
+    ports:
+      - "17777:17777"
+    command: >
+      -v=1
+      mq.broker 
+      -master=master0:9333,master1:9334,master2:9335 
+      -port=17777 
+      -ip=127.0.0.1
+      -dataCenter=dc1
+      -rack=rack1
+    networks:
+      - seaweedmq-test
+    depends_on:
+      filer1:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "nc", "-z", "localhost", "17777"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  broker2:
+    image: chrislusf/seaweedfs:local
+    container_name: test-broker2
+    ports:
+      - "17778:17778"
+    command: >
+      -v=1
+      mq.broker 
+      -master=master0:9333,master1:9334,master2:9335 
+      -port=17778 
+      -ip=127.0.0.1
+      -dataCenter=dc1
+      -rack=rack2
+    networks:
+      - seaweedmq-test
+    depends_on:
+      broker1:
+        condition: service_healthy
+
+  broker3:
+    image: chrislusf/seaweedfs:local
+    container_name: test-broker3
+    ports:
+      - "17779:17779"
+    command: >
+      -v=1
+      mq.broker 
+      -master=master0:9333,master1:9334,master2:9335 
+      -port=17779 
+      -ip=127.0.0.1
+      -dataCenter=dc2
+      -rack=rack1
+    networks:
+      - seaweedmq-test
+    depends_on:
+      broker1:
+        condition: service_healthy
+
+  # Test runner container
+  test-runner:
+    build:
+      context: ../../
+      dockerfile: test/mq/Dockerfile.test
+    container_name: test-runner
+    volumes:
+      - ../../:/app
+      - /tmp/test-results:/test-results
+    working_dir: /app
+    environment:
+      - SEAWEED_MASTERS=master0:9333,master1:9334,master2:9335
+      - SEAWEED_BROKERS=broker1:17777,broker2:17778,broker3:17779
+      - SEAWEED_FILERS=filer1:8888,filer2:8889
+      - TEST_RESULTS_DIR=/test-results
+      - GO_TEST_TIMEOUT=30m
+    networks:
+      - seaweedmq-test
+    depends_on:
+      broker1:
+        condition: service_healthy
+      broker2:
+        condition: service_started
+      broker3:
+        condition: service_started
+    command: >
+      sh -c "
+        echo 'Waiting for cluster to be ready...' &&
+        sleep 30 &&
+        echo 'Running integration tests...' &&
+        go test -v -timeout=30m ./test/mq/integration/... -args -test.parallel=4
+      "
+
+  # Monitoring and metrics
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: test-prometheus
+    ports:
+      - "19090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    networks:
+      - seaweedmq-test
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.enable-lifecycle'
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: test-grafana
+    ports:
+      - "13000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    volumes:
+      - grafana-storage:/var/lib/grafana
+      - ./grafana/dashboards:/etc/grafana/provisioning/dashboards
+      - ./grafana/datasources:/etc/grafana/provisioning/datasources
+    networks:
+      - seaweedmq-test
+
+networks:
+  seaweedmq-test:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.20.0.0/16
+
+volumes:
+  grafana-storage:
+  volume1-data:
+  volume2-data:
+  volume3-data: 

test/mq/integration/basic_pubsub_test.go

@@ -0,0 +1,339 @@
+package integration
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBasicPublishSubscribe(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	namespace := "test"
+	topicName := fmt.Sprintf("basic-pubsub-%d", time.Now().UnixNano()) // Unique topic name per run
+	testSchema := CreateTestSchema()
+	messageCount := 10
+
+	// Create publisher
+	pubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName,
+		PartitionCount: 1,
+		PublisherName:  "basic-publisher",
+		RecordType:     testSchema,
+	}
+
+	publisher, err := suite.CreatePublisher(pubConfig)
+	require.NoError(t, err)
+
+	// Create subscriber
+	subConfig := &SubscriberTestConfig{
+		Namespace:          namespace,
+		TopicName:          topicName,
+		ConsumerGroup:      "test-group",
+		ConsumerInstanceId: "consumer-1",
+		MaxPartitionCount:  1,
+		SlidingWindowSize:  10,
+		OffsetType:         schema_pb.OffsetType_RESET_TO_EARLIEST,
+	}
+
+	subscriber, err := suite.CreateSubscriber(subConfig)
+	require.NoError(t, err, "Failed to create subscriber")
+
+	// Set up message collector
+	collector := NewMessageCollector(messageCount)
+	subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
+		t.Logf("[Subscriber] Received message with key: %s, ts: %d", string(m.Data.Key), m.Data.TsNs)
+		collector.AddMessage(TestMessage{
+			ID:        fmt.Sprintf("msg-%d", len(collector.GetMessages())),
+			Content:   m.Data.Value,
+			Timestamp: time.Unix(0, m.Data.TsNs),
+			Key:       m.Data.Key,
+		})
+	})
+
+	// Start subscriber
+	go func() {
+		err := subscriber.Subscribe()
+		if err != nil {
+			t.Logf("Subscriber error: %v", err)
+		}
+	}()
+
+	// Wait for subscriber to be ready
+	t.Logf("[Test] Waiting for subscriber to be ready...")
+	time.Sleep(2 * time.Second)
+
+	// Publish test messages
+	for i := 0; i < messageCount; i++ {
+		record := schema.RecordBegin().
+			SetString("id", fmt.Sprintf("msg-%d", i)).
+			SetInt64("timestamp", time.Now().UnixNano()).
+			SetString("content", fmt.Sprintf("Test message %d", i)).
+			SetInt32("sequence", int32(i)).
+			RecordEnd()
+
+		key := []byte(fmt.Sprintf("key-%d", i))
+		t.Logf("[Publisher] Publishing message %d with key: %s", i, string(key))
+		err := publisher.PublishRecord(key, record)
+		require.NoError(t, err, "Failed to publish message %d", i)
+	}
+
+	t.Logf("[Test] Waiting for messages to be received...")
+	messages := collector.WaitForMessages(30 * time.Second)
+	t.Logf("[Test] WaitForMessages returned. Received %d messages.", len(messages))
+
+	// Verify all messages were received
+	assert.Len(t, messages, messageCount, "Expected %d messages, got %d", messageCount, len(messages))
+
+	// Verify message content
+	for i, msg := range messages {
+		assert.NotEmpty(t, msg.Content, "Message %d should have content", i)
+		assert.NotEmpty(t, msg.Key, "Message %d should have key", i)
+	}
+
+	t.Logf("[Test] TestBasicPublishSubscribe completed.")
+}
+
+func TestMultipleConsumers(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	namespace := "test"
+	topicName := "multi-consumer"
+	testSchema := CreateTestSchema()
+	messageCount := 20
+	consumerCount := 3
+
+	// Create publisher
+	pubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName,
+		PartitionCount: 3, // Multiple partitions for load distribution
+		PublisherName:  "multi-publisher",
+		RecordType:     testSchema,
+	}
+
+	publisher, err := suite.CreatePublisher(pubConfig)
+	require.NoError(t, err)
+
+	// Create multiple consumers
+	collectors := make([]*MessageCollector, consumerCount)
+	for i := 0; i < consumerCount; i++ {
+		collectors[i] = NewMessageCollector(messageCount / consumerCount) // Expect roughly equal distribution
+
+		subConfig := &SubscriberTestConfig{
+			Namespace:          namespace,
+			TopicName:          topicName,
+			ConsumerGroup:      "multi-consumer-group", // Same group for load balancing
+			ConsumerInstanceId: fmt.Sprintf("consumer-%d", i),
+			MaxPartitionCount:  1,
+			SlidingWindowSize:  10,
+			OffsetType:         schema_pb.OffsetType_RESET_TO_EARLIEST,
+		}
+
+		subscriber, err := suite.CreateSubscriber(subConfig)
+		require.NoError(t, err)
+
+		// Set up message collection for this consumer
+		collectorIndex := i
+		subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
+			collectors[collectorIndex].AddMessage(TestMessage{
+				ID:        fmt.Sprintf("consumer-%d-msg-%d", collectorIndex, len(collectors[collectorIndex].GetMessages())),
+				Content:   m.Data.Value,
+				Timestamp: time.Unix(0, m.Data.TsNs),
+				Key:       m.Data.Key,
+			})
+		})
+
+		// Start subscriber
+		go func() {
+			subscriber.Subscribe()
+		}()
+	}
+
+	// Wait for subscribers to be ready
+	time.Sleep(3 * time.Second)
+
+	// Publish messages with different keys to distribute across partitions
+	for i := 0; i < messageCount; i++ {
+		record := schema.RecordBegin().
+			SetString("id", fmt.Sprintf("multi-msg-%d", i)).
+			SetInt64("timestamp", time.Now().UnixNano()).
+			SetString("content", fmt.Sprintf("Multi consumer test message %d", i)).
+			SetInt32("sequence", int32(i)).
+			RecordEnd()
+
+		key := []byte(fmt.Sprintf("partition-key-%d", i%3)) // Distribute across 3 partitions
+		err := publisher.PublishRecord(key, record)
+		require.NoError(t, err)
+	}
+
+	// Wait for all messages to be consumed
+	time.Sleep(10 * time.Second)
+
+	// Verify message distribution
+	totalReceived := 0
+	for i, collector := range collectors {
+		messages := collector.GetMessages()
+		t.Logf("Consumer %d received %d messages", i, len(messages))
+		totalReceived += len(messages)
+	}
+
+	// All messages should be consumed across all consumers
+	assert.Equal(t, messageCount, totalReceived, "Total messages received should equal messages sent")
+}
+
+func TestMessageOrdering(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	namespace := "test"
+	topicName := "ordering-test"
+	testSchema := CreateTestSchema()
+	messageCount := 15
+
+	// Create publisher
+	pubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName,
+		PartitionCount: 1, // Single partition to guarantee ordering
+		PublisherName:  "ordering-publisher",
+		RecordType:     testSchema,
+	}
+
+	publisher, err := suite.CreatePublisher(pubConfig)
+	require.NoError(t, err)
+
+	// Create subscriber
+	subConfig := &SubscriberTestConfig{
+		Namespace:          namespace,
+		TopicName:          topicName,
+		ConsumerGroup:      "ordering-group",
+		ConsumerInstanceId: "ordering-consumer",
+		MaxPartitionCount:  1,
+		SlidingWindowSize:  5,
+		OffsetType:         schema_pb.OffsetType_RESET_TO_EARLIEST,
+	}
+
+	subscriber, err := suite.CreateSubscriber(subConfig)
+	require.NoError(t, err)
+
+	// Set up message collector
+	collector := NewMessageCollector(messageCount)
+	subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
+		collector.AddMessage(TestMessage{
+			ID:        fmt.Sprintf("ordered-msg"),
+			Content:   m.Data.Value,
+			Timestamp: time.Unix(0, m.Data.TsNs),
+			Key:       m.Data.Key,
+		})
+	})
+
+	// Start subscriber
+	go func() {
+		subscriber.Subscribe()
+	}()
+
+	// Wait for consumer to be ready
+	time.Sleep(2 * time.Second)
+
+	// Publish messages with same key to ensure they go to same partition
+	publishTimes := make([]time.Time, messageCount)
+	for i := 0; i < messageCount; i++ {
+		publishTimes[i] = time.Now()
+
+		record := schema.RecordBegin().
+			SetString("id", fmt.Sprintf("ordered-%d", i)).
+			SetInt64("timestamp", publishTimes[i].UnixNano()).
+			SetString("content", fmt.Sprintf("Ordered message %d", i)).
+			SetInt32("sequence", int32(i)).
+			RecordEnd()
+
+		key := []byte("same-partition-key") // Same key ensures same partition
+		err := publisher.PublishRecord(key, record)
+		require.NoError(t, err)
+
+		// Small delay to ensure different timestamps
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	// Wait for all messages
+	messages := collector.WaitForMessages(30 * time.Second)
+	require.Len(t, messages, messageCount)
+
+	// Verify ordering within the partition
+	suite.AssertMessageOrdering(t, messages)
+}
+
+func TestSchemaValidation(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	namespace := "test"
+	topicName := "schema-validation"
+
+	// Test with simple schema
+	simpleSchema := CreateTestSchema()
+
+	pubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName,
+		PartitionCount: 1,
+		PublisherName:  "schema-publisher",
+		RecordType:     simpleSchema,
+	}
+
+	publisher, err := suite.CreatePublisher(pubConfig)
+	require.NoError(t, err)
+
+	// Test valid record
+	validRecord := schema.RecordBegin().
+		SetString("id", "valid-msg").
+		SetInt64("timestamp", time.Now().UnixNano()).
+		SetString("content", "Valid message").
+		SetInt32("sequence", 1).
+		RecordEnd()
+
+	err = publisher.PublishRecord([]byte("test-key"), validRecord)
+	assert.NoError(t, err, "Valid record should be published successfully")
+
+	// Test with complex nested schema
+	complexSchema := CreateComplexTestSchema()
+
+	complexPubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName + "-complex",
+		PartitionCount: 1,
+		PublisherName:  "complex-publisher",
+		RecordType:     complexSchema,
+	}
+
+	complexPublisher, err := suite.CreatePublisher(complexPubConfig)
+	require.NoError(t, err)
+
+	// Test complex nested record
+	complexRecord := schema.RecordBegin().
+		SetString("user_id", "user123").
+		SetString("name", "John Doe").
+		SetInt32("age", 30).
+		SetStringList("emails", "john@example.com", "john.doe@company.com").
+		SetRecord("address",
+			schema.RecordBegin().
+				SetString("street", "123 Main St").
+				SetString("city", "New York").
+				SetString("zipcode", "10001").
+				RecordEnd()).
+		SetInt64("created_at", time.Now().UnixNano()).
+		RecordEnd()
+
+	err = complexPublisher.PublishRecord([]byte("complex-key"), complexRecord)
+	assert.NoError(t, err, "Complex nested record should be published successfully")
+}

test/mq/integration/framework.go

@@ -0,0 +1,379 @@
+package integration
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/agent"
+	"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/client/sub_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// TestEnvironment holds the configuration for the test environment
+type TestEnvironment struct {
+	Masters      []string
+	Brokers      []string
+	Filers       []string
+	TestTimeout  time.Duration
+	CleanupFuncs []func()
+	mutex        sync.Mutex
+}
+
+// IntegrationTestSuite provides the base test framework
+type IntegrationTestSuite struct {
+	env         *TestEnvironment
+	agents      map[string]*agent.MessageQueueAgent
+	publishers  map[string]*pub_client.TopicPublisher
+	subscribers map[string]*sub_client.TopicSubscriber
+	subCancels  map[string]context.CancelFunc
+	cleanupOnce sync.Once
+	t           *testing.T
+}
+
+// NewIntegrationTestSuite creates a new test suite instance
+func NewIntegrationTestSuite(t *testing.T) *IntegrationTestSuite {
+	env := &TestEnvironment{
+		Masters:     getEnvList("SEAWEED_MASTERS", []string{"localhost:19333"}),
+		Brokers:     getEnvList("SEAWEED_BROKERS", []string{"localhost:17777"}),
+		Filers:      getEnvList("SEAWEED_FILERS", []string{"localhost:18888"}),
+		TestTimeout: getEnvDuration("GO_TEST_TIMEOUT", 30*time.Minute),
+	}
+
+	return &IntegrationTestSuite{
+		env:         env,
+		agents:      make(map[string]*agent.MessageQueueAgent),
+		publishers:  make(map[string]*pub_client.TopicPublisher),
+		subscribers: make(map[string]*sub_client.TopicSubscriber),
+		subCancels:  make(map[string]context.CancelFunc),
+		t:           t,
+	}
+}
+
+// Setup initializes the test environment
+func (its *IntegrationTestSuite) Setup() error {
+	// Wait for cluster to be ready
+	if err := its.waitForClusterReady(); err != nil {
+		return fmt.Errorf("cluster not ready: %v", err)
+	}
+
+	// Register cleanup
+	its.t.Cleanup(its.Cleanup)
+
+	return nil
+}
+
+// Cleanup performs cleanup operations
+func (its *IntegrationTestSuite) Cleanup() {
+	its.cleanupOnce.Do(func() {
+		// Close all subscribers first (they use context cancellation)
+		for name := range its.subscribers {
+			if cancel, ok := its.subCancels[name]; ok && cancel != nil {
+				cancel()
+				its.t.Logf("Cancelled subscriber context: %s", name)
+			}
+			its.t.Logf("Cleaned up subscriber: %s", name)
+		}
+
+		// Wait a moment for gRPC connections to close gracefully
+		time.Sleep(1 * time.Second)
+
+		// Close all publishers
+		for name, publisher := range its.publishers {
+			if publisher != nil {
+				// Add timeout to prevent deadlock during shutdown
+				done := make(chan bool, 1)
+				go func(p *pub_client.TopicPublisher, n string) {
+					p.Shutdown()
+					done <- true
+				}(publisher, name)
+
+				select {
+				case <-done:
+					its.t.Logf("Cleaned up publisher: %s", name)
+				case <-time.After(5 * time.Second):
+					its.t.Logf("Publisher shutdown timed out: %s", name)
+				}
+			}
+		}
+
+		// Execute additional cleanup functions
+		its.env.mutex.Lock()
+		for _, cleanup := range its.env.CleanupFuncs {
+			cleanup()
+		}
+		its.env.mutex.Unlock()
+	})
+}
+
+// CreatePublisher creates a new topic publisher
+func (its *IntegrationTestSuite) CreatePublisher(config *PublisherTestConfig) (*pub_client.TopicPublisher, error) {
+	publisherConfig := &pub_client.PublisherConfiguration{
+		Topic:          topic.NewTopic(config.Namespace, config.TopicName),
+		PartitionCount: config.PartitionCount,
+		Brokers:        its.env.Brokers,
+		PublisherName:  config.PublisherName,
+		RecordType:     config.RecordType,
+	}
+
+	publisher, err := pub_client.NewTopicPublisher(publisherConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create publisher: %v", err)
+	}
+
+	its.publishers[config.PublisherName] = publisher
+	return publisher, nil
+}
+
+// CreateSubscriber creates a new topic subscriber
+func (its *IntegrationTestSuite) CreateSubscriber(config *SubscriberTestConfig) (*sub_client.TopicSubscriber, error) {
+	subscriberConfig := &sub_client.SubscriberConfiguration{
+		ConsumerGroup:           config.ConsumerGroup,
+		ConsumerGroupInstanceId: config.ConsumerInstanceId,
+		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
+		MaxPartitionCount:       config.MaxPartitionCount,
+		SlidingWindowSize:       config.SlidingWindowSize,
+	}
+
+	contentConfig := &sub_client.ContentConfiguration{
+		Topic:            topic.NewTopic(config.Namespace, config.TopicName),
+		Filter:           config.Filter,
+		PartitionOffsets: config.PartitionOffsets,
+		OffsetType:       config.OffsetType,
+		OffsetTsNs:       config.OffsetTsNs,
+	}
+
+	offsetChan := make(chan sub_client.KeyedOffset, 1024)
+	ctx, cancel := context.WithCancel(context.Background())
+	subscriber := sub_client.NewTopicSubscriber(
+		ctx,
+		its.env.Brokers,
+		subscriberConfig,
+		contentConfig,
+		offsetChan,
+	)
+
+	its.subscribers[config.ConsumerInstanceId] = subscriber
+	its.subCancels[config.ConsumerInstanceId] = cancel
+	return subscriber, nil
+}
+
+// CreateAgent creates a new message queue agent
+func (its *IntegrationTestSuite) CreateAgent(name string) (*agent.MessageQueueAgent, error) {
+	var brokerAddresses []pb.ServerAddress
+	for _, broker := range its.env.Brokers {
+		brokerAddresses = append(brokerAddresses, pb.ServerAddress(broker))
+	}
+
+	agentOptions := &agent.MessageQueueAgentOptions{
+		SeedBrokers: brokerAddresses,
+	}
+
+	mqAgent := agent.NewMessageQueueAgent(
+		agentOptions,
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+	)
+
+	its.agents[name] = mqAgent
+	return mqAgent, nil
+}
+
+// PublisherTestConfig holds configuration for creating test publishers
+type PublisherTestConfig struct {
+	Namespace      string
+	TopicName      string
+	PartitionCount int32
+	PublisherName  string
+	RecordType     *schema_pb.RecordType
+}
+
+// SubscriberTestConfig holds configuration for creating test subscribers
+type SubscriberTestConfig struct {
+	Namespace          string
+	TopicName          string
+	ConsumerGroup      string
+	ConsumerInstanceId string
+	MaxPartitionCount  int32
+	SlidingWindowSize  int32
+	Filter             string
+	PartitionOffsets   []*schema_pb.PartitionOffset
+	OffsetType         schema_pb.OffsetType
+	OffsetTsNs         int64
+}
+
+// TestMessage represents a test message with metadata
+type TestMessage struct {
+	ID        string
+	Content   []byte
+	Timestamp time.Time
+	Key       []byte
+}
+
+// MessageCollector collects received messages for verification
+type MessageCollector struct {
+	messages []TestMessage
+	mutex    sync.RWMutex
+	waitCh   chan struct{}
+	expected int
+	closed   bool // protect against closing waitCh multiple times
+}
+
+// NewMessageCollector creates a new message collector
+func NewMessageCollector(expectedCount int) *MessageCollector {
+	return &MessageCollector{
+		messages: make([]TestMessage, 0),
+		waitCh:   make(chan struct{}),
+		expected: expectedCount,
+	}
+}
+
+// AddMessage adds a received message to the collector
+func (mc *MessageCollector) AddMessage(msg TestMessage) {
+	mc.mutex.Lock()
+	defer mc.mutex.Unlock()
+
+	mc.messages = append(mc.messages, msg)
+	if len(mc.messages) >= mc.expected && !mc.closed {
+		close(mc.waitCh)
+		mc.closed = true
+	}
+}
+
+// WaitForMessages waits for the expected number of messages or timeout
+func (mc *MessageCollector) WaitForMessages(timeout time.Duration) []TestMessage {
+	select {
+	case <-mc.waitCh:
+	case <-time.After(timeout):
+	}
+
+	mc.mutex.RLock()
+	defer mc.mutex.RUnlock()
+
+	result := make([]TestMessage, len(mc.messages))
+	copy(result, mc.messages)
+	return result
+}
+
+// GetMessages returns all collected messages
+func (mc *MessageCollector) GetMessages() []TestMessage {
+	mc.mutex.RLock()
+	defer mc.mutex.RUnlock()
+
+	result := make([]TestMessage, len(mc.messages))
+	copy(result, mc.messages)
+	return result
+}
+
+// CreateTestSchema creates a simple test schema
+func CreateTestSchema() *schema_pb.RecordType {
+	return schema.RecordTypeBegin().
+		WithField("id", schema.TypeString).
+		WithField("timestamp", schema.TypeInt64).
+		WithField("content", schema.TypeString).
+		WithField("sequence", schema.TypeInt32).
+		RecordTypeEnd()
+}
+
+// CreateComplexTestSchema creates a complex test schema with nested structures
+func CreateComplexTestSchema() *schema_pb.RecordType {
+	addressType := schema.RecordTypeBegin().
+		WithField("street", schema.TypeString).
+		WithField("city", schema.TypeString).
+		WithField("zipcode", schema.TypeString).
+		RecordTypeEnd()
+
+	return schema.RecordTypeBegin().
+		WithField("user_id", schema.TypeString).
+		WithField("name", schema.TypeString).
+		WithField("age", schema.TypeInt32).
+		WithField("emails", schema.ListOf(schema.TypeString)).
+		WithRecordField("address", addressType).
+		WithField("created_at", schema.TypeInt64).
+		RecordTypeEnd()
+}
+
+// Helper functions
+
+func getEnvList(key string, defaultValue []string) []string {
+	value := os.Getenv(key)
+	if value == "" {
+		return defaultValue
+	}
+	return strings.Split(value, ",")
+}
+
+func getEnvDuration(key string, defaultValue time.Duration) time.Duration {
+	value := os.Getenv(key)
+	if value == "" {
+		return defaultValue
+	}
+
+	duration, err := time.ParseDuration(value)
+	if err != nil {
+		return defaultValue
+	}
+	return duration
+}
+
+func (its *IntegrationTestSuite) waitForClusterReady() error {
+	maxRetries := 30
+	retryInterval := 2 * time.Second
+
+	for i := 0; i < maxRetries; i++ {
+		if its.isClusterReady() {
+			return nil
+		}
+		its.t.Logf("Waiting for cluster to be ready... attempt %d/%d", i+1, maxRetries)
+		time.Sleep(retryInterval)
+	}
+
+	return fmt.Errorf("cluster not ready after %d attempts", maxRetries)
+}
+
+func (its *IntegrationTestSuite) isClusterReady() bool {
+	// Check if at least one broker is accessible
+	for _, broker := range its.env.Brokers {
+		if its.isBrokerReady(broker) {
+			return true
+		}
+	}
+	return false
+}
+
+func (its *IntegrationTestSuite) isBrokerReady(broker string) bool {
+	// Simple connection test
+	conn, err := grpc.NewClient(broker, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false
+	}
+	defer conn.Close()
+
+	// TODO: Add actual health check call here
+	return true
+}
+
+// AssertMessagesReceived verifies that expected messages were received
+func (its *IntegrationTestSuite) AssertMessagesReceived(t *testing.T, collector *MessageCollector, expectedCount int, timeout time.Duration) {
+	messages := collector.WaitForMessages(timeout)
+	require.Len(t, messages, expectedCount, "Expected %d messages, got %d", expectedCount, len(messages))
+}
+
+// AssertMessageOrdering verifies that messages are received in the expected order
+func (its *IntegrationTestSuite) AssertMessageOrdering(t *testing.T, messages []TestMessage) {
+	for i := 1; i < len(messages); i++ {
+		require.True(t, messages[i].Timestamp.After(messages[i-1].Timestamp) || messages[i].Timestamp.Equal(messages[i-1].Timestamp),
+			"Messages not in chronological order: message %d timestamp %v should be >= message %d timestamp %v",
+			i, messages[i].Timestamp, i-1, messages[i-1].Timestamp)
+	}
+}

test/mq/integration/performance_test.go

@@ -0,0 +1,666 @@
+package integration
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// PerformanceMetrics holds test metrics
+type PerformanceMetrics struct {
+	MessagesPublished int64
+	MessagesConsumed  int64
+	PublishLatencies  []time.Duration
+	ConsumeLatencies  []time.Duration
+	StartTime         time.Time
+	EndTime           time.Time
+	ErrorCount        int64
+	mu                sync.RWMutex
+}
+
+func (m *PerformanceMetrics) AddPublishLatency(d time.Duration) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.PublishLatencies = append(m.PublishLatencies, d)
+}
+
+func (m *PerformanceMetrics) AddConsumeLatency(d time.Duration) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.ConsumeLatencies = append(m.ConsumeLatencies, d)
+}
+
+func (m *PerformanceMetrics) GetThroughput() float64 {
+	duration := m.EndTime.Sub(m.StartTime).Seconds()
+	if duration == 0 {
+		return 0
+	}
+	return float64(atomic.LoadInt64(&m.MessagesPublished)) / duration
+}
+
+func (m *PerformanceMetrics) GetP95Latency(latencies []time.Duration) time.Duration {
+	if len(latencies) == 0 {
+		return 0
+	}
+
+	// Simple P95 calculation - in production use proper percentile library
+	index := int(float64(len(latencies)) * 0.95)
+	if index >= len(latencies) {
+		index = len(latencies) - 1
+	}
+
+	// Sort latencies (simplified)
+	for i := 0; i < len(latencies)-1; i++ {
+		for j := 0; j < len(latencies)-i-1; j++ {
+			if latencies[j] > latencies[j+1] {
+				latencies[j], latencies[j+1] = latencies[j+1], latencies[j]
+			}
+		}
+	}
+
+	return latencies[index]
+}
+
+// Enhanced performance metrics with connection error tracking
+type EnhancedPerformanceMetrics struct {
+	MessagesPublished int64
+	MessagesConsumed  int64
+	PublishLatencies  []time.Duration
+	ConsumeLatencies  []time.Duration
+	StartTime         time.Time
+	EndTime           time.Time
+	ErrorCount        int64
+	ConnectionErrors  int64
+	ApplicationErrors int64
+	RetryAttempts     int64
+	mu                sync.RWMutex
+}
+
+func (m *EnhancedPerformanceMetrics) AddPublishLatency(d time.Duration) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.PublishLatencies = append(m.PublishLatencies, d)
+}
+
+func (m *EnhancedPerformanceMetrics) AddConsumeLatency(d time.Duration) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.ConsumeLatencies = append(m.ConsumeLatencies, d)
+}
+
+func (m *EnhancedPerformanceMetrics) GetThroughput() float64 {
+	duration := m.EndTime.Sub(m.StartTime).Seconds()
+	if duration == 0 {
+		return 0
+	}
+	return float64(atomic.LoadInt64(&m.MessagesPublished)) / duration
+}
+
+func (m *EnhancedPerformanceMetrics) GetP95Latency(latencies []time.Duration) time.Duration {
+	if len(latencies) == 0 {
+		return 0
+	}
+
+	// Simple P95 calculation
+	index := int(float64(len(latencies)) * 0.95)
+	if index >= len(latencies) {
+		index = len(latencies) - 1
+	}
+
+	// Sort latencies (simplified bubble sort for small datasets)
+	sorted := make([]time.Duration, len(latencies))
+	copy(sorted, latencies)
+	for i := 0; i < len(sorted)-1; i++ {
+		for j := 0; j < len(sorted)-i-1; j++ {
+			if sorted[j] > sorted[j+1] {
+				sorted[j], sorted[j+1] = sorted[j+1], sorted[j]
+			}
+		}
+	}
+
+	return sorted[index]
+}
+
+// isConnectionError determines if an error is a connection-level error
+func isConnectionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errStr := err.Error()
+
+	// Check for gRPC status codes
+	if st, ok := status.FromError(err); ok {
+		switch st.Code() {
+		case codes.Unavailable, codes.DeadlineExceeded, codes.Canceled, codes.Unknown:
+			return true
+		}
+	}
+
+	// Check for common connection error strings
+	connectionErrorPatterns := []string{
+		"EOF",
+		"error reading server preface",
+		"connection refused",
+		"connection reset",
+		"broken pipe",
+		"network is unreachable",
+		"no route to host",
+		"transport is closing",
+		"connection error",
+		"dial tcp",
+		"context deadline exceeded",
+	}
+
+	for _, pattern := range connectionErrorPatterns {
+		if containsString(errStr, pattern) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func containsString(s, substr string) bool {
+	for i := 0; i <= len(s)-len(substr); i++ {
+		if s[i:i+len(substr)] == substr {
+			return true
+		}
+	}
+	return false
+}
+
+func TestPerformanceThroughput(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	topicName := "performance-throughput-test"
+	namespace := "perf-test"
+
+	metrics := &PerformanceMetrics{
+		StartTime: time.Now(),
+	}
+
+	// Test parameters
+	numMessages := 50000
+	numPublishers := 5
+	messageSize := 1024 // 1KB messages
+
+	// Create message payload
+	payload := make([]byte, messageSize)
+	for i := range payload {
+		payload[i] = byte(i % 256)
+	}
+
+	t.Logf("Starting throughput test: %d messages, %d publishers, %d bytes per message",
+		numMessages, numPublishers, messageSize)
+
+	// Start publishers
+	var publishWg sync.WaitGroup
+	messagesPerPublisher := numMessages / numPublishers
+
+	for i := 0; i < numPublishers; i++ {
+		publishWg.Add(1)
+		go func(publisherID int) {
+			defer publishWg.Done()
+
+			// Create publisher for this goroutine
+			pubConfig := &PublisherTestConfig{
+				Namespace:      namespace,
+				TopicName:      topicName,
+				PartitionCount: 4, // Multiple partitions for better throughput
+				PublisherName:  fmt.Sprintf("perf-publisher-%d", publisherID),
+				RecordType:     nil, // Use raw publish
+			}
+
+			publisher, err := suite.CreatePublisher(pubConfig)
+			if err != nil {
+				atomic.AddInt64(&metrics.ErrorCount, 1)
+				t.Errorf("Failed to create publisher %d: %v", publisherID, err)
+				return
+			}
+
+			for j := 0; j < messagesPerPublisher; j++ {
+				messageKey := fmt.Sprintf("publisher-%d-msg-%d", publisherID, j)
+
+				start := time.Now()
+				err := publisher.Publish([]byte(messageKey), payload)
+				latency := time.Since(start)
+
+				if err != nil {
+					atomic.AddInt64(&metrics.ErrorCount, 1)
+					continue
+				}
+
+				atomic.AddInt64(&metrics.MessagesPublished, 1)
+				metrics.AddPublishLatency(latency)
+
+				// Small delay to prevent overwhelming the system
+				if j%1000 == 0 {
+					time.Sleep(1 * time.Millisecond)
+				}
+			}
+		}(i)
+	}
+
+	// Wait for publishing to complete
+	publishWg.Wait()
+	metrics.EndTime = time.Now()
+
+	// Verify results
+	publishedCount := atomic.LoadInt64(&metrics.MessagesPublished)
+	errorCount := atomic.LoadInt64(&metrics.ErrorCount)
+	throughput := metrics.GetThroughput()
+
+	t.Logf("Performance Results:")
+	t.Logf("  Messages Published: %d", publishedCount)
+	t.Logf("  Errors: %d", errorCount)
+	t.Logf("  Throughput: %.2f messages/second", throughput)
+	t.Logf("  Duration: %v", metrics.EndTime.Sub(metrics.StartTime))
+
+	if len(metrics.PublishLatencies) > 0 {
+		p95Latency := metrics.GetP95Latency(metrics.PublishLatencies)
+		t.Logf("  P95 Publish Latency: %v", p95Latency)
+
+		// Performance assertions
+		assert.Less(t, p95Latency, 100*time.Millisecond, "P95 publish latency should be under 100ms")
+	}
+
+	// Throughput requirements
+	expectedMinThroughput := 5000.0 // 5K messages/sec minimum (relaxed from 10K for initial testing)
+	assert.Greater(t, throughput, expectedMinThroughput,
+		"Throughput should exceed %.0f messages/second", expectedMinThroughput)
+
+	// Error rate should be low
+	errorRate := float64(errorCount) / float64(publishedCount+errorCount)
+	assert.Less(t, errorRate, 0.05, "Error rate should be less than 5%")
+}
+
+func TestPerformanceLatency(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	topicName := "performance-latency-test"
+	namespace := "perf-test"
+
+	// Create publisher
+	pubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName,
+		PartitionCount: 1, // Single partition for latency testing
+		PublisherName:  "latency-publisher",
+		RecordType:     nil,
+	}
+
+	publisher, err := suite.CreatePublisher(pubConfig)
+	require.NoError(t, err)
+
+	// Start consumer first
+	subConfig := &SubscriberTestConfig{
+		Namespace:          namespace,
+		TopicName:          topicName,
+		ConsumerGroup:      "latency-test-group",
+		ConsumerInstanceId: "latency-consumer-1",
+		MaxPartitionCount:  1,
+		SlidingWindowSize:  10,
+		OffsetType:         schema_pb.OffsetType_RESET_TO_EARLIEST,
+	}
+
+	subscriber, err := suite.CreateSubscriber(subConfig)
+	require.NoError(t, err)
+
+	numMessages := 5000
+	collector := NewMessageCollector(numMessages)
+
+	// Set up message handler
+	subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
+		collector.AddMessage(TestMessage{
+			ID:        string(m.Data.Key),
+			Content:   m.Data.Value,
+			Timestamp: time.Unix(0, m.Data.TsNs),
+			Key:       m.Data.Key,
+		})
+	})
+
+	// Start subscriber
+	go func() {
+		err := subscriber.Subscribe()
+		if err != nil {
+			t.Logf("Subscriber error: %v", err)
+		}
+	}()
+
+	// Wait for consumer to be ready
+	time.Sleep(2 * time.Second)
+
+	metrics := &PerformanceMetrics{
+		StartTime: time.Now(),
+	}
+
+	t.Logf("Starting latency test with %d messages", numMessages)
+
+	// Publish messages with controlled timing
+	for i := 0; i < numMessages; i++ {
+		messageKey := fmt.Sprintf("latency-msg-%d", i)
+		payload := fmt.Sprintf("test-payload-data-%d", i)
+
+		start := time.Now()
+		err := publisher.Publish([]byte(messageKey), []byte(payload))
+		publishLatency := time.Since(start)
+
+		require.NoError(t, err)
+		metrics.AddPublishLatency(publishLatency)
+
+		// Controlled rate for latency measurement
+		if i%100 == 0 {
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+	metrics.EndTime = time.Now()
+
+	// Wait for messages to be consumed
+	messages := collector.WaitForMessages(30 * time.Second)
+
+	// Analyze latency results
+	t.Logf("Latency Test Results:")
+	t.Logf("  Messages Published: %d", numMessages)
+	t.Logf("  Messages Consumed: %d", len(messages))
+
+	if len(metrics.PublishLatencies) > 0 {
+		p95PublishLatency := metrics.GetP95Latency(metrics.PublishLatencies)
+		t.Logf("  P95 Publish Latency: %v", p95PublishLatency)
+
+		// Latency assertions
+		assert.Less(t, p95PublishLatency, 50*time.Millisecond,
+			"P95 publish latency should be under 50ms")
+	}
+
+	// Verify message delivery
+	deliveryRate := float64(len(messages)) / float64(numMessages)
+	assert.Greater(t, deliveryRate, 0.80, "Should deliver at least 80% of messages")
+}
+
+func TestPerformanceConcurrentConsumers(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	topicName := "performance-concurrent-test"
+	namespace := "perf-test"
+	numPartitions := int32(4)
+
+	// Create publisher first
+	pubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName,
+		PartitionCount: numPartitions,
+		PublisherName:  "concurrent-publisher",
+		RecordType:     nil,
+	}
+
+	publisher, err := suite.CreatePublisher(pubConfig)
+	require.NoError(t, err)
+
+	// Test parameters
+	numConsumers := 8
+	numMessages := 20000
+	consumerGroup := "concurrent-perf-group"
+
+	t.Logf("Starting concurrent consumer test: %d consumers, %d messages, %d partitions",
+		numConsumers, numMessages, numPartitions)
+
+	// Start multiple consumers
+	var collectors []*MessageCollector
+
+	for i := 0; i < numConsumers; i++ {
+		collector := NewMessageCollector(numMessages / numConsumers) // Expected per consumer
+		collectors = append(collectors, collector)
+
+		subConfig := &SubscriberTestConfig{
+			Namespace:          namespace,
+			TopicName:          topicName,
+			ConsumerGroup:      consumerGroup,
+			ConsumerInstanceId: fmt.Sprintf("consumer-%d", i),
+			MaxPartitionCount:  numPartitions,
+			SlidingWindowSize:  10,
+			OffsetType:         schema_pb.OffsetType_RESET_TO_EARLIEST,
+		}
+
+		subscriber, err := suite.CreateSubscriber(subConfig)
+		require.NoError(t, err)
+
+		// Set up message handler for this consumer
+		func(consumerID int, c *MessageCollector) {
+			subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
+				c.AddMessage(TestMessage{
+					ID:        fmt.Sprintf("%d-%s", consumerID, string(m.Data.Key)),
+					Content:   m.Data.Value,
+					Timestamp: time.Unix(0, m.Data.TsNs),
+					Key:       m.Data.Key,
+				})
+			})
+		}(i, collector)
+
+		// Start subscriber
+		go func(consumerID int, s *SubscriberTestConfig) {
+			sub, _ := suite.CreateSubscriber(s)
+			err := sub.Subscribe()
+			if err != nil {
+				t.Logf("Consumer %d error: %v", consumerID, err)
+			}
+		}(i, subConfig)
+	}
+
+	// Wait for consumers to initialize
+	time.Sleep(3 * time.Second)
+
+	// Start publishing
+	startTime := time.Now()
+
+	var publishWg sync.WaitGroup
+	numPublishers := 3
+	messagesPerPublisher := numMessages / numPublishers
+
+	for i := 0; i < numPublishers; i++ {
+		publishWg.Add(1)
+		go func(publisherID int) {
+			defer publishWg.Done()
+
+			for j := 0; j < messagesPerPublisher; j++ {
+				messageKey := fmt.Sprintf("concurrent-msg-%d-%d", publisherID, j)
+				payload := fmt.Sprintf("publisher-%d-message-%d-data", publisherID, j)
+
+				err := publisher.Publish([]byte(messageKey), []byte(payload))
+
+				if err != nil {
+					t.Logf("Publish error: %v", err)
+				}
+
+				// Rate limiting
+				if j%1000 == 0 {
+					time.Sleep(2 * time.Millisecond)
+				}
+			}
+		}(i)
+	}
+
+	publishWg.Wait()
+	publishDuration := time.Since(startTime)
+
+	// Allow time for message consumption
+	time.Sleep(10 * time.Second)
+
+	// Analyze results
+	totalConsumed := int64(0)
+	for i, collector := range collectors {
+		messages := collector.GetMessages()
+		consumed := int64(len(messages))
+		totalConsumed += consumed
+		t.Logf("Consumer %d consumed %d messages", i, consumed)
+	}
+
+	publishThroughput := float64(numMessages) / publishDuration.Seconds()
+	consumeThroughput := float64(totalConsumed) / publishDuration.Seconds()
+
+	t.Logf("Concurrent Consumer Test Results:")
+	t.Logf("  Total Published: %d", numMessages)
+	t.Logf("  Total Consumed: %d", totalConsumed)
+	t.Logf("  Publish Throughput: %.2f msg/sec", publishThroughput)
+	t.Logf("  Consume Throughput: %.2f msg/sec", consumeThroughput)
+	t.Logf("  Test Duration: %v", publishDuration)
+
+	// Performance assertions (relaxed for initial testing)
+	deliveryRate := float64(totalConsumed) / float64(numMessages)
+	assert.Greater(t, deliveryRate, 0.70, "Should consume at least 70% of messages")
+
+	expectedMinThroughput := 2000.0 // 2K messages/sec minimum for concurrent consumption
+	assert.Greater(t, consumeThroughput, expectedMinThroughput,
+		"Consume throughput should exceed %.0f messages/second", expectedMinThroughput)
+}
+
+func TestPerformanceWithErrorHandling(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	topicName := "performance-error-handling-test"
+	namespace := "perf-test"
+
+	metrics := &EnhancedPerformanceMetrics{
+		StartTime: time.Now(),
+	}
+
+	// Test parameters
+	numMessages := 50000
+	numPublishers := 5
+	messageSize := 1024 // 1KB messages
+
+	// Create message payload
+	payload := make([]byte, messageSize)
+	for i := range payload {
+		payload[i] = byte(i % 256)
+	}
+
+	t.Logf("Starting performance test with enhanced error handling: %d messages, %d publishers, %d bytes per message",
+		numMessages, numPublishers, messageSize)
+
+	// Start publishers
+	var publishWg sync.WaitGroup
+	messagesPerPublisher := numMessages / numPublishers
+
+	for i := 0; i < numPublishers; i++ {
+		publishWg.Add(1)
+		go func(publisherID int) {
+			defer publishWg.Done()
+
+			// Create publisher for this goroutine
+			pubConfig := &PublisherTestConfig{
+				Namespace:      namespace,
+				TopicName:      topicName,
+				PartitionCount: 4, // Multiple partitions for better throughput
+				PublisherName:  fmt.Sprintf("error-aware-publisher-%d", publisherID),
+				RecordType:     nil, // Use raw publish
+			}
+
+			publisher, err := suite.CreatePublisher(pubConfig)
+			if err != nil {
+				atomic.AddInt64(&metrics.ErrorCount, 1)
+				t.Errorf("Failed to create publisher %d: %v", publisherID, err)
+				return
+			}
+
+			for j := 0; j < messagesPerPublisher; j++ {
+				messageKey := fmt.Sprintf("publisher-%d-msg-%d", publisherID, j)
+
+				start := time.Now()
+				err := publisher.Publish([]byte(messageKey), payload)
+				latency := time.Since(start)
+
+				if err != nil {
+					atomic.AddInt64(&metrics.ErrorCount, 1)
+
+					// Classify the error type
+					if isConnectionError(err) {
+						atomic.AddInt64(&metrics.ConnectionErrors, 1)
+						t.Logf("Connection error (publisher %d, msg %d): %v", publisherID, j, err)
+					} else {
+						atomic.AddInt64(&metrics.ApplicationErrors, 1)
+						t.Logf("Application error (publisher %d, msg %d): %v", publisherID, j, err)
+					}
+					continue
+				}
+
+				atomic.AddInt64(&metrics.MessagesPublished, 1)
+				metrics.AddPublishLatency(latency)
+
+				// Small delay to prevent overwhelming the system
+				if j%1000 == 0 {
+					time.Sleep(1 * time.Millisecond)
+				}
+			}
+		}(i)
+	}
+
+	// Wait for publishing to complete
+	publishWg.Wait()
+	metrics.EndTime = time.Now()
+
+	// Analyze results with enhanced error reporting
+	publishedCount := atomic.LoadInt64(&metrics.MessagesPublished)
+	totalErrors := atomic.LoadInt64(&metrics.ErrorCount)
+	connectionErrors := atomic.LoadInt64(&metrics.ConnectionErrors)
+	applicationErrors := atomic.LoadInt64(&metrics.ApplicationErrors)
+	throughput := metrics.GetThroughput()
+
+	t.Logf("Enhanced Performance Results:")
+	t.Logf("  Messages Successfully Published: %d", publishedCount)
+	t.Logf("  Total Errors: %d", totalErrors)
+	t.Logf("  Connection-Level Errors: %d", connectionErrors)
+	t.Logf("  Application-Level Errors: %d", applicationErrors)
+	t.Logf("  Throughput: %.2f messages/second", throughput)
+	t.Logf("  Duration: %v", metrics.EndTime.Sub(metrics.StartTime))
+
+	if len(metrics.PublishLatencies) > 0 {
+		p95Latency := metrics.GetP95Latency(metrics.PublishLatencies)
+		t.Logf("  P95 Publish Latency: %v", p95Latency)
+
+		// Performance assertions (adjusted for error handling overhead)
+		assert.Less(t, p95Latency, 100*time.Millisecond, "P95 publish latency should be under 100ms")
+	}
+
+	// Enhanced error analysis
+	totalAttempts := publishedCount + totalErrors
+	if totalAttempts > 0 {
+		successRate := float64(publishedCount) / float64(totalAttempts)
+		connectionErrorRate := float64(connectionErrors) / float64(totalAttempts)
+		applicationErrorRate := float64(applicationErrors) / float64(totalAttempts)
+
+		t.Logf("Error Analysis:")
+		t.Logf("  Success Rate: %.2f%%", successRate*100)
+		t.Logf("  Connection Error Rate: %.2f%%", connectionErrorRate*100)
+		t.Logf("  Application Error Rate: %.2f%%", applicationErrorRate*100)
+
+		// Assertions based on error types
+		assert.Greater(t, successRate, 0.80, "Success rate should be greater than 80%")
+		assert.Less(t, applicationErrorRate, 0.01, "Application error rate should be less than 1%")
+
+		// Connection errors are expected under high load but should be handled
+		if connectionErrors > 0 {
+			t.Logf("Note: %d connection errors detected - this indicates the test is successfully stressing the system", connectionErrors)
+			t.Logf("Recommendation: Implement retry logic for production applications to handle these connection errors")
+		}
+	}
+
+	// Throughput requirements (adjusted for error handling)
+	expectedMinThroughput := 5000.0 // 5K messages/sec minimum
+	assert.Greater(t, throughput, expectedMinThroughput,
+		"Throughput should exceed %.0f messages/second", expectedMinThroughput)
+}

test/mq/integration/resilient_publisher.go

@@ -0,0 +1,336 @@
+package integration
+
+import (
+	"fmt"
+	"math"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// ResilientPublisher wraps TopicPublisher with enhanced error handling
+type ResilientPublisher struct {
+	publisher *pub_client.TopicPublisher
+	config    *PublisherTestConfig
+	suite     *IntegrationTestSuite
+
+	// Error tracking
+	connectionErrors  int64
+	applicationErrors int64
+	retryAttempts     int64
+	totalPublishes    int64
+
+	// Retry configuration
+	maxRetries    int
+	baseDelay     time.Duration
+	maxDelay      time.Duration
+	backoffFactor float64
+
+	// Circuit breaker
+	circuitOpen     bool
+	circuitOpenTime time.Time
+	circuitTimeout  time.Duration
+
+	mu sync.RWMutex
+}
+
+// RetryConfig holds retry configuration
+type RetryConfig struct {
+	MaxRetries     int
+	BaseDelay      time.Duration
+	MaxDelay       time.Duration
+	BackoffFactor  float64
+	CircuitTimeout time.Duration
+}
+
+// DefaultRetryConfig returns sensible defaults for retry configuration
+func DefaultRetryConfig() *RetryConfig {
+	return &RetryConfig{
+		MaxRetries:     5,
+		BaseDelay:      10 * time.Millisecond,
+		MaxDelay:       5 * time.Second,
+		BackoffFactor:  2.0,
+		CircuitTimeout: 30 * time.Second,
+	}
+}
+
+// NewResilientPublisher creates a new resilient publisher
+func (its *IntegrationTestSuite) CreateResilientPublisher(config *PublisherTestConfig, retryConfig *RetryConfig) (*ResilientPublisher, error) {
+	if retryConfig == nil {
+		retryConfig = DefaultRetryConfig()
+	}
+
+	publisher, err := its.CreatePublisher(config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create base publisher: %v", err)
+	}
+
+	return &ResilientPublisher{
+		publisher:      publisher,
+		config:         config,
+		suite:          its,
+		maxRetries:     retryConfig.MaxRetries,
+		baseDelay:      retryConfig.BaseDelay,
+		maxDelay:       retryConfig.MaxDelay,
+		backoffFactor:  retryConfig.BackoffFactor,
+		circuitTimeout: retryConfig.CircuitTimeout,
+	}, nil
+}
+
+// PublishWithRetry publishes a message with retry logic and error handling
+func (rp *ResilientPublisher) PublishWithRetry(key, value []byte) error {
+	atomic.AddInt64(&rp.totalPublishes, 1)
+
+	// Check circuit breaker
+	if rp.isCircuitOpen() {
+		atomic.AddInt64(&rp.applicationErrors, 1)
+		return fmt.Errorf("circuit breaker is open")
+	}
+
+	var lastErr error
+	for attempt := 0; attempt <= rp.maxRetries; attempt++ {
+		if attempt > 0 {
+			atomic.AddInt64(&rp.retryAttempts, 1)
+			delay := rp.calculateDelay(attempt)
+			glog.V(1).Infof("Retrying publish after %v (attempt %d/%d)", delay, attempt, rp.maxRetries)
+			time.Sleep(delay)
+		}
+
+		err := rp.publisher.Publish(key, value)
+		if err == nil {
+			// Success - reset circuit breaker if it was open
+			rp.resetCircuitBreaker()
+			return nil
+		}
+
+		lastErr = err
+
+		// Classify error type
+		if rp.isConnectionError(err) {
+			atomic.AddInt64(&rp.connectionErrors, 1)
+			glog.V(1).Infof("Connection error on attempt %d: %v", attempt+1, err)
+
+			// For connection errors, try to recreate the publisher
+			if attempt < rp.maxRetries {
+				if recreateErr := rp.recreatePublisher(); recreateErr != nil {
+					glog.Warningf("Failed to recreate publisher: %v", recreateErr)
+				}
+			}
+			continue
+		} else {
+			// Application error - don't retry
+			atomic.AddInt64(&rp.applicationErrors, 1)
+			glog.Warningf("Application error (not retrying): %v", err)
+			break
+		}
+	}
+
+	// All retries exhausted or non-retryable error
+	rp.openCircuitBreaker()
+	return fmt.Errorf("publish failed after %d attempts, last error: %v", rp.maxRetries+1, lastErr)
+}
+
+// PublishRecord publishes a record with retry logic
+func (rp *ResilientPublisher) PublishRecord(key []byte, record *schema_pb.RecordValue) error {
+	atomic.AddInt64(&rp.totalPublishes, 1)
+
+	if rp.isCircuitOpen() {
+		atomic.AddInt64(&rp.applicationErrors, 1)
+		return fmt.Errorf("circuit breaker is open")
+	}
+
+	var lastErr error
+	for attempt := 0; attempt <= rp.maxRetries; attempt++ {
+		if attempt > 0 {
+			atomic.AddInt64(&rp.retryAttempts, 1)
+			delay := rp.calculateDelay(attempt)
+			time.Sleep(delay)
+		}
+
+		err := rp.publisher.PublishRecord(key, record)
+		if err == nil {
+			rp.resetCircuitBreaker()
+			return nil
+		}
+
+		lastErr = err
+
+		if rp.isConnectionError(err) {
+			atomic.AddInt64(&rp.connectionErrors, 1)
+			if attempt < rp.maxRetries {
+				if recreateErr := rp.recreatePublisher(); recreateErr != nil {
+					glog.Warningf("Failed to recreate publisher: %v", recreateErr)
+				}
+			}
+			continue
+		} else {
+			atomic.AddInt64(&rp.applicationErrors, 1)
+			break
+		}
+	}
+
+	rp.openCircuitBreaker()
+	return fmt.Errorf("publish record failed after %d attempts, last error: %v", rp.maxRetries+1, lastErr)
+}
+
+// recreatePublisher attempts to recreate the underlying publisher
+func (rp *ResilientPublisher) recreatePublisher() error {
+	rp.mu.Lock()
+	defer rp.mu.Unlock()
+
+	// Shutdown old publisher
+	if rp.publisher != nil {
+		rp.publisher.Shutdown()
+	}
+
+	// Create new publisher
+	newPublisher, err := rp.suite.CreatePublisher(rp.config)
+	if err != nil {
+		return fmt.Errorf("failed to recreate publisher: %v", err)
+	}
+
+	rp.publisher = newPublisher
+	glog.V(1).Infof("Successfully recreated publisher")
+	return nil
+}
+
+// isConnectionError determines if an error is a connection-level error that should be retried
+func (rp *ResilientPublisher) isConnectionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errStr := err.Error()
+
+	// Check for gRPC status codes
+	if st, ok := status.FromError(err); ok {
+		switch st.Code() {
+		case codes.Unavailable, codes.DeadlineExceeded, codes.Canceled, codes.Unknown:
+			return true
+		}
+	}
+
+	// Check for common connection error strings
+	connectionErrorPatterns := []string{
+		"EOF",
+		"error reading server preface",
+		"connection refused",
+		"connection reset",
+		"broken pipe",
+		"network is unreachable",
+		"no route to host",
+		"transport is closing",
+		"connection error",
+		"dial tcp",
+		"context deadline exceeded",
+	}
+
+	for _, pattern := range connectionErrorPatterns {
+		if contains(errStr, pattern) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// contains checks if a string contains a substring (case-insensitive)
+func contains(s, substr string) bool {
+	return len(s) >= len(substr) &&
+		(s == substr ||
+			len(s) > len(substr) &&
+				(s[:len(substr)] == substr ||
+					s[len(s)-len(substr):] == substr ||
+					containsSubstring(s, substr)))
+}
+
+func containsSubstring(s, substr string) bool {
+	for i := 0; i <= len(s)-len(substr); i++ {
+		if s[i:i+len(substr)] == substr {
+			return true
+		}
+	}
+	return false
+}
+
+// calculateDelay calculates exponential backoff delay
+func (rp *ResilientPublisher) calculateDelay(attempt int) time.Duration {
+	delay := float64(rp.baseDelay) * math.Pow(rp.backoffFactor, float64(attempt-1))
+	if delay > float64(rp.maxDelay) {
+		delay = float64(rp.maxDelay)
+	}
+	return time.Duration(delay)
+}
+
+// Circuit breaker methods
+func (rp *ResilientPublisher) isCircuitOpen() bool {
+	rp.mu.RLock()
+	defer rp.mu.RUnlock()
+
+	if !rp.circuitOpen {
+		return false
+	}
+
+	// Check if circuit should be reset
+	if time.Since(rp.circuitOpenTime) > rp.circuitTimeout {
+		return false
+	}
+
+	return true
+}
+
+func (rp *ResilientPublisher) openCircuitBreaker() {
+	rp.mu.Lock()
+	defer rp.mu.Unlock()
+
+	rp.circuitOpen = true
+	rp.circuitOpenTime = time.Now()
+	glog.Warningf("Circuit breaker opened due to repeated failures")
+}
+
+func (rp *ResilientPublisher) resetCircuitBreaker() {
+	rp.mu.Lock()
+	defer rp.mu.Unlock()
+
+	if rp.circuitOpen {
+		rp.circuitOpen = false
+		glog.V(1).Infof("Circuit breaker reset")
+	}
+}
+
+// GetErrorStats returns error statistics
+func (rp *ResilientPublisher) GetErrorStats() ErrorStats {
+	return ErrorStats{
+		ConnectionErrors:  atomic.LoadInt64(&rp.connectionErrors),
+		ApplicationErrors: atomic.LoadInt64(&rp.applicationErrors),
+		RetryAttempts:     atomic.LoadInt64(&rp.retryAttempts),
+		TotalPublishes:    atomic.LoadInt64(&rp.totalPublishes),
+		CircuitOpen:       rp.isCircuitOpen(),
+	}
+}
+
+// ErrorStats holds error statistics
+type ErrorStats struct {
+	ConnectionErrors  int64
+	ApplicationErrors int64
+	RetryAttempts     int64
+	TotalPublishes    int64
+	CircuitOpen       bool
+}
+
+// Shutdown gracefully shuts down the resilient publisher
+func (rp *ResilientPublisher) Shutdown() error {
+	rp.mu.Lock()
+	defer rp.mu.Unlock()
+
+	if rp.publisher != nil {
+		return rp.publisher.Shutdown()
+	}
+	return nil
+}

test/mq/integration_test_design.md

@@ -0,0 +1,286 @@
+# SeaweedMQ Integration Test Design
+
+## Overview
+
+This document outlines the comprehensive integration test strategy for SeaweedMQ, covering all critical functionalities from basic pub/sub operations to advanced features like auto-scaling, failover, and performance testing.
+
+## Architecture Under Test
+
+SeaweedMQ consists of:
+- **Masters**: Cluster coordination and metadata management
+- **Volume Servers**: Storage layer for persistent messages
+- **Filers**: File system interface for metadata storage
+- **Brokers**: Message processing and routing (stateless)
+- **Agents**: Client interface for pub/sub operations
+- **Schema System**: Protobuf-based message schema management
+
+## Test Categories
+
+### 1. Basic Functionality Tests
+
+#### 1.1 Basic Pub/Sub Operations
+- **Test**: `TestBasicPublishSubscribe`
+  - Publish messages to a topic
+  - Subscribe and receive messages
+  - Verify message content and ordering
+  - Test with different data types (string, int, bytes, records)
+
+- **Test**: `TestMultipleConsumers`
+  - Multiple subscribers on same topic
+  - Verify message distribution
+  - Test consumer group functionality
+
+- **Test**: `TestMessageOrdering`
+  - Publish messages in sequence
+  - Verify FIFO ordering within partitions
+  - Test with different partition keys
+
+#### 1.2 Schema Management
+- **Test**: `TestSchemaValidation`
+  - Publish with valid schemas
+  - Reject invalid schema messages
+  - Test schema evolution scenarios
+
+- **Test**: `TestRecordTypes`
+  - Nested record structures
+  - List types and complex schemas
+  - Schema-to-Parquet conversion
+
+### 2. Partitioning and Scaling Tests
+
+#### 2.1 Partition Management
+- **Test**: `TestPartitionDistribution`
+  - Messages distributed across partitions based on keys
+  - Verify partition assignment logic
+  - Test partition rebalancing
+
+- **Test**: `TestAutoSplitMerge`
+  - Simulate high load to trigger auto-split
+  - Simulate low load to trigger auto-merge
+  - Verify data consistency during splits/merges
+
+#### 2.2 Broker Scaling
+- **Test**: `TestBrokerAddRemove`
+  - Add brokers during operation
+  - Remove brokers gracefully
+  - Verify partition reassignment
+
+- **Test**: `TestLoadBalancing`
+  - Verify even load distribution across brokers
+  - Test with varying message sizes and rates
+  - Monitor broker resource utilization
+
+### 3. Failover and Reliability Tests
+
+#### 3.1 Broker Failover
+- **Test**: `TestBrokerFailover`
+  - Kill leader broker during publishing
+  - Verify seamless failover to follower
+  - Test data consistency after failover
+
+- **Test**: `TestBrokerRecovery`
+  - Broker restart scenarios
+  - State recovery from storage
+  - Partition reassignment after recovery
+
+#### 3.2 Data Durability
+- **Test**: `TestMessagePersistence`
+  - Publish messages and restart cluster
+  - Verify all messages are recovered
+  - Test with different replication settings
+
+- **Test**: `TestFollowerReplication`
+  - Leader-follower message replication
+  - Verify consistency between replicas
+  - Test follower promotion scenarios
+
+### 4. Agent Functionality Tests
+
+#### 4.1 Session Management
+- **Test**: `TestPublishSessions`
+  - Create/close publish sessions
+  - Concurrent session management
+  - Session cleanup after failures
+
+- **Test**: `TestSubscribeSessions`
+  - Subscribe session lifecycle
+  - Consumer group management
+  - Offset tracking and acknowledgments
+
+#### 4.2 Error Handling
+- **Test**: `TestConnectionFailures`
+  - Network partitions between agent and broker
+  - Automatic reconnection logic
+  - Message buffering during outages
+
+### 5. Performance and Load Tests
+
+#### 5.1 Throughput Tests
+- **Test**: `TestHighThroughputPublish`
+  - Publish 100K+ messages/second
+  - Monitor system resources
+  - Verify no message loss
+
+- **Test**: `TestHighThroughputSubscribe`
+  - Multiple consumers processing high volume
+  - Monitor processing latency
+  - Test backpressure handling
+
+#### 5.2 Spike Traffic Tests
+- **Test**: `TestTrafficSpikes`
+  - Sudden increase in message volume
+  - Auto-scaling behavior verification
+  - Resource utilization patterns
+
+- **Test**: `TestLargeMessages`
+  - Messages with large payloads (MB size)
+  - Memory usage monitoring
+  - Storage efficiency testing
+
+### 6. End-to-End Scenarios
+
+#### 6.1 Complete Workflow Tests
+- **Test**: `TestProducerConsumerWorkflow`
+  - Multi-stage data processing pipeline
+  - Producer → Topic → Multiple Consumers
+  - Data transformation and aggregation
+
+- **Test**: `TestMultiTopicOperations`
+  - Multiple topics with different schemas
+  - Cross-topic message routing
+  - Topic management operations
+
+## Test Infrastructure
+
+### Environment Setup
+
+#### Docker Compose Configuration
+```yaml
+# test-environment.yml
+version: '3.9'
+services:
+  master-cluster:
+    # 3 master nodes for HA
+  volume-cluster:
+    # 3 volume servers for data storage
+  filer-cluster:
+    # 2 filers for metadata
+  broker-cluster:
+    # 3 brokers for message processing
+  test-runner:
+    # Container to run integration tests
+```
+
+#### Test Data Management
+- Pre-defined test schemas
+- Sample message datasets
+- Performance benchmarking data
+
+### Test Framework Structure
+
+```go
+// Base test framework
+type IntegrationTestSuite struct {
+    masters     []string
+    brokers     []string
+    filers      []string
+    testClient  *TestClient
+    cleanup     []func()
+}
+
+// Test utilities
+type TestClient struct {
+    publishers  map[string]*pub_client.TopicPublisher
+    subscribers map[string]*sub_client.TopicSubscriber
+    agents      []*agent.MessageQueueAgent
+}
+```
+
+### Monitoring and Metrics
+
+#### Health Checks
+- Broker connectivity status
+- Master cluster health
+- Storage system availability
+- Network connectivity between components
+
+#### Performance Metrics
+- Message throughput (msgs/sec)
+- End-to-end latency
+- Resource utilization (CPU, Memory, Disk)
+- Network bandwidth usage
+
+## Test Execution Strategy
+
+### Parallel Test Execution
+- Categorize tests by resource requirements
+- Run independent tests in parallel
+- Serialize tests that modify cluster state
+
+### Continuous Integration
+- Automated test runs on PR submissions
+- Performance regression detection
+- Multi-platform testing (Linux, macOS, Windows)
+
+### Test Environment Management
+- Docker-based isolated environments
+- Automatic cleanup after test completion
+- Resource monitoring and alerts
+
+## Success Criteria
+
+### Functional Requirements
+- ✅ All messages published are received by subscribers
+- ✅ Message ordering preserved within partitions
+- ✅ Schema validation works correctly
+- ✅ Auto-scaling triggers at expected thresholds
+- ✅ Failover completes within 30 seconds
+- ✅ No data loss during normal operations
+
+### Performance Requirements
+- ✅ Throughput: 50K+ messages/second/broker
+- ✅ Latency: P95 < 100ms end-to-end
+- ✅ Memory usage: < 1GB per broker under normal load
+- ✅ Storage efficiency: < 20% overhead vs raw message size
+
+### Reliability Requirements
+- ✅ 99.9% uptime during normal operations
+- ✅ Automatic recovery from single component failures
+- ✅ Data consistency maintained across all scenarios
+- ✅ Graceful degradation under resource constraints
+
+## Implementation Timeline
+
+### Phase 1: Core Functionality (Week 1-2)
+- Basic pub/sub tests
+- Schema validation tests
+- Simple failover scenarios
+
+### Phase 2: Advanced Features (Week 3-4)
+- Auto-scaling tests
+- Complex failover scenarios
+- Agent functionality tests
+
+### Phase 3: Performance & Load (Week 5-6)
+- Throughput and latency tests
+- Spike traffic handling
+- Resource utilization monitoring
+
+### Phase 4: End-to-End (Week 7-8)
+- Complete workflow tests
+- Multi-component integration
+- Performance regression testing
+
+## Maintenance and Updates
+
+### Regular Updates
+- Add tests for new features
+- Update performance baselines
+- Enhance error scenarios coverage
+
+### Test Data Refresh
+- Generate new test datasets quarterly
+- Update schema examples
+- Refresh performance benchmarks
+
+This comprehensive test design ensures SeaweedMQ's reliability, performance, and functionality across all critical use cases and failure scenarios. 

test/mq/prometheus.yml

@@ -0,0 +1,54 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+scrape_configs:
+  # SeaweedFS Masters
+  - job_name: 'seaweedfs-master'
+    static_configs:
+      - targets: 
+        - 'master0:9333'
+        - 'master1:9334'
+        - 'master2:9335'
+    metrics_path: '/metrics'
+    scrape_interval: 10s
+
+  # SeaweedFS Volume Servers
+  - job_name: 'seaweedfs-volume'
+    static_configs:
+      - targets:
+        - 'volume1:8080'
+        - 'volume2:8081'
+        - 'volume3:8082'
+    metrics_path: '/metrics'
+    scrape_interval: 10s
+
+  # SeaweedFS Filers
+  - job_name: 'seaweedfs-filer'
+    static_configs:
+      - targets:
+        - 'filer1:8888'
+        - 'filer2:8889'
+    metrics_path: '/metrics'
+    scrape_interval: 10s
+
+  # SeaweedMQ Brokers
+  - job_name: 'seaweedmq-broker'
+    static_configs:
+      - targets:
+        - 'broker1:17777'
+        - 'broker2:17778'
+        - 'broker3:17779'
+    metrics_path: '/metrics'
+    scrape_interval: 5s
+
+  # Docker containers
+  - job_name: 'docker'
+    static_configs:
+      - targets: ['localhost:9323']
+    metrics_path: '/metrics'
+    scrape_interval: 30s 

test/mq/simple_demo.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+)
+
+func main() {
+	log.Println("Starting SeaweedMQ logging test...")
+
+	// Create publisher configuration
+	config := &pub_client.PublisherConfiguration{
+		Topic:          topic.NewTopic("test", "logging-demo"),
+		PartitionCount: 3,
+		Brokers:        []string{"127.0.0.1:17777"},
+		PublisherName:  "logging-test-client",
+	}
+
+	log.Println("Creating topic publisher...")
+	publisher, err := pub_client.NewTopicPublisher(config)
+	if err != nil {
+		log.Printf("Failed to create publisher: %v", err)
+		return
+	}
+	defer publisher.Shutdown()
+
+	log.Println("Publishing test messages...")
+
+	// Publish some test messages
+	for i := 0; i < 100; i++ {
+		key := fmt.Sprintf("key-%d", i)
+		value := fmt.Sprintf("message-%d-timestamp-%d", i, time.Now().Unix())
+
+		err := publisher.Publish([]byte(key), []byte(value))
+		if err != nil {
+			log.Printf("Failed to publish message %d: %v", i, err)
+		}
+
+		// Small delay to create some connection stress
+		if i%10 == 0 {
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+	log.Println("Finishing publish...")
+	err = publisher.FinishPublish()
+	if err != nil {
+		log.Printf("Failed to finish publish: %v", err)
+	}
+
+	log.Println("Test completed successfully!")
+}

weed/cluster/lock_manager/lock_ring.go

@@ -1,12 +1,13 @@
 package lock_manager
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"sort"
 	"sync"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 type LockRingSnapshot struct {
@@ -22,6 +23,7 @@ type LockRing struct {
 	lastCompactTime  time.Time
 	snapshotInterval time.Duration
 	onTakeSnapshot   func(snapshot []pb.ServerAddress)
+	cleanupWg        sync.WaitGroup
 }
 
 func NewLockRing(snapshotInterval time.Duration) *LockRing {
@@ -87,7 +89,9 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
 
 	r.addOneSnapshot(servers)
 
+	r.cleanupWg.Add(1)
 	go func() {
+		defer r.cleanupWg.Done()
 		<-time.After(r.snapshotInterval)
 		r.compactSnapshots()
 	}()
@@ -96,7 +100,9 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
 func (r *LockRing) takeSnapshotWithDelayedCompaction() {
 	r.doTakeSnapshot()
 
+	r.cleanupWg.Add(1)
 	go func() {
+		defer r.cleanupWg.Done()
 		<-time.After(r.snapshotInterval)
 		r.compactSnapshots()
 	}()
@@ -172,6 +178,12 @@ func (r *LockRing) GetSnapshot() (servers []pb.ServerAddress) {
 	return r.snapshots[0].servers
 }
 
+// WaitForCleanup waits for all pending cleanup operations to complete
+// This is useful for testing to ensure deterministic behavior
+func (r *LockRing) WaitForCleanup() {
+	r.cleanupWg.Wait()
+}
+
 func hashKeyToServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
 	if len(servers) == 0 {
 		return ""

weed/cluster/lock_manager/lock_ring_test.go

@@ -1,10 +1,11 @@
 package lock_manager
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/stretchr/testify/assert"
 	"testing"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestAddServer(t *testing.T) {
@@ -21,7 +22,9 @@ func TestAddServer(t *testing.T) {
 
 	assert.Equal(t, 8, len(r.snapshots))
 
-	time.Sleep(110 * time.Millisecond)
+	// Wait for all cleanup operations to complete instead of using time.Sleep
+	time.Sleep(110 * time.Millisecond) // Still need to wait for the cleanup interval
+	r.WaitForCleanup()                 // Ensure all cleanup goroutines have finished
 
 	assert.Equal(t, 2, len(r.snapshots))
 

weed/command/backup.go

@@ -115,7 +115,10 @@ func runBackup(cmd *Command, args []string) bool {
 			return true
 		}
 	}
-	v, err := storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0)
+
+	ver := needle.Version(stats.Version)
+
+	v, err := storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, ver, 0, 0)
 	if err != nil {
 		fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
 		return true
@@ -142,7 +145,7 @@ func runBackup(cmd *Command, args []string) bool {
 			fmt.Printf("Error destroying volume: %v\n", err)
 		}
 		// recreate an empty volume
-		v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0)
+		v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, ver, 0, 0)
 		if err != nil {
 			fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
 			return true

weed/command/compact.go

@@ -41,7 +41,7 @@ func runCompact(cmd *Command, args []string) bool {
 	preallocate := *compactVolumePreallocate * (1 << 20)
 
 	vid := needle.VolumeId(*compactVolumeId)
-	v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, 0, 0)
+	v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, needle.GetCurrentVersion(), 0, 0)
 	if err != nil {
 		glog.Fatalf("Load Volume [ERROR] %s\n", err)
 	}

weed/command/fuse_std.go

@@ -193,6 +193,13 @@ func runFuse(cmd *Command, args []string) bool {
 			} else {
 				panic(fmt.Errorf("readOnly: %s", err))
 			}
+		case "disableXAttr":
+			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
+
+				mountOptions.disableXAttr = &parsed
+			} else {
+				panic(fmt.Errorf("disableXAttr: %s", err))
+			}
 		case "cpuprofile":
 			mountCpuProfile = &parameter.value
 		case "memprofile":

weed/command/mount_freebsd.go

@@ -0,0 +1,5 @@
+package command
+
+func checkMountPointAvailable(dir string) bool {
+	return true
+}

weed/command/mount_notsupported.go

@@ -1,5 +1,5 @@
-//go:build !linux && !darwin
-// +build !linux,!darwin
+//go:build !linux && !darwin && !freebsd
+// +build !linux,!darwin,!freebsd
 
 package command
 

weed/command/mount_std.go

@@ -1,12 +1,11 @@
-//go:build linux || darwin
-// +build linux darwin
+//go:build linux || darwin || freebsd
+// +build linux darwin freebsd
 
 package command
 
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net"
 	"net/http"
 	"os"
@@ -17,6 +16,8 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+
 	"github.com/hanwen/go-fuse/v2/fuse"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mount"

weed/command/mq_agent.go

@@ -32,7 +32,7 @@ func init() {
 
 var cmdMqAgent = &Command{
 	UsageLine: "mq.agent [-port=16777] [-broker=<ip:port>]",
-	Short:     "<WIP> start a message queue agent",
+	Short:     "start a message queue agent",
 	Long: `start a message queue agent
 
 	The agent runs on local server to accept gRPC calls to write or read messages. 

weed/command/mq_broker.go

@@ -43,7 +43,7 @@ func init() {
 
 var cmdMqBroker = &Command{
 	UsageLine: "mq.broker [-port=17777] [-master=<ip:port>]",
-	Short:     "<WIP> start a message queue broker",
+	Short:     "start a message queue broker",
 	Long: `start a message queue broker
 
 	The broker can accept gRPC calls to write or read messages. The messages are stored via filer.

weed/filer/mongodb/mongodb_store.go

@@ -234,16 +234,24 @@ func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dir
 		"directory": string(dirPath),
 	}
 
+	nameQuery := bson.M{}
+
 	if len(prefix) > 0 {
-		where["name"].(bson.M)["$regex"] = "^" + regexp.QuoteMeta(prefix)
+		nameQuery["$regex"] = "^" + regexp.QuoteMeta(prefix)
 	}
 
-	if includeStartFile {
-		where["name"].(bson.M)["$gte"] = startFileName
-	} else {
-		where["name"].(bson.M)["$gt"] = startFileName
+	if len(startFileName) > 0 {
+		if includeStartFile {
+			nameQuery["$gte"] = startFileName
+		} else {
+			nameQuery["$gt"] = startFileName
+		}
 	}
-		
+
+	if len(nameQuery) > 0 {
+		where["name"] = nameQuery
+	}
+
 	optLimit := int64(limit)
 	opts := &options.FindOptions{Limit: &optLimit, Sort: bson.M{"name": 1}}
 	cur, err := store.connect.Database(store.database).Collection(store.collectionName).Find(ctx, where, opts)

weed/filer/rocksdb/rocksdb_store.go

@@ -169,7 +169,7 @@ func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath we
 
 	iter := store.db.NewIterator(ro)
 	defer iter.Close()
-	err = enumerate(iter, directoryPrefix, nil, false, -1, func(key, value []byte) bool {
+	err = enumerate(iter, directoryPrefix, nil, false, -1, "", func(key, value []byte) bool {
 		batch.Delete(key)
 		return true
 	})
@@ -186,23 +186,16 @@ func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath we
 	return nil
 }
 
-func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int64, fn func(key, value []byte) bool) (err error) {
+func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int64, startFileName string, fn func(key, value []byte) bool) (err error) {
 
 	if len(lastKey) == 0 {
 		iter.Seek(prefix)
 	} else {
 		iter.Seek(lastKey)
-		if !includeLastKey {
-			if iter.Valid() {
-				if bytes.Equal(iter.Key().Data(), lastKey) {
-					iter.Next()
-				}
-			}
-		}
 	}
 
 	i := int64(0)
-	for ; iter.Valid(); iter.Next() {
+	for iter.Valid() {
 
 		if limit > 0 {
 			i++
@@ -217,12 +210,23 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey
 			break
 		}
 
+		fileName := getNameFromKey(key)
+		if fileName == "" {
+			iter.Next()
+			continue
+		}
+		if fileName == startFileName && !includeLastKey {
+			iter.Next()
+			continue
+		}
+
 		ret := fn(key, iter.Value().Data())
 
 		if !ret {
 			break
 		}
 
+		iter.Next()
 	}
 
 	if err := iter.Err(); err != nil {
@@ -249,7 +253,7 @@ func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dir
 
 	iter := store.db.NewIterator(ro)
 	defer iter.Close()
-	err = enumerate(iter, directoryPrefix, lastFileStart, includeStartFile, limit, func(key, value []byte) bool {
+	err = enumerate(iter, directoryPrefix, lastFileStart, includeStartFile, limit, startFileName, func(key, value []byte) bool {
 		fileName := getNameFromKey(key)
 		if fileName == "" {
 			return true

weed/filer/rocksdb/rocksdb_store_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
@@ -112,3 +113,73 @@ func BenchmarkInsertEntry(b *testing.B) {
 		store.InsertEntry(ctx, entry)
 	}
 }
+
+func TestListDirectoryWithPrefix(t *testing.T) {
+	testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
+	dir := t.TempDir()
+	store := &RocksDBStore{}
+	store.initialize(dir)
+	testFiler.SetStore(store)
+
+	ctx := context.Background()
+
+	files := []string{
+		"/bucket1/test-prefix1/file1.txt",
+		"/bucket1/test-prefix1/file2.txt",
+		"/bucket1/test-prefix1-extra.txt",
+	}
+
+	expected1 := []string{
+		"/bucket1/test-prefix1",
+		"/bucket1/test-prefix1-extra.txt",
+	}
+
+	expected2 := []string{
+		"/bucket1/test-prefix1/file1.txt",
+		"/bucket1/test-prefix1/file2.txt",
+	}
+
+	for _, file := range files {
+		fullpath := util.FullPath(file)
+		entry := &filer.Entry{
+			FullPath: fullpath,
+			Attr: filer.Attr{
+				Mode: 0644,
+				Uid:  1,
+				Gid:  1,
+			},
+		}
+		if err := testFiler.CreateEntry(ctx, entry, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {
+			t.Fatalf("Failed to create entry %s: %v", fullpath, err)
+		}
+	}
+
+	prefix1 := "test-prefix1"
+	entries1, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/bucket1"), "", false, 100, prefix1, "", "")
+	if err != nil {
+		t.Fatalf("Failed to list entries with prefix %s: %v", prefix1, err)
+	}
+	if len(entries1) != 2 {
+		t.Errorf("Expected 2 entries with prefix %s, got %d", prefix1, len(entries1))
+	} else {
+		t.Logf("Found %d entries with prefix %s", len(entries1), prefix1)
+	}
+	for i, entry := range entries1 {
+		if string(entry.FullPath) != expected1[i] {
+			t.Errorf("Expected entry %s, got %s", expected1[i], entry.FullPath)
+		}
+	}
+
+	entries2, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/bucket1/test-prefix1"), "", false, 100, "", "", "")
+	if err != nil {
+		t.Fatalf("Failed to list entries with prefix %s: %v", prefix1, err)
+	}
+	if len(entries2) != 2 {
+		t.Errorf("Expected 2 entries with prefix %s, got %d", prefix1, len(entries1))
+	}
+	for i, entry := range entries2 {
+		if string(entry.FullPath) != expected2[i] {
+			t.Errorf("Expected entry %s, got %s", expected2[i], entry.FullPath)
+		}
+	}
+}

weed/filer/ydb/ydb_queries.go

@@ -22,19 +22,21 @@ const (
 	deleteQuery = `
 		PRAGMA TablePathPrefix("%v");
 		DECLARE $dir_hash AS int64;
+		DECLARE $directory AS Utf8;
 		DECLARE $name AS Utf8;
 
 		DELETE FROM ` + asql.DEFAULT_TABLE + ` 
-		WHERE dir_hash = $dir_hash AND name = $name;`
+		WHERE dir_hash = $dir_hash AND directory = $directory AND name = $name;`
 
 	findQuery = `
 		PRAGMA TablePathPrefix("%v");
 		DECLARE $dir_hash AS int64;
+		DECLARE $directory AS Utf8;
 		DECLARE $name AS Utf8;
 		
 		SELECT meta
 		FROM ` + asql.DEFAULT_TABLE + ` 
-		WHERE dir_hash = $dir_hash AND name = $name;`
+		WHERE dir_hash = $dir_hash AND directory = $directory AND name = $name;`
 
 	deleteFolderChildrenQuery = `
 		PRAGMA TablePathPrefix("%v");

weed/filer/ydb/ydb_store.go

@@ -6,6 +6,8 @@ package ydb
 import (
 	"context"
 	"fmt"
+	"github.com/ydb-platform/ydb-go-sdk/v3/query"
+	"github.com/ydb-platform/ydb-go-sdk/v3/table/options"
 	"os"
 	"path"
 	"strings"
@@ -20,30 +22,37 @@ import (
 	environ "github.com/ydb-platform/ydb-go-sdk-auth-environ"
 	"github.com/ydb-platform/ydb-go-sdk/v3"
 	"github.com/ydb-platform/ydb-go-sdk/v3/table"
-	"github.com/ydb-platform/ydb-go-sdk/v3/table/result"
-	"github.com/ydb-platform/ydb-go-sdk/v3/table/result/named"
 	"github.com/ydb-platform/ydb-go-sdk/v3/table/types"
 )
 
 const (
-	defaultDialTimeOut = 10
+	defaultDialTimeOut            = 10
+	defaultPartitionBySizeEnabled = true
+	defaultPartitionSizeMb        = 200
+	defaultPartitionByLoadEnabled = true
+	defaultMinPartitionsCount     = 5
+	defaultMaxPartitionsCount     = 1000
+	defaultMaxListChunk           = 2000
 )
 
 var (
-	roTX = table.TxControl(
-		table.BeginTx(table.WithOnlineReadOnly()),
-		table.CommitTx(),
-	)
-	rwTX = table.DefaultTxControl()
+	roQC = query.WithTxControl(query.OnlineReadOnlyTxControl())
+	rwQC = query.WithTxControl(query.DefaultTxControl())
 )
 
 type YdbStore struct {
-	DB                 ydb.Connection
-	dirBuckets         string
-	tablePathPrefix    string
-	SupportBucketTable bool
-	dbs                map[string]bool
-	dbsLock            sync.Mutex
+	DB                     *ydb.Driver
+	dirBuckets             string
+	tablePathPrefix        string
+	SupportBucketTable     bool
+	partitionBySizeEnabled options.FeatureFlag
+	partitionSizeMb        uint64
+	partitionByLoadEnabled options.FeatureFlag
+	minPartitionsCount     uint64
+	maxPartitionsCount     uint64
+	maxListChunk           int
+	dbs                    map[string]bool
+	dbsLock                sync.Mutex
 }
 
 func init() {
@@ -55,6 +64,12 @@ func (store *YdbStore) GetName() string {
 }
 
 func (store *YdbStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	configuration.SetDefault(prefix+"partitionBySizeEnabled", defaultPartitionBySizeEnabled)
+	configuration.SetDefault(prefix+"partitionSizeMb", defaultPartitionSizeMb)
+	configuration.SetDefault(prefix+"partitionByLoadEnabled", defaultPartitionByLoadEnabled)
+	configuration.SetDefault(prefix+"minPartitionsCount", defaultMinPartitionsCount)
+	configuration.SetDefault(prefix+"maxPartitionsCount", defaultMaxPartitionsCount)
+	configuration.SetDefault(prefix+"maxListChunk", defaultMaxListChunk)
 	return store.initialize(
 		configuration.GetString("filer.options.buckets_folder"),
 		configuration.GetString(prefix+"dsn"),
@@ -62,18 +77,37 @@ func (store *YdbStore) Initialize(configuration util.Configuration, prefix strin
 		configuration.GetBool(prefix+"useBucketPrefix"),
 		configuration.GetInt(prefix+"dialTimeOut"),
 		configuration.GetInt(prefix+"poolSizeLimit"),
+		configuration.GetBool(prefix+"partitionBySizeEnabled"),
+		uint64(configuration.GetInt(prefix+"partitionSizeMb")),
+		configuration.GetBool(prefix+"partitionByLoadEnabled"),
+		uint64(configuration.GetInt(prefix+"minPartitionsCount")),
+		uint64(configuration.GetInt(prefix+"maxPartitionsCount")),
+		configuration.GetInt(prefix+"maxListChunk"),
 	)
 }
 
-func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix string, useBucketPrefix bool, dialTimeOut int, poolSizeLimit int) (err error) {
+func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix string, useBucketPrefix bool, dialTimeOut int, poolSizeLimit int, partitionBySizeEnabled bool, partitionSizeMb uint64, partitionByLoadEnabled bool, minPartitionsCount uint64, maxPartitionsCount uint64, maxListChunk int) (err error) {
 	store.dirBuckets = dirBuckets
 	store.SupportBucketTable = useBucketPrefix
+	if partitionBySizeEnabled {
+		store.partitionBySizeEnabled = options.FeatureEnabled
+	} else {
+		store.partitionBySizeEnabled = options.FeatureDisabled
+	}
+	if partitionByLoadEnabled {
+		store.partitionByLoadEnabled = options.FeatureEnabled
+	} else {
+		store.partitionByLoadEnabled = options.FeatureDisabled
+	}
+	store.partitionSizeMb = partitionSizeMb
+	store.minPartitionsCount = minPartitionsCount
+	store.maxPartitionsCount = maxPartitionsCount
+	store.maxListChunk = maxListChunk
 	if store.SupportBucketTable {
 		glog.V(0).Infof("enabled BucketPrefix")
 	}
 	store.dbs = make(map[string]bool)
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
+	ctx := context.Background()
 	if dialTimeOut == 0 {
 		dialTimeOut = defaultDialTimeOut
 	}
@@ -89,11 +123,7 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix
 	}
 	store.DB, err = ydb.Open(ctx, dsn, opts...)
 	if err != nil {
-		if store.DB != nil {
-			_ = store.DB.Close(ctx)
-			store.DB = nil
-		}
-		return fmt.Errorf("can not connect to %s error: %v", dsn, err)
+		return fmt.Errorf("can not connect to %s: %w", dsn, err)
 	}
 
 	store.tablePathPrefix = path.Join(store.DB.Name(), tablePathPrefix)
@@ -104,29 +134,27 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix
 	return err
 }
 
-func (store *YdbStore) doTxOrDB(ctx context.Context, query *string, params *table.QueryParameters, tc *table.TransactionControl, processResultFunc func(res result.Result) error) (err error) {
-	var res result.Result
-	if tx, ok := ctx.Value("tx").(table.Transaction); ok {
-		res, err = tx.Execute(ctx, *query, params)
+func (store *YdbStore) doTxOrDB(ctx context.Context, q *string, params *table.QueryParameters, ts query.ExecuteOption, processResultFunc func(res query.Result) error) (err error) {
+	var res query.Result
+	if tx, ok := ctx.Value("tx").(query.Transaction); ok {
+		res, err = tx.Query(ctx, *q, query.WithParameters(params))
 		if err != nil {
 			return fmt.Errorf("execute transaction: %v", err)
 		}
 	} else {
-		err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) {
-			_, res, err = s.Execute(ctx, tc, *query, params)
+		err = store.DB.Query().Do(ctx, func(ctx context.Context, s query.Session) (err error) {
+			res, err = s.Query(ctx, *q, query.WithParameters(params), ts)
 			if err != nil {
 				return fmt.Errorf("execute statement: %v", err)
 			}
 			return nil
-		},
-			table.WithIdempotent(),
-		)
+		}, query.WithIdempotent())
 	}
 	if err != nil {
 		return err
 	}
 	if res != nil {
-		defer func() { _ = res.Close() }()
+		defer func() { _ = res.Close(ctx) }()
 		if processResultFunc != nil {
 			if err = processResultFunc(res); err != nil {
 				return fmt.Errorf("process result: %v", err)
@@ -148,7 +176,7 @@ func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Ent
 	}
 	tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
 	fileMeta := FileMeta{util.HashStringToLong(dir), name, *shortDir, meta}
-	return store.doTxOrDB(ctx, withPragma(tablePathPrefix, upsertQuery), fileMeta.queryParameters(entry.TtlSec), rwTX, nil)
+	return store.doTxOrDB(ctx, withPragma(tablePathPrefix, upsertQuery), fileMeta.queryParameters(entry.TtlSec), rwQC, nil)
 }
 
 func (store *YdbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
@@ -164,23 +192,29 @@ func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (e
 	var data []byte
 	entryFound := false
 	tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
-	query := withPragma(tablePathPrefix, findQuery)
+	q := withPragma(tablePathPrefix, findQuery)
 	queryParams := table.NewQueryParameters(
 		table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
+		table.ValueParam("$directory", types.UTF8Value(*shortDir)),
 		table.ValueParam("$name", types.UTF8Value(name)))
 
-	err = store.doTxOrDB(ctx, query, queryParams, roTX, func(res result.Result) error {
-		if !res.NextResultSet(ctx) || !res.HasNextRow() {
-			return nil
-		}
-		for res.NextRow() {
-			if err = res.ScanNamed(named.OptionalWithDefault("meta", &data)); err != nil {
-				return fmt.Errorf("scanNamed %s : %v", fullpath, err)
+	err = store.doTxOrDB(ctx, q, queryParams, roQC, func(res query.Result) error {
+		for rs, err := range res.ResultSets(ctx) {
+			if err != nil {
+				return err
+			}
+			for row, err := range rs.Rows(ctx) {
+				if err != nil {
+					return err
+				}
+				if scanErr := row.Scan(&data); scanErr != nil {
+					return fmt.Errorf("scan %s: %v", fullpath, scanErr)
+				}
+				entryFound = true
+				return nil
 			}
-			entryFound = true
-			return nil
 		}
-		return res.Err()
+		return nil
 	})
 	if err != nil {
 		return nil, err
@@ -189,37 +223,35 @@ func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (e
 		return nil, filer_pb.ErrNotFound
 	}
 
-	entry = &filer.Entry{
-		FullPath: fullpath,
+	entry = &filer.Entry{FullPath: fullpath}
+	if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); decodeErr != nil {
+		return nil, fmt.Errorf("decode %s: %v", fullpath, decodeErr)
 	}
-	if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
-		return nil, fmt.Errorf("decode %s : %v", fullpath, err)
-	}
-
 	return entry, nil
 }
 
 func (store *YdbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
 	dir, name := fullpath.DirAndName()
 	tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
-	query := withPragma(tablePathPrefix, deleteQuery)
+	q := withPragma(tablePathPrefix, deleteQuery)
 	glog.V(4).Infof("DeleteEntry %s, tablePathPrefix %s, shortDir %s", fullpath, *tablePathPrefix, *shortDir)
 	queryParams := table.NewQueryParameters(
 		table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
+		table.ValueParam("$directory", types.UTF8Value(*shortDir)),
 		table.ValueParam("$name", types.UTF8Value(name)))
 
-	return store.doTxOrDB(ctx, query, queryParams, rwTX, nil)
+	return store.doTxOrDB(ctx, q, queryParams, rwQC, nil)
 }
 
 func (store *YdbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
 	dir := string(fullpath)
 	tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
-	query := withPragma(tablePathPrefix, deleteFolderChildrenQuery)
+	q := withPragma(tablePathPrefix, deleteFolderChildrenQuery)
 	queryParams := table.NewQueryParameters(
 		table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
 		table.ValueParam("$directory", types.UTF8Value(*shortDir)))
 
-	return store.doTxOrDB(ctx, query, queryParams, rwTX, nil)
+	return store.doTxOrDB(ctx, q, queryParams, rwQC, nil)
 }
 
 func (store *YdbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
@@ -229,71 +261,79 @@ func (store *YdbStore) ListDirectoryEntries(ctx context.Context, dirPath util.Fu
 func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
 	dir := string(dirPath)
 	tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
-	var query *string
-	if includeStartFile {
-		query = withPragma(tablePathPrefix, listInclusiveDirectoryQuery)
-	} else {
-		query = withPragma(tablePathPrefix, listDirectoryQuery)
-	}
-	truncated := true
-	eachEntryFuncIsNotBreake := true
-	entryCount := int64(0)
-	for truncated && eachEntryFuncIsNotBreake {
-		if lastFileName != "" {
-			startFileName = lastFileName
-			if includeStartFile {
-				query = withPragma(tablePathPrefix, listDirectoryQuery)
-			}
+	baseInclusive := withPragma(tablePathPrefix, listInclusiveDirectoryQuery)
+	baseExclusive := withPragma(tablePathPrefix, listDirectoryQuery)
+	var entryCount int64
+	var prevFetchedLessThanChunk bool
+	for entryCount < limit {
+		if prevFetchedLessThanChunk {
+			break
 		}
-		restLimit := limit - entryCount
-		const maxChunk = int64(1000)
-		chunkLimit := restLimit
-		if chunkLimit > maxChunk {
-			chunkLimit = maxChunk
+		var q *string
+		if entryCount == 0 && includeStartFile {
+			q = baseInclusive
+		} else {
+			q = baseExclusive
 		}
-		glog.V(4).Infof("startFileName %s, restLimit %d, chunkLimit %d", startFileName, restLimit, chunkLimit)
+		rest := limit - entryCount
+		chunkLimit := rest
+		if chunkLimit > int64(store.maxListChunk) {
+			chunkLimit = int64(store.maxListChunk)
+		}
+		var rowCount int64
 
-		queryParams := table.NewQueryParameters(
+		params := table.NewQueryParameters(
 			table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
 			table.ValueParam("$directory", types.UTF8Value(*shortDir)),
 			table.ValueParam("$start_name", types.UTF8Value(startFileName)),
 			table.ValueParam("$prefix", types.UTF8Value(prefix+"%")),
 			table.ValueParam("$limit", types.Uint64Value(uint64(chunkLimit))),
 		)
-		err = store.doTxOrDB(ctx, query, queryParams, roTX, func(res result.Result) error {
-			var name string
-			var data []byte
-			if !res.NextResultSet(ctx) || !res.HasNextRow() {
-				truncated = false
-				return nil
+
+		err := store.doTxOrDB(ctx, q, params, roQC, func(res query.Result) error {
+			for rs, err := range res.ResultSets(ctx) {
+				if err != nil {
+					return err
+				}
+				for row, err := range rs.Rows(ctx) {
+					if err != nil {
+						return err
+					}
+
+					var name string
+					var data []byte
+					if scanErr := row.Scan(&name, &data); scanErr != nil {
+						return fmt.Errorf("scan %s: %w", dir, scanErr)
+					}
+
+					lastFileName = name
+					entry := &filer.Entry{FullPath: util.NewFullPath(dir, name)}
+					if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); decodeErr != nil {
+						return fmt.Errorf("decode entry %s: %w", entry.FullPath, decodeErr)
+					}
+
+					if !eachEntryFunc(entry) {
+						return nil
+					}
+
+					rowCount++
+					entryCount++
+					startFileName = lastFileName
+
+					if entryCount >= limit {
+						return nil
+					}
+				}
 			}
-			truncated = res.CurrentResultSet().Truncated()
-			glog.V(4).Infof("truncated %v, entryCount %d", truncated, entryCount)
-			for res.NextRow() {
-				if err := res.ScanNamed(
-					named.OptionalWithDefault("name", &name),
-					named.OptionalWithDefault("meta", &data)); err != nil {
-					return fmt.Errorf("list scanNamed %s : %v", dir, err)
-				}
-				glog.V(8).Infof("name %s, fullpath %s", name, util.NewFullPath(dir, name))
-				lastFileName = name
-				entry := &filer.Entry{
-					FullPath: util.NewFullPath(dir, name),
-				}
-				if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
-					return fmt.Errorf("scan decode %s : %v", entry.FullPath, err)
-				}
-				if !eachEntryFunc(entry) {
-					eachEntryFuncIsNotBreake = false
-					break
-				}
-				entryCount += 1
-			}
-			return res.Err()
+			return nil
 		})
-	}
-	if err != nil {
-		return lastFileName, err
+		if err != nil {
+			return lastFileName, err
+		}
+
+		if rowCount < chunkLimit {
+			prevFetchedLessThanChunk = true
+		}
 	}
 	return lastFileName, nil
 }
@@ -380,7 +420,7 @@ func (store *YdbStore) OnBucketDeletion(bucket string) {
 
 func (store *YdbStore) createTable(ctx context.Context, prefix string) error {
 	return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
-		return s.CreateTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE), createTableOptions()...)
+		return s.CreateTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE), store.createTableOptions()...)
 	})
 }
 
@@ -424,16 +464,22 @@ func (store *YdbStore) getPrefix(ctx context.Context, dir *string) (tablePathPre
 		store.dbsLock.Lock()
 		defer store.dbsLock.Unlock()
 
-		tablePathPrefixWithBucket := path.Join(store.tablePathPrefix, bucket)
 		if _, found := store.dbs[bucket]; !found {
-			if err := store.createTable(ctx, tablePathPrefixWithBucket); err == nil {
-				store.dbs[bucket] = true
-				glog.V(4).Infof("created table %s", tablePathPrefixWithBucket)
-			} else {
-				glog.Errorf("createTable %s: %v", tablePathPrefixWithBucket, err)
+			glog.V(4).Infof("bucket %q not in cache, verifying existence via DescribeTable", bucket)
+			tablePath := path.Join(store.tablePathPrefix, bucket, abstract_sql.DEFAULT_TABLE)
+			err2 := store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
+				_, err3 := s.DescribeTable(ctx, tablePath)
+				return err3
+			})
+			if err2 != nil {
+				glog.V(4).Infof("bucket %q not found (DescribeTable %s failed)", bucket, tablePath)
+				return
 			}
+			glog.V(4).Infof("bucket %q exists, adding to cache", bucket)
+			store.dbs[bucket] = true
 		}
-		tablePathPrefix = &tablePathPrefixWithBucket
+		bucketPrefix := path.Join(store.tablePathPrefix, bucket)
+		tablePathPrefix = &bucketPrefix
 	}
 	return
 }
@@ -444,7 +490,7 @@ func (store *YdbStore) ensureTables(ctx context.Context) error {
 	glog.V(4).Infof("creating base table %s", prefixFull)
 	baseTable := path.Join(prefixFull, abstract_sql.DEFAULT_TABLE)
 	if err := store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
-		return s.CreateTable(ctx, baseTable, createTableOptions()...)
+		return s.CreateTable(ctx, baseTable, store.createTableOptions()...)
 	}); err != nil {
 		return fmt.Errorf("failed to create base table %s: %v", baseTable, err)
 	}
@@ -457,7 +503,7 @@ func (store *YdbStore) ensureTables(ctx context.Context) error {
 			glog.V(4).Infof("creating bucket table %s", bucket)
 			bucketTable := path.Join(prefixFull, bucket, abstract_sql.DEFAULT_TABLE)
 			if err := store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
-				return s.CreateTable(ctx, bucketTable, createTableOptions()...)
+				return s.CreateTable(ctx, bucketTable, store.createTableOptions()...)
 			}); err != nil {
 				glog.Errorf("failed to create bucket table %s: %v", bucketTable, err)
 			}

weed/filer/ydb/ydb_store_kv.go

@@ -9,48 +9,54 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/ydb-platform/ydb-go-sdk/v3/query"
 	"github.com/ydb-platform/ydb-go-sdk/v3/table"
-	"github.com/ydb-platform/ydb-go-sdk/v3/table/result/named"
 	"github.com/ydb-platform/ydb-go-sdk/v3/table/types"
 )
 
 func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
 	dirStr, dirHash, name := abstract_sql.GenDirAndName(key)
 	fileMeta := FileMeta{dirHash, name, dirStr, value}
-	return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) {
-		_, _, err = s.Execute(ctx, rwTX, *withPragma(&store.tablePathPrefix, upsertQuery),
-			fileMeta.queryParameters(0))
+	return store.DB.Query().Do(ctx, func(ctx context.Context, s query.Session) (err error) {
+		_, err = s.Query(ctx, *withPragma(&store.tablePathPrefix, upsertQuery),
+			query.WithParameters(fileMeta.queryParameters(0)), rwQC)
 		if err != nil {
 			return fmt.Errorf("kv put execute %s: %v", util.NewFullPath(dirStr, name).Name(), err)
 		}
 		return nil
-	})
+	}, query.WithIdempotent())
 }
 
 func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
 	dirStr, dirHash, name := abstract_sql.GenDirAndName(key)
 	valueFound := false
-	err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
-		_, res, err := s.Execute(ctx, roTX, *withPragma(&store.tablePathPrefix, findQuery),
-			table.NewQueryParameters(
+	err = store.DB.Query().Do(ctx, func(ctx context.Context, s query.Session) error {
+		res, err := s.Query(ctx, *withPragma(&store.tablePathPrefix, findQuery),
+			query.WithParameters(table.NewQueryParameters(
 				table.ValueParam("$dir_hash", types.Int64Value(dirHash)),
-				table.ValueParam("$name", types.UTF8Value(name))))
+				table.ValueParam("$directory", types.UTF8Value(dirStr)),
+				table.ValueParam("$name", types.UTF8Value(name)))), roQC)
 		if err != nil {
 			return fmt.Errorf("kv get execute %s: %v", util.NewFullPath(dirStr, name).Name(), err)
 		}
-		defer func() { _ = res.Close() }()
-		if !res.NextResultSet(ctx) || !res.HasNextRow() {
-			return nil
-		}
-		for res.NextRow() {
-			if err := res.ScanNamed(named.OptionalWithDefault("meta", &value)); err != nil {
-				return fmt.Errorf("scanNamed %s : %v", util.NewFullPath(dirStr, name).Name(), err)
+		defer func() { _ = res.Close(ctx) }()
+		for rs, err := range res.ResultSets(ctx) {
+			if err != nil {
+				return err
+			}
+			for row, err := range rs.Rows(ctx) {
+				if err != nil {
+					return err
+				}
+				if err := row.Scan(&value); err != nil {
+					return fmt.Errorf("scan %s : %v", util.NewFullPath(dirStr, name).Name(), err)
+				}
+				valueFound = true
+				return nil
 			}
-			valueFound = true
-			return nil
 		}
-		return res.Err()
-	})
+		return nil
+	}, query.WithIdempotent())
 
 	if !valueFound {
 		return nil, filer.ErrKvNotFound
@@ -61,15 +67,16 @@ func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err
 
 func (store *YdbStore) KvDelete(ctx context.Context, key []byte) (err error) {
 	dirStr, dirHash, name := abstract_sql.GenDirAndName(key)
-	return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) {
-		_, _, err = s.Execute(ctx, rwTX, *withPragma(&store.tablePathPrefix, deleteQuery),
-			table.NewQueryParameters(
+	return store.DB.Query().Do(ctx, func(ctx context.Context, s query.Session) (err error) {
+		_, err = s.Query(ctx, *withPragma(&store.tablePathPrefix, deleteQuery),
+			query.WithParameters(table.NewQueryParameters(
 				table.ValueParam("$dir_hash", types.Int64Value(dirHash)),
-				table.ValueParam("$name", types.UTF8Value(name))))
+				table.ValueParam("$directory", types.UTF8Value(dirStr)),
+				table.ValueParam("$name", types.UTF8Value(name)))), rwQC)
 		if err != nil {
 			return fmt.Errorf("kv delete %s: %v", util.NewFullPath(dirStr, name).Name(), err)
 		}
 		return nil
-	})
+	}, query.WithIdempotent())
 
 }

weed/filer/ydb/ydb_store_test.go

@@ -13,7 +13,8 @@ func TestStore(t *testing.T) {
 	// to set up local env
 	if false {
 		store := &YdbStore{}
-		store.initialize("/buckets", "grpc://localhost:2136/?database=local", "seaweedfs", true, 10, 50)
+		store.initialize("/buckets", "grpc://localhost:2136/?database=local", "seaweedfs", true, 10, 50,
+			true, 200, true, 5, 1000, 2000)
 		store_test.TestFilerStore(t, store)
 	}
 }

weed/filer/ydb/ydb_types.go

@@ -30,26 +30,35 @@ func (fm *FileMeta) queryParameters(ttlSec int32) *table.QueryParameters {
 		table.ValueParam("$dir_hash", types.Int64Value(fm.DirHash)),
 		table.ValueParam("$directory", types.UTF8Value(fm.Directory)),
 		table.ValueParam("$name", types.UTF8Value(fm.Name)),
-		table.ValueParam("$meta", types.StringValue(fm.Meta)),
+		table.ValueParam("$meta", types.BytesValue(fm.Meta)),
 		table.ValueParam("$expire_at", expireAtValue))
 }
 
-func createTableOptions() []options.CreateTableOption {
+func (store *YdbStore) createTableOptions() []options.CreateTableOption {
 	columnUnit := options.TimeToLiveUnitSeconds
 	return []options.CreateTableOption{
-		options.WithColumn("dir_hash", types.Optional(types.TypeInt64)),
-		options.WithColumn("directory", types.Optional(types.TypeUTF8)),
-		options.WithColumn("name", types.Optional(types.TypeUTF8)),
-		options.WithColumn("meta", types.Optional(types.TypeString)),
+		options.WithColumn("dir_hash", types.TypeInt64),
+		options.WithColumn("directory", types.TypeUTF8),
+		options.WithColumn("name", types.TypeUTF8),
+		options.WithColumn("meta", types.TypeString),
 		options.WithColumn("expire_at", types.Optional(types.TypeUint32)),
-		options.WithPrimaryKeyColumn("dir_hash", "name"),
+		options.WithPrimaryKeyColumn("dir_hash", "directory", "name"),
 		options.WithTimeToLiveSettings(options.TimeToLiveSettings{
 			ColumnName: "expire_at",
 			ColumnUnit: &columnUnit,
 			Mode:       options.TimeToLiveModeValueSinceUnixEpoch},
 		),
+		options.WithPartitioningSettings(
+			options.WithPartitioningBy([]string{"dir_hash", "name"}),
+			options.WithPartitioningBySize(store.partitionBySizeEnabled),
+			options.WithPartitionSizeMb(store.partitionSizeMb),
+			options.WithPartitioningByLoad(store.partitionByLoadEnabled),
+			options.WithMinPartitionsCount(store.minPartitionsCount),
+			options.WithMaxPartitionsCount(store.maxPartitionsCount),
+		),
 	}
 }
+
 func withPragma(prefix *string, query string) *string {
 	queryWithPragma := fmt.Sprintf(query, *prefix)
 	return &queryWithPragma

weed/glog/glog_ctx.go

@@ -0,0 +1,246 @@
+package glog
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+
+	reqid "github.com/seaweedfs/seaweedfs/weed/util/request_id"
+)
+
+const requestIDField = "request_id"
+
+// formatMetaTag returns a formatted request ID tag from the context,
+// like "request_id:abc123". Returns an empty string if no request ID is found.
+func formatMetaTag(ctx context.Context) string {
+	if requestID := reqid.Get(ctx); requestID != "" {
+		return fmt.Sprintf("%s:%s", requestIDField, requestID)
+	}
+	return ""
+}
+
+// InfoCtx is a context-aware alternative to Verbose.Info.
+// Logs to the INFO log, guarded by the value of v, and prepends a request ID from the context if present.
+// Arguments are handled in the manner of fmt.Print.
+func (v Verbose) InfoCtx(ctx context.Context, args ...interface{}) {
+	if !v {
+		return
+	}
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.print(infoLog, args...)
+}
+
+// InfolnCtx is a context-aware alternative to Verbose.Infoln.
+// Logs to the INFO log, prepending a request ID from the context if it exists.
+// Arguments are handled in the manner of fmt.Println.
+func (v Verbose) InfolnCtx(ctx context.Context, args ...interface{}) {
+	if !v {
+		return
+	}
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.println(infoLog, args...)
+}
+
+// InfofCtx is a context-aware alternative to Verbose.Infof.
+// Logs to the INFO log, guarded by the value of v, and prepends a request ID from the context if present.
+// Arguments are handled in the manner of fmt.Printf.
+func (v Verbose) InfofCtx(ctx context.Context, format string, args ...interface{}) {
+	if !v {
+		return
+	}
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		format = metaTag + " " + format
+	}
+	logging.printf(infoLog, format, args...)
+}
+
+// InfofCtx logs a formatted message at info level, prepending a request ID from
+// the context if it exists. This is a context-aware alternative to Infof.
+func InfofCtx(ctx context.Context, format string, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		format = metaTag + " " + format
+	}
+	logging.printf(infoLog, format, args...)
+}
+
+// InfoCtx logs a message at info level, prepending a request ID from the context
+// if it exists. This is a context-aware alternative to Info.
+func InfoCtx(ctx context.Context, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.print(infoLog, args...)
+}
+
+// WarningCtx logs to the WARNING and INFO logs.
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
+// This is a context-aware alternative to Warning.
+func WarningCtx(ctx context.Context, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.print(warningLog, args...)
+}
+
+// WarningDepthCtx logs to the WARNING and INFO logs with a custom call depth.
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
+// This is a context-aware alternative to WarningDepth.
+func WarningDepthCtx(ctx context.Context, depth int, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.printDepth(warningLog, depth, args...)
+}
+
+// WarninglnCtx logs to the WARNING and INFO logs.
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Println.
+// This is a context-aware alternative to Warningln.
+func WarninglnCtx(ctx context.Context, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.println(warningLog, args...)
+}
+
+// WarningfCtx logs to the WARNING and INFO logs.
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Printf.
+// This is a context-aware alternative to Warningf.
+func WarningfCtx(ctx context.Context, format string, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		format = metaTag + " " + format
+	}
+	logging.printf(warningLog, format, args...)
+}
+
+// ErrorCtx logs to the ERROR, WARNING, and INFO logs.
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
+// This is a context-aware alternative to Error.
+func ErrorCtx(ctx context.Context, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.print(errorLog, args...)
+}
+
+// ErrorDepthCtx logs to the ERROR, WARNING, and INFO logs with a custom call depth.
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
+// This is a context-aware alternative to ErrorDepth.
+func ErrorDepthCtx(ctx context.Context, depth int, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.printDepth(errorLog, depth, args...)
+}
+
+// ErrorlnCtx logs to the ERROR, WARNING, and INFO logs.
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Println.
+// This is a context-aware alternative to Errorln.
+func ErrorlnCtx(ctx context.Context, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.println(errorLog, args...)
+}
+
+// ErrorfCtx logs to the ERROR, WARNING, and INFO logs.
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Printf.
+// This is a context-aware alternative to Errorf.
+func ErrorfCtx(ctx context.Context, format string, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		format = metaTag + " " + format
+	}
+	logging.printf(errorLog, format, args...)
+}
+
+// FatalCtx logs to the FATAL, ERROR, WARNING, and INFO logs,
+// including a stack trace of all running goroutines, then calls os.Exit(255).
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
+// This is a context-aware alternative to Fatal.
+func FatalCtx(ctx context.Context, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.print(fatalLog, args...)
+}
+
+// FatalDepthCtx logs to the FATAL, ERROR, WARNING, and INFO logs with a custom call depth,
+// including a stack trace of all running goroutines, then calls os.Exit(255).
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
+// This is a context-aware alternative to FatalDepth.
+func FatalDepthCtx(ctx context.Context, depth int, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.printDepth(fatalLog, depth, args...)
+}
+
+// FatallnCtx logs to the FATAL, ERROR, WARNING, and INFO logs,
+// including a stack trace of all running goroutines, then calls os.Exit(255).
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Println.
+// This is a context-aware alternative to Fatalln.
+func FatallnCtx(ctx context.Context, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.println(fatalLog, args...)
+}
+
+// FatalfCtx logs to the FATAL, ERROR, WARNING, and INFO logs,
+// including a stack trace of all running goroutines, then calls os.Exit(255).
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Printf.
+// This is a context-aware alternative to Fatalf.
+func FatalfCtx(ctx context.Context, format string, args ...interface{}) {
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		format = metaTag + " " + format
+	}
+	logging.printf(fatalLog, format, args...)
+}
+
+// ExitCtx logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1).
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
+// This is a context-aware alternative to ExitCtx
+func ExitCtx(ctx context.Context, args ...interface{}) {
+	atomic.StoreUint32(&fatalNoStacks, 1)
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.print(fatalLog, args...)
+}
+
+// ExitDepthCtx logs to the FATAL, ERROR, WARNING, and INFO logs with a custom call depth,
+// then calls os.Exit(1). Prepends a request ID from the context if it exists.
+// Arguments are handled in the manner of fmt.Print.
+// This is a context-aware alternative to ExitDepth.
+func ExitDepthCtx(ctx context.Context, depth int, args ...interface{}) {
+	atomic.StoreUint32(&fatalNoStacks, 1)
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.printDepth(fatalLog, depth, args...)
+}
+
+// ExitlnCtx logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1).
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Println.
+// This is a context-aware alternative to Exitln.
+func ExitlnCtx(ctx context.Context, args ...interface{}) {
+	atomic.StoreUint32(&fatalNoStacks, 1)
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		args = append([]interface{}{metaTag}, args...)
+	}
+	logging.println(fatalLog, args...)
+}
+
+// ExitfCtx logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1).
+// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Printf.
+// This is a context-aware alternative to Exitf.
+func ExitfCtx(ctx context.Context, format string, args ...interface{}) {
+	atomic.StoreUint32(&fatalNoStacks, 1)
+	if metaTag := formatMetaTag(ctx); metaTag != "" {
+		format = metaTag + " " + format
+	}
+	logging.printf(fatalLog, format, args...)
+}

weed/mount/weedfs.go

@@ -3,11 +3,11 @@ package mount
 import (
 	"context"
 	"errors"
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"math/rand"
 	"os"
 	"path"
 	"path/filepath"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -23,6 +23,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 
 	"github.com/hanwen/go-fuse/v2/fs"
@@ -71,19 +72,21 @@ type WFS struct {
 	fuse.RawFileSystem
 	mount_pb.UnimplementedSeaweedMountServer
 	fs.Inode
-	option            *Option
-	metaCache         *meta_cache.MetaCache
-	stats             statsCache
-	chunkCache        *chunk_cache.TieredChunkCache
-	signature         int32
-	concurrentWriters *util.LimitedConcurrentExecutor
-	inodeToPath       *InodeToPath
-	fhMap             *FileHandleToInode
-	dhMap             *DirectoryHandleToInode
-	fuseServer        *fuse.Server
-	IsOverQuota       bool
-	fhLockTable       *util.LockTable[FileHandleId]
-	FilerConf         *filer.FilerConf
+	option               *Option
+	metaCache            *meta_cache.MetaCache
+	stats                statsCache
+	chunkCache           *chunk_cache.TieredChunkCache
+	signature            int32
+	concurrentWriters    *util.LimitedConcurrentExecutor
+	copyBufferPool       sync.Pool
+	concurrentCopiersSem chan struct{}
+	inodeToPath          *InodeToPath
+	fhMap                *FileHandleToInode
+	dhMap                *DirectoryHandleToInode
+	fuseServer           *fuse.Server
+	IsOverQuota          bool
+	fhLockTable          *util.LockTable[FileHandleId]
+	FilerConf            *filer.FilerConf
 }
 
 func NewSeaweedFileSystem(option *Option) *WFS {
@@ -139,6 +142,10 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 
 	if wfs.option.ConcurrentWriters > 0 {
 		wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(wfs.option.ConcurrentWriters)
+		wfs.concurrentCopiersSem = make(chan struct{}, wfs.option.ConcurrentWriters)
+	}
+	wfs.copyBufferPool.New = func() any {
+		return make([]byte, option.ChunkSizeLimit)
 	}
 	return wfs
 }
@@ -183,7 +190,6 @@ func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle
 }
 
 func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, fuse.Status) {
-
 	// glog.V(3).Infof("read entry cache miss %s", fullpath)
 	dir, name := fullpath.DirAndName()
 

weed/mount/weedfs_attr_freebsd.go

@@ -0,0 +1,8 @@
+package mount
+
+import (
+	"github.com/hanwen/go-fuse/v2/fuse"
+)
+
+func setBlksize(out *fuse.Attr, size uint32) {
+}

weed/mount/weedfs_file_copy_range.go

@@ -1,13 +1,13 @@
 package mount
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"net/http"
 	"time"
 
 	"github.com/hanwen/go-fuse/v2/fuse"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 // CopyFileRange copies data from one file to another from and to specified offsets.
@@ -70,30 +70,85 @@ func (wfs *WFS) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn)
 		in.OffOut, in.OffOut+in.Len,
 	)
 
-	data := make([]byte, in.Len)
-	totalRead, err := readDataByFileHandle(data, fhIn, int64(in.OffIn))
-	if err != nil {
-		glog.Warningf("file handle read %s %d: %v", fhIn.FullPath(), totalRead, err)
-		return 0, fuse.EIO
+	// Concurrent copy operations could allocate too much memory, so we want to
+	// throttle our concurrency, scaling with the number of writers the mount
+	// was configured with.
+	if wfs.concurrentCopiersSem != nil {
+		wfs.concurrentCopiersSem <- struct{}{}
+		defer func() { <-wfs.concurrentCopiersSem }()
 	}
-	data = data[:totalRead]
 
-	if totalRead == 0 {
+	// We want to stream the copy operation to avoid allocating massive buffers.
+	nowUnixNano := time.Now().UnixNano()
+	totalCopied := int64(0)
+	buff := wfs.copyBufferPool.Get().([]byte)
+	defer wfs.copyBufferPool.Put(buff)
+	for {
+		// Comply with cancellation as best as we can, given that the underlying
+		// IO functions aren't cancellation-aware.
+		select {
+		case <-cancel:
+			glog.Warningf("canceled CopyFileRange for %s (copied %d)",
+				fhIn.FullPath(), totalCopied)
+			return uint32(totalCopied), fuse.EINTR
+		default: // keep going
+		}
+
+		// We can save one IO by breaking early if we already know the next read
+		// will result in zero bytes.
+		remaining := int64(in.Len) - totalCopied
+		readLen := min(remaining, int64(len(buff)))
+		if readLen == 0 {
+			break
+		}
+
+		// Perform the read
+		offsetIn := totalCopied + int64(in.OffIn)
+		numBytesRead, err := readDataByFileHandle(
+			buff[:readLen], fhIn, offsetIn)
+		if err != nil {
+			glog.Warningf("file handle read %s %d (total %d): %v",
+				fhIn.FullPath(), numBytesRead, totalCopied, err)
+			return 0, fuse.EIO
+		}
+
+		// Break if we're done copying (no more bytes to read)
+		if numBytesRead == 0 {
+			break
+		}
+
+		offsetOut := int64(in.OffOut) + totalCopied
+
+		// Detect mime type only during the beginning of our stream, since
+		// DetectContentType is expecting some of the first 512 bytes of the
+		// file. See [http.DetectContentType] for details.
+		if offsetOut <= 512 {
+			fhOut.contentType = http.DetectContentType(buff[:numBytesRead])
+		}
+
+		// Perform the write
+		fhOut.dirtyPages.writerPattern.MonitorWriteAt(offsetOut, int(numBytesRead))
+		fhOut.dirtyPages.AddPage(
+			offsetOut,
+			buff[:numBytesRead],
+			fhOut.dirtyPages.writerPattern.IsSequentialMode(),
+			nowUnixNano)
+
+		// Accumulate for the next loop iteration
+		totalCopied += numBytesRead
+	}
+
+	if totalCopied == 0 {
 		return 0, fuse.OK
 	}
 
-	// put data at the specified offset in target file
-	fhOut.dirtyPages.writerPattern.MonitorWriteAt(int64(in.OffOut), int(in.Len))
+	fhOut.entry.Attributes.FileSize = uint64(max(
+		totalCopied+int64(in.OffOut),
+		int64(fhOut.entry.Attributes.FileSize),
+	))
 	fhOut.entry.Content = nil
-	fhOut.dirtyPages.AddPage(int64(in.OffOut), data, fhOut.dirtyPages.writerPattern.IsSequentialMode(), time.Now().UnixNano())
-	fhOut.entry.Attributes.FileSize = uint64(max(int64(in.OffOut)+totalRead, int64(fhOut.entry.Attributes.FileSize)))
 	fhOut.dirtyMetadata = true
-	written = uint32(totalRead)
-
-	// detect mime type
-	if written > 0 && in.OffOut <= 512 {
-		fhOut.contentType = http.DetectContentType(data)
-	}
 
+	written = uint32(totalCopied)
 	return written, fuse.OK
 }

weed/mount/weedfs_xattr.go

@@ -1,3 +1,6 @@
+//go:build !freebsd
+// +build !freebsd
+
 package mount
 
 import (

weed/mount/weedfs_xattr_freebsd.go

@@ -0,0 +1,27 @@
+package mount
+
+import (
+	"syscall"
+
+	"github.com/hanwen/go-fuse/v2/fuse"
+)
+
+func (wfs *WFS) GetXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr string, dest []byte) (size uint32, code fuse.Status) {
+
+	return 0, fuse.Status(syscall.ENOTSUP)
+}
+
+func (wfs *WFS) SetXAttr(cancel <-chan struct{}, input *fuse.SetXAttrIn, attr string, data []byte) fuse.Status {
+
+	return fuse.Status(syscall.ENOTSUP)
+}
+
+func (wfs *WFS) ListXAttr(cancel <-chan struct{}, header *fuse.InHeader, dest []byte) (n uint32, code fuse.Status) {
+
+	return 0, fuse.Status(syscall.ENOTSUP)
+}
+
+func (wfs *WFS) RemoveXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr string) fuse.Status {
+
+	return fuse.Status(syscall.ENOTSUP)
+}

weed/mq/broker/broker_grpc_pub.go

@@ -3,15 +3,19 @@ package broker
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
-	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
-	"google.golang.org/grpc/peer"
 	"io"
 	"math/rand"
 	"net"
+	"strings"
 	"sync/atomic"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
 )
 
 // PUB
@@ -126,7 +130,12 @@ func (b *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_Publis
 			if err == io.EOF {
 				break
 			}
-			glog.V(0).Infof("topic %v partition %v publish stream from %s error: %v", initMessage.Topic, initMessage.Partition, initMessage.PublisherName, err)
+			// Use appropriate logging level based on error type
+			if isConnectionError(err) {
+				glog.V(1).Infof("topic %v partition %v publish stream from %s connection error (client disconnected): %v", initMessage.Topic, initMessage.Partition, initMessage.PublisherName, err)
+			} else {
+				glog.V(0).Infof("topic %v partition %v publish stream from %s error: %v", initMessage.Topic, initMessage.Partition, initMessage.PublisherName, err)
+			}
 			break
 		}
 
@@ -145,7 +154,7 @@ func (b *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_Publis
 		}
 	}
 
-	glog.V(0).Infof("topic %v partition %v publish stream from %s closed.", initMessage.Topic, initMessage.Partition, initMessage.PublisherName)
+	glog.V(1).Infof("topic %v partition %v publish stream from %s closed.", initMessage.Topic, initMessage.Partition, initMessage.PublisherName)
 
 	return nil
 }
@@ -164,3 +173,40 @@ func findClientAddress(ctx context.Context) string {
 	}
 	return pr.Addr.String()
 }
+
+// isConnectionError checks if an error is a connection-level error
+func isConnectionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	// Check gRPC status codes for connection errors
+	if grpcStatus, ok := status.FromError(err); ok {
+		switch grpcStatus.Code() {
+		case codes.Unavailable, codes.DeadlineExceeded, codes.Canceled, codes.Unknown:
+			return true
+		}
+	}
+
+	// Check for common connection error patterns
+	errStr := strings.ToLower(err.Error())
+	connectionErrorPatterns := []string{
+		"eof",
+		"error reading server preface",
+		"connection refused",
+		"connection reset",
+		"broken pipe",
+		"no such host",
+		"network is unreachable",
+		"context deadline exceeded",
+		"context canceled",
+	}
+
+	for _, pattern := range connectionErrorPatterns {
+		if strings.Contains(errStr, pattern) {
+			return true
+		}
+	}
+
+	return false
+}

weed/mq/broker/broker_grpc_pub_follow.go

@@ -2,13 +2,14 @@ package broker
 
 import (
 	"fmt"
+	"io"
+	"time"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util/buffered_queue"
 	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
-	"io"
-	"time"
 )
 
 type memBuffer struct {
@@ -43,7 +44,12 @@ func (b *MessageQueueBroker) PublishFollowMe(stream mq_pb.SeaweedMessaging_Publi
 				err = nil
 				break
 			}
-			glog.V(0).Infof("topic %v partition %v publish stream error: %v", initMessage.Topic, initMessage.Partition, err)
+			// Use appropriate logging level based on error type
+			if isConnectionError(err) {
+				glog.V(1).Infof("topic %v partition %v publish follower stream connection error (leader disconnected): %v", initMessage.Topic, initMessage.Partition, err)
+			} else {
+				glog.V(0).Infof("topic %v partition %v publish stream error: %v", initMessage.Topic, initMessage.Partition, err)
+			}
 			break
 		}
 
@@ -62,10 +68,10 @@ func (b *MessageQueueBroker) PublishFollowMe(stream mq_pb.SeaweedMessaging_Publi
 			}
 			// println("ack", string(dataMessage.Key), dataMessage.TsNs)
 		} else if closeMessage := req.GetClose(); closeMessage != nil {
-			glog.V(0).Infof("topic %v partition %v publish stream closed: %v", initMessage.Topic, initMessage.Partition, closeMessage)
+			glog.V(1).Infof("topic %v partition %v publish stream closed: %v", initMessage.Topic, initMessage.Partition, closeMessage)
 			break
 		} else if flushMessage := req.GetFlush(); flushMessage != nil {
-			glog.V(0).Infof("topic %v partition %v publish stream flushed: %v", initMessage.Topic, initMessage.Partition, flushMessage)
+			glog.V(1).Infof("topic %v partition %v publish stream flushed: %v", initMessage.Topic, initMessage.Partition, flushMessage)
 
 			lastFlushTsNs = flushMessage.TsNs
 
@@ -124,7 +130,7 @@ func (b *MessageQueueBroker) PublishFollowMe(stream mq_pb.SeaweedMessaging_Publi
 		glog.V(0).Infof("flushed remaining data at %v to %s size %d", mem.stopTime.UnixNano(), targetFile, len(mem.buf))
 	}
 
-	glog.V(0).Infof("shut down follower for %v %v", t, p)
+	glog.V(1).Infof("shut down follower for %v %v", t, p)
 
 	return err
 }

weed/mq/client/pub_client/scheduler.go

@@ -3,6 +3,13 @@ package pub_client
 import (
 	"context"
 	"fmt"
+	"log"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
@@ -11,11 +18,6 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/status"
-	"log"
-	"sort"
-	"sync"
-	"sync/atomic"
-	"time"
 )
 
 type EachPartitionError struct {
@@ -39,7 +41,7 @@ func (p *TopicPublisher) startSchedulerThread(wg *sync.WaitGroup) error {
 		return fmt.Errorf("configure topic %s: %v", p.config.Topic, err)
 	}
 
-	log.Printf("start scheduler thread for topic %s", p.config.Topic)
+	glog.V(0).Infof("start scheduler thread for topic %s", p.config.Topic)
 
 	generation := 0
 	var errChan chan EachPartitionError
@@ -66,7 +68,12 @@ func (p *TopicPublisher) startSchedulerThread(wg *sync.WaitGroup) error {
 		for {
 			select {
 			case eachErr := <-errChan:
-				glog.Errorf("gen %d publish to topic %s partition %v: %v", eachErr.generation, p.config.Topic, eachErr.Partition, eachErr.Err)
+				// Use appropriate logging level based on error type
+				if isConnectionError(eachErr.Err) {
+					glog.V(1).Infof("gen %d connection error for topic %s partition %v (will retry): %v", eachErr.generation, p.config.Topic, eachErr.Partition, eachErr.Err)
+				} else {
+					glog.Errorf("gen %d publish to topic %s partition %v: %v", eachErr.generation, p.config.Topic, eachErr.Partition, eachErr.Err)
+				}
 				if eachErr.generation < generation {
 					continue
 				}
@@ -114,7 +121,12 @@ func (p *TopicPublisher) onEachAssignments(generation int, assignments []*mq_pb.
 		go func(job *EachPartitionPublishJob) {
 			defer job.wg.Done()
 			if err := p.doPublishToPartition(job); err != nil {
-				log.Printf("publish to %s partition %v: %v", p.config.Topic, job.Partition, err)
+				// Use appropriate logging level based on error type
+				if isConnectionError(err) {
+					glog.V(1).Infof("connection error publishing to %s partition %v (will retry): %v", p.config.Topic, job.Partition, err)
+				} else {
+					log.Printf("publish to %s partition %v: %v", p.config.Topic, job.Partition, err)
+				}
 				errChan <- EachPartitionError{assignment, err, generation}
 			}
 		}(job)
@@ -128,7 +140,7 @@ func (p *TopicPublisher) onEachAssignments(generation int, assignments []*mq_pb.
 
 func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) error {
 
-	log.Printf("connecting to %v for topic partition %+v", job.LeaderBroker, job.Partition)
+	glog.V(1).Infof("connecting to %v for topic partition %+v", job.LeaderBroker, job.Partition)
 
 	grpcConnection, err := grpc.NewClient(job.LeaderBroker, grpc.WithTransportCredentials(insecure.NewCredentials()), p.grpcDialOption)
 	if err != nil {
@@ -176,20 +188,25 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
 			if err != nil {
 				e, _ := status.FromError(err)
 				if e.Code() == codes.Unknown && e.Message() == "EOF" {
-					log.Printf("publish to %s EOF", publishClient.Broker)
+					glog.V(1).Infof("publish stream to %s closed (EOF)", publishClient.Broker)
 					return
 				}
 				publishClient.Err = err
-				log.Printf("publish1 to %s error: %v\n", publishClient.Broker, err)
+				// Use appropriate logging level based on error type
+				if isConnectionError(err) {
+					glog.V(1).Infof("connection error receiving from %s (will retry): %v", publishClient.Broker, err)
+				} else {
+					log.Printf("publish receive error from %s: %v", publishClient.Broker, err)
+				}
 				return
 			}
 			if ackResp.Error != "" {
 				publishClient.Err = fmt.Errorf("ack error: %v", ackResp.Error)
-				log.Printf("publish2 to %s error: %v\n", publishClient.Broker, ackResp.Error)
+				log.Printf("publish ack error from %s: %v", publishClient.Broker, ackResp.Error)
 				return
 			}
 			if ackResp.AckSequence > 0 {
-				log.Printf("ack %d published %d hasMoreData:%d", ackResp.AckSequence, atomic.LoadInt64(&publishedTsNs), atomic.LoadInt32(&hasMoreData))
+				glog.V(2).Infof("ack %d published %d hasMoreData:%d", ackResp.AckSequence, atomic.LoadInt64(&publishedTsNs), atomic.LoadInt32(&hasMoreData))
 			}
 			if atomic.LoadInt64(&publishedTsNs) <= ackResp.AckSequence && atomic.LoadInt32(&hasMoreData) == 0 {
 				return
@@ -222,7 +239,7 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
 		}
 	}
 
-	log.Printf("published %d messages to %v for topic partition %+v", publishCounter, job.LeaderBroker, job.Partition)
+	glog.V(1).Infof("published %d messages to %v for topic partition %+v", publishCounter, job.LeaderBroker, job.Partition)
 
 	return nil
 }
@@ -296,3 +313,40 @@ func (p *TopicPublisher) doLookupTopicPartitions() (assignments []*mq_pb.BrokerP
 	return nil, fmt.Errorf("lookup topic %s: %v", p.config.Topic, lastErr)
 
 }
+
+// isConnectionError checks if an error is a connection-level error that is handled automatically
+func isConnectionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	// Check gRPC status codes for connection errors
+	if grpcStatus, ok := status.FromError(err); ok {
+		switch grpcStatus.Code() {
+		case codes.Unavailable, codes.DeadlineExceeded, codes.Canceled, codes.Unknown:
+			return true
+		}
+	}
+
+	// Check for common connection error patterns
+	errStr := strings.ToLower(err.Error())
+	connectionErrorPatterns := []string{
+		"eof",
+		"error reading server preface",
+		"connection refused",
+		"connection reset",
+		"broken pipe",
+		"no such host",
+		"network is unreachable",
+		"context deadline exceeded",
+		"context canceled",
+	}
+
+	for _, pattern := range connectionErrorPatterns {
+		if strings.Contains(errStr, pattern) {
+			return true
+		}
+	}
+
+	return false
+}

weed/mq/logstore/read_parquet_to_log.go

@@ -4,6 +4,10 @@ import (
 	"context"
 	"encoding/binary"
 	"fmt"
+	"io"
+	"math"
+	"strings"
+
 	"github.com/parquet-go/parquet-go"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
@@ -13,9 +17,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
 	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
 	"google.golang.org/protobuf/proto"
-	"io"
-	"math"
-	"strings"
 )
 
 var (
@@ -42,10 +43,6 @@ func GenParquetReadFunc(filerClient filer_pb.FilerClient, t topic.Topic, p topic
 		WithField(SW_COLUMN_NAME_KEY, schema.TypeBytes).
 		RecordTypeEnd()
 
-	parquetSchema, err := schema.ToParquetSchema(t.Name, recordType)
-	if err != nil {
-		return nil
-	}
 	parquetLevels, err := schema.ToParquetLevels(recordType)
 	if err != nil {
 		return nil
@@ -61,11 +58,12 @@ func GenParquetReadFunc(filerClient filer_pb.FilerClient, t topic.Topic, p topic
 		readerAt := filer.NewChunkReaderAtFromClient(readerCache, chunkViews, int64(fileSize))
 
 		// create parquet reader
-		parquetReader := parquet.NewReader(readerAt, parquetSchema)
+		parquetReader := parquet.NewReader(readerAt)
 		rows := make([]parquet.Row, 128)
 		for {
 			rowCount, readErr := parquetReader.ReadRows(rows)
 
+			// Process the rows first, even if EOF is returned
 			for i := 0; i < rowCount; i++ {
 				row := rows[i]
 				// convert parquet row to schema_pb.RecordValue
@@ -99,12 +97,16 @@ func GenParquetReadFunc(filerClient filer_pb.FilerClient, t topic.Topic, p topic
 				}
 			}
 
+			// Check for end conditions after processing rows
 			if readErr != nil {
 				if readErr == io.EOF {
 					return processedTsNs, nil
 				}
 				return processedTsNs, readErr
 			}
+			if rowCount == 0 {
+				return processedTsNs, nil
+			}
 		}
 		return
 	}

weed/mq/schema/write_parquet_test.go

@@ -2,12 +2,13 @@ package schema
 
 import (
 	"fmt"
-	"github.com/parquet-go/parquet-go"
-	"github.com/parquet-go/parquet-go/compress/zstd"
-	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
 	"io"
 	"os"
 	"testing"
+
+	"github.com/parquet-go/parquet-go"
+	"github.com/parquet-go/parquet-go/compress/zstd"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
 )
 
 func TestWriteReadParquet(t *testing.T) {
@@ -125,16 +126,25 @@ func testReadingParquetFile(t *testing.T, filename string, parquetSchema *parque
 		t.Fatalf("os.Open failed: %v", err)
 	}
 	defer file.Close()
-	reader := parquet.NewReader(file, parquetSchema)
+
+	// Get file info to determine size
+	fileInfo, err := file.Stat()
+	if err != nil {
+		t.Fatalf("file.Stat failed: %v", err)
+	}
+
+	// Create a parquet file from the opened file
+	parquetFile, err := parquet.OpenFile(file, fileInfo.Size())
+	if err != nil {
+		t.Fatalf("parquet.OpenFile failed: %v", err)
+	}
+
+	reader := parquet.NewReader(parquetFile)
 	rows := make([]parquet.Row, 128)
 	for {
 		rowCount, err := reader.ReadRows(rows)
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			t.Fatalf("reader.Read failed: %v", err)
-		}
+
+		// Process the rows first, even if EOF is returned
 		for i := 0; i < rowCount; i++ {
 			row := rows[i]
 			// convert parquet row to schema_pb.RecordValue
@@ -147,6 +157,17 @@ func testReadingParquetFile(t *testing.T, filename string, parquetSchema *parque
 			}
 		}
 		total += rowCount
+
+		// Check for end conditions after processing rows
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			t.Fatalf("reader.Read failed: %v", err)
+		}
+		if rowCount == 0 {
+			break
+		}
 	}
 	fmt.Printf("total: %v\n", total)
 	return

weed/mq/topic/local_partition.go

@@ -3,6 +3,11 @@ package topic
 import (
 	"context"
 	"fmt"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
@@ -10,9 +15,6 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"sync"
-	"sync/atomic"
-	"time"
 )
 
 type LocalPartition struct {
@@ -182,7 +184,12 @@ func (p *LocalPartition) MaybeConnectToFollowers(initMessage *mq_pb.PublishMessa
 					glog.V(0).Infof("local partition %v follower %v stopped", p.Partition, p.Follower)
 					return
 				}
-				glog.Errorf("Receiving local partition %v  follower %s ack: %v", p.Partition, p.Follower, err)
+				// Use appropriate logging level based on error type
+				if isConnectionError(err) {
+					glog.V(1).Infof("local partition %v follower %s connection error (will retry): %v", p.Partition, p.Follower, err)
+				} else {
+					glog.Errorf("Receiving local partition %v  follower %s ack: %v", p.Partition, p.Follower, err)
+				}
 				return
 			}
 			atomic.StoreInt64(&p.AckTsNs, ack.AckTsNs)
@@ -242,3 +249,40 @@ func (p *LocalPartition) NotifyLogFlushed(flushTsNs int64) {
 		// println("notifying", p.Follower, "flushed at", flushTsNs)
 	}
 }
+
+// isConnectionError checks if an error is a connection-level error
+func isConnectionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	// Check gRPC status codes for connection errors
+	if grpcStatus, ok := status.FromError(err); ok {
+		switch grpcStatus.Code() {
+		case codes.Unavailable, codes.DeadlineExceeded, codes.Canceled, codes.Unknown:
+			return true
+		}
+	}
+
+	// Check for common connection error patterns
+	errStr := strings.ToLower(err.Error())
+	connectionErrorPatterns := []string{
+		"eof",
+		"error reading server preface",
+		"connection refused",
+		"connection reset",
+		"broken pipe",
+		"no such host",
+		"network is unreachable",
+		"context deadline exceeded",
+		"context canceled",
+	}
+
+	for _, pattern := range connectionErrorPatterns {
+		if strings.Contains(errStr, pattern) {
+			return true
+		}
+	}
+
+	return false
+}

weed/operation/tail_volume.go

@@ -3,9 +3,10 @@ package operation
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"io"
 
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+
 	"google.golang.org/grpc"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -53,6 +54,10 @@ func TailVolumeFromSource(volumeServer pb.ServerAddress, grpcDialOption grpc.Dia
 
 			needleHeader := resp.NeedleHeader
 			needleBody := resp.NeedleBody
+			version := needle.Version(resp.Version)
+			if version == 0 {
+				version = needle.GetCurrentVersion()
+			}
 
 			if len(needleHeader) == 0 {
 				continue
@@ -72,7 +77,7 @@ func TailVolumeFromSource(volumeServer pb.ServerAddress, grpcDialOption grpc.Dia
 
 			n := new(needle.Needle)
 			n.ParseNeedleHeader(needleHeader)
-			err = n.ReadNeedleBodyBytes(needleBody, needle.CurrentVersion)
+			err = n.ReadNeedleBodyBytes(needleBody, version)
 			if err != nil {
 				return err
 			}

weed/operation/upload_content.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/valyala/bytebufferpool"
 	"io"
 	"mime"
 	"mime/multipart"
@@ -16,6 +15,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
+	"github.com/valyala/bytebufferpool"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
@@ -359,7 +361,7 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction
 		req.Header.Set("Authorization", "BEARER "+string(option.Jwt))
 	}
 
-	util.ReqWithRequestId(req, ctx)
+	request_id.InjectToRequest(ctx, req)
 
 	// print("+")
 	resp, post_err := uploader.httpClient.Do(req)

weed/pb/filer_pb/filer_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.5.1
-// - protoc             v5.28.3
+// - protoc             v5.29.3
 // source: filer.proto
 
 package filer_pb

weed/pb/grpc_client_server.go

@@ -3,8 +3,6 @@ package pb
 import (
 	"context"
 	"fmt"
-	"github.com/google/uuid"
-	"google.golang.org/grpc/metadata"
 	"math/rand/v2"
 	"net/http"
 	"strconv"
@@ -12,6 +10,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/google/uuid"
+	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
+	"google.golang.org/grpc/metadata"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -128,7 +130,7 @@ func requestIDUnaryInterceptor() grpc.UnaryServerInterceptor {
 		handler grpc.UnaryHandler,
 	) (interface{}, error) {
 		incomingMd, _ := metadata.FromIncomingContext(ctx)
-		idList := incomingMd.Get(util.RequestIDKey)
+		idList := incomingMd.Get(request_id.AmzRequestIDHeader)
 		var reqID string
 		if len(idList) > 0 {
 			reqID = idList[0]
@@ -139,11 +141,12 @@ func requestIDUnaryInterceptor() grpc.UnaryServerInterceptor {
 
 		ctx = metadata.NewOutgoingContext(ctx,
 			metadata.New(map[string]string{
-				util.RequestIDKey: reqID,
+				request_id.AmzRequestIDHeader: reqID,
 			}))
 
-		ctx = util.WithRequestID(ctx, reqID)
-		grpc.SetTrailer(ctx, metadata.Pairs(util.RequestIDKey, reqID))
+		ctx = request_id.Set(ctx, reqID)
+
+		grpc.SetTrailer(ctx, metadata.Pairs(request_id.AmzRequestIDHeader, reqID))
 
 		return handler(ctx, req)
 	}

weed/pb/iam_pb/iam.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.34.2
-// 	protoc        v5.28.3
+// 	protoc-gen-go v1.36.6
+// 	protoc        v5.29.3
 // source: iam.proto
 
 package iam_pb
@@ -11,6 +11,7 @@ import (
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	reflect "reflect"
 	sync "sync"
+	unsafe "unsafe"
 )
 
 const (
@@ -21,21 +22,18 @@ const (
 )
 
 type S3ApiConfiguration struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Identities    []*Identity            `protobuf:"bytes,1,rep,name=identities,proto3" json:"identities,omitempty"`
+	Accounts      []*Account             `protobuf:"bytes,2,rep,name=accounts,proto3" json:"accounts,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	Identities []*Identity `protobuf:"bytes,1,rep,name=identities,proto3" json:"identities,omitempty"`
-	Accounts   []*Account  `protobuf:"bytes,2,rep,name=accounts,proto3" json:"accounts,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *S3ApiConfiguration) Reset() {
 	*x = S3ApiConfiguration{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_iam_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_iam_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *S3ApiConfiguration) String() string {
@@ -46,7 +44,7 @@ func (*S3ApiConfiguration) ProtoMessage() {}
 
 func (x *S3ApiConfiguration) ProtoReflect() protoreflect.Message {
 	mi := &file_iam_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -76,23 +74,20 @@ func (x *S3ApiConfiguration) GetAccounts() []*Account {
 }
 
 type Identity struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Name          string                 `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Credentials   []*Credential          `protobuf:"bytes,2,rep,name=credentials,proto3" json:"credentials,omitempty"`
+	Actions       []string               `protobuf:"bytes,3,rep,name=actions,proto3" json:"actions,omitempty"`
+	Account       *Account               `protobuf:"bytes,4,opt,name=account,proto3" json:"account,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	Name        string        `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	Credentials []*Credential `protobuf:"bytes,2,rep,name=credentials,proto3" json:"credentials,omitempty"`
-	Actions     []string      `protobuf:"bytes,3,rep,name=actions,proto3" json:"actions,omitempty"`
-	Account     *Account      `protobuf:"bytes,4,opt,name=account,proto3" json:"account,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *Identity) Reset() {
 	*x = Identity{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_iam_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_iam_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *Identity) String() string {
@@ -103,7 +98,7 @@ func (*Identity) ProtoMessage() {}
 
 func (x *Identity) ProtoReflect() protoreflect.Message {
 	mi := &file_iam_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -147,21 +142,18 @@ func (x *Identity) GetAccount() *Account {
 }
 
 type Credential struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	AccessKey     string                 `protobuf:"bytes,1,opt,name=access_key,json=accessKey,proto3" json:"access_key,omitempty"`
+	SecretKey     string                 `protobuf:"bytes,2,opt,name=secret_key,json=secretKey,proto3" json:"secret_key,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	AccessKey string `protobuf:"bytes,1,opt,name=access_key,json=accessKey,proto3" json:"access_key,omitempty"`
-	SecretKey string `protobuf:"bytes,2,opt,name=secret_key,json=secretKey,proto3" json:"secret_key,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *Credential) Reset() {
 	*x = Credential{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_iam_proto_msgTypes[2]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_iam_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *Credential) String() string {
@@ -172,7 +164,7 @@ func (*Credential) ProtoMessage() {}
 
 func (x *Credential) ProtoReflect() protoreflect.Message {
 	mi := &file_iam_proto_msgTypes[2]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -202,22 +194,19 @@ func (x *Credential) GetSecretKey() string {
 }
 
 type Account struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Id            string                 `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+	DisplayName   string                 `protobuf:"bytes,2,opt,name=display_name,json=displayName,proto3" json:"display_name,omitempty"`
+	EmailAddress  string                 `protobuf:"bytes,3,opt,name=email_address,json=emailAddress,proto3" json:"email_address,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	Id           string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
-	DisplayName  string `protobuf:"bytes,2,opt,name=display_name,json=displayName,proto3" json:"display_name,omitempty"`
-	EmailAddress string `protobuf:"bytes,3,opt,name=email_address,json=emailAddress,proto3" json:"email_address,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *Account) Reset() {
 	*x = Account{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_iam_proto_msgTypes[3]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_iam_proto_msgTypes[3]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *Account) String() string {
@@ -228,7 +217,7 @@ func (*Account) ProtoMessage() {}
 
 func (x *Account) ProtoReflect() protoreflect.Message {
 	mi := &file_iam_proto_msgTypes[3]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -266,54 +255,40 @@ func (x *Account) GetEmailAddress() string {
 
 var File_iam_proto protoreflect.FileDescriptor
 
-var file_iam_proto_rawDesc = []byte{
-	0x0a, 0x09, 0x69, 0x61, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x69, 0x61, 0x6d,
-	0x5f, 0x70, 0x62, 0x22, 0x73, 0x0a, 0x12, 0x53, 0x33, 0x41, 0x70, 0x69, 0x43, 0x6f, 0x6e, 0x66,
-	0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x0a, 0x69, 0x64, 0x65,
-	0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e,
-	0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52,
-	0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x2b, 0x0a, 0x08, 0x61,
-	0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e,
-	0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x08,
-	0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x22, 0x99, 0x01, 0x0a, 0x08, 0x49, 0x64, 0x65,
-	0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b, 0x63, 0x72, 0x65,
-	0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12,
-	0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
-	0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x12,
-	0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09,
-	0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x29, 0x0a, 0x07, 0x61, 0x63, 0x63,
-	0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x69, 0x61, 0x6d,
-	0x5f, 0x70, 0x62, 0x2e, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x07, 0x61, 0x63, 0x63,
-	0x6f, 0x75, 0x6e, 0x74, 0x22, 0x4a, 0x0a, 0x0a, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
-	0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65,
-	0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18,
-	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79,
-	0x22, 0x61, 0x0a, 0x07, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69,
-	0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x64,
-	0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23,
-	0x0a, 0x0d, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18,
-	0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x64, 0x64, 0x72,
-	0x65, 0x73, 0x73, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x49, 0x64,
-	0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4d, 0x61, 0x6e, 0x61,
-	0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65,
-	0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08, 0x49, 0x61, 0x6d, 0x50,
-	0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
-	0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65,
-	0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x69, 0x61, 0x6d,
-	0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-}
+const file_iam_proto_rawDesc = "" +
+	"\n" +
+	"\tiam.proto\x12\x06iam_pb\"s\n" +
+	"\x12S3ApiConfiguration\x120\n" +
+	"\n" +
+	"identities\x18\x01 \x03(\v2\x10.iam_pb.IdentityR\n" +
+	"identities\x12+\n" +
+	"\baccounts\x18\x02 \x03(\v2\x0f.iam_pb.AccountR\baccounts\"\x99\x01\n" +
+	"\bIdentity\x12\x12\n" +
+	"\x04name\x18\x01 \x01(\tR\x04name\x124\n" +
+	"\vcredentials\x18\x02 \x03(\v2\x12.iam_pb.CredentialR\vcredentials\x12\x18\n" +
+	"\aactions\x18\x03 \x03(\tR\aactions\x12)\n" +
+	"\aaccount\x18\x04 \x01(\v2\x0f.iam_pb.AccountR\aaccount\"J\n" +
+	"\n" +
+	"Credential\x12\x1d\n" +
+	"\n" +
+	"access_key\x18\x01 \x01(\tR\taccessKey\x12\x1d\n" +
+	"\n" +
+	"secret_key\x18\x02 \x01(\tR\tsecretKey\"a\n" +
+	"\aAccount\x12\x0e\n" +
+	"\x02id\x18\x01 \x01(\tR\x02id\x12!\n" +
+	"\fdisplay_name\x18\x02 \x01(\tR\vdisplayName\x12#\n" +
+	"\remail_address\x18\x03 \x01(\tR\femailAddress2!\n" +
+	"\x1fSeaweedIdentityAccessManagementBK\n" +
+	"\x10seaweedfs.clientB\bIamProtoZ-github.com/seaweedfs/seaweedfs/weed/pb/iam_pbb\x06proto3"
 
 var (
 	file_iam_proto_rawDescOnce sync.Once
-	file_iam_proto_rawDescData = file_iam_proto_rawDesc
+	file_iam_proto_rawDescData []byte
 )
 
 func file_iam_proto_rawDescGZIP() []byte {
 	file_iam_proto_rawDescOnce.Do(func() {
-		file_iam_proto_rawDescData = protoimpl.X.CompressGZIP(file_iam_proto_rawDescData)
+		file_iam_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_iam_proto_rawDesc), len(file_iam_proto_rawDesc)))
 	})
 	return file_iam_proto_rawDescData
 }
@@ -342,61 +317,11 @@ func file_iam_proto_init() {
 	if File_iam_proto != nil {
 		return
 	}
-	if !protoimpl.UnsafeEnabled {
-		file_iam_proto_msgTypes[0].Exporter = func(v any, i int) any {
-			switch v := v.(*S3ApiConfiguration); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_iam_proto_msgTypes[1].Exporter = func(v any, i int) any {
-			switch v := v.(*Identity); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_iam_proto_msgTypes[2].Exporter = func(v any, i int) any {
-			switch v := v.(*Credential); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_iam_proto_msgTypes[3].Exporter = func(v any, i int) any {
-			switch v := v.(*Account); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_iam_proto_rawDesc,
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_iam_proto_rawDesc), len(file_iam_proto_rawDesc)),
 			NumEnums:      0,
 			NumMessages:   4,
 			NumExtensions: 0,
@@ -407,7 +332,6 @@ func file_iam_proto_init() {
 		MessageInfos:      file_iam_proto_msgTypes,
 	}.Build()
 	File_iam_proto = out.File
-	file_iam_proto_rawDesc = nil
 	file_iam_proto_goTypes = nil
 	file_iam_proto_depIdxs = nil
 }

weed/pb/iam_pb/iam_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.5.1
-// - protoc             v5.28.3
+// - protoc             v5.29.3
 // source: iam.proto
 
 package iam_pb

weed/pb/master_pb/master_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.5.1
-// - protoc             v5.28.3
+// - protoc             v5.29.3
 // source: master.proto
 
 package master_pb

weed/pb/mount_pb/mount.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.34.2
-// 	protoc        v5.28.3
+// 	protoc-gen-go v1.36.6
+// 	protoc        v5.29.3
 // source: mount.proto
 
 package mount_pb
@@ -11,6 +11,7 @@ import (
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	reflect "reflect"
 	sync "sync"
+	unsafe "unsafe"
 )
 
 const (
@@ -21,20 +22,17 @@ const (
 )
 
 type ConfigureRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	CollectionCapacity int64 `protobuf:"varint,1,opt,name=collection_capacity,json=collectionCapacity,proto3" json:"collection_capacity,omitempty"`
+	state              protoimpl.MessageState `protogen:"open.v1"`
+	CollectionCapacity int64                  `protobuf:"varint,1,opt,name=collection_capacity,json=collectionCapacity,proto3" json:"collection_capacity,omitempty"`
+	unknownFields      protoimpl.UnknownFields
+	sizeCache          protoimpl.SizeCache
 }
 
 func (x *ConfigureRequest) Reset() {
 	*x = ConfigureRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mount_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mount_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *ConfigureRequest) String() string {
@@ -45,7 +43,7 @@ func (*ConfigureRequest) ProtoMessage() {}
 
 func (x *ConfigureRequest) ProtoReflect() protoreflect.Message {
 	mi := &file_mount_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -68,18 +66,16 @@ func (x *ConfigureRequest) GetCollectionCapacity() int64 {
 }
 
 type ConfigureResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
 	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *ConfigureResponse) Reset() {
 	*x = ConfigureResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mount_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mount_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *ConfigureResponse) String() string {
@@ -90,7 +86,7 @@ func (*ConfigureResponse) ProtoMessage() {}
 
 func (x *ConfigureResponse) ProtoReflect() protoreflect.Message {
 	mi := &file_mount_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -107,36 +103,25 @@ func (*ConfigureResponse) Descriptor() ([]byte, []int) {
 
 var File_mount_proto protoreflect.FileDescriptor
 
-var file_mount_proto_rawDesc = []byte{
-	0x0a, 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d,
-	0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x22, 0x43, 0x0a, 0x10, 0x43,
-	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
-	0x2f, 0x0a, 0x13, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x61,
-	0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x63, 0x6f,
-	0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79,
-	0x22, 0x13, 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x5e, 0x0a, 0x0c, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64,
-	0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x4e, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
-	0x72, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70,
-	0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70,
-	0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64,
-	0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x4d, 0x6f, 0x75, 0x6e, 0x74,
-	0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
-	0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77,
-	0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x6f,
-	0x75, 0x6e, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-}
+const file_mount_proto_rawDesc = "" +
+	"\n" +
+	"\vmount.proto\x12\fmessaging_pb\"C\n" +
+	"\x10ConfigureRequest\x12/\n" +
+	"\x13collection_capacity\x18\x01 \x01(\x03R\x12collectionCapacity\"\x13\n" +
+	"\x11ConfigureResponse2^\n" +
+	"\fSeaweedMount\x12N\n" +
+	"\tConfigure\x12\x1e.messaging_pb.ConfigureRequest\x1a\x1f.messaging_pb.ConfigureResponse\"\x00BO\n" +
+	"\x10seaweedfs.clientB\n" +
+	"MountProtoZ/github.com/seaweedfs/seaweedfs/weed/pb/mount_pbb\x06proto3"
 
 var (
 	file_mount_proto_rawDescOnce sync.Once
-	file_mount_proto_rawDescData = file_mount_proto_rawDesc
+	file_mount_proto_rawDescData []byte
 )
 
 func file_mount_proto_rawDescGZIP() []byte {
 	file_mount_proto_rawDescOnce.Do(func() {
-		file_mount_proto_rawDescData = protoimpl.X.CompressGZIP(file_mount_proto_rawDescData)
+		file_mount_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mount_proto_rawDesc), len(file_mount_proto_rawDesc)))
 	})
 	return file_mount_proto_rawDescData
 }
@@ -161,37 +146,11 @@ func file_mount_proto_init() {
 	if File_mount_proto != nil {
 		return
 	}
-	if !protoimpl.UnsafeEnabled {
-		file_mount_proto_msgTypes[0].Exporter = func(v any, i int) any {
-			switch v := v.(*ConfigureRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mount_proto_msgTypes[1].Exporter = func(v any, i int) any {
-			switch v := v.(*ConfigureResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_mount_proto_rawDesc,
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_mount_proto_rawDesc), len(file_mount_proto_rawDesc)),
 			NumEnums:      0,
 			NumMessages:   2,
 			NumExtensions: 0,
@@ -202,7 +161,6 @@ func file_mount_proto_init() {
 		MessageInfos:      file_mount_proto_msgTypes,
 	}.Build()
 	File_mount_proto = out.File
-	file_mount_proto_rawDesc = nil
 	file_mount_proto_goTypes = nil
 	file_mount_proto_depIdxs = nil
 }

weed/pb/mount_pb/mount_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.5.1
-// - protoc             v5.28.3
+// - protoc             v5.29.3
 // source: mount.proto
 
 package mount_pb

weed/pb/mq_agent_pb/mq_agent.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.34.2
-// 	protoc        v5.28.3
+// 	protoc-gen-go v1.36.6
+// 	protoc        v5.29.3
 // source: mq_agent.proto
 
 package mq_agent_pb
@@ -12,6 +12,7 @@ import (
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	reflect "reflect"
 	sync "sync"
+	unsafe "unsafe"
 )
 
 const (
@@ -23,23 +24,20 @@ const (
 
 // ////////////////////////////////////////////////
 type StartPublishSessionRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Topic          *schema_pb.Topic      `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"`
-	PartitionCount int32                 `protobuf:"varint,2,opt,name=partition_count,json=partitionCount,proto3" json:"partition_count,omitempty"`
-	RecordType     *schema_pb.RecordType `protobuf:"bytes,3,opt,name=record_type,json=recordType,proto3" json:"record_type,omitempty"`
-	PublisherName  string                `protobuf:"bytes,4,opt,name=publisher_name,json=publisherName,proto3" json:"publisher_name,omitempty"`
+	state          protoimpl.MessageState `protogen:"open.v1"`
+	Topic          *schema_pb.Topic       `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"`
+	PartitionCount int32                  `protobuf:"varint,2,opt,name=partition_count,json=partitionCount,proto3" json:"partition_count,omitempty"`
+	RecordType     *schema_pb.RecordType  `protobuf:"bytes,3,opt,name=record_type,json=recordType,proto3" json:"record_type,omitempty"`
+	PublisherName  string                 `protobuf:"bytes,4,opt,name=publisher_name,json=publisherName,proto3" json:"publisher_name,omitempty"`
+	unknownFields  protoimpl.UnknownFields
+	sizeCache      protoimpl.SizeCache
 }
 
 func (x *StartPublishSessionRequest) Reset() {
 	*x = StartPublishSessionRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_agent_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_agent_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *StartPublishSessionRequest) String() string {
@@ -50,7 +48,7 @@ func (*StartPublishSessionRequest) ProtoMessage() {}
 
 func (x *StartPublishSessionRequest) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_agent_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -94,21 +92,18 @@ func (x *StartPublishSessionRequest) GetPublisherName() string {
 }
 
 type StartPublishSessionResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Error         string                 `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
+	SessionId     int64                  `protobuf:"varint,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	Error     string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
-	SessionId int64  `protobuf:"varint,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *StartPublishSessionResponse) Reset() {
 	*x = StartPublishSessionResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_agent_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_agent_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *StartPublishSessionResponse) String() string {
@@ -119,7 +114,7 @@ func (*StartPublishSessionResponse) ProtoMessage() {}
 
 func (x *StartPublishSessionResponse) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_agent_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -149,20 +144,17 @@ func (x *StartPublishSessionResponse) GetSessionId() int64 {
 }
 
 type ClosePublishSessionRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	SessionId     int64                  `protobuf:"varint,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	SessionId int64 `protobuf:"varint,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *ClosePublishSessionRequest) Reset() {
 	*x = ClosePublishSessionRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_agent_proto_msgTypes[2]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_agent_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *ClosePublishSessionRequest) String() string {
@@ -173,7 +165,7 @@ func (*ClosePublishSessionRequest) ProtoMessage() {}
 
 func (x *ClosePublishSessionRequest) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_agent_proto_msgTypes[2]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -196,20 +188,17 @@ func (x *ClosePublishSessionRequest) GetSessionId() int64 {
 }
 
 type ClosePublishSessionResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Error         string                 `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *ClosePublishSessionResponse) Reset() {
 	*x = ClosePublishSessionResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_agent_proto_msgTypes[3]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_agent_proto_msgTypes[3]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *ClosePublishSessionResponse) String() string {
@@ -220,7 +209,7 @@ func (*ClosePublishSessionResponse) ProtoMessage() {}
 
 func (x *ClosePublishSessionResponse) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_agent_proto_msgTypes[3]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -244,22 +233,19 @@ func (x *ClosePublishSessionResponse) GetError() string {
 
 // ////////////////////////////////////////////////
 type PublishRecordRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	SessionId     int64                  `protobuf:"varint,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // session_id is required for the first record
+	Key           []byte                 `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
+	Value         *schema_pb.RecordValue `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	SessionId int64                  `protobuf:"varint,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // session_id is required for the first record
-	Key       []byte                 `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
-	Value     *schema_pb.RecordValue `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *PublishRecordRequest) Reset() {
 	*x = PublishRecordRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_agent_proto_msgTypes[4]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_agent_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *PublishRecordRequest) String() string {
@@ -270,7 +256,7 @@ func (*PublishRecordRequest) ProtoMessage() {}
 
 func (x *PublishRecordRequest) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_agent_proto_msgTypes[4]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -307,21 +293,18 @@ func (x *PublishRecordRequest) GetValue() *schema_pb.RecordValue {
 }
 
 type PublishRecordResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	AckSequence   int64                  `protobuf:"varint,1,opt,name=ack_sequence,json=ackSequence,proto3" json:"ack_sequence,omitempty"`
+	Error         string                 `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	AckSequence int64  `protobuf:"varint,1,opt,name=ack_sequence,json=ackSequence,proto3" json:"ack_sequence,omitempty"`
-	Error       string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *PublishRecordResponse) Reset() {
 	*x = PublishRecordResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_agent_proto_msgTypes[5]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_agent_proto_msgTypes[5]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *PublishRecordResponse) String() string {
@@ -332,7 +315,7 @@ func (*PublishRecordResponse) ProtoMessage() {}
 
 func (x *PublishRecordResponse) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_agent_proto_msgTypes[5]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -363,22 +346,19 @@ func (x *PublishRecordResponse) GetError() string {
 
 // ////////////////////////////////////////////////
 type SubscribeRecordRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState                             `protogen:"open.v1"`
+	Init          *SubscribeRecordRequest_InitSubscribeRecordRequest `protobuf:"bytes,1,opt,name=init,proto3" json:"init,omitempty"`
+	AckSequence   int64                                              `protobuf:"varint,2,opt,name=ack_sequence,json=ackSequence,proto3" json:"ack_sequence,omitempty"`
+	AckKey        []byte                                             `protobuf:"bytes,3,opt,name=ack_key,json=ackKey,proto3" json:"ack_key,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	Init        *SubscribeRecordRequest_InitSubscribeRecordRequest `protobuf:"bytes,1,opt,name=init,proto3" json:"init,omitempty"`
-	AckSequence int64                                              `protobuf:"varint,2,opt,name=ack_sequence,json=ackSequence,proto3" json:"ack_sequence,omitempty"`
-	AckKey      []byte                                             `protobuf:"bytes,3,opt,name=ack_key,json=ackKey,proto3" json:"ack_key,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *SubscribeRecordRequest) Reset() {
 	*x = SubscribeRecordRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_agent_proto_msgTypes[6]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_agent_proto_msgTypes[6]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *SubscribeRecordRequest) String() string {
@@ -389,7 +369,7 @@ func (*SubscribeRecordRequest) ProtoMessage() {}
 
 func (x *SubscribeRecordRequest) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_agent_proto_msgTypes[6]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -426,25 +406,22 @@ func (x *SubscribeRecordRequest) GetAckKey() []byte {
 }
 
 type SubscribeRecordResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
+	state         protoimpl.MessageState `protogen:"open.v1"`
 	Key           []byte                 `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
 	Value         *schema_pb.RecordValue `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"`
 	TsNs          int64                  `protobuf:"varint,4,opt,name=ts_ns,json=tsNs,proto3" json:"ts_ns,omitempty"`
 	Error         string                 `protobuf:"bytes,5,opt,name=error,proto3" json:"error,omitempty"`
 	IsEndOfStream bool                   `protobuf:"varint,6,opt,name=is_end_of_stream,json=isEndOfStream,proto3" json:"is_end_of_stream,omitempty"`
 	IsEndOfTopic  bool                   `protobuf:"varint,7,opt,name=is_end_of_topic,json=isEndOfTopic,proto3" json:"is_end_of_topic,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *SubscribeRecordResponse) Reset() {
 	*x = SubscribeRecordResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_agent_proto_msgTypes[7]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_agent_proto_msgTypes[7]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *SubscribeRecordResponse) String() string {
@@ -455,7 +432,7 @@ func (*SubscribeRecordResponse) ProtoMessage() {}
 
 func (x *SubscribeRecordResponse) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_agent_proto_msgTypes[7]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -513,10 +490,7 @@ func (x *SubscribeRecordResponse) GetIsEndOfTopic() bool {
 }
 
 type SubscribeRecordRequest_InitSubscribeRecordRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
+	state                   protoimpl.MessageState       `protogen:"open.v1"`
 	ConsumerGroup           string                       `protobuf:"bytes,1,opt,name=consumer_group,json=consumerGroup,proto3" json:"consumer_group,omitempty"`
 	ConsumerGroupInstanceId string                       `protobuf:"bytes,2,opt,name=consumer_group_instance_id,json=consumerGroupInstanceId,proto3" json:"consumer_group_instance_id,omitempty"`
 	Topic                   *schema_pb.Topic             `protobuf:"bytes,4,opt,name=topic,proto3" json:"topic,omitempty"`
@@ -526,15 +500,15 @@ type SubscribeRecordRequest_InitSubscribeRecordRequest struct {
 	Filter                  string                       `protobuf:"bytes,10,opt,name=filter,proto3" json:"filter,omitempty"`
 	MaxSubscribedPartitions int32                        `protobuf:"varint,11,opt,name=max_subscribed_partitions,json=maxSubscribedPartitions,proto3" json:"max_subscribed_partitions,omitempty"`
 	SlidingWindowSize       int32                        `protobuf:"varint,12,opt,name=sliding_window_size,json=slidingWindowSize,proto3" json:"sliding_window_size,omitempty"`
+	unknownFields           protoimpl.UnknownFields
+	sizeCache               protoimpl.SizeCache
 }
 
 func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) Reset() {
 	*x = SubscribeRecordRequest_InitSubscribeRecordRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_agent_proto_msgTypes[8]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_agent_proto_msgTypes[8]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) String() string {
@@ -545,7 +519,7 @@ func (*SubscribeRecordRequest_InitSubscribeRecordRequest) ProtoMessage() {}
 
 func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_agent_proto_msgTypes[8]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -625,146 +599,71 @@ func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) GetSlidingWindowSize
 
 var File_mq_agent_proto protoreflect.FileDescriptor
 
-var file_mq_agent_proto_rawDesc = []byte{
-	0x0a, 0x0e, 0x6d, 0x71, 0x5f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x12, 0x0c, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x1a, 0x0f,
-	0x6d, 0x71, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22,
-	0xcc, 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x61, 0x72, 0x74, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68,
-	0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26,
-	0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e,
-	0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x52,
-	0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74,
-	0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52,
-	0x0e, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12,
-	0x36, 0x0a, 0x0b, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62,
-	0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x72, 0x65, 0x63,
-	0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x75, 0x62, 0x6c, 0x69,
-	0x73, 0x68, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x0d, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x52,
-	0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x53, 0x65,
-	0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a,
-	0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72,
-	0x72, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69,
-	0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e,
-	0x49, 0x64, 0x22, 0x3b, 0x0a, 0x1a, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69,
-	0x73, 0x68, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22,
-	0x33, 0x0a, 0x1b, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x53,
-	0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14,
-	0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65,
-	0x72, 0x72, 0x6f, 0x72, 0x22, 0x75, 0x0a, 0x14, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52,
-	0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a,
-	0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03,
-	0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b,
-	0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2c, 0x0a,
-	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73,
-	0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x56,
-	0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x50, 0x0a, 0x15, 0x50,
-	0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x63, 0x6b, 0x5f, 0x73, 0x65, 0x71, 0x75,
-	0x65, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x61, 0x63, 0x6b, 0x53,
-	0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xfb, 0x04,
-	0x0a, 0x16, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72,
-	0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x53, 0x0a, 0x04, 0x69, 0x6e, 0x69, 0x74,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69,
-	0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x52,
-	0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x49, 0x6e, 0x69,
-	0x74, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x12, 0x21, 0x0a,
-	0x0c, 0x61, 0x63, 0x6b, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x03, 0x52, 0x0b, 0x61, 0x63, 0x6b, 0x53, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65,
-	0x12, 0x17, 0x0a, 0x07, 0x61, 0x63, 0x6b, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28,
-	0x0c, 0x52, 0x06, 0x61, 0x63, 0x6b, 0x4b, 0x65, 0x79, 0x1a, 0xcf, 0x03, 0x0a, 0x1a, 0x49, 0x6e,
-	0x69, 0x74, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72,
-	0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x73,
-	0x75, 0x6d, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12,
-	0x3b, 0x0a, 0x1a, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75,
-	0x70, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x17, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x47, 0x72, 0x6f,
-	0x75, 0x70, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x26, 0x0a, 0x05,
-	0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x73, 0x63,
-	0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x52, 0x05, 0x74,
-	0x6f, 0x70, 0x69, 0x63, 0x12, 0x47, 0x0a, 0x11, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f,
-	0x6e, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32,
-	0x1a, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x61, 0x72, 0x74,
-	0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x52, 0x10, 0x70, 0x61, 0x72,
-	0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x73, 0x12, 0x36, 0x0a,
-	0x0b, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01,
-	0x28, 0x0e, 0x32, 0x15, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x4f,
-	0x66, 0x66, 0x73, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x6f, 0x66, 0x66, 0x73, 0x65,
-	0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0c, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x5f,
-	0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6f, 0x66, 0x66,
-	0x73, 0x65, 0x74, 0x54, 0x73, 0x4e, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65,
-	0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12,
-	0x3a, 0x0a, 0x19, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65,
-	0x64, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0b, 0x20, 0x01,
-	0x28, 0x05, 0x52, 0x17, 0x6d, 0x61, 0x78, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65,
-	0x64, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x73,
-	0x6c, 0x69, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x73, 0x69,
-	0x7a, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x73, 0x6c, 0x69, 0x64, 0x69, 0x6e,
-	0x67, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x53, 0x69, 0x7a, 0x65, 0x22, 0xd4, 0x01, 0x0a, 0x17,
-	0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x05, 0x76, 0x61, 0x6c,
-	0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d,
-	0x61, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65,
-	0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73,
-	0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x74, 0x73, 0x4e, 0x73, 0x12, 0x14, 0x0a, 0x05,
-	0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72,
-	0x6f, 0x72, 0x12, 0x27, 0x0a, 0x10, 0x69, 0x73, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x6f, 0x66, 0x5f,
-	0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x69, 0x73,
-	0x45, 0x6e, 0x64, 0x4f, 0x66, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x25, 0x0a, 0x0f, 0x69,
-	0x73, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x6f, 0x66, 0x5f, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x07,
-	0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x73, 0x45, 0x6e, 0x64, 0x4f, 0x66, 0x54, 0x6f, 0x70,
-	0x69, 0x63, 0x32, 0xb9, 0x03, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x4d, 0x65,
-	0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x6c, 0x0a, 0x13,
-	0x53, 0x74, 0x61, 0x72, 0x74, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x53, 0x65, 0x73, 0x73,
-	0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f,
-	0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x53,
-	0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e,
-	0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61,
-	0x72, 0x74, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6c, 0x0a, 0x13, 0x43, 0x6c,
-	0x6f, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f,
-	0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62,
-	0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x53, 0x65, 0x73,
-	0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x65,
-	0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65,
-	0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, 0x0a, 0x0d, 0x50, 0x75, 0x62, 0x6c,
-	0x69, 0x73, 0x68, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x22, 0x2e, 0x6d, 0x65, 0x73, 0x73,
-	0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68,
-	0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e,
-	0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x62,
-	0x6c, 0x69, 0x73, 0x68, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x64, 0x0a, 0x0f, 0x53, 0x75, 0x62, 0x73,
-	0x63, 0x72, 0x69, 0x62, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x24, 0x2e, 0x6d, 0x65,
-	0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63,
-	0x72, 0x69, 0x62, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62,
-	0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x60,
-	0x0a, 0x12, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x6d, 0x71, 0x5f, 0x61,
-	0x67, 0x65, 0x6e, 0x74, 0x42, 0x16, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x51, 0x75, 0x65,
-	0x75, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x32, 0x67, 0x69,
-	0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64,
-	0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65,
-	0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x71, 0x5f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x62,
-	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-}
+const file_mq_agent_proto_rawDesc = "" +
+	"\n" +
+	"\x0emq_agent.proto\x12\fmessaging_pb\x1a\x0fmq_schema.proto\"\xcc\x01\n" +
+	"\x1aStartPublishSessionRequest\x12&\n" +
+	"\x05topic\x18\x01 \x01(\v2\x10.schema_pb.TopicR\x05topic\x12'\n" +
+	"\x0fpartition_count\x18\x02 \x01(\x05R\x0epartitionCount\x126\n" +
+	"\vrecord_type\x18\x03 \x01(\v2\x15.schema_pb.RecordTypeR\n" +
+	"recordType\x12%\n" +
+	"\x0epublisher_name\x18\x04 \x01(\tR\rpublisherName\"R\n" +
+	"\x1bStartPublishSessionResponse\x12\x14\n" +
+	"\x05error\x18\x01 \x01(\tR\x05error\x12\x1d\n" +
+	"\n" +
+	"session_id\x18\x02 \x01(\x03R\tsessionId\";\n" +
+	"\x1aClosePublishSessionRequest\x12\x1d\n" +
+	"\n" +
+	"session_id\x18\x01 \x01(\x03R\tsessionId\"3\n" +
+	"\x1bClosePublishSessionResponse\x12\x14\n" +
+	"\x05error\x18\x01 \x01(\tR\x05error\"u\n" +
+	"\x14PublishRecordRequest\x12\x1d\n" +
+	"\n" +
+	"session_id\x18\x01 \x01(\x03R\tsessionId\x12\x10\n" +
+	"\x03key\x18\x02 \x01(\fR\x03key\x12,\n" +
+	"\x05value\x18\x03 \x01(\v2\x16.schema_pb.RecordValueR\x05value\"P\n" +
+	"\x15PublishRecordResponse\x12!\n" +
+	"\fack_sequence\x18\x01 \x01(\x03R\vackSequence\x12\x14\n" +
+	"\x05error\x18\x02 \x01(\tR\x05error\"\xfb\x04\n" +
+	"\x16SubscribeRecordRequest\x12S\n" +
+	"\x04init\x18\x01 \x01(\v2?.messaging_pb.SubscribeRecordRequest.InitSubscribeRecordRequestR\x04init\x12!\n" +
+	"\fack_sequence\x18\x02 \x01(\x03R\vackSequence\x12\x17\n" +
+	"\aack_key\x18\x03 \x01(\fR\x06ackKey\x1a\xcf\x03\n" +
+	"\x1aInitSubscribeRecordRequest\x12%\n" +
+	"\x0econsumer_group\x18\x01 \x01(\tR\rconsumerGroup\x12;\n" +
+	"\x1aconsumer_group_instance_id\x18\x02 \x01(\tR\x17consumerGroupInstanceId\x12&\n" +
+	"\x05topic\x18\x04 \x01(\v2\x10.schema_pb.TopicR\x05topic\x12G\n" +
+	"\x11partition_offsets\x18\x05 \x03(\v2\x1a.schema_pb.PartitionOffsetR\x10partitionOffsets\x126\n" +
+	"\voffset_type\x18\x06 \x01(\x0e2\x15.schema_pb.OffsetTypeR\n" +
+	"offsetType\x12 \n" +
+	"\foffset_ts_ns\x18\a \x01(\x03R\n" +
+	"offsetTsNs\x12\x16\n" +
+	"\x06filter\x18\n" +
+	" \x01(\tR\x06filter\x12:\n" +
+	"\x19max_subscribed_partitions\x18\v \x01(\x05R\x17maxSubscribedPartitions\x12.\n" +
+	"\x13sliding_window_size\x18\f \x01(\x05R\x11slidingWindowSize\"\xd4\x01\n" +
+	"\x17SubscribeRecordResponse\x12\x10\n" +
+	"\x03key\x18\x02 \x01(\fR\x03key\x12,\n" +
+	"\x05value\x18\x03 \x01(\v2\x16.schema_pb.RecordValueR\x05value\x12\x13\n" +
+	"\x05ts_ns\x18\x04 \x01(\x03R\x04tsNs\x12\x14\n" +
+	"\x05error\x18\x05 \x01(\tR\x05error\x12'\n" +
+	"\x10is_end_of_stream\x18\x06 \x01(\bR\risEndOfStream\x12%\n" +
+	"\x0fis_end_of_topic\x18\a \x01(\bR\fisEndOfTopic2\xb9\x03\n" +
+	"\x15SeaweedMessagingAgent\x12l\n" +
+	"\x13StartPublishSession\x12(.messaging_pb.StartPublishSessionRequest\x1a).messaging_pb.StartPublishSessionResponse\"\x00\x12l\n" +
+	"\x13ClosePublishSession\x12(.messaging_pb.ClosePublishSessionRequest\x1a).messaging_pb.ClosePublishSessionResponse\"\x00\x12^\n" +
+	"\rPublishRecord\x12\".messaging_pb.PublishRecordRequest\x1a#.messaging_pb.PublishRecordResponse\"\x00(\x010\x01\x12d\n" +
+	"\x0fSubscribeRecord\x12$.messaging_pb.SubscribeRecordRequest\x1a%.messaging_pb.SubscribeRecordResponse\"\x00(\x010\x01B`\n" +
+	"\x12seaweedfs.mq_agentB\x16MessageQueueAgentProtoZ2github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pbb\x06proto3"
 
 var (
 	file_mq_agent_proto_rawDescOnce sync.Once
-	file_mq_agent_proto_rawDescData = file_mq_agent_proto_rawDesc
+	file_mq_agent_proto_rawDescData []byte
 )
 
 func file_mq_agent_proto_rawDescGZIP() []byte {
 	file_mq_agent_proto_rawDescOnce.Do(func() {
-		file_mq_agent_proto_rawDescData = protoimpl.X.CompressGZIP(file_mq_agent_proto_rawDescData)
+		file_mq_agent_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mq_agent_proto_rawDesc), len(file_mq_agent_proto_rawDesc)))
 	})
 	return file_mq_agent_proto_rawDescData
 }
@@ -815,121 +714,11 @@ func file_mq_agent_proto_init() {
 	if File_mq_agent_proto != nil {
 		return
 	}
-	if !protoimpl.UnsafeEnabled {
-		file_mq_agent_proto_msgTypes[0].Exporter = func(v any, i int) any {
-			switch v := v.(*StartPublishSessionRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_agent_proto_msgTypes[1].Exporter = func(v any, i int) any {
-			switch v := v.(*StartPublishSessionResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_agent_proto_msgTypes[2].Exporter = func(v any, i int) any {
-			switch v := v.(*ClosePublishSessionRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_agent_proto_msgTypes[3].Exporter = func(v any, i int) any {
-			switch v := v.(*ClosePublishSessionResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_agent_proto_msgTypes[4].Exporter = func(v any, i int) any {
-			switch v := v.(*PublishRecordRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_agent_proto_msgTypes[5].Exporter = func(v any, i int) any {
-			switch v := v.(*PublishRecordResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_agent_proto_msgTypes[6].Exporter = func(v any, i int) any {
-			switch v := v.(*SubscribeRecordRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_agent_proto_msgTypes[7].Exporter = func(v any, i int) any {
-			switch v := v.(*SubscribeRecordResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_agent_proto_msgTypes[8].Exporter = func(v any, i int) any {
-			switch v := v.(*SubscribeRecordRequest_InitSubscribeRecordRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_mq_agent_proto_rawDesc,
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_mq_agent_proto_rawDesc), len(file_mq_agent_proto_rawDesc)),
 			NumEnums:      0,
 			NumMessages:   9,
 			NumExtensions: 0,
@@ -940,7 +729,6 @@ func file_mq_agent_proto_init() {
 		MessageInfos:      file_mq_agent_proto_msgTypes,
 	}.Build()
 	File_mq_agent_proto = out.File
-	file_mq_agent_proto_rawDesc = nil
 	file_mq_agent_proto_goTypes = nil
 	file_mq_agent_proto_depIdxs = nil
 }

weed/pb/mq_agent_pb/mq_agent_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.5.1
-// - protoc             v5.28.3
+// - protoc             v5.29.3
 // source: mq_agent.proto
 
 package mq_agent_pb

weed/pb/mq_pb/mq_broker_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.5.1
-// - protoc             v5.28.3
+// - protoc             v5.29.3
 // source: mq_broker.proto
 
 package mq_pb

weed/pb/remote_pb/remote.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.34.2
-// 	protoc        v5.28.3
+// 	protoc-gen-go v1.36.6
+// 	protoc        v5.29.3
 // source: remote.proto
 
 package remote_pb
@@ -11,6 +11,7 @@ import (
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	reflect "reflect"
 	sync "sync"
+	unsafe "unsafe"
 )
 
 const (
@@ -24,62 +25,59 @@ const (
 // Remote Storage related
 // ///////////////////////
 type RemoteConf struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Type                            string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"`
-	Name                            string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
-	S3AccessKey                     string `protobuf:"bytes,4,opt,name=s3_access_key,json=s3AccessKey,proto3" json:"s3_access_key,omitempty"`
-	S3SecretKey                     string `protobuf:"bytes,5,opt,name=s3_secret_key,json=s3SecretKey,proto3" json:"s3_secret_key,omitempty"`
-	S3Region                        string `protobuf:"bytes,6,opt,name=s3_region,json=s3Region,proto3" json:"s3_region,omitempty"`
-	S3Endpoint                      string `protobuf:"bytes,7,opt,name=s3_endpoint,json=s3Endpoint,proto3" json:"s3_endpoint,omitempty"`
-	S3StorageClass                  string `protobuf:"bytes,8,opt,name=s3_storage_class,json=s3StorageClass,proto3" json:"s3_storage_class,omitempty"`
-	S3ForcePathStyle                bool   `protobuf:"varint,9,opt,name=s3_force_path_style,json=s3ForcePathStyle,proto3" json:"s3_force_path_style,omitempty"`
-	S3SupportTagging                bool   `protobuf:"varint,13,opt,name=s3_support_tagging,json=s3SupportTagging,proto3" json:"s3_support_tagging,omitempty"`
-	S3V4Signature                   bool   `protobuf:"varint,11,opt,name=s3_v4_signature,json=s3V4Signature,proto3" json:"s3_v4_signature,omitempty"`
-	GcsGoogleApplicationCredentials string `protobuf:"bytes,10,opt,name=gcs_google_application_credentials,json=gcsGoogleApplicationCredentials,proto3" json:"gcs_google_application_credentials,omitempty"`
-	GcsProjectId                    string `protobuf:"bytes,12,opt,name=gcs_project_id,json=gcsProjectId,proto3" json:"gcs_project_id,omitempty"`
-	AzureAccountName                string `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"`
-	AzureAccountKey                 string `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"`
-	BackblazeKeyId                  string `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"`
-	BackblazeApplicationKey         string `protobuf:"bytes,21,opt,name=backblaze_application_key,json=backblazeApplicationKey,proto3" json:"backblaze_application_key,omitempty"`
-	BackblazeEndpoint               string `protobuf:"bytes,22,opt,name=backblaze_endpoint,json=backblazeEndpoint,proto3" json:"backblaze_endpoint,omitempty"`
-	BackblazeRegion                 string `protobuf:"bytes,23,opt,name=backblaze_region,json=backblazeRegion,proto3" json:"backblaze_region,omitempty"`
-	AliyunAccessKey                 string `protobuf:"bytes,25,opt,name=aliyun_access_key,json=aliyunAccessKey,proto3" json:"aliyun_access_key,omitempty"`
-	AliyunSecretKey                 string `protobuf:"bytes,26,opt,name=aliyun_secret_key,json=aliyunSecretKey,proto3" json:"aliyun_secret_key,omitempty"`
-	AliyunEndpoint                  string `protobuf:"bytes,27,opt,name=aliyun_endpoint,json=aliyunEndpoint,proto3" json:"aliyun_endpoint,omitempty"`
-	AliyunRegion                    string `protobuf:"bytes,28,opt,name=aliyun_region,json=aliyunRegion,proto3" json:"aliyun_region,omitempty"`
-	TencentSecretId                 string `protobuf:"bytes,30,opt,name=tencent_secret_id,json=tencentSecretId,proto3" json:"tencent_secret_id,omitempty"`
-	TencentSecretKey                string `protobuf:"bytes,31,opt,name=tencent_secret_key,json=tencentSecretKey,proto3" json:"tencent_secret_key,omitempty"`
-	TencentEndpoint                 string `protobuf:"bytes,32,opt,name=tencent_endpoint,json=tencentEndpoint,proto3" json:"tencent_endpoint,omitempty"`
-	BaiduAccessKey                  string `protobuf:"bytes,35,opt,name=baidu_access_key,json=baiduAccessKey,proto3" json:"baidu_access_key,omitempty"`
-	BaiduSecretKey                  string `protobuf:"bytes,36,opt,name=baidu_secret_key,json=baiduSecretKey,proto3" json:"baidu_secret_key,omitempty"`
-	BaiduEndpoint                   string `protobuf:"bytes,37,opt,name=baidu_endpoint,json=baiduEndpoint,proto3" json:"baidu_endpoint,omitempty"`
-	BaiduRegion                     string `protobuf:"bytes,38,opt,name=baidu_region,json=baiduRegion,proto3" json:"baidu_region,omitempty"`
-	WasabiAccessKey                 string `protobuf:"bytes,40,opt,name=wasabi_access_key,json=wasabiAccessKey,proto3" json:"wasabi_access_key,omitempty"`
-	WasabiSecretKey                 string `protobuf:"bytes,41,opt,name=wasabi_secret_key,json=wasabiSecretKey,proto3" json:"wasabi_secret_key,omitempty"`
-	WasabiEndpoint                  string `protobuf:"bytes,42,opt,name=wasabi_endpoint,json=wasabiEndpoint,proto3" json:"wasabi_endpoint,omitempty"`
-	WasabiRegion                    string `protobuf:"bytes,43,opt,name=wasabi_region,json=wasabiRegion,proto3" json:"wasabi_region,omitempty"`
-	FilebaseAccessKey               string `protobuf:"bytes,60,opt,name=filebase_access_key,json=filebaseAccessKey,proto3" json:"filebase_access_key,omitempty"`
-	FilebaseSecretKey               string `protobuf:"bytes,61,opt,name=filebase_secret_key,json=filebaseSecretKey,proto3" json:"filebase_secret_key,omitempty"`
-	FilebaseEndpoint                string `protobuf:"bytes,62,opt,name=filebase_endpoint,json=filebaseEndpoint,proto3" json:"filebase_endpoint,omitempty"`
-	StorjAccessKey                  string `protobuf:"bytes,65,opt,name=storj_access_key,json=storjAccessKey,proto3" json:"storj_access_key,omitempty"`
-	StorjSecretKey                  string `protobuf:"bytes,66,opt,name=storj_secret_key,json=storjSecretKey,proto3" json:"storj_secret_key,omitempty"`
-	StorjEndpoint                   string `protobuf:"bytes,67,opt,name=storj_endpoint,json=storjEndpoint,proto3" json:"storj_endpoint,omitempty"`
-	ContaboAccessKey                string `protobuf:"bytes,68,opt,name=contabo_access_key,json=contaboAccessKey,proto3" json:"contabo_access_key,omitempty"`
-	ContaboSecretKey                string `protobuf:"bytes,69,opt,name=contabo_secret_key,json=contaboSecretKey,proto3" json:"contabo_secret_key,omitempty"`
-	ContaboEndpoint                 string `protobuf:"bytes,70,opt,name=contabo_endpoint,json=contaboEndpoint,proto3" json:"contabo_endpoint,omitempty"`
-	ContaboRegion                   string `protobuf:"bytes,71,opt,name=contabo_region,json=contaboRegion,proto3" json:"contabo_region,omitempty"`
+	state                           protoimpl.MessageState `protogen:"open.v1"`
+	Type                            string                 `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"`
+	Name                            string                 `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
+	S3AccessKey                     string                 `protobuf:"bytes,4,opt,name=s3_access_key,json=s3AccessKey,proto3" json:"s3_access_key,omitempty"`
+	S3SecretKey                     string                 `protobuf:"bytes,5,opt,name=s3_secret_key,json=s3SecretKey,proto3" json:"s3_secret_key,omitempty"`
+	S3Region                        string                 `protobuf:"bytes,6,opt,name=s3_region,json=s3Region,proto3" json:"s3_region,omitempty"`
+	S3Endpoint                      string                 `protobuf:"bytes,7,opt,name=s3_endpoint,json=s3Endpoint,proto3" json:"s3_endpoint,omitempty"`
+	S3StorageClass                  string                 `protobuf:"bytes,8,opt,name=s3_storage_class,json=s3StorageClass,proto3" json:"s3_storage_class,omitempty"`
+	S3ForcePathStyle                bool                   `protobuf:"varint,9,opt,name=s3_force_path_style,json=s3ForcePathStyle,proto3" json:"s3_force_path_style,omitempty"`
+	S3SupportTagging                bool                   `protobuf:"varint,13,opt,name=s3_support_tagging,json=s3SupportTagging,proto3" json:"s3_support_tagging,omitempty"`
+	S3V4Signature                   bool                   `protobuf:"varint,11,opt,name=s3_v4_signature,json=s3V4Signature,proto3" json:"s3_v4_signature,omitempty"`
+	GcsGoogleApplicationCredentials string                 `protobuf:"bytes,10,opt,name=gcs_google_application_credentials,json=gcsGoogleApplicationCredentials,proto3" json:"gcs_google_application_credentials,omitempty"`
+	GcsProjectId                    string                 `protobuf:"bytes,12,opt,name=gcs_project_id,json=gcsProjectId,proto3" json:"gcs_project_id,omitempty"`
+	AzureAccountName                string                 `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"`
+	AzureAccountKey                 string                 `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"`
+	BackblazeKeyId                  string                 `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"`
+	BackblazeApplicationKey         string                 `protobuf:"bytes,21,opt,name=backblaze_application_key,json=backblazeApplicationKey,proto3" json:"backblaze_application_key,omitempty"`
+	BackblazeEndpoint               string                 `protobuf:"bytes,22,opt,name=backblaze_endpoint,json=backblazeEndpoint,proto3" json:"backblaze_endpoint,omitempty"`
+	BackblazeRegion                 string                 `protobuf:"bytes,23,opt,name=backblaze_region,json=backblazeRegion,proto3" json:"backblaze_region,omitempty"`
+	AliyunAccessKey                 string                 `protobuf:"bytes,25,opt,name=aliyun_access_key,json=aliyunAccessKey,proto3" json:"aliyun_access_key,omitempty"`
+	AliyunSecretKey                 string                 `protobuf:"bytes,26,opt,name=aliyun_secret_key,json=aliyunSecretKey,proto3" json:"aliyun_secret_key,omitempty"`
+	AliyunEndpoint                  string                 `protobuf:"bytes,27,opt,name=aliyun_endpoint,json=aliyunEndpoint,proto3" json:"aliyun_endpoint,omitempty"`
+	AliyunRegion                    string                 `protobuf:"bytes,28,opt,name=aliyun_region,json=aliyunRegion,proto3" json:"aliyun_region,omitempty"`
+	TencentSecretId                 string                 `protobuf:"bytes,30,opt,name=tencent_secret_id,json=tencentSecretId,proto3" json:"tencent_secret_id,omitempty"`
+	TencentSecretKey                string                 `protobuf:"bytes,31,opt,name=tencent_secret_key,json=tencentSecretKey,proto3" json:"tencent_secret_key,omitempty"`
+	TencentEndpoint                 string                 `protobuf:"bytes,32,opt,name=tencent_endpoint,json=tencentEndpoint,proto3" json:"tencent_endpoint,omitempty"`
+	BaiduAccessKey                  string                 `protobuf:"bytes,35,opt,name=baidu_access_key,json=baiduAccessKey,proto3" json:"baidu_access_key,omitempty"`
+	BaiduSecretKey                  string                 `protobuf:"bytes,36,opt,name=baidu_secret_key,json=baiduSecretKey,proto3" json:"baidu_secret_key,omitempty"`
+	BaiduEndpoint                   string                 `protobuf:"bytes,37,opt,name=baidu_endpoint,json=baiduEndpoint,proto3" json:"baidu_endpoint,omitempty"`
+	BaiduRegion                     string                 `protobuf:"bytes,38,opt,name=baidu_region,json=baiduRegion,proto3" json:"baidu_region,omitempty"`
+	WasabiAccessKey                 string                 `protobuf:"bytes,40,opt,name=wasabi_access_key,json=wasabiAccessKey,proto3" json:"wasabi_access_key,omitempty"`
+	WasabiSecretKey                 string                 `protobuf:"bytes,41,opt,name=wasabi_secret_key,json=wasabiSecretKey,proto3" json:"wasabi_secret_key,omitempty"`
+	WasabiEndpoint                  string                 `protobuf:"bytes,42,opt,name=wasabi_endpoint,json=wasabiEndpoint,proto3" json:"wasabi_endpoint,omitempty"`
+	WasabiRegion                    string                 `protobuf:"bytes,43,opt,name=wasabi_region,json=wasabiRegion,proto3" json:"wasabi_region,omitempty"`
+	FilebaseAccessKey               string                 `protobuf:"bytes,60,opt,name=filebase_access_key,json=filebaseAccessKey,proto3" json:"filebase_access_key,omitempty"`
+	FilebaseSecretKey               string                 `protobuf:"bytes,61,opt,name=filebase_secret_key,json=filebaseSecretKey,proto3" json:"filebase_secret_key,omitempty"`
+	FilebaseEndpoint                string                 `protobuf:"bytes,62,opt,name=filebase_endpoint,json=filebaseEndpoint,proto3" json:"filebase_endpoint,omitempty"`
+	StorjAccessKey                  string                 `protobuf:"bytes,65,opt,name=storj_access_key,json=storjAccessKey,proto3" json:"storj_access_key,omitempty"`
+	StorjSecretKey                  string                 `protobuf:"bytes,66,opt,name=storj_secret_key,json=storjSecretKey,proto3" json:"storj_secret_key,omitempty"`
+	StorjEndpoint                   string                 `protobuf:"bytes,67,opt,name=storj_endpoint,json=storjEndpoint,proto3" json:"storj_endpoint,omitempty"`
+	ContaboAccessKey                string                 `protobuf:"bytes,68,opt,name=contabo_access_key,json=contaboAccessKey,proto3" json:"contabo_access_key,omitempty"`
+	ContaboSecretKey                string                 `protobuf:"bytes,69,opt,name=contabo_secret_key,json=contaboSecretKey,proto3" json:"contabo_secret_key,omitempty"`
+	ContaboEndpoint                 string                 `protobuf:"bytes,70,opt,name=contabo_endpoint,json=contaboEndpoint,proto3" json:"contabo_endpoint,omitempty"`
+	ContaboRegion                   string                 `protobuf:"bytes,71,opt,name=contabo_region,json=contaboRegion,proto3" json:"contabo_region,omitempty"`
+	unknownFields                   protoimpl.UnknownFields
+	sizeCache                       protoimpl.SizeCache
 }
 
 func (x *RemoteConf) Reset() {
 	*x = RemoteConf{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_remote_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_remote_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *RemoteConf) String() string {
@@ -90,7 +88,7 @@ func (*RemoteConf) ProtoMessage() {}
 
 func (x *RemoteConf) ProtoReflect() protoreflect.Message {
 	mi := &file_remote_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -407,21 +405,18 @@ func (x *RemoteConf) GetContaboRegion() string {
 }
 
 type RemoteStorageMapping struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Mappings                 map[string]*RemoteStorageLocation `protobuf:"bytes,1,rep,name=mappings,proto3" json:"mappings,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	state                    protoimpl.MessageState            `protogen:"open.v1"`
+	Mappings                 map[string]*RemoteStorageLocation `protobuf:"bytes,1,rep,name=mappings,proto3" json:"mappings,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
 	PrimaryBucketStorageName string                            `protobuf:"bytes,2,opt,name=primary_bucket_storage_name,json=primaryBucketStorageName,proto3" json:"primary_bucket_storage_name,omitempty"`
+	unknownFields            protoimpl.UnknownFields
+	sizeCache                protoimpl.SizeCache
 }
 
 func (x *RemoteStorageMapping) Reset() {
 	*x = RemoteStorageMapping{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_remote_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_remote_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *RemoteStorageMapping) String() string {
@@ -432,7 +427,7 @@ func (*RemoteStorageMapping) ProtoMessage() {}
 
 func (x *RemoteStorageMapping) ProtoReflect() protoreflect.Message {
 	mi := &file_remote_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -462,22 +457,19 @@ func (x *RemoteStorageMapping) GetPrimaryBucketStorageName() string {
 }
 
 type RemoteStorageLocation struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Name          string                 `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Bucket        string                 `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"`
+	Path          string                 `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	Name   string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"`
-	Path   string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *RemoteStorageLocation) Reset() {
 	*x = RemoteStorageLocation{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_remote_proto_msgTypes[2]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_remote_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *RemoteStorageLocation) String() string {
@@ -488,7 +480,7 @@ func (*RemoteStorageLocation) ProtoMessage() {}
 
 func (x *RemoteStorageLocation) ProtoReflect() protoreflect.Message {
 	mi := &file_remote_proto_msgTypes[2]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -526,160 +518,77 @@ func (x *RemoteStorageLocation) GetPath() string {
 
 var File_remote_proto protoreflect.FileDescriptor
 
-var file_remote_proto_rawDesc = []byte{
-	0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09,
-	0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0x9b, 0x0e, 0x0a, 0x0a, 0x52, 0x65,
-	0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04,
-	0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
-	0x12, 0x22, 0x0a, 0x0d, 0x73, 0x33, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65,
-	0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x33, 0x41, 0x63, 0x63, 0x65, 0x73,
-	0x73, 0x4b, 0x65, 0x79, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x33, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65,
-	0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x33, 0x53,
-	0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x33, 0x5f, 0x72,
-	0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x33, 0x52,
-	0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x33, 0x5f, 0x65, 0x6e, 0x64, 0x70,
-	0x6f, 0x69, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x33, 0x45, 0x6e,
-	0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x33, 0x5f, 0x73, 0x74, 0x6f,
-	0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x0e, 0x73, 0x33, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73,
-	0x12, 0x2d, 0x0a, 0x13, 0x73, 0x33, 0x5f, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x70, 0x61, 0x74,
-	0x68, 0x5f, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x73,
-	0x33, 0x46, 0x6f, 0x72, 0x63, 0x65, 0x50, 0x61, 0x74, 0x68, 0x53, 0x74, 0x79, 0x6c, 0x65, 0x12,
-	0x2c, 0x0a, 0x12, 0x73, 0x33, 0x5f, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x74, 0x61,
-	0x67, 0x67, 0x69, 0x6e, 0x67, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x73, 0x33, 0x53,
-	0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x12, 0x26, 0x0a,
-	0x0f, 0x73, 0x33, 0x5f, 0x76, 0x34, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65,
-	0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x73, 0x33, 0x56, 0x34, 0x53, 0x69, 0x67, 0x6e,
-	0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x4b, 0x0a, 0x22, 0x67, 0x63, 0x73, 0x5f, 0x67, 0x6f, 0x6f,
-	0x67, 0x6c, 0x65, 0x5f, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f,
-	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x1f, 0x67, 0x63, 0x73, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c,
-	0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
-	0x6c, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x67, 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63,
-	0x74, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x67, 0x63, 0x73, 0x50,
-	0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x7a, 0x75, 0x72,
-	0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0f,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75,
-	0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x5f,
-	0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x10, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x0f, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x4b,
-	0x65, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f,
-	0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x14, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x62, 0x61,
-	0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x49, 0x64, 0x12, 0x3a, 0x0a, 0x19,
-	0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63,
-	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x15, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x17, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63,
-	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x12, 0x62, 0x61, 0x63, 0x6b,
-	0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x16,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x45,
-	0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x62, 0x61, 0x63, 0x6b, 0x62,
-	0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x17, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x0f, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x52, 0x65, 0x67, 0x69,
-	0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x61, 0x63, 0x63,
-	0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x19, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61,
-	0x6c, 0x69, 0x79, 0x75, 0x6e, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2a,
-	0x0a, 0x11, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f,
-	0x6b, 0x65, 0x79, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75,
-	0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x61, 0x6c,
-	0x69, 0x79, 0x75, 0x6e, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x1b, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f,
-	0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x72, 0x65,
-	0x67, 0x69, 0x6f, 0x6e, 0x18, 0x1c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x6c, 0x69, 0x79,
-	0x75, 0x6e, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x65, 0x6e, 0x63,
-	0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x1e, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72,
-	0x65, 0x74, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x12, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f,
-	0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x10, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b,
-	0x65, 0x79, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x6e,
-	0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x20, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65,
-	0x6e, 0x63, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a,
-	0x10, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65,
-	0x79, 0x18, 0x23, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x62, 0x61, 0x69, 0x64, 0x75, 0x41, 0x63,
-	0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x69, 0x64, 0x75,
-	0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x24, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x0e, 0x62, 0x61, 0x69, 0x64, 0x75, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65,
-	0x79, 0x12, 0x25, 0x0a, 0x0e, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f,
-	0x69, 0x6e, 0x74, 0x18, 0x25, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x62, 0x61, 0x69, 0x64, 0x75,
-	0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x61, 0x69, 0x64,
-	0x75, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x26, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b,
-	0x62, 0x61, 0x69, 0x64, 0x75, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x77,
-	0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79,
-	0x18, 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x41, 0x63,
-	0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x11, 0x77, 0x61, 0x73, 0x61, 0x62,
-	0x69, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x29, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x0f, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74,
-	0x4b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x65, 0x6e,
-	0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x2a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x77, 0x61,
-	0x73, 0x61, 0x62, 0x69, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d,
-	0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x2b, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x52, 0x65, 0x67, 0x69, 0x6f,
-	0x6e, 0x12, 0x2e, 0x0a, 0x13, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x61, 0x63,
-	0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x3c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11,
-	0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, 0x73, 0x65, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65,
-	0x79, 0x12, 0x2e, 0x0a, 0x13, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x73, 0x65,
-	0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x3d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11,
-	0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, 0x73, 0x65, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65,
-	0x79, 0x12, 0x2b, 0x0a, 0x11, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x65, 0x6e,
-	0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x3e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x66, 0x69,
-	0x6c, 0x65, 0x62, 0x61, 0x73, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x28,
-	0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b,
-	0x65, 0x79, 0x18, 0x41, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x41,
-	0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72,
-	0x6a, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x42, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b,
-	0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x5f, 0x65, 0x6e, 0x64, 0x70,
-	0x6f, 0x69, 0x6e, 0x74, 0x18, 0x43, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x74, 0x6f, 0x72,
-	0x6a, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x63, 0x6f, 0x6e,
-	0x74, 0x61, 0x62, 0x6f, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18,
-	0x44, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x41, 0x63,
-	0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x61,
-	0x62, 0x6f, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x45, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x53, 0x65, 0x63, 0x72,
-	0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f,
-	0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x46, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
-	0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x5f, 0x72, 0x65, 0x67, 0x69,
-	0x6f, 0x6e, 0x18, 0x47, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62,
-	0x6f, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x22, 0xff, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x6d, 0x6f,
-	0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67,
-	0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03,
-	0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52,
-	0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x70,
-	0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72,
-	0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x3d, 0x0a, 0x1b, 0x70,
-	0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x73, 0x74,
-	0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x18, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53,
-	0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0x5d, 0x0a, 0x0d, 0x4d, 0x61,
-	0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
-	0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x36, 0x0a,
-	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72,
-	0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53,
-	0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05,
-	0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, 0x15, 0x52, 0x65, 0x6d,
-	0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69,
-	0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12,
-	0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61,
-	0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e,
-	0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x50, 0x72, 0x6f,
-	0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73,
-	0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64,
-	0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74,
-	0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-}
+const file_remote_proto_rawDesc = "" +
+	"\n" +
+	"\fremote.proto\x12\tremote_pb\"\x9b\x0e\n" +
+	"\n" +
+	"RemoteConf\x12\x12\n" +
+	"\x04type\x18\x01 \x01(\tR\x04type\x12\x12\n" +
+	"\x04name\x18\x02 \x01(\tR\x04name\x12\"\n" +
+	"\rs3_access_key\x18\x04 \x01(\tR\vs3AccessKey\x12\"\n" +
+	"\rs3_secret_key\x18\x05 \x01(\tR\vs3SecretKey\x12\x1b\n" +
+	"\ts3_region\x18\x06 \x01(\tR\bs3Region\x12\x1f\n" +
+	"\vs3_endpoint\x18\a \x01(\tR\n" +
+	"s3Endpoint\x12(\n" +
+	"\x10s3_storage_class\x18\b \x01(\tR\x0es3StorageClass\x12-\n" +
+	"\x13s3_force_path_style\x18\t \x01(\bR\x10s3ForcePathStyle\x12,\n" +
+	"\x12s3_support_tagging\x18\r \x01(\bR\x10s3SupportTagging\x12&\n" +
+	"\x0fs3_v4_signature\x18\v \x01(\bR\rs3V4Signature\x12K\n" +
+	"\"gcs_google_application_credentials\x18\n" +
+	" \x01(\tR\x1fgcsGoogleApplicationCredentials\x12$\n" +
+	"\x0egcs_project_id\x18\f \x01(\tR\fgcsProjectId\x12,\n" +
+	"\x12azure_account_name\x18\x0f \x01(\tR\x10azureAccountName\x12*\n" +
+	"\x11azure_account_key\x18\x10 \x01(\tR\x0fazureAccountKey\x12(\n" +
+	"\x10backblaze_key_id\x18\x14 \x01(\tR\x0ebackblazeKeyId\x12:\n" +
+	"\x19backblaze_application_key\x18\x15 \x01(\tR\x17backblazeApplicationKey\x12-\n" +
+	"\x12backblaze_endpoint\x18\x16 \x01(\tR\x11backblazeEndpoint\x12)\n" +
+	"\x10backblaze_region\x18\x17 \x01(\tR\x0fbackblazeRegion\x12*\n" +
+	"\x11aliyun_access_key\x18\x19 \x01(\tR\x0faliyunAccessKey\x12*\n" +
+	"\x11aliyun_secret_key\x18\x1a \x01(\tR\x0faliyunSecretKey\x12'\n" +
+	"\x0faliyun_endpoint\x18\x1b \x01(\tR\x0ealiyunEndpoint\x12#\n" +
+	"\raliyun_region\x18\x1c \x01(\tR\faliyunRegion\x12*\n" +
+	"\x11tencent_secret_id\x18\x1e \x01(\tR\x0ftencentSecretId\x12,\n" +
+	"\x12tencent_secret_key\x18\x1f \x01(\tR\x10tencentSecretKey\x12)\n" +
+	"\x10tencent_endpoint\x18  \x01(\tR\x0ftencentEndpoint\x12(\n" +
+	"\x10baidu_access_key\x18# \x01(\tR\x0ebaiduAccessKey\x12(\n" +
+	"\x10baidu_secret_key\x18$ \x01(\tR\x0ebaiduSecretKey\x12%\n" +
+	"\x0ebaidu_endpoint\x18% \x01(\tR\rbaiduEndpoint\x12!\n" +
+	"\fbaidu_region\x18& \x01(\tR\vbaiduRegion\x12*\n" +
+	"\x11wasabi_access_key\x18( \x01(\tR\x0fwasabiAccessKey\x12*\n" +
+	"\x11wasabi_secret_key\x18) \x01(\tR\x0fwasabiSecretKey\x12'\n" +
+	"\x0fwasabi_endpoint\x18* \x01(\tR\x0ewasabiEndpoint\x12#\n" +
+	"\rwasabi_region\x18+ \x01(\tR\fwasabiRegion\x12.\n" +
+	"\x13filebase_access_key\x18< \x01(\tR\x11filebaseAccessKey\x12.\n" +
+	"\x13filebase_secret_key\x18= \x01(\tR\x11filebaseSecretKey\x12+\n" +
+	"\x11filebase_endpoint\x18> \x01(\tR\x10filebaseEndpoint\x12(\n" +
+	"\x10storj_access_key\x18A \x01(\tR\x0estorjAccessKey\x12(\n" +
+	"\x10storj_secret_key\x18B \x01(\tR\x0estorjSecretKey\x12%\n" +
+	"\x0estorj_endpoint\x18C \x01(\tR\rstorjEndpoint\x12,\n" +
+	"\x12contabo_access_key\x18D \x01(\tR\x10contaboAccessKey\x12,\n" +
+	"\x12contabo_secret_key\x18E \x01(\tR\x10contaboSecretKey\x12)\n" +
+	"\x10contabo_endpoint\x18F \x01(\tR\x0fcontaboEndpoint\x12%\n" +
+	"\x0econtabo_region\x18G \x01(\tR\rcontaboRegion\"\xff\x01\n" +
+	"\x14RemoteStorageMapping\x12I\n" +
+	"\bmappings\x18\x01 \x03(\v2-.remote_pb.RemoteStorageMapping.MappingsEntryR\bmappings\x12=\n" +
+	"\x1bprimary_bucket_storage_name\x18\x02 \x01(\tR\x18primaryBucketStorageName\x1a]\n" +
+	"\rMappingsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x126\n" +
+	"\x05value\x18\x02 \x01(\v2 .remote_pb.RemoteStorageLocationR\x05value:\x028\x01\"W\n" +
+	"\x15RemoteStorageLocation\x12\x12\n" +
+	"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
+	"\x06bucket\x18\x02 \x01(\tR\x06bucket\x12\x12\n" +
+	"\x04path\x18\x03 \x01(\tR\x04pathBP\n" +
+	"\x10seaweedfs.clientB\n" +
+	"FilerProtoZ0github.com/seaweedfs/seaweedfs/weed/pb/remote_pbb\x06proto3"
 
 var (
 	file_remote_proto_rawDescOnce sync.Once
-	file_remote_proto_rawDescData = file_remote_proto_rawDesc
+	file_remote_proto_rawDescData []byte
 )
 
 func file_remote_proto_rawDescGZIP() []byte {
 	file_remote_proto_rawDescOnce.Do(func() {
-		file_remote_proto_rawDescData = protoimpl.X.CompressGZIP(file_remote_proto_rawDescData)
+		file_remote_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_remote_proto_rawDesc), len(file_remote_proto_rawDesc)))
 	})
 	return file_remote_proto_rawDescData
 }
@@ -706,49 +615,11 @@ func file_remote_proto_init() {
 	if File_remote_proto != nil {
 		return
 	}
-	if !protoimpl.UnsafeEnabled {
-		file_remote_proto_msgTypes[0].Exporter = func(v any, i int) any {
-			switch v := v.(*RemoteConf); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_remote_proto_msgTypes[1].Exporter = func(v any, i int) any {
-			switch v := v.(*RemoteStorageMapping); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_remote_proto_msgTypes[2].Exporter = func(v any, i int) any {
-			switch v := v.(*RemoteStorageLocation); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_remote_proto_rawDesc,
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_remote_proto_rawDesc), len(file_remote_proto_rawDesc)),
 			NumEnums:      0,
 			NumMessages:   4,
 			NumExtensions: 0,
@@ -759,7 +630,6 @@ func file_remote_proto_init() {
 		MessageInfos:      file_remote_proto_msgTypes,
 	}.Build()
 	File_remote_proto = out.File
-	file_remote_proto_rawDesc = nil
 	file_remote_proto_goTypes = nil
 	file_remote_proto_depIdxs = nil
 }

weed/pb/s3_pb/s3.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.34.2
-// 	protoc        v5.28.3
+// 	protoc-gen-go v1.36.6
+// 	protoc        v5.29.3
 // source: s3.proto
 
 package s3_pb
@@ -11,6 +11,7 @@ import (
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	reflect "reflect"
 	sync "sync"
+	unsafe "unsafe"
 )
 
 const (
@@ -21,20 +22,17 @@ const (
 )
 
 type S3ConfigureRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	S3ConfigurationFileContent []byte `protobuf:"bytes,1,opt,name=s3_configuration_file_content,json=s3ConfigurationFileContent,proto3" json:"s3_configuration_file_content,omitempty"`
+	state                      protoimpl.MessageState `protogen:"open.v1"`
+	S3ConfigurationFileContent []byte                 `protobuf:"bytes,1,opt,name=s3_configuration_file_content,json=s3ConfigurationFileContent,proto3" json:"s3_configuration_file_content,omitempty"`
+	unknownFields              protoimpl.UnknownFields
+	sizeCache                  protoimpl.SizeCache
 }
 
 func (x *S3ConfigureRequest) Reset() {
 	*x = S3ConfigureRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_s3_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_s3_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *S3ConfigureRequest) String() string {
@@ -45,7 +43,7 @@ func (*S3ConfigureRequest) ProtoMessage() {}
 
 func (x *S3ConfigureRequest) ProtoReflect() protoreflect.Message {
 	mi := &file_s3_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -68,18 +66,16 @@ func (x *S3ConfigureRequest) GetS3ConfigurationFileContent() []byte {
 }
 
 type S3ConfigureResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
 	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *S3ConfigureResponse) Reset() {
 	*x = S3ConfigureResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_s3_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_s3_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *S3ConfigureResponse) String() string {
@@ -90,7 +86,7 @@ func (*S3ConfigureResponse) ProtoMessage() {}
 
 func (x *S3ConfigureResponse) ProtoReflect() protoreflect.Message {
 	mi := &file_s3_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -106,21 +102,18 @@ func (*S3ConfigureResponse) Descriptor() ([]byte, []int) {
 }
 
 type S3CircuitBreakerConfig struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState              `protogen:"open.v1"`
+	Global        *S3CircuitBreakerOptions            `protobuf:"bytes,1,opt,name=global,proto3" json:"global,omitempty"`
+	Buckets       map[string]*S3CircuitBreakerOptions `protobuf:"bytes,2,rep,name=buckets,proto3" json:"buckets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
 	unknownFields protoimpl.UnknownFields
-
-	Global  *S3CircuitBreakerOptions            `protobuf:"bytes,1,opt,name=global,proto3" json:"global,omitempty"`
-	Buckets map[string]*S3CircuitBreakerOptions `protobuf:"bytes,2,rep,name=buckets,proto3" json:"buckets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *S3CircuitBreakerConfig) Reset() {
 	*x = S3CircuitBreakerConfig{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_s3_proto_msgTypes[2]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_s3_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *S3CircuitBreakerConfig) String() string {
@@ -131,7 +124,7 @@ func (*S3CircuitBreakerConfig) ProtoMessage() {}
 
 func (x *S3CircuitBreakerConfig) ProtoReflect() protoreflect.Message {
 	mi := &file_s3_proto_msgTypes[2]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -161,21 +154,18 @@ func (x *S3CircuitBreakerConfig) GetBuckets() map[string]*S3CircuitBreakerOption
 }
 
 type S3CircuitBreakerOptions struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Enabled       bool                   `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"`
+	Actions       map[string]int64       `protobuf:"bytes,2,rep,name=actions,proto3" json:"actions,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
 	unknownFields protoimpl.UnknownFields
-
-	Enabled bool             `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"`
-	Actions map[string]int64 `protobuf:"bytes,2,rep,name=actions,proto3" json:"actions,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *S3CircuitBreakerOptions) Reset() {
 	*x = S3CircuitBreakerOptions{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_s3_proto_msgTypes[3]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_s3_proto_msgTypes[3]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *S3CircuitBreakerOptions) String() string {
@@ -186,7 +176,7 @@ func (*S3CircuitBreakerOptions) ProtoMessage() {}
 
 func (x *S3CircuitBreakerOptions) ProtoReflect() protoreflect.Message {
 	mi := &file_s3_proto_msgTypes[3]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -217,66 +207,36 @@ func (x *S3CircuitBreakerOptions) GetActions() map[string]int64 {
 
 var File_s3_proto protoreflect.FileDescriptor
 
-var file_s3_proto_rawDesc = []byte{
-	0x0a, 0x08, 0x73, 0x33, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d, 0x65, 0x73, 0x73,
-	0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x22, 0x57, 0x0a, 0x12, 0x53, 0x33, 0x43, 0x6f,
-	0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41,
-	0x0a, 0x1d, 0x73, 0x33, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69,
-	0x6f, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x1a, 0x73, 0x33, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
-	0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e,
-	0x74, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x33, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x87, 0x02, 0x0a, 0x16, 0x53, 0x33, 0x43,
-	0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x43, 0x6f, 0x6e,
-	0x66, 0x69, 0x67, 0x12, 0x3d, 0x0a, 0x06, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f,
-	0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61,
-	0x6b, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x06, 0x67, 0x6c, 0x6f, 0x62,
-	0x61, 0x6c, 0x12, 0x4b, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20,
-	0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f,
-	0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61,
-	0x6b, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74,
-	0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x1a,
-	0x61, 0x0a, 0x0c, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12,
-	0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65,
-	0x79, 0x12, 0x3b, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
-	0x32, 0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e,
-	0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72,
-	0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02,
-	0x38, 0x01, 0x22, 0xbd, 0x01, 0x0a, 0x17, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74,
-	0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18,
-	0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52,
-	0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69,
-	0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x6d, 0x65, 0x73, 0x73,
-	0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75,
-	0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73,
-	0x2e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x61,
-	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3a, 0x0a, 0x0c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e,
-	0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75,
-	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02,
-	0x38, 0x01, 0x32, 0x5f, 0x0a, 0x09, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x53, 0x33, 0x12,
-	0x52, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x20, 0x2e, 0x6d,
-	0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x6f,
-	0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21,
-	0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33,
-	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x22, 0x00, 0x42, 0x49, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73,
-	0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x07, 0x53, 0x33, 0x50, 0x72, 0x6f, 0x74, 0x6f,
-	0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61,
-	0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73,
-	0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x33, 0x5f, 0x70, 0x62, 0x62, 0x06,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-}
+const file_s3_proto_rawDesc = "" +
+	"\n" +
+	"\bs3.proto\x12\fmessaging_pb\"W\n" +
+	"\x12S3ConfigureRequest\x12A\n" +
+	"\x1ds3_configuration_file_content\x18\x01 \x01(\fR\x1as3ConfigurationFileContent\"\x15\n" +
+	"\x13S3ConfigureResponse\"\x87\x02\n" +
+	"\x16S3CircuitBreakerConfig\x12=\n" +
+	"\x06global\x18\x01 \x01(\v2%.messaging_pb.S3CircuitBreakerOptionsR\x06global\x12K\n" +
+	"\abuckets\x18\x02 \x03(\v21.messaging_pb.S3CircuitBreakerConfig.BucketsEntryR\abuckets\x1aa\n" +
+	"\fBucketsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12;\n" +
+	"\x05value\x18\x02 \x01(\v2%.messaging_pb.S3CircuitBreakerOptionsR\x05value:\x028\x01\"\xbd\x01\n" +
+	"\x17S3CircuitBreakerOptions\x12\x18\n" +
+	"\aenabled\x18\x01 \x01(\bR\aenabled\x12L\n" +
+	"\aactions\x18\x02 \x03(\v22.messaging_pb.S3CircuitBreakerOptions.ActionsEntryR\aactions\x1a:\n" +
+	"\fActionsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\x03R\x05value:\x028\x012_\n" +
+	"\tSeaweedS3\x12R\n" +
+	"\tConfigure\x12 .messaging_pb.S3ConfigureRequest\x1a!.messaging_pb.S3ConfigureResponse\"\x00BI\n" +
+	"\x10seaweedfs.clientB\aS3ProtoZ,github.com/seaweedfs/seaweedfs/weed/pb/s3_pbb\x06proto3"
 
 var (
 	file_s3_proto_rawDescOnce sync.Once
-	file_s3_proto_rawDescData = file_s3_proto_rawDesc
+	file_s3_proto_rawDescData []byte
 )
 
 func file_s3_proto_rawDescGZIP() []byte {
 	file_s3_proto_rawDescOnce.Do(func() {
-		file_s3_proto_rawDescData = protoimpl.X.CompressGZIP(file_s3_proto_rawDescData)
+		file_s3_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_s3_proto_rawDesc), len(file_s3_proto_rawDesc)))
 	})
 	return file_s3_proto_rawDescData
 }
@@ -309,61 +269,11 @@ func file_s3_proto_init() {
 	if File_s3_proto != nil {
 		return
 	}
-	if !protoimpl.UnsafeEnabled {
-		file_s3_proto_msgTypes[0].Exporter = func(v any, i int) any {
-			switch v := v.(*S3ConfigureRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_s3_proto_msgTypes[1].Exporter = func(v any, i int) any {
-			switch v := v.(*S3ConfigureResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_s3_proto_msgTypes[2].Exporter = func(v any, i int) any {
-			switch v := v.(*S3CircuitBreakerConfig); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_s3_proto_msgTypes[3].Exporter = func(v any, i int) any {
-			switch v := v.(*S3CircuitBreakerOptions); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_s3_proto_rawDesc,
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_s3_proto_rawDesc), len(file_s3_proto_rawDesc)),
 			NumEnums:      0,
 			NumMessages:   6,
 			NumExtensions: 0,
@@ -374,7 +284,6 @@ func file_s3_proto_init() {
 		MessageInfos:      file_s3_proto_msgTypes,
 	}.Build()
 	File_s3_proto = out.File
-	file_s3_proto_rawDesc = nil
 	file_s3_proto_goTypes = nil
 	file_s3_proto_depIdxs = nil
 }

weed/pb/s3_pb/s3_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.5.1
-// - protoc             v5.28.3
+// - protoc             v5.29.3
 // source: s3.proto
 
 package s3_pb

weed/pb/schema_pb/mq_schema.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.34.2
-// 	protoc        v5.28.3
+// 	protoc-gen-go v1.36.6
+// 	protoc        v5.29.3
 // source: mq_schema.proto
 
 package schema_pb
@@ -11,6 +11,7 @@ import (
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	reflect "reflect"
 	sync "sync"
+	unsafe "unsafe"
 )
 
 const (
@@ -137,21 +138,18 @@ func (ScalarType) EnumDescriptor() ([]byte, []int) {
 }
 
 type Topic struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Namespace     string                 `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"`
+	Name          string                 `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"`
-	Name      string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *Topic) Reset() {
 	*x = Topic{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_schema_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_schema_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *Topic) String() string {
@@ -162,7 +160,7 @@ func (*Topic) ProtoMessage() {}
 
 func (x *Topic) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_schema_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -192,23 +190,20 @@ func (x *Topic) GetName() string {
 }
 
 type Partition struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	RingSize      int32                  `protobuf:"varint,1,opt,name=ring_size,json=ringSize,proto3" json:"ring_size,omitempty"`
+	RangeStart    int32                  `protobuf:"varint,2,opt,name=range_start,json=rangeStart,proto3" json:"range_start,omitempty"`
+	RangeStop     int32                  `protobuf:"varint,3,opt,name=range_stop,json=rangeStop,proto3" json:"range_stop,omitempty"`
+	UnixTimeNs    int64                  `protobuf:"varint,4,opt,name=unix_time_ns,json=unixTimeNs,proto3" json:"unix_time_ns,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	RingSize   int32 `protobuf:"varint,1,opt,name=ring_size,json=ringSize,proto3" json:"ring_size,omitempty"`
-	RangeStart int32 `protobuf:"varint,2,opt,name=range_start,json=rangeStart,proto3" json:"range_start,omitempty"`
-	RangeStop  int32 `protobuf:"varint,3,opt,name=range_stop,json=rangeStop,proto3" json:"range_stop,omitempty"`
-	UnixTimeNs int64 `protobuf:"varint,4,opt,name=unix_time_ns,json=unixTimeNs,proto3" json:"unix_time_ns,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *Partition) Reset() {
 	*x = Partition{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_schema_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_schema_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *Partition) String() string {
@@ -219,7 +214,7 @@ func (*Partition) ProtoMessage() {}
 
 func (x *Partition) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_schema_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -263,21 +258,18 @@ func (x *Partition) GetUnixTimeNs() int64 {
 }
 
 type Offset struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Topic            *Topic             `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"`
-	PartitionOffsets []*PartitionOffset `protobuf:"bytes,2,rep,name=partition_offsets,json=partitionOffsets,proto3" json:"partition_offsets,omitempty"`
+	state            protoimpl.MessageState `protogen:"open.v1"`
+	Topic            *Topic                 `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"`
+	PartitionOffsets []*PartitionOffset     `protobuf:"bytes,2,rep,name=partition_offsets,json=partitionOffsets,proto3" json:"partition_offsets,omitempty"`
+	unknownFields    protoimpl.UnknownFields
+	sizeCache        protoimpl.SizeCache
 }
 
 func (x *Offset) Reset() {
 	*x = Offset{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_schema_proto_msgTypes[2]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_schema_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *Offset) String() string {
@@ -288,7 +280,7 @@ func (*Offset) ProtoMessage() {}
 
 func (x *Offset) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_schema_proto_msgTypes[2]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -318,21 +310,18 @@ func (x *Offset) GetPartitionOffsets() []*PartitionOffset {
 }
 
 type PartitionOffset struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Partition     *Partition             `protobuf:"bytes,1,opt,name=partition,proto3" json:"partition,omitempty"`
+	StartTsNs     int64                  `protobuf:"varint,2,opt,name=start_ts_ns,json=startTsNs,proto3" json:"start_ts_ns,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	Partition *Partition `protobuf:"bytes,1,opt,name=partition,proto3" json:"partition,omitempty"`
-	StartTsNs int64      `protobuf:"varint,2,opt,name=start_ts_ns,json=startTsNs,proto3" json:"start_ts_ns,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *PartitionOffset) Reset() {
 	*x = PartitionOffset{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_schema_proto_msgTypes[3]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_schema_proto_msgTypes[3]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *PartitionOffset) String() string {
@@ -343,7 +332,7 @@ func (*PartitionOffset) ProtoMessage() {}
 
 func (x *PartitionOffset) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_schema_proto_msgTypes[3]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -373,20 +362,17 @@ func (x *PartitionOffset) GetStartTsNs() int64 {
 }
 
 type RecordType struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Fields        []*Field               `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	Fields []*Field `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *RecordType) Reset() {
 	*x = RecordType{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_schema_proto_msgTypes[4]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_schema_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *RecordType) String() string {
@@ -397,7 +383,7 @@ func (*RecordType) ProtoMessage() {}
 
 func (x *RecordType) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_schema_proto_msgTypes[4]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -420,24 +406,21 @@ func (x *RecordType) GetFields() []*Field {
 }
 
 type Field struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Name          string                 `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	FieldIndex    int32                  `protobuf:"varint,2,opt,name=field_index,json=fieldIndex,proto3" json:"field_index,omitempty"`
+	Type          *Type                  `protobuf:"bytes,3,opt,name=type,proto3" json:"type,omitempty"`
+	IsRepeated    bool                   `protobuf:"varint,4,opt,name=is_repeated,json=isRepeated,proto3" json:"is_repeated,omitempty"`
+	IsRequired    bool                   `protobuf:"varint,5,opt,name=is_required,json=isRequired,proto3" json:"is_required,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	Name       string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	FieldIndex int32  `protobuf:"varint,2,opt,name=field_index,json=fieldIndex,proto3" json:"field_index,omitempty"`
-	Type       *Type  `protobuf:"bytes,3,opt,name=type,proto3" json:"type,omitempty"`
-	IsRepeated bool   `protobuf:"varint,4,opt,name=is_repeated,json=isRepeated,proto3" json:"is_repeated,omitempty"`
-	IsRequired bool   `protobuf:"varint,5,opt,name=is_required,json=isRequired,proto3" json:"is_required,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *Field) Reset() {
 	*x = Field{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_schema_proto_msgTypes[5]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_schema_proto_msgTypes[5]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *Field) String() string {
@@ -448,7 +431,7 @@ func (*Field) ProtoMessage() {}
 
 func (x *Field) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_schema_proto_msgTypes[5]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -499,25 +482,22 @@ func (x *Field) GetIsRequired() bool {
 }
 
 type Type struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Types that are assignable to Kind:
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// Types that are valid to be assigned to Kind:
 	//
 	//	*Type_ScalarType
 	//	*Type_RecordType
 	//	*Type_ListType
-	Kind isType_Kind `protobuf_oneof:"kind"`
+	Kind          isType_Kind `protobuf_oneof:"kind"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *Type) Reset() {
 	*x = Type{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_schema_proto_msgTypes[6]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_schema_proto_msgTypes[6]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *Type) String() string {
@@ -528,7 +508,7 @@ func (*Type) ProtoMessage() {}
 
 func (x *Type) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_schema_proto_msgTypes[6]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -543,30 +523,36 @@ func (*Type) Descriptor() ([]byte, []int) {
 	return file_mq_schema_proto_rawDescGZIP(), []int{6}
 }
 
-func (m *Type) GetKind() isType_Kind {
-	if m != nil {
-		return m.Kind
+func (x *Type) GetKind() isType_Kind {
+	if x != nil {
+		return x.Kind
 	}
 	return nil
 }
 
 func (x *Type) GetScalarType() ScalarType {
-	if x, ok := x.GetKind().(*Type_ScalarType); ok {
-		return x.ScalarType
+	if x != nil {
+		if x, ok := x.Kind.(*Type_ScalarType); ok {
+			return x.ScalarType
+		}
 	}
 	return ScalarType_BOOL
 }
 
 func (x *Type) GetRecordType() *RecordType {
-	if x, ok := x.GetKind().(*Type_RecordType); ok {
-		return x.RecordType
+	if x != nil {
+		if x, ok := x.Kind.(*Type_RecordType); ok {
+			return x.RecordType
+		}
 	}
 	return nil
 }
 
 func (x *Type) GetListType() *ListType {
-	if x, ok := x.GetKind().(*Type_ListType); ok {
-		return x.ListType
+	if x != nil {
+		if x, ok := x.Kind.(*Type_ListType); ok {
+			return x.ListType
+		}
 	}
 	return nil
 }
@@ -594,20 +580,17 @@ func (*Type_RecordType) isType_Kind() {}
 func (*Type_ListType) isType_Kind() {}
 
 type ListType struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	ElementType   *Type                  `protobuf:"bytes,1,opt,name=element_type,json=elementType,proto3" json:"element_type,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	ElementType *Type `protobuf:"bytes,1,opt,name=element_type,json=elementType,proto3" json:"element_type,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *ListType) Reset() {
 	*x = ListType{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_schema_proto_msgTypes[7]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_schema_proto_msgTypes[7]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *ListType) String() string {
@@ -618,7 +601,7 @@ func (*ListType) ProtoMessage() {}
 
 func (x *ListType) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_schema_proto_msgTypes[7]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -644,20 +627,17 @@ func (x *ListType) GetElementType() *Type {
 // value definition
 // /////////////////////////
 type RecordValue struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Fields        map[string]*Value      `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
 	unknownFields protoimpl.UnknownFields
-
-	Fields map[string]*Value `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *RecordValue) Reset() {
 	*x = RecordValue{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_schema_proto_msgTypes[8]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_schema_proto_msgTypes[8]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *RecordValue) String() string {
@@ -668,7 +648,7 @@ func (*RecordValue) ProtoMessage() {}
 
 func (x *RecordValue) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_schema_proto_msgTypes[8]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -691,11 +671,8 @@ func (x *RecordValue) GetFields() map[string]*Value {
 }
 
 type Value struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Types that are assignable to Kind:
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// Types that are valid to be assigned to Kind:
 	//
 	//	*Value_BoolValue
 	//	*Value_Int32Value
@@ -706,16 +683,16 @@ type Value struct {
 	//	*Value_StringValue
 	//	*Value_ListValue
 	//	*Value_RecordValue
-	Kind isValue_Kind `protobuf_oneof:"kind"`
+	Kind          isValue_Kind `protobuf_oneof:"kind"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *Value) Reset() {
 	*x = Value{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_schema_proto_msgTypes[9]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_schema_proto_msgTypes[9]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *Value) String() string {
@@ -726,7 +703,7 @@ func (*Value) ProtoMessage() {}
 
 func (x *Value) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_schema_proto_msgTypes[9]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -741,72 +718,90 @@ func (*Value) Descriptor() ([]byte, []int) {
 	return file_mq_schema_proto_rawDescGZIP(), []int{9}
 }
 
-func (m *Value) GetKind() isValue_Kind {
-	if m != nil {
-		return m.Kind
+func (x *Value) GetKind() isValue_Kind {
+	if x != nil {
+		return x.Kind
 	}
 	return nil
 }
 
 func (x *Value) GetBoolValue() bool {
-	if x, ok := x.GetKind().(*Value_BoolValue); ok {
-		return x.BoolValue
+	if x != nil {
+		if x, ok := x.Kind.(*Value_BoolValue); ok {
+			return x.BoolValue
+		}
 	}
 	return false
 }
 
 func (x *Value) GetInt32Value() int32 {
-	if x, ok := x.GetKind().(*Value_Int32Value); ok {
-		return x.Int32Value
+	if x != nil {
+		if x, ok := x.Kind.(*Value_Int32Value); ok {
+			return x.Int32Value
+		}
 	}
 	return 0
 }
 
 func (x *Value) GetInt64Value() int64 {
-	if x, ok := x.GetKind().(*Value_Int64Value); ok {
-		return x.Int64Value
+	if x != nil {
+		if x, ok := x.Kind.(*Value_Int64Value); ok {
+			return x.Int64Value
+		}
 	}
 	return 0
 }
 
 func (x *Value) GetFloatValue() float32 {
-	if x, ok := x.GetKind().(*Value_FloatValue); ok {
-		return x.FloatValue
+	if x != nil {
+		if x, ok := x.Kind.(*Value_FloatValue); ok {
+			return x.FloatValue
+		}
 	}
 	return 0
 }
 
 func (x *Value) GetDoubleValue() float64 {
-	if x, ok := x.GetKind().(*Value_DoubleValue); ok {
-		return x.DoubleValue
+	if x != nil {
+		if x, ok := x.Kind.(*Value_DoubleValue); ok {
+			return x.DoubleValue
+		}
 	}
 	return 0
 }
 
 func (x *Value) GetBytesValue() []byte {
-	if x, ok := x.GetKind().(*Value_BytesValue); ok {
-		return x.BytesValue
+	if x != nil {
+		if x, ok := x.Kind.(*Value_BytesValue); ok {
+			return x.BytesValue
+		}
 	}
 	return nil
 }
 
 func (x *Value) GetStringValue() string {
-	if x, ok := x.GetKind().(*Value_StringValue); ok {
-		return x.StringValue
+	if x != nil {
+		if x, ok := x.Kind.(*Value_StringValue); ok {
+			return x.StringValue
+		}
 	}
 	return ""
 }
 
 func (x *Value) GetListValue() *ListValue {
-	if x, ok := x.GetKind().(*Value_ListValue); ok {
-		return x.ListValue
+	if x != nil {
+		if x, ok := x.Kind.(*Value_ListValue); ok {
+			return x.ListValue
+		}
 	}
 	return nil
 }
 
 func (x *Value) GetRecordValue() *RecordValue {
-	if x, ok := x.GetKind().(*Value_RecordValue); ok {
-		return x.RecordValue
+	if x != nil {
+		if x, ok := x.Kind.(*Value_RecordValue); ok {
+			return x.RecordValue
+		}
 	}
 	return nil
 }
@@ -870,20 +865,17 @@ func (*Value_ListValue) isValue_Kind() {}
 func (*Value_RecordValue) isValue_Kind() {}
 
 type ListValue struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Values        []*Value               `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"`
 	unknownFields protoimpl.UnknownFields
-
-	Values []*Value `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"`
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *ListValue) Reset() {
 	*x = ListValue{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_schema_proto_msgTypes[10]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
+	mi := &file_mq_schema_proto_msgTypes[10]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
 }
 
 func (x *ListValue) String() string {
@@ -894,7 +886,7 @@ func (*ListValue) ProtoMessage() {}
 
 func (x *ListValue) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_schema_proto_msgTypes[10]
-	if protoimpl.UnsafeEnabled && x != nil {
+	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
 			ms.StoreMessageInfo(mi)
@@ -918,129 +910,99 @@ func (x *ListValue) GetValues() []*Value {
 
 var File_mq_schema_proto protoreflect.FileDescriptor
 
-var file_mq_schema_proto_rawDesc = []byte{
-	0x0a, 0x0f, 0x6d, 0x71, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74,
-	0x6f, 0x12, 0x09, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x22, 0x39, 0x0a, 0x05,
-	0x54, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61,
-	0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70,
-	0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x8a, 0x01, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74,
-	0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x69,
-	0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x69, 0x6e, 0x67, 0x53, 0x69,
-	0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x72,
-	0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x53, 0x74,
-	0x61, 0x72, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x73, 0x74, 0x6f,
-	0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x53, 0x74,
-	0x6f, 0x70, 0x12, 0x20, 0x0a, 0x0c, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f,
-	0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x75, 0x6e, 0x69, 0x78, 0x54, 0x69,
-	0x6d, 0x65, 0x4e, 0x73, 0x22, 0x79, 0x0a, 0x06, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x26,
-	0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e,
-	0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x52,
-	0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x47, 0x0a, 0x11, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74,
-	0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
-	0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x61,
-	0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x52, 0x10, 0x70,
-	0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x73, 0x22,
-	0x65, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x66, 0x66, 0x73,
-	0x65, 0x74, 0x12, 0x32, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70,
-	0x62, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72,
-	0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f,
-	0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x74, 0x61,
-	0x72, 0x74, 0x54, 0x73, 0x4e, 0x73, 0x22, 0x36, 0x0a, 0x0a, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
-	0x54, 0x79, 0x70, 0x65, 0x12, 0x28, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01,
-	0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62,
-	0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x22, 0xa3,
-	0x01, 0x0a, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b,
-	0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x05, 0x52, 0x0a, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x23, 0x0a,
-	0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x73, 0x63,
-	0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79,
-	0x70, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x73, 0x5f, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65,
-	0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x73, 0x52, 0x65, 0x70, 0x65, 0x61,
-	0x74, 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x73, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72,
-	0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x73, 0x52, 0x65, 0x71, 0x75,
-	0x69, 0x72, 0x65, 0x64, 0x22, 0xb6, 0x01, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x38, 0x0a,
-	0x0b, 0x73, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01,
-	0x28, 0x0e, 0x32, 0x15, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x53,
-	0x63, 0x61, 0x6c, 0x61, 0x72, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x63, 0x61,
-	0x6c, 0x61, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x38, 0x0a, 0x0b, 0x72, 0x65, 0x63, 0x6f, 0x72,
-	0x64, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73,
-	0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54,
-	0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70,
-	0x65, 0x12, 0x32, 0x0a, 0x09, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62,
-	0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x08, 0x6c, 0x69, 0x73,
-	0x74, 0x54, 0x79, 0x70, 0x65, 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x22, 0x3e, 0x0a,
-	0x08, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x32, 0x0a, 0x0c, 0x65, 0x6c, 0x65,
-	0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x0f, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x79, 0x70, 0x65,
-	0x52, 0x0b, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x96, 0x01,
-	0x0a, 0x0b, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x3a, 0x0a,
-	0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e,
-	0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
-	0x56, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72,
-	0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x4b, 0x0a, 0x0b, 0x46, 0x69, 0x65,
-	0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x05, 0x76, 0x61,
-	0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x73, 0x63, 0x68, 0x65,
-	0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c,
-	0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xfa, 0x02, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65,
-	0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75,
-	0x65, 0x12, 0x21, 0x0a, 0x0b, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x0a, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x56,
-	0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x5f, 0x76, 0x61,
-	0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x0a, 0x69, 0x6e, 0x74,
-	0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, 0x66, 0x6c, 0x6f, 0x61, 0x74,
-	0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x02, 0x48, 0x00, 0x52, 0x0a,
-	0x66, 0x6c, 0x6f, 0x61, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f,
-	0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01,
-	0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12,
-	0x21, 0x0a, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x06,
-	0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x56, 0x61, 0x6c,
-	0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c,
-	0x75, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69,
-	0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x35, 0x0a, 0x0a, 0x6c, 0x69, 0x73, 0x74, 0x5f,
-	0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x63,
-	0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x75,
-	0x65, 0x48, 0x00, 0x52, 0x09, 0x6c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x3b,
-	0x0a, 0x0c, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x0f,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62,
-	0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x48, 0x00, 0x52, 0x0b,
-	0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x06, 0x0a, 0x04, 0x6b,
-	0x69, 0x6e, 0x64, 0x22, 0x35, 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65,
-	0x12, 0x28, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b,
-	0x32, 0x10, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x6c,
-	0x75, 0x65, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2a, 0x77, 0x0a, 0x0a, 0x4f, 0x66,
-	0x66, 0x73, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x52, 0x45, 0x53, 0x55,
-	0x4d, 0x45, 0x5f, 0x4f, 0x52, 0x5f, 0x45, 0x41, 0x52, 0x4c, 0x49, 0x45, 0x53, 0x54, 0x10, 0x00,
-	0x12, 0x15, 0x0a, 0x11, 0x52, 0x45, 0x53, 0x45, 0x54, 0x5f, 0x54, 0x4f, 0x5f, 0x45, 0x41, 0x52,
-	0x4c, 0x49, 0x45, 0x53, 0x54, 0x10, 0x05, 0x12, 0x0f, 0x0a, 0x0b, 0x45, 0x58, 0x41, 0x43, 0x54,
-	0x5f, 0x54, 0x53, 0x5f, 0x4e, 0x53, 0x10, 0x0a, 0x12, 0x13, 0x0a, 0x0f, 0x52, 0x45, 0x53, 0x45,
-	0x54, 0x5f, 0x54, 0x4f, 0x5f, 0x4c, 0x41, 0x54, 0x45, 0x53, 0x54, 0x10, 0x0f, 0x12, 0x14, 0x0a,
-	0x10, 0x52, 0x45, 0x53, 0x55, 0x4d, 0x45, 0x5f, 0x4f, 0x52, 0x5f, 0x4c, 0x41, 0x54, 0x45, 0x53,
-	0x54, 0x10, 0x14, 0x2a, 0x5a, 0x0a, 0x0a, 0x53, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x54, 0x79, 0x70,
-	0x65, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x49,
-	0x4e, 0x54, 0x33, 0x32, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x10,
-	0x03, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06,
-	0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x59, 0x54, 0x45,
-	0x53, 0x10, 0x06, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x07, 0x42,
-	0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65,
-	0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66,
-	0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61,
-	0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-}
+const file_mq_schema_proto_rawDesc = "" +
+	"\n" +
+	"\x0fmq_schema.proto\x12\tschema_pb\"9\n" +
+	"\x05Topic\x12\x1c\n" +
+	"\tnamespace\x18\x01 \x01(\tR\tnamespace\x12\x12\n" +
+	"\x04name\x18\x02 \x01(\tR\x04name\"\x8a\x01\n" +
+	"\tPartition\x12\x1b\n" +
+	"\tring_size\x18\x01 \x01(\x05R\bringSize\x12\x1f\n" +
+	"\vrange_start\x18\x02 \x01(\x05R\n" +
+	"rangeStart\x12\x1d\n" +
+	"\n" +
+	"range_stop\x18\x03 \x01(\x05R\trangeStop\x12 \n" +
+	"\funix_time_ns\x18\x04 \x01(\x03R\n" +
+	"unixTimeNs\"y\n" +
+	"\x06Offset\x12&\n" +
+	"\x05topic\x18\x01 \x01(\v2\x10.schema_pb.TopicR\x05topic\x12G\n" +
+	"\x11partition_offsets\x18\x02 \x03(\v2\x1a.schema_pb.PartitionOffsetR\x10partitionOffsets\"e\n" +
+	"\x0fPartitionOffset\x122\n" +
+	"\tpartition\x18\x01 \x01(\v2\x14.schema_pb.PartitionR\tpartition\x12\x1e\n" +
+	"\vstart_ts_ns\x18\x02 \x01(\x03R\tstartTsNs\"6\n" +
+	"\n" +
+	"RecordType\x12(\n" +
+	"\x06fields\x18\x01 \x03(\v2\x10.schema_pb.FieldR\x06fields\"\xa3\x01\n" +
+	"\x05Field\x12\x12\n" +
+	"\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n" +
+	"\vfield_index\x18\x02 \x01(\x05R\n" +
+	"fieldIndex\x12#\n" +
+	"\x04type\x18\x03 \x01(\v2\x0f.schema_pb.TypeR\x04type\x12\x1f\n" +
+	"\vis_repeated\x18\x04 \x01(\bR\n" +
+	"isRepeated\x12\x1f\n" +
+	"\vis_required\x18\x05 \x01(\bR\n" +
+	"isRequired\"\xb6\x01\n" +
+	"\x04Type\x128\n" +
+	"\vscalar_type\x18\x01 \x01(\x0e2\x15.schema_pb.ScalarTypeH\x00R\n" +
+	"scalarType\x128\n" +
+	"\vrecord_type\x18\x02 \x01(\v2\x15.schema_pb.RecordTypeH\x00R\n" +
+	"recordType\x122\n" +
+	"\tlist_type\x18\x03 \x01(\v2\x13.schema_pb.ListTypeH\x00R\blistTypeB\x06\n" +
+	"\x04kind\">\n" +
+	"\bListType\x122\n" +
+	"\felement_type\x18\x01 \x01(\v2\x0f.schema_pb.TypeR\velementType\"\x96\x01\n" +
+	"\vRecordValue\x12:\n" +
+	"\x06fields\x18\x01 \x03(\v2\".schema_pb.RecordValue.FieldsEntryR\x06fields\x1aK\n" +
+	"\vFieldsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12&\n" +
+	"\x05value\x18\x02 \x01(\v2\x10.schema_pb.ValueR\x05value:\x028\x01\"\xfa\x02\n" +
+	"\x05Value\x12\x1f\n" +
+	"\n" +
+	"bool_value\x18\x01 \x01(\bH\x00R\tboolValue\x12!\n" +
+	"\vint32_value\x18\x02 \x01(\x05H\x00R\n" +
+	"int32Value\x12!\n" +
+	"\vint64_value\x18\x03 \x01(\x03H\x00R\n" +
+	"int64Value\x12!\n" +
+	"\vfloat_value\x18\x04 \x01(\x02H\x00R\n" +
+	"floatValue\x12#\n" +
+	"\fdouble_value\x18\x05 \x01(\x01H\x00R\vdoubleValue\x12!\n" +
+	"\vbytes_value\x18\x06 \x01(\fH\x00R\n" +
+	"bytesValue\x12#\n" +
+	"\fstring_value\x18\a \x01(\tH\x00R\vstringValue\x125\n" +
+	"\n" +
+	"list_value\x18\x0e \x01(\v2\x14.schema_pb.ListValueH\x00R\tlistValue\x12;\n" +
+	"\frecord_value\x18\x0f \x01(\v2\x16.schema_pb.RecordValueH\x00R\vrecordValueB\x06\n" +
+	"\x04kind\"5\n" +
+	"\tListValue\x12(\n" +
+	"\x06values\x18\x01 \x03(\v2\x10.schema_pb.ValueR\x06values*w\n" +
+	"\n" +
+	"OffsetType\x12\x16\n" +
+	"\x12RESUME_OR_EARLIEST\x10\x00\x12\x15\n" +
+	"\x11RESET_TO_EARLIEST\x10\x05\x12\x0f\n" +
+	"\vEXACT_TS_NS\x10\n" +
+	"\x12\x13\n" +
+	"\x0fRESET_TO_LATEST\x10\x0f\x12\x14\n" +
+	"\x10RESUME_OR_LATEST\x10\x14*Z\n" +
+	"\n" +
+	"ScalarType\x12\b\n" +
+	"\x04BOOL\x10\x00\x12\t\n" +
+	"\x05INT32\x10\x01\x12\t\n" +
+	"\x05INT64\x10\x03\x12\t\n" +
+	"\x05FLOAT\x10\x04\x12\n" +
+	"\n" +
+	"\x06DOUBLE\x10\x05\x12\t\n" +
+	"\x05BYTES\x10\x06\x12\n" +
+	"\n" +
+	"\x06STRING\x10\aB2Z0github.com/seaweedfs/seaweedfs/weed/pb/schema_pbb\x06proto3"
 
 var (
 	file_mq_schema_proto_rawDescOnce sync.Once
-	file_mq_schema_proto_rawDescData = file_mq_schema_proto_rawDesc
+	file_mq_schema_proto_rawDescData []byte
 )
 
 func file_mq_schema_proto_rawDescGZIP() []byte {
 	file_mq_schema_proto_rawDescOnce.Do(func() {
-		file_mq_schema_proto_rawDescData = protoimpl.X.CompressGZIP(file_mq_schema_proto_rawDescData)
+		file_mq_schema_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mq_schema_proto_rawDesc), len(file_mq_schema_proto_rawDesc)))
 	})
 	return file_mq_schema_proto_rawDescData
 }
@@ -1090,140 +1052,6 @@ func file_mq_schema_proto_init() {
 	if File_mq_schema_proto != nil {
 		return
 	}
-	if !protoimpl.UnsafeEnabled {
-		file_mq_schema_proto_msgTypes[0].Exporter = func(v any, i int) any {
-			switch v := v.(*Topic); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_schema_proto_msgTypes[1].Exporter = func(v any, i int) any {
-			switch v := v.(*Partition); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_schema_proto_msgTypes[2].Exporter = func(v any, i int) any {
-			switch v := v.(*Offset); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_schema_proto_msgTypes[3].Exporter = func(v any, i int) any {
-			switch v := v.(*PartitionOffset); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_schema_proto_msgTypes[4].Exporter = func(v any, i int) any {
-			switch v := v.(*RecordType); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_schema_proto_msgTypes[5].Exporter = func(v any, i int) any {
-			switch v := v.(*Field); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_schema_proto_msgTypes[6].Exporter = func(v any, i int) any {
-			switch v := v.(*Type); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_schema_proto_msgTypes[7].Exporter = func(v any, i int) any {
-			switch v := v.(*ListType); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_schema_proto_msgTypes[8].Exporter = func(v any, i int) any {
-			switch v := v.(*RecordValue); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_schema_proto_msgTypes[9].Exporter = func(v any, i int) any {
-			switch v := v.(*Value); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_schema_proto_msgTypes[10].Exporter = func(v any, i int) any {
-			switch v := v.(*ListValue); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
 	file_mq_schema_proto_msgTypes[6].OneofWrappers = []any{
 		(*Type_ScalarType)(nil),
 		(*Type_RecordType)(nil),
@@ -1244,7 +1072,7 @@ func file_mq_schema_proto_init() {
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_mq_schema_proto_rawDesc,
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_mq_schema_proto_rawDesc), len(file_mq_schema_proto_rawDesc)),
 			NumEnums:      2,
 			NumMessages:   12,
 			NumExtensions: 0,
@@ -1256,7 +1084,6 @@ func file_mq_schema_proto_init() {
 		MessageInfos:      file_mq_schema_proto_msgTypes,
 	}.Build()
 	File_mq_schema_proto = out.File
-	file_mq_schema_proto_rawDesc = nil
 	file_mq_schema_proto_goTypes = nil
 	file_mq_schema_proto_depIdxs = nil
 }

weed/pb/volume_server.proto

@@ -180,6 +180,7 @@ message AllocateVolumeRequest {
     string ttl = 5;
     uint32 memory_map_max_size_mb = 6;
     string disk_type = 7;
+    uint32 version = 8;
 }
 message AllocateVolumeResponse {
 }
@@ -195,6 +196,7 @@ message VolumeSyncStatusResponse {
     uint64 tail_offset = 6;
     uint32 compact_revision = 7;
     uint64 idx_file_size = 8;
+    uint32 version = 9;
 }
 
 message VolumeIncrementalCopyRequest {
@@ -339,6 +341,7 @@ message VolumeTailSenderResponse {
     bytes needle_header = 1;
     bytes needle_body = 2;
     bool is_last_chunk = 3;
+    uint32 version = 4;
 }
 
 message VolumeTailReceiverRequest {
@@ -442,6 +445,7 @@ message ReadVolumeFileStatusResponse {
     string collection = 8;
     string disk_type = 9;
     VolumeInfo volume_info = 10;
+    uint32 version = 11;
 }
 
 message DiskStatus {

weed/pb/volume_server_pb/volume_server_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.5.1
-// - protoc             v5.28.3
+// - protoc             v5.29.3
 // source: volume_server.proto
 
 package volume_server_pb

weed/s3api/auth_signature_v4.go

@@ -719,9 +719,11 @@ func extractHostHeader(r *http.Request) string {
 
 	forwardedHost := r.Header.Get("X-Forwarded-Host")
 	forwardedPort := r.Header.Get("X-Forwarded-Port")
+	forwardedProto := r.Header.Get("X-Forwarded-Proto")
 
 	// If X-Forwarded-Host is set, use that as the host.
 	// If X-Forwarded-Port is set, use that too to form the host.
+	// If X-Forwarded-Proto is set, check if is it default to omit the port.
 	if forwardedHost != "" {
 		extractedHost := forwardedHost
 		host, port, err := net.SplitHostPort(extractedHost)
@@ -731,7 +733,11 @@ func extractHostHeader(r *http.Request) string {
 				forwardedPort = port
 			}
 		}
-		if !isDefaultPort(r.URL.Scheme, forwardedPort) {
+		scheme := r.URL.Scheme
+		if forwardedProto != "" {
+			scheme = forwardedProto
+		}
+		if !isDefaultPort(scheme, forwardedPort) {
 			extractedHost = net.JoinHostPort(extractedHost, forwardedPort)
 		}
 		return extractedHost

weed/s3api/s3api_bucket_handlers.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"math"
 	"net/http"
+	"sort"
 	"strings"
 	"time"
 
@@ -329,7 +330,15 @@ func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWr
 	}
 
 	response := Lifecycle{}
-	for locationPrefix, internalTtl := range ttls {
+	// Sort locationPrefixes to ensure consistent ordering of lifecycle rules
+	var locationPrefixes []string
+	for locationPrefix := range ttls {
+		locationPrefixes = append(locationPrefixes, locationPrefix)
+	}
+	sort.Strings(locationPrefixes)
+
+	for _, locationPrefix := range locationPrefixes {
+		internalTtl := ttls[locationPrefix]
 		ttl, _ := needle.ReadTTL(internalTtl)
 		days := int(ttl.Minutes() / 60 / 24)
 		if days == 0 {

weed/s3api/s3api_put_object_helper.go

@@ -7,6 +7,11 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
 
+// getRequestDataReader returns the appropriate reader for the request body.
+// When IAM is disabled, it still processes chunked transfer encoding for
+// authTypeStreamingUnsigned to strip checksum headers and extract the actual data.
+// This fixes issues where chunked data with checksums would be stored incorrectly
+// when IAM is not enabled.
 func getRequestDataReader(s3a *S3ApiServer, r *http.Request) (io.ReadCloser, s3err.ErrorCode) {
 	var s3ErrCode s3err.ErrorCode
 	dataReader := r.Body
@@ -21,8 +26,13 @@ func getRequestDataReader(s3a *S3ApiServer, r *http.Request) (io.ReadCloser, s3e
 			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
 		}
 	} else {
-		if authTypeStreamingSigned == rAuthType {
+		switch rAuthType {
+		case authTypeStreamingSigned:
 			s3ErrCode = s3err.ErrAuthNotSetup
+		case authTypeStreamingUnsigned:
+			// Even when IAM is disabled, we still need to handle chunked transfer encoding
+			// to strip checksum headers and process the data correctly
+			dataReader, s3ErrCode = s3a.iam.newChunkedReader(r)
 		}
 	}
 

weed/s3api/s3api_put_object_helper_test.go

@@ -0,0 +1,183 @@
+package s3api
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+)
+
+func TestGetRequestDataReader_ChunkedEncodingWithoutIAM(t *testing.T) {
+	// Create an S3ApiServer with IAM disabled
+	s3a := &S3ApiServer{
+		iam: NewIdentityAccessManagement(&S3ApiServerOption{}),
+	}
+	// Ensure IAM is disabled for this test
+	s3a.iam.isAuthEnabled = false
+
+	tests := []struct {
+		name          string
+		contentSha256 string
+		expectedError s3err.ErrorCode
+		shouldProcess bool
+		description   string
+	}{
+		{
+			name:          "RegularRequest",
+			contentSha256: "",
+			expectedError: s3err.ErrNone,
+			shouldProcess: false,
+			description:   "Regular requests without chunked encoding should pass through unchanged",
+		},
+		{
+			name:          "StreamingSignedWithoutIAM",
+			contentSha256: "STREAMING-AWS4-HMAC-SHA256-PAYLOAD",
+			expectedError: s3err.ErrAuthNotSetup,
+			shouldProcess: false,
+			description:   "Streaming signed requests should fail when IAM is disabled",
+		},
+		{
+			name:          "StreamingUnsignedWithoutIAM",
+			contentSha256: "STREAMING-UNSIGNED-PAYLOAD-TRAILER",
+			expectedError: s3err.ErrNone,
+			shouldProcess: true,
+			description:   "Streaming unsigned requests should be processed even when IAM is disabled",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			body := strings.NewReader("test data")
+			req, _ := http.NewRequest("PUT", "/bucket/key", body)
+
+			if tt.contentSha256 != "" {
+				req.Header.Set("x-amz-content-sha256", tt.contentSha256)
+			}
+
+			dataReader, errCode := getRequestDataReader(s3a, req)
+
+			// Check error code
+			if errCode != tt.expectedError {
+				t.Errorf("Expected error code %v, got %v", tt.expectedError, errCode)
+			}
+
+			// For successful cases, check if processing occurred
+			if errCode == s3err.ErrNone {
+				if tt.shouldProcess {
+					// For chunked requests, the reader should be different from the original body
+					if dataReader == req.Body {
+						t.Error("Expected dataReader to be processed by newChunkedReader, but got raw request body")
+					}
+				} else {
+					// For regular requests, the reader should be the same as the original body
+					if dataReader != req.Body {
+						t.Error("Expected dataReader to be the same as request body for regular requests")
+					}
+				}
+			}
+
+			t.Logf("Test case: %s - %s", tt.name, tt.description)
+		})
+	}
+}
+
+func TestGetRequestDataReader_AuthTypeDetection(t *testing.T) {
+	// Create an S3ApiServer with IAM disabled
+	s3a := &S3ApiServer{
+		iam: NewIdentityAccessManagement(&S3ApiServerOption{}),
+	}
+	s3a.iam.isAuthEnabled = false
+
+	// Test the specific case mentioned in the issue where chunked data
+	// with checksum headers would be stored incorrectly
+	t.Run("ChunkedDataWithChecksum", func(t *testing.T) {
+		// Simulate a request with chunked data and checksum trailer
+		body := strings.NewReader("test content")
+		req, _ := http.NewRequest("PUT", "/bucket/key", body)
+		req.Header.Set("x-amz-content-sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER")
+		req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32")
+
+		// Verify the auth type is detected correctly
+		authType := getRequestAuthType(req)
+		if authType != authTypeStreamingUnsigned {
+			t.Errorf("Expected authTypeStreamingUnsigned, got %v", authType)
+		}
+
+		// Verify the request is processed correctly
+		dataReader, errCode := getRequestDataReader(s3a, req)
+		if errCode != s3err.ErrNone {
+			t.Errorf("Expected no error, got %v", errCode)
+		}
+
+		// The dataReader should be processed by newChunkedReader
+		if dataReader == req.Body {
+			t.Error("Expected dataReader to be processed by newChunkedReader to handle chunked encoding")
+		}
+	})
+}
+
+func TestGetRequestDataReader_IAMEnabled(t *testing.T) {
+	// Create an S3ApiServer with IAM enabled
+	s3a := &S3ApiServer{
+		iam: NewIdentityAccessManagement(&S3ApiServerOption{}),
+	}
+	s3a.iam.isAuthEnabled = true
+
+	t.Run("StreamingUnsignedWithIAMEnabled", func(t *testing.T) {
+		body := strings.NewReader("test data")
+		req, _ := http.NewRequest("PUT", "/bucket/key", body)
+		req.Header.Set("x-amz-content-sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER")
+
+		dataReader, errCode := getRequestDataReader(s3a, req)
+
+		// Should succeed and be processed
+		if errCode != s3err.ErrNone {
+			t.Errorf("Expected no error, got %v", errCode)
+		}
+
+		// Should be processed by newChunkedReader
+		if dataReader == req.Body {
+			t.Error("Expected dataReader to be processed by newChunkedReader")
+		}
+	})
+}
+
+// Test helper to verify auth type detection works correctly
+func TestAuthTypeDetection(t *testing.T) {
+	tests := []struct {
+		name         string
+		headers      map[string]string
+		expectedType authType
+	}{
+		{
+			name:         "StreamingUnsigned",
+			headers:      map[string]string{"x-amz-content-sha256": "STREAMING-UNSIGNED-PAYLOAD-TRAILER"},
+			expectedType: authTypeStreamingUnsigned,
+		},
+		{
+			name:         "StreamingSigned",
+			headers:      map[string]string{"x-amz-content-sha256": "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"},
+			expectedType: authTypeStreamingSigned,
+		},
+		{
+			name:         "Regular",
+			headers:      map[string]string{},
+			expectedType: authTypeAnonymous,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			req, _ := http.NewRequest("PUT", "/bucket/key", strings.NewReader("test"))
+			for key, value := range tt.headers {
+				req.Header.Set(key, value)
+			}
+
+			authType := getRequestAuthType(req)
+			if authType != tt.expectedType {
+				t.Errorf("Expected auth type %v, got %v", tt.expectedType, authType)
+			}
+		})
+	}
+}

weed/server/common.go

@@ -7,9 +7,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/google/uuid"
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
-	"google.golang.org/grpc/metadata"
 	"io"
 	"io/fs"
 	"mime/multipart"
@@ -21,18 +18,21 @@ import (
 	"sync"
 	"time"
 
+	"github.com/google/uuid"
+	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+	"google.golang.org/grpc/metadata"
+
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 
 	"google.golang.org/grpc"
 
+	"github.com/gorilla/mux"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-
-	"github.com/gorilla/mux"
 )
 
 var serverStats *stats.ServerStats
@@ -429,18 +429,18 @@ func ProcessRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
 
 func requestIDMiddleware(h http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		reqID := r.Header.Get(util.RequestIdHttpHeader)
+		reqID := r.Header.Get(request_id.AmzRequestIDHeader)
 		if reqID == "" {
 			reqID = uuid.New().String()
 		}
 
-		ctx := context.WithValue(r.Context(), util.RequestIDKey, reqID)
+		ctx := context.WithValue(r.Context(), request_id.AmzRequestIDHeader, reqID)
 		ctx = metadata.NewOutgoingContext(ctx,
 			metadata.New(map[string]string{
-				util.RequestIDKey: reqID,
+				request_id.AmzRequestIDHeader: reqID,
 			}))
 
-		w.Header().Set(util.RequestIdHttpHeader, reqID)
+		w.Header().Set(request_id.AmzRequestIDHeader, reqID)
 		h(w, r.WithContext(ctx))
 	}
 }

weed/server/filer_server.go

@@ -25,6 +25,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/arangodb"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/cassandra"
+	_ "github.com/seaweedfs/seaweedfs/weed/filer/cassandra2"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/elastic/v7"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/etcd"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/hbase"

weed/server/filer_server_handlers_proxy.go

@@ -3,9 +3,10 @@ package weed_server
 import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/security"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 	"github.com/seaweedfs/seaweedfs/weed/util/mem"
+	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
+
 	"io"
 	"math/rand/v2"
 	"net/http"
@@ -54,7 +55,7 @@ func (fs *FilerServer) proxyToVolumeServer(w http.ResponseWriter, r *http.Reques
 
 	proxyReq.Header.Set("Host", r.Host)
 	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)
-	util.ReqWithRequestId(proxyReq, ctx)
+	request_id.InjectToRequest(ctx, proxyReq)
 
 	for header, values := range r.Header {
 		for _, value := range values {

weed/server/master_grpc_server_assign.go

@@ -3,11 +3,12 @@ package weed_server
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"strings"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+
 	"github.com/seaweedfs/raft"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
@@ -59,6 +60,7 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
 	}
 	diskType := types.ToDiskType(req.DiskType)
 
+	ver := needle.GetCurrentVersion()
 	option := &topology.VolumeGrowOption{
 		Collection:         req.Collection,
 		ReplicaPlacement:   replicaPlacement,
@@ -69,6 +71,7 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
 		Rack:               req.Rack,
 		DataNode:           req.DataNode,
 		MemoryMapMaxSizeMb: req.MemoryMapMaxSizeMb,
+		Version:            uint32(ver),
 	}
 
 	if !ms.Topo.DataCenterExists(option.DataCenter) {

weed/server/master_grpc_server_volume.go

@@ -333,6 +333,8 @@ func (ms *MasterServer) VolumeGrow(ctx context.Context, req *master_pb.VolumeGro
 	if req.DataCenter != "" && !ms.Topo.DataCenterExists(req.DataCenter) {
 		return nil, fmt.Errorf("data center not exists")
 	}
+
+	ver := needle.GetCurrentVersion()
 	volumeGrowOption := topology.VolumeGrowOption{
 		Collection:         req.Collection,
 		ReplicaPlacement:   replicaPlacement,
@@ -343,6 +345,7 @@ func (ms *MasterServer) VolumeGrow(ctx context.Context, req *master_pb.VolumeGro
 		Rack:               req.Rack,
 		DataNode:           req.DataNode,
 		MemoryMapMaxSizeMb: req.MemoryMapMaxSizeMb,
+		Version:            uint32(ver),
 	}
 	volumeGrowRequest := topology.VolumeGrowRequest{
 		Option: &volumeGrowOption,

weed/server/master_server_handlers_admin.go

@@ -3,11 +3,12 @@ package weed_server
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"math/rand/v2"
 	"net/http"
 	"strconv"
 
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 
@@ -166,6 +167,7 @@ func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGr
 			return nil, fmt.Errorf("Failed to parse int64 preallocate = %s: %v", r.FormValue("preallocate"), err)
 		}
 	}
+	ver := needle.GetCurrentVersion()
 	volumeGrowOption := &topology.VolumeGrowOption{
 		Collection:         r.FormValue("collection"),
 		ReplicaPlacement:   replicaPlacement,
@@ -176,6 +178,7 @@ func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGr
 		Rack:               r.FormValue("rack"),
 		DataNode:           r.FormValue("dataNode"),
 		MemoryMapMaxSizeMb: memoryMapMaxSizeMb,
+		Version:            uint32(ver),
 	}
 	return volumeGrowOption, nil
 }

weed/server/volume_grpc_admin.go

@@ -3,10 +3,11 @@ package weed_server
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"path/filepath"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+
 	"github.com/seaweedfs/seaweedfs/weed/storage"
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
@@ -48,6 +49,7 @@ func (vs *VolumeServer) AllocateVolume(ctx context.Context, req *volume_server_p
 		req.Replication,
 		req.Ttl,
 		req.Preallocate,
+		needle.Version(req.Version),
 		req.MemoryMapMaxSizeMb,
 		types.ToDiskType(req.DiskType),
 		vs.ldbTimout,

weed/server/volume_grpc_copy.go

@@ -310,6 +310,7 @@ func (vs *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_se
 	resp.Collection = v.Collection
 	resp.DiskType = string(v.DiskType())
 	resp.VolumeInfo = v.GetVolumeInfo()
+	resp.Version = uint32(v.Version())
 	return resp, nil
 }
 

weed/server/volume_grpc_tail.go

@@ -3,9 +3,10 @@ package weed_server
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -65,12 +66,13 @@ func sendNeedlesSince(stream volume_server_pb.VolumeServer_VolumeTailSenderServe
 
 	if isLastOne {
 		// need to heart beat to the client to ensure the connection health
-		sendErr := stream.Send(&volume_server_pb.VolumeTailSenderResponse{IsLastChunk: true})
+		sendErr := stream.Send(&volume_server_pb.VolumeTailSenderResponse{IsLastChunk: true, Version: uint32(v.Version())})
 		return lastTimestampNs, sendErr
 	}
 
 	scanner := &VolumeFileScanner4Tailing{
-		stream: stream,
+		stream:  stream,
+		version: uint32(v.Version()),
 	}
 
 	err = storage.ScanVolumeFileFrom(v.Version(), v.DataBackend, foundOffset.ToActualOffset(), scanner)
@@ -101,6 +103,7 @@ func (vs *VolumeServer) VolumeTailReceiver(ctx context.Context, req *volume_serv
 type VolumeFileScanner4Tailing struct {
 	stream                   volume_server_pb.VolumeServer_VolumeTailSenderServer
 	lastProcessedTimestampNs uint64
+	version                  uint32
 }
 
 func (scanner *VolumeFileScanner4Tailing) VisitSuperBlock(superBlock super_block.SuperBlock) error {
@@ -126,6 +129,7 @@ func (scanner *VolumeFileScanner4Tailing) VisitNeedle(n *needle.Needle, offset i
 			NeedleHeader: needleHeader,
 			NeedleBody:   needleBody[i:stopOffset],
 			IsLastChunk:  isLastChunk,
+			Version:      scanner.version,
 		})
 		if sendErr != nil {
 			return sendErr

weed/server/volume_server_ui/volume.html

@@ -114,6 +114,7 @@
                 <th>Trash</th>
                 <th>TTL</th>
                 <th>ReadOnly</th>
+                <th>Version</th>
             </tr>
             </thead>
             <tbody>
@@ -127,6 +128,7 @@
                 <td>{{ .DeleteCount }} / {{bytesToHumanReadable .DeletedByteCount}}</td>
                 <td>{{ .Ttl }}</td>
                 <td>{{ .ReadOnly }}</td>
+                <td>{{ .Version }}</td>
             </tr>
             {{ end }}
             </tbody>

weed/shell/command_ec_encode.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"io"
 	"time"
 
@@ -107,7 +108,7 @@ func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 		balanceCollections = collectCollectionsForVolumeIds(topologyInfo, volumeIds)
 	} else {
 		// apply to all volumes for the given collection
-		volumeIds, err = collectVolumeIdsForEcEncode(commandEnv, *collection, *fullPercentage, *quietPeriod)
+		volumeIds, err = collectVolumeIdsForEcEncode(commandEnv, *collection, nil, *fullPercentage, *quietPeriod)
 		if err != nil {
 			return err
 		}
@@ -250,7 +251,7 @@ func generateEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId,
 
 }
 
-func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, selectedCollection string, fullPercentage float64, quietPeriod time.Duration) (vids []needle.VolumeId, err error) {
+func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, selectedCollection string, sourceDiskType *types.DiskType, fullPercentage float64, quietPeriod time.Duration) (vids []needle.VolumeId, err error) {
 	// collect topology information
 	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
@@ -270,7 +271,8 @@ func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, selectedCollection stri
 				if v.RemoteStorageName != "" && v.RemoteStorageKey != "" {
 					continue
 				}
-				if v.Collection == selectedCollection && v.ModifiedAtSecond+quietSeconds < nowUnixSeconds {
+				if v.Collection == selectedCollection && v.ModifiedAtSecond+quietSeconds < nowUnixSeconds &&
+					(sourceDiskType == nil || types.ToDiskType(v.DiskType) == *sourceDiskType) {
 					if float64(v.Size) > fullPercentage/100*float64(volumeSizeLimitMb)*1024*1024 {
 						if good, found := vidMap[v.Id]; found {
 							if good {

weed/shell/command_volume_copy.go

@@ -41,11 +41,14 @@ func (c *commandVolumeCopy) Do(args []string, commandEnv *CommandEnv, writer io.
 	volumeIdInt := volCopyCommand.Int("volumeId", 0, "the volume id")
 	sourceNodeStr := volCopyCommand.String("source", "", "the source volume server <host>:<port>")
 	targetNodeStr := volCopyCommand.String("target", "", "the target volume server <host>:<port>")
+	noLock := volCopyCommand.Bool("noLock", false, "do not lock the admin shell at one's own risk")
 	if err = volCopyCommand.Parse(args); err != nil {
 		return nil
 	}
 
-	if err = commandEnv.confirmIsLocked(args); err != nil {
+	if *noLock {
+		commandEnv.noLock = true
+	} else if err = commandEnv.confirmIsLocked(args); err != nil {
 		return
 	}
 

weed/shell/command_volume_list.go

@@ -9,6 +9,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"path/filepath"
 	"slices"
+	"sort"
 	"strings"
 	"time"
 
@@ -73,9 +74,34 @@ func (c *commandVolumeList) Do(args []string, commandEnv *CommandEnv, writer io.
 	return nil
 }
 
+func sortMapKey[T1 comparable, T2 any](m map[T1]T2) []T1 {
+	keys := make([]T1, 0, len(m))
+	for k, _ := range m {
+		keys = append(keys, k)
+	}
+
+	sort.Slice(keys, func(i, j int) bool {
+		switch v1 := any(keys[i]).(type) {
+		case int:
+			return v1 < any(keys[j]).(int)
+		case string:
+			if strings.Compare(v1, any(keys[j]).(string)) < 0 {
+				return true
+			}
+			return false
+		default:
+			return false
+		}
+	})
+	return keys
+}
+
 func diskInfosToString(diskInfos map[string]*master_pb.DiskInfo) string {
 	var buf bytes.Buffer
-	for diskType, diskInfo := range diskInfos {
+
+	for _, diskType := range sortMapKey(diskInfos) {
+		diskInfo := diskInfos[diskType]
+
 		if diskType == "" {
 			diskType = types.HddType
 		}
@@ -152,7 +178,8 @@ func (c *commandVolumeList) writeRackInfo(writer io.Writer, t *master_pb.RackInf
 func (c *commandVolumeList) writeDataNodeInfo(writer io.Writer, t *master_pb.DataNodeInfo, verbosityLevel int, outRackInfo func()) statistics {
 	var s statistics
 	diskInfoFound := false
-	for _, diskInfo := range t.DiskInfos {
+	for _, diskType := range sortMapKey(t.DiskInfos) {
+		diskInfo := t.DiskInfos[diskType]
 		s = s.plus(c.writeDiskInfo(writer, diskInfo, verbosityLevel, func() {
 			outRackInfo()
 			output(verbosityLevel >= 3, writer, "      DataNode %s%s\n", t.Id, diskInfosToString(t.DiskInfos))

weed/shell/command_volume_tier_upload.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"io"
 	"time"
 
@@ -69,6 +70,7 @@ func (c *commandVolumeTierUpload) Do(args []string, commandEnv *CommandEnv, writ
 	quietPeriod := tierCommand.Duration("quietFor", 24*time.Hour, "select volumes without no writes for this period")
 	dest := tierCommand.String("dest", "", "the target tier name")
 	keepLocalDatFile := tierCommand.Bool("keepLocalDatFile", false, "whether keep local dat file")
+	disk := tierCommand.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	if err = tierCommand.Parse(args); err != nil {
 		return nil
 	}
@@ -84,9 +86,15 @@ func (c *commandVolumeTierUpload) Do(args []string, commandEnv *CommandEnv, writ
 		return doVolumeTierUpload(commandEnv, writer, *collection, vid, *dest, *keepLocalDatFile)
 	}
 
+	var diskType *types.DiskType
+	if disk != nil {
+		_diskType := types.ToDiskType(*disk)
+		diskType = &_diskType
+	}
+
 	// apply to all volumes in the collection
 	// reusing collectVolumeIdsForEcEncode for now
-	volumeIds, err := collectVolumeIdsForEcEncode(commandEnv, *collection, *fullPercentage, *quietPeriod)
+	volumeIds, err := collectVolumeIdsForEcEncode(commandEnv, *collection, diskType, *fullPercentage, *quietPeriod)
 	if err != nil {
 		return err
 	}
@@ -147,11 +155,15 @@ func uploadDatToRemoteTier(grpcDialOption grpc.DialOption, writer io.Writer, vol
 			KeepLocalDatFile:       keepLocalDatFile,
 		})
 
-		if stream == nil && copyErr == nil {
-			// when the volume is already uploaded, VolumeTierMoveDatToRemote will return nil stream and nil error
-			// so we should directly return in this case
-			fmt.Fprintf(writer, "volume %v already uploaded", volumeId)
-			return nil
+		if stream == nil {
+			if copyErr == nil {
+				// when the volume is already uploaded, VolumeTierMoveDatToRemote will return nil stream and nil error
+				// so we should directly return in this caseAdd commentMore actions
+				fmt.Fprintf(writer, "volume %v already uploaded", volumeId)
+				return nil
+			} else {
+				return copyErr
+			}
 		}
 		var lastProcessed int64
 		for {

weed/storage/disk_location.go

@@ -178,7 +178,7 @@ func (l *DiskLocation) loadExistingVolume(dirEntry os.DirEntry, needleMapKind Ne
 	}
 
 	// load the volume
-	v, e := NewVolume(l.Directory, l.IdxDirectory, collection, vid, needleMapKind, nil, nil, 0, 0, ldbTimeout)
+	v, e := NewVolume(l.Directory, l.IdxDirectory, collection, vid, needleMapKind, nil, nil, 0, needle.GetCurrentVersion(), 0, ldbTimeout)
 	if e != nil {
 		glog.V(0).Infof("new volume %s error %s", volumeName, e)
 		return false

weed/storage/erasure_coding/ec_volume_test.go

@@ -44,7 +44,7 @@ func TestPositioning(t *testing.T) {
 	fmt.Printf("offset: %d size: %d\n", offset.ToActualOffset(), size)
 
 	var shardEcdFileSize int64 = 1118830592 // 1024*1024*1024*3
-	intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shardEcdFileSize, offset.ToActualOffset(), types.Size(needle.GetActualSize(size, needle.CurrentVersion)))
+	intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shardEcdFileSize, offset.ToActualOffset(), types.Size(needle.GetActualSize(size, needle.GetCurrentVersion())))
 
 	for _, interval := range intervals {
 		shardId, shardOffset := interval.ToShardIdAndOffset(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize)

weed/storage/idx_binary_search_test.go

@@ -14,7 +14,7 @@ import (
 func TestFirstInvalidIndex(t *testing.T) {
 	dir := t.TempDir()
 
-	v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, 0, 0)
+	v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
 	if err != nil {
 		t.Fatalf("volume creation: %v", err)
 	}