perf tests run fine

Bumps [github.com/rclone/rclone](https://github.com/rclone/rclone) from 1.69.3 to 1.70.1.
- [Release notes](https://github.com/rclone/rclone/releases)
- [Changelog](https://github.com/rclone/rclone/blob/master/RELEASE.md)
- [Commits](https://github.com/rclone/rclone/compare/v1.69.3...v1.70.1)

---
updated-dependencies:
- dependency-name: github.com/rclone/rclone
  dependency-version: 1.70.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/binaries_dev.yml

@@ -53,7 +53,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed-large-disk
@@ -68,7 +68,7 @@ jobs:
           release_tag: dev
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed-normal-disk
@@ -102,7 +102,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed-large-disk
@@ -117,7 +117,7 @@ jobs:
           release_tag: dev
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed-normal-disk

.github/workflows/binaries_release0.yml

@@ -38,7 +38,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed
@@ -52,7 +52,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed

.github/workflows/binaries_release1.yml

@@ -38,7 +38,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed
@@ -52,7 +52,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed

.github/workflows/binaries_release2.yml

@@ -38,7 +38,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed
@@ -52,7 +52,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed

.github/workflows/binaries_release3.yml

@@ -38,7 +38,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed
@@ -52,7 +52,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed

.github/workflows/binaries_release4.yml

@@ -39,7 +39,7 @@ jobs:
           build_flags: -tags elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed
@@ -53,7 +53,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed

.github/workflows/binaries_release5.yml

@@ -38,7 +38,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed
@@ -52,7 +52,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed

.github/workflows/container_dev.yml

@@ -36,7 +36,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
         with:
           buildkitd-flags: "--debug"
       -
@@ -56,7 +56,7 @@ jobs:
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
         with:
           buildkitd-flags: "--debug"
       -
@@ -57,7 +57,7 @@ jobs:
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
@@ -47,7 +47,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -38,7 +38,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
@@ -48,7 +48,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -38,7 +38,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
@@ -48,7 +48,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release4.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
@@ -47,7 +47,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release5.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
@@ -47,7 +47,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/depsreview.yml

@@ -11,4 +11,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@0c155c5e8556a497adf53f2c18edabf945ed8e70
+        uses: actions/dependency-review-action@da24556b548a50705dd671f47852072ea4c105d9

README.md

@@ -46,6 +46,7 @@ Your support will be really appreciated by me and other supporters!
 - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [Wiki Documentation](https://github.com/seaweedfs/seaweedfs/wiki)
 - [SeaweedFS White Paper](https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
+- [SeaweedFS Introduction Slides 2025.5](https://docs.google.com/presentation/d/1tdkp45J01oRV68dIm4yoTXKJDof-EhainlA0LMXexQE/edit?usp=sharing)
 - [SeaweedFS Introduction Slides 2021.5](https://docs.google.com/presentation/d/1DcxKWlINc-HNCjhYeERkpGXXm6nTCES8mi2W5G0Z4Ts/edit?usp=sharing)
 - [SeaweedFS Introduction Slides 2019.3](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
 

docker/Dockerfile.go_build

@@ -6,7 +6,7 @@ ARG BRANCH=${BRANCH:-master}
 ARG TAGS
 RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
 RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
-  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
   && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
 
 FROM alpine AS final

docker/Dockerfile.rocksdb_dev_env

@@ -3,7 +3,7 @@ FROM golang:1.24 as builder
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v9.10.0
+ENV ROCKSDB_VERSION v10.2.1
 
 # build RocksDB
 RUN cd /tmp && \

docker/Dockerfile.rocksdb_large

@@ -3,7 +3,7 @@ FROM golang:1.24 as builder
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v9.10.0
+ENV ROCKSDB_VERSION v10.2.1
 
 # build RocksDB
 RUN cd /tmp && \
@@ -21,7 +21,7 @@ RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedf
 ARG BRANCH=${BRANCH:-master}
 RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
 RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
-  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
   && go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
 
 

docker/Dockerfile.rocksdb_large_local

@@ -5,7 +5,7 @@ RUN mkdir -p /go/src/github.com/seaweedfs/
 ADD . /go/src/github.com/seaweedfs/seaweedfs
 RUN ls -al /go/src/github.com/seaweedfs/ && \
   cd /go/src/github.com/seaweedfs/seaweedfs/weed \
-  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
   && go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
 
 

docker/Makefile

@@ -7,7 +7,7 @@ gen: dev
 cgo ?= 0
 binary:
 	export SWCOMMIT=$(shell git rev-parse --short HEAD)
-	export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
+	export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(SWCOMMIT)"
 	cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w -extldflags -static $(SWLDFLAGS)" && mv weed ../docker/
 	cd ../other/mq_client_example/agent_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_pub_record ../../../docker/
 	cd ../other/mq_client_example/agent_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_sub_record ../../../docker/

docker/compose/userstore.json

@@ -0,0 +1,37 @@
+[
+  {
+    "Username": "admin",
+    "Password": "myadminpassword",
+    "PublicKeys": [
+    ],
+    "HomeDir": "/",
+    "Permissions": {
+      "/": ["*"]
+    },
+    "Uid": 0,
+    "Gid": 0
+  },
+  {
+    "Username": "user1",
+    "Password": "myuser1password",
+    "PublicKeys": [""],
+    "HomeDir": "/user1",
+    "Permissions": {
+      "/user1": ["*"],
+      "/public": ["read", "list","write"]
+    },
+    "Uid": 1111,
+    "Gid": 1111
+  },
+  {
+    "Username": "readonly",
+    "Password": "myreadonlypassword",
+    "PublicKeys": [],
+    "HomeDir": "/public",
+    "Permissions": {
+      "/public": ["read", "list"]
+    },
+    "Uid": 1112,
+    "Gid": 1112
+  }
+]

go.mod

@@ -5,13 +5,13 @@ go 1.24
 toolchain go1.24.1
 
 require (
-	cloud.google.com/go v0.120.0 // indirect
+	cloud.google.com/go v0.121.0 // indirect
 	cloud.google.com/go/pubsub v1.49.0
-	cloud.google.com/go/storage v1.51.0
+	cloud.google.com/go/storage v1.54.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Shopify/sarama v1.38.1
-	github.com/aws/aws-sdk-go v1.55.6
+	github.com/aws/aws-sdk-go v1.55.7
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cenkalti/backoff/v4 v4.3.0
@@ -29,10 +29,9 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
-	github.com/fclairamb/ftpserverlib v0.25.0
 	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/go-redsync/redsync/v4 v4.13.0
-	github.com/go-sql-driver/mysql v1.9.1
+	github.com/go-sql-driver/mysql v1.9.2
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.7.0
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
@@ -41,7 +40,7 @@ require (
 	github.com/google/btree v1.1.3
 	github.com/google/uuid v1.6.0
 	github.com/google/wire v0.6.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.14.2 // indirect
 	github.com/gorilla/mux v1.8.1
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -57,7 +56,7 @@ require (
 	github.com/klauspost/reedsolomon v1.12.4
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.9
-	github.com/linxGnu/grocksdb v1.9.9
+	github.com/linxGnu/grocksdb v1.10.1
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.11 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
@@ -69,10 +68,10 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/posener/complete v1.2.3
 	github.com/pquerna/cachecontrol v0.2.0
-	github.com/prometheus/client_golang v1.21.1
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.62.0 // indirect
-	github.com/prometheus/procfs v0.16.0
+	github.com/prometheus/client_golang v1.22.0
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.64.0 // indirect
+	github.com/prometheus/procfs v0.16.1
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
@@ -95,30 +94,30 @@ require (
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	go.etcd.io/etcd/client/v3 v3.5.18
-	go.mongodb.org/mongo-driver v1.17.3
+	go.etcd.io/etcd/client/v3 v3.6.1
+	go.mongodb.org/mongo-driver v1.17.4
 	go.opencensus.io v0.24.0 // indirect
 	gocloud.dev v0.41.0
 	gocloud.dev/pubsub/natspubsub v0.41.0
 	gocloud.dev/pubsub/rabbitpubsub v0.41.0
-	golang.org/x/crypto v0.37.0 // indirect
-	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394
-	golang.org/x/image v0.24.0
-	golang.org/x/net v0.39.0
-	golang.org/x/oauth2 v0.29.0 // indirect
-	golang.org/x/sys v0.32.0
-	golang.org/x/text v0.24.0 // indirect
-	golang.org/x/tools v0.31.0
+	golang.org/x/crypto v0.39.0
+	golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476
+	golang.org/x/image v0.28.0
+	golang.org/x/net v0.41.0
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/sys v0.33.0
+	golang.org/x/text v0.26.0 // indirect
+	golang.org/x/tools v0.34.0
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
-	google.golang.org/api v0.230.0
-	google.golang.org/genproto v0.0.0-20250324211829-b45e905df463 // indirect
-	google.golang.org/grpc v1.72.0
+	google.golang.org/api v0.238.0
+	google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect
+	google.golang.org/grpc v1.73.0
 	google.golang.org/protobuf v1.36.6
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
 	modernc.org/mathutil v1.7.1
-	modernc.org/memory v1.9.1 // indirect
-	modernc.org/sqlite v1.37.0
+	modernc.org/memory v1.11.0 // indirect
+	modernc.org/sqlite v1.38.0
 	modernc.org/strutil v1.2.1
 )
 
@@ -126,132 +125,140 @@ require (
 	github.com/Jille/raft-grpc-transport v1.6.1
 	github.com/arangodb/go-driver v1.6.6
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.36.3
-	github.com/aws/aws-sdk-go-v2/config v1.29.13
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.66
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2
-	github.com/cognusion/imaging v1.0.1
-	github.com/fluent/fluent-logger-golang v1.9.0
-	github.com/getsentry/sentry-go v0.31.1
+	github.com/aws/aws-sdk-go-v2 v1.36.5
+	github.com/aws/aws-sdk-go-v2/config v1.29.17
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.70
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.80.1
+	github.com/cognusion/imaging v1.0.2
+	github.com/fluent/fluent-logger-golang v1.10.0
+	github.com/getsentry/sentry-go v0.33.0
 	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
-	github.com/hanwen/go-fuse/v2 v2.7.2
+	github.com/hanwen/go-fuse/v2 v2.8.0
 	github.com/hashicorp/raft v1.7.3
 	github.com/hashicorp/raft-boltdb/v2 v2.3.1
-	github.com/minio/crc64nvme v1.0.1
+	github.com/minio/crc64nvme v1.0.2
 	github.com/orcaman/concurrent-map/v2 v2.0.1
-	github.com/parquet-go/parquet-go v0.24.0
+	github.com/parquet-go/parquet-go v0.25.1
+	github.com/pkg/sftp v1.13.9
 	github.com/rabbitmq/amqp091-go v1.10.0
-	github.com/rclone/rclone v1.69.1
-	github.com/rdleal/intervalst v1.4.1
-	github.com/redis/go-redis/v9 v9.7.3
+	github.com/rclone/rclone v1.70.1
+	github.com/rdleal/intervalst v1.5.0
+	github.com/redis/go-redis/v9 v9.10.0
 	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/shirou/gopsutil/v3 v3.24.5
-	github.com/tarantool/go-tarantool/v2 v2.3.0
+	github.com/tarantool/go-tarantool/v2 v2.3.2
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.106.1
-	go.etcd.io/etcd/client/pkg/v3 v3.5.21
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.110.1
+	go.etcd.io/etcd/client/pkg/v3 v3.6.1
 	go.uber.org/atomic v1.11.0
-	golang.org/x/sync v0.13.0
+	golang.org/x/sync v0.15.0
 	google.golang.org/grpc/security/advancedtls v1.0.0
 )
 
 require (
-	cel.dev/expr v0.22.1 // indirect
-	cloud.google.com/go/auth v0.16.0 // indirect
+	cel.dev/expr v0.23.0 // indirect
+	cloud.google.com/go/auth v0.16.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
-	cloud.google.com/go/compute/metadata v0.6.0 // indirect
-	cloud.google.com/go/iam v1.4.2 // indirect
-	cloud.google.com/go/monitoring v1.24.1 // indirect
+	cloud.google.com/go/compute/metadata v0.7.0 // indirect
+	cloud.google.com/go/iam v1.5.2 // indirect
+	cloud.google.com/go/monitoring v1.24.2 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.4.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.1 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
-	github.com/Files-com/files-sdk-go/v3 v3.2.97 // indirect
+	github.com/Files-com/files-sdk-go/v3 v3.2.173 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
+	github.com/IBM/go-sdk-core/v5 v5.20.0 // indirect
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
 	github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
-	github.com/ProtonMail/go-crypto v1.1.3 // indirect
+	github.com/ProtonMail/go-crypto v1.3.0 // indirect
 	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
 	github.com/ProtonMail/go-srp v0.0.7 // indirect
-	github.com/ProtonMail/gopenpgp/v2 v2.7.4 // indirect
-	github.com/PuerkitoBio/goquery v1.8.1 // indirect
+	github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
+	github.com/PuerkitoBio/goquery v1.10.3 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
+	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.69 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.77 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.34.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 // indirect
-	github.com/aws/smithy-go v1.22.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect
+	github.com/aws/smithy-go v1.22.4 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
-	github.com/bradenaw/juniper v0.15.2 // indirect
+	github.com/bradenaw/juniper v0.15.3 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
 	github.com/buengese/sgzip v0.1.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
 	github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
-	github.com/cloudflare/circl v1.3.7 // indirect
-	github.com/cloudinary/cloudinary-go/v2 v2.9.0 // indirect
-	github.com/cloudsoda/go-smb2 v0.0.0-20241223203758-52b943b88fd6 // indirect
+	github.com/cloudflare/circl v1.6.1 // indirect
+	github.com/cloudinary/cloudinary-go/v2 v2.10.0 // indirect
+	github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
+	github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
 	github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect
 	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
-	github.com/creasty/defaults v1.7.0 // indirect
+	github.com/creasty/defaults v1.8.0 // indirect
 	github.com/cronokirby/saferith v0.33.0 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
-	github.com/ebitengine/purego v0.8.1 // indirect
+	github.com/ebitengine/purego v0.8.4 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
-	github.com/emersion/go-message v0.18.0 // indirect
-	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
-	github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
+	github.com/emersion/go-message v0.18.2 // indirect
+	github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
 	github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/fatih/color v1.16.0 // indirect
-	github.com/fclairamb/go-log v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/flynn/noise v1.0.1 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.7 // indirect
-	github.com/geoffgarside/ber v1.1.0 // indirect
-	github.com/go-chi/chi/v5 v5.1.0 // indirect
+	github.com/flynn/noise v1.1.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.9 // indirect
+	github.com/geoffgarside/ber v1.2.0 // indirect
+	github.com/go-chi/chi/v5 v5.2.2 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
-	github.com/go-resty/resty/v2 v2.11.0 // indirect
+	github.com/go-openapi/errors v0.22.1 // indirect
+	github.com/go-openapi/strfmt v0.23.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.26.0 // indirect
+	github.com/go-resty/resty/v2 v2.16.5 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
-	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/gofrs/flock v0.12.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
 	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -266,76 +273,77 @@ require (
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 // indirect
-	github.com/jonboulle/clockwork v0.3.0 // indirect
+	github.com/jonboulle/clockwork v0.5.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/lanrat/extsort v1.0.2 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lpar/date v1.0.0 // indirect
-	github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nats-io/nats.go v1.40.1 // indirect
 	github.com/nats-io/nkeys v0.4.10 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
-	github.com/ncw/swift/v2 v2.0.3 // indirect
+	github.com/ncw/swift/v2 v2.0.4 // indirect
 	github.com/nxadm/tail v1.4.11 // indirect
-	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/onsi/ginkgo/v2 v2.19.0 // indirect
-	github.com/onsi/gomega v1.34.1 // indirect
+	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/onsi/ginkgo/v2 v2.23.3 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.80.0 // indirect
-	github.com/panjf2000/ants/v2 v2.9.1 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.93.0 // indirect
+	github.com/panjf2000/ants/v2 v2.11.3 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
-	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
 	github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/pkg/sftp v1.13.7 // indirect
 	github.com/pkg/xattr v0.4.10 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
-	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
+	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
-	github.com/relvacode/iso8601 v1.3.0 // indirect
+	github.com/relvacode/iso8601 v1.6.0 // indirect
 	github.com/rfjakob/eme v1.1.2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
 	github.com/sagikazarmark/locafero v0.7.0 // indirect
-	github.com/samber/lo v1.47.0 // indirect
-	github.com/shirou/gopsutil/v4 v4.24.12 // indirect
+	github.com/samber/lo v1.50.0 // indirect
+	github.com/shirou/gopsutil/v4 v4.25.5 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
-	github.com/sony/gobreaker v0.5.0 // indirect
+	github.com/sony/gobreaker v1.0.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
-	github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
+	github.com/spacemonkeygo/monkit/v3 v3.0.24 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/t3rm1n4l/go-mega v0.0.0-20241213150454-ec0027fb0002 // indirect
+	github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 // indirect
 	github.com/tarantool/go-iproto v1.1.0 // indirect
 	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
 	github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
-	github.com/tinylib/msgp v1.1.8 // indirect
-	github.com/tklauser/go-sysconf v0.3.13 // indirect
-	github.com/tklauser/numcpus v0.7.0 // indirect
+	github.com/tinylib/msgp v1.3.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.15 // indirect
+	github.com/tklauser/numcpus v0.10.0 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
 	github.com/unknwon/goconfig v1.0.0 // indirect
-	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
@@ -344,36 +352,37 @@ require (
 	github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	github.com/zeebo/blake3 v0.2.3 // indirect
+	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
-	go.etcd.io/bbolt v1.3.10 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.18 // indirect
+	go.etcd.io/bbolt v1.4.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.6.1 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
-	go.opentelemetry.io/otel v1.35.0 // indirect
-	go.opentelemetry.io/otel/metric v1.35.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.35.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
-	go.opentelemetry.io/otel/trace v1.35.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+	go.opentelemetry.io/otel v1.36.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.36.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.36.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/term v0.31.0 // indirect
-	golang.org/x/time v0.11.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250414145226-207652e42e2e // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/time v0.12.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	modernc.org/libc v1.62.1 // indirect
+	modernc.org/libc v1.65.10 // indirect
 	moul.io/http2curl/v2 v2.3.0 // indirect
-	storj.io/common v0.0.0-20240812101423-26b53789c348 // indirect
-	storj.io/drpc v0.0.35-0.20240709171858-0075ac871661 // indirect
-	storj.io/eventkit v0.0.0-20240415002644-1d9596fee086 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+	storj.io/common v0.0.0-20250605163628-70ca83b6228e // indirect
+	storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
+	storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
 	storj.io/infectious v0.0.2 // indirect
-	storj.io/picobuf v0.0.3 // indirect
+	storj.io/picobuf v0.0.4 // indirect
 	storj.io/uplink v1.13.1 // indirect
 )
 

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.87"
+appVersion: "3.92"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.387
+version: 4.0.392

k8s/charts/seaweedfs/templates/_helpers.tpl

@@ -73,6 +73,16 @@ Inject extra environment vars in the format key:value, if populated
 {{- end -}}
 {{- end -}}
 
+{{/* Return the proper sftp image */}}
+{{- define "sftp.image" -}}
+{{- if .Values.sftp.imageOverride -}}
+{{- $imageOverride := .Values.sftp.imageOverride -}}
+{{- printf "%s" $imageOverride -}}
+{{- else -}}
+{{- include "common.image" . }}
+{{- end -}}
+{{- end -}}
+
 {{/* Return the proper volume image */}}
 {{- define "volume.image" -}}
 {{- if .Values.volume.imageOverride -}}
@@ -88,7 +98,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- $registryName := default .Values.image.registry .Values.global.registry | toString -}}
 {{- $repositoryName := .Values.image.repository | toString -}}
 {{- $name := .Values.global.imageName | toString -}}
-{{- $tag := .Chart.AppVersion | toString -}}
+{{- $tag := default .Chart.AppVersion .Values.image.tag  | toString -}}
 {{- if $registryName -}}
 {{- printf "%s/%s%s:%s" $registryName $repositoryName $name $tag -}}
 {{- else -}}
@@ -168,3 +178,45 @@ Usage:
     {{- $value }}
 {{- end }}
 {{- end -}}
+
+
+{{/*
+getOrGeneratePassword will check if a password exists in a secret and return it,
+or generate a new random password if it doesn't exist.
+*/}}
+{{- define "getOrGeneratePassword" -}}
+{{- $params := . -}}
+{{- $namespace := $params.namespace -}}
+{{- $secretName := $params.secretName -}}
+{{- $key := $params.key -}}
+{{- $length := default 16 $params.length -}}
+
+{{- $existingSecret := lookup "v1" "Secret" $namespace $secretName -}}
+{{- if and $existingSecret (index $existingSecret.data $key) -}}
+  {{- index $existingSecret.data $key | b64dec -}}
+{{- else -}}
+  {{- randAlphaNum $length -}}
+{{- end -}}
+{{- end -}}
+
+{{- /*
+Render a component’s topologySpreadConstraints exactly as given in values,
+respecting string vs. list, and providing the component name for tpl lookups.
+
+Usage:
+  {{ include "seaweedfs.topologySpreadConstraints" (dict "Values" .Values "component" "filer") | nindent 8 }}
+*/ -}}
+{{- define "seaweedfs.topologySpreadConstraints" -}}
+  {{- $vals := .Values -}}
+  {{- $comp := .component -}}
+  {{- $section := index $vals $comp | default dict -}}
+  {{- $tsp := index $section "topologySpreadConstraints" -}}
+  {{- with $tsp }}
+topologySpreadConstraints:
+{{- if kindIs "string" $tsp }}
+{{ tpl $tsp (dict "Values" $vals "component" $comp) }}
+{{- else }}
+{{ toYaml $tsp }}
+{{- end }}
+  {{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/all-in-one-deployment.yaml

@@ -0,0 +1,427 @@
+{{- if .Values.allInOne.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "seaweedfs.name" . }}-all-in-one
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: seaweedfs-all-in-one
+  {{- if .Values.allInOne.annotations }}
+  annotations:
+    {{- toYaml .Values.allInOne.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: seaweedfs-all-in-one
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+        helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: seaweedfs-all-in-one
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.allInOne.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      annotations:
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.allInOne.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      restartPolicy: {{ default .Values.global.restartPolicy .Values.allInOne.restartPolicy }}
+      {{- if .Values.allInOne.affinity }}
+      affinity:
+        {{ tpl .Values.allInOne.affinity . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.allInOne.topologySpreadConstraints }}
+      {{- include "seaweedfs.topologySpreadConstraints" (dict "Values" .Values "component" "all-in-one") | nindent 6 }}
+      {{- end }}
+      {{- if .Values.allInOne.tolerations }}
+      tolerations:
+        {{- tpl .Values.allInOne.tolerations . | nindent 8 }}
+      {{- end }}
+      {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
+      terminationGracePeriodSeconds: 60
+      enableServiceLinks: false
+      {{- if .Values.allInOne.priorityClassName }}
+      priorityClassName: {{ .Values.allInOne.priorityClassName | quote }}
+      {{- end }}
+      {{- if .Values.allInOne.serviceAccountName }}
+      serviceAccountName: {{ .Values.allInOne.serviceAccountName | quote }}
+      {{- end }}
+      {{- if .Values.allInOne.initContainers }}
+      initContainers:
+        {{- tpl .Values.allInOne.initContainers . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.allInOne.podSecurityContext.enabled }}
+      securityContext:
+        {{- omit .Values.allInOne.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: seaweedfs
+          image: {{ template "master.image" . }}
+          imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: SEAWEEDFS_FULLNAME
+              value: "{{ template "seaweedfs.name" . }}"
+            {{- if .Values.allInOne.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.allInOne.extraEnvironmentVars }}
+            - name: {{ $key }}
+              {{- if kindIs "string" $value }}
+              value: {{ $value | quote }}
+              {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 }}
+              {{- end }}
+            {{- end }}
+            {{- end }}
+            {{- if .Values.global.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
+            - name: {{ $key }}
+              {{- if kindIs "string" $value }}
+              value: {{ $value | quote }}
+              {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 }}
+              {{- end }}
+            {{- end }}
+            {{- end }}
+          command:
+            - "/bin/sh"
+            - "-ec"
+            - |
+              /usr/bin/weed \
+              -v={{ .Values.global.loggingLevel }} \
+              server \
+              -dir=/data \
+              -master \
+              -volume \
+              -ip=${POD_IP} \
+              -ip.bind=0.0.0.0 \
+              {{- if .Values.allInOne.idleTimeout }}
+              -idleTimeout={{ .Values.allInOne.idleTimeout }} \
+              {{- end }}
+              {{- if .Values.allInOne.dataCenter }}
+              -dataCenter={{ .Values.allInOne.dataCenter }} \
+              {{- end }}
+              {{- if .Values.allInOne.rack }}
+              -rack={{ .Values.allInOne.rack }} \
+              {{- end }}
+              {{- if .Values.allInOne.whiteList }}
+              -whiteList={{ .Values.allInOne.whiteList }} \
+              {{- end }}
+              {{- if .Values.allInOne.disableHttp }}
+              -disableHttp={{ .Values.allInOne.disableHttp }} \
+              {{- end }}
+              -master.port={{ .Values.master.port }} \
+              {{- if .Values.global.enableReplication }}
+              -master.defaultReplication={{ .Values.global.replicationPlacement }} \
+              {{- else }}
+              -master.defaultReplication={{ .Values.master.defaultReplication }} \
+              {{- end }}
+              {{- if .Values.master.volumePreallocate }}
+              -master.volumePreallocate \
+              {{- end }}
+              -master.volumeSizeLimitMB={{ .Values.master.volumeSizeLimitMB }} \
+              {{- if .Values.master.garbageThreshold }}
+              -master.garbageThreshold={{ .Values.master.garbageThreshold }} \
+              {{- end }}
+              -volume.port={{ .Values.volume.port }} \
+              -volume.readMode={{ .Values.volume.readMode }} \
+              {{- if .Values.volume.imagesFixOrientation }}
+              -volume.images.fix.orientation \
+              {{- end }}
+              {{- if .Values.volume.index }}
+              -volume.index={{ .Values.volume.index }} \
+              {{- end }}
+              {{- if .Values.volume.fileSizeLimitMB }}
+              -volume.fileSizeLimitMB={{ .Values.volume.fileSizeLimitMB }} \
+              {{- end }}
+              -volume.minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \
+              -volume.compactionMBps={{ .Values.volume.compactionMBps }} \
+              {{- if .Values.allInOne.metricsPort }}
+              -metricsPort={{ .Values.allInOne.metricsPort }} \
+              {{- else if .Values.master.metricsPort }}
+              -metricsPort={{ .Values.master.metricsPort }} \
+              {{- end }}
+              -filer \
+              -filer.port={{ .Values.filer.port }} \
+              {{- if .Values.filer.disableDirListing }}
+              -filer.disableDirListing \
+              {{- end }}
+              -filer.dirListLimit={{ .Values.filer.dirListLimit }} \
+              {{- if .Values.global.enableReplication }}
+              -filer.defaultReplicaPlacement={{ .Values.global.replicationPlacement }} \
+              {{- else }}
+              -filer.defaultReplicaPlacement={{ .Values.filer.defaultReplicaPlacement }} \
+              {{- end }}
+              {{- if .Values.filer.maxMB }}
+              -filer.maxMB={{ .Values.filer.maxMB }} \
+              {{- end }}
+              {{- if .Values.filer.encryptVolumeData }}
+              -filer.encryptVolumeData \
+              {{- end }}
+              {{- if .Values.filer.filerGroup}}
+              -filer.filerGroup={{ .Values.filer.filerGroup}} \
+              {{- end }}
+              {{- if .Values.filer.rack }}
+              -filer.rack={{ .Values.filer.rack }} \
+              {{- end }}
+              {{- if .Values.filer.dataCenter }}
+              -filer.dataCenter={{ .Values.filer.dataCenter }} \
+              {{- end }}
+              {{- if .Values.allInOne.s3.enabled }}
+              -s3 \
+              -s3.port={{ .Values.s3.port }} \
+              {{- if .Values.s3.domainName }}
+              -s3.domainName={{ .Values.s3.domainName }} \
+              {{- end }}
+              {{- if .Values.global.enableSecurity }}
+              {{- if .Values.s3.httpsPort }}
+              -s3.port.https={{ .Values.s3.httpsPort }} \
+              {{- end }}
+              -s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
+              -s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
+              {{- end }}
+              {{- if eq (typeOf .Values.s3.allowEmptyFolder) "bool" }}
+              -s3.allowEmptyFolder={{ .Values.s3.allowEmptyFolder }} \
+              {{- end }}
+              {{- if .Values.s3.enableAuth }}
+              -s3.config=/etc/sw/s3/seaweedfs_s3_config \
+              {{- end }}
+              {{- if .Values.s3.auditLogConfig }}
+              -s3.auditLogConfig=/etc/sw/s3/s3_auditLogConfig.json \
+              {{- end }}
+              {{- end }}
+              {{- if .Values.allInOne.sftp.enabled }}
+              -sftp \
+              -sftp.port={{ .Values.sftp.port }} \
+              {{- if .Values.sftp.sshPrivateKey }}
+              -sftp.sshPrivateKey={{ .Values.sftp.sshPrivateKey }} \
+              {{- end }}
+              {{- if .Values.sftp.hostKeysFolder }}
+              -sftp.hostKeysFolder={{ .Values.sftp.hostKeysFolder }} \
+              {{- end }}
+              {{- if .Values.sftp.authMethods }}
+              -sftp.authMethods={{ .Values.sftp.authMethods }} \
+              {{- end }}
+              {{- if .Values.sftp.maxAuthTries }}
+              -sftp.maxAuthTries={{ .Values.sftp.maxAuthTries }} \
+              {{- end }}
+              {{- if .Values.sftp.bannerMessage }}
+              -sftp.bannerMessage="{{ .Values.sftp.bannerMessage }}" \
+              {{- end }}
+              {{- if .Values.sftp.loginGraceTime }}
+              -sftp.loginGraceTime={{ .Values.sftp.loginGraceTime }} \
+              {{- end }}
+              {{- if .Values.sftp.clientAliveInterval }}
+              -sftp.clientAliveInterval={{ .Values.sftp.clientAliveInterval }} \
+              {{- end }}
+              {{- if .Values.sftp.clientAliveCountMax }}
+              -sftp.clientAliveCountMax={{ .Values.sftp.clientAliveCountMax }} \
+              {{- end }}
+              -sftp.userStoreFile=/etc/sw/sftp/seaweedfs_sftp_config \
+              {{- end }}
+
+          volumeMounts:
+            - name: data
+              mountPath: /data
+            {{- if and .Values.allInOne.s3.enabled (or .Values.s3.enableAuth .Values.filer.s3.enableAuth) }}
+            - name: config-s3-users
+              mountPath: /etc/sw/s3
+              readOnly: true
+            {{- end }}
+            {{- if .Values.allInOne.sftp.enabled }}
+            - name: config-ssh
+              mountPath: /etc/sw/ssh
+              readOnly: true
+            - mountPath: /etc/sw/sftp
+              name: config-users
+              readOnly: true
+            {{- end }}
+            {{- if .Values.filer.notificationConfig }}
+            - name: notification-config
+              mountPath: /etc/seaweedfs/notification.toml
+              subPath: notification.toml
+              readOnly: true
+            {{- end }}
+            - name: master-config
+              mountPath: /etc/seaweedfs/master.toml
+              subPath: master.toml
+              readOnly: true
+            {{- if .Values.global.enableSecurity }}
+            - name: security-config
+              mountPath: /etc/seaweedfs/security.toml
+              subPath: security.toml
+              readOnly: true
+            - name: ca-cert
+              mountPath: /usr/local/share/ca-certificates/ca/
+              readOnly: true
+            - name: master-cert
+              mountPath: /usr/local/share/ca-certificates/master/
+              readOnly: true
+            - name: volume-cert
+              mountPath: /usr/local/share/ca-certificates/volume/
+              readOnly: true
+            - name: filer-cert
+              mountPath: /usr/local/share/ca-certificates/filer/
+              readOnly: true
+            - name: client-cert
+              mountPath: /usr/local/share/ca-certificates/client/
+              readOnly: true
+            {{- end }}
+            {{ tpl .Values.allInOne.extraVolumeMounts . | nindent 12 }}
+          ports:
+            - containerPort: {{ .Values.master.port }}
+              name: swfs-mas
+            - containerPort: {{ .Values.master.grpcPort }}
+              name: swfs-mas-grpc
+            - containerPort: {{ .Values.volume.port }}
+              name: swfs-vol
+            - containerPort: {{ .Values.volume.grpcPort }}
+              name: swfs-vol-grpc
+            - containerPort: {{ .Values.filer.port }}
+              name: swfs-fil
+            - containerPort: {{ .Values.filer.grpcPort }}
+              name: swfs-fil-grpc
+            {{- if .Values.allInOne.s3.enabled }}
+            - containerPort: {{ .Values.s3.port }}
+              name: swfs-s3
+              {{- if .Values.s3.httpsPort }}
+            - containerPort: {{ .Values.s3.httpsPort }}
+              name: swfs-s3-tls
+              {{- end }}
+            {{- end }}
+            {{- if .Values.allInOne.sftp.enabled }}
+            - containerPort: {{ .Values.sftp.port }}
+              name: swfs-sftp
+            {{- end }}
+            {{- if .Values.allInOne.metricsPort }}
+            - containerPort: {{ .Values.allInOne.metricsPort }}
+              name: server-metrics
+            {{- end }}
+          {{- if .Values.allInOne.readinessProbe.enabled }}
+          readinessProbe:
+            httpGet:
+              path: {{ .Values.allInOne.readinessProbe.httpGet.path }}
+              port: {{ .Values.master.port }}
+              scheme: {{ .Values.allInOne.readinessProbe.scheme }}
+            initialDelaySeconds: {{ .Values.allInOne.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.allInOne.readinessProbe.periodSeconds }}
+            successThreshold: {{ .Values.allInOne.readinessProbe.successThreshold }}
+            failureThreshold: {{ .Values.allInOne.readinessProbe.failureThreshold }}
+            timeoutSeconds: {{ .Values.allInOne.readinessProbe.timeoutSeconds }}
+          {{- end }}
+          {{- if .Values.allInOne.livenessProbe.enabled }}
+          livenessProbe:
+            httpGet:
+              path: {{ .Values.allInOne.livenessProbe.httpGet.path }}
+              port: {{ .Values.master.port }}
+              scheme: {{ .Values.allInOne.livenessProbe.scheme }}
+            initialDelaySeconds: {{ .Values.allInOne.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.allInOne.livenessProbe.periodSeconds }}
+            successThreshold: {{ .Values.allInOne.livenessProbe.successThreshold }}
+            failureThreshold: {{ .Values.allInOne.livenessProbe.failureThreshold }}
+            timeoutSeconds: {{ .Values.allInOne.livenessProbe.timeoutSeconds }}
+          {{- end }}
+          {{- with .Values.allInOne.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- if .Values.allInOne.containerSecurityContext.enabled }}
+          securityContext:
+            {{- omit .Values.allInOne.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
+      {{- if .Values.allInOne.sidecars }}
+      {{- include "common.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: data
+          {{- if eq .Values.allInOne.data.type "hostPath" }}
+          hostPath:
+            path: {{ .Values.allInOne.data.hostPathPrefix }}/seaweedfs-all-in-one-data/
+            type: DirectoryOrCreate
+          {{- else if eq .Values.allInOne.data.type "persistentVolumeClaim" }}
+          persistentVolumeClaim:
+            claimName: {{ .Values.allInOne.data.claimName }}
+          {{- else if eq .Values.allInOne.data.type "emptyDir" }}
+          emptyDir: {}
+          {{- end }}
+        {{- if and .Values.allInOne.s3.enabled (or .Values.s3.enableAuth .Values.filer.s3.enableAuth) }}
+        - name: config-s3-users
+          secret:
+            defaultMode: 420
+            secretName: {{ default (printf "%s-s3-secret" (include "seaweedfs.name" .)) (or .Values.s3.existingConfigSecret .Values.filer.s3.existingConfigSecret) }}
+        {{- end }}
+        {{- if .Values.allInOne.sftp.enabled }}
+        - name: config-ssh
+          secret:
+            defaultMode: 420
+            secretName: {{ default (printf "%s-sftp-ssh-secret" (include "seaweedfs.name" .)) .Values.sftp.existingSshConfigSecret }}
+        - name: config-users
+          secret:
+            defaultMode: 420
+            secretName: {{ default (printf "%s-sftp-secret" (include "seaweedfs.name" .)) .Values.sftp.existingConfigSecret }}
+        {{- end }}
+        {{- if .Values.filer.notificationConfig }}
+        - name: notification-config
+          configMap:
+            name: {{ template "seaweedfs.name" . }}-notification-config
+        {{- end }}
+        - name: master-config
+          configMap:
+            name: {{ template "seaweedfs.name" . }}-master-config
+        {{- if .Values.global.enableSecurity }}
+        - name: security-config
+          configMap:
+            name: {{ template "seaweedfs.name" . }}-security-config
+        - name: ca-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-ca-cert
+        - name: master-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-master-cert
+        - name: volume-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-volume-cert
+        - name: filer-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-filer-cert
+        - name: client-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-client-cert
+        {{- end }}
+        {{ tpl .Values.allInOne.extraVolumes . | nindent 8 }}
+      {{- if .Values.allInOne.nodeSelector }}
+      nodeSelector:
+        {{ tpl .Values.allInOne.nodeSelector . | nindent 8 }}
+      {{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/all-in-one-pvc.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.allInOne.enabled (eq .Values.allInOne.data.type "persistentVolumeClaim") }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ .Values.allInOne.data.claimName }}
+  labels:
+    app.kubernetes.io/component: seaweedfs-all-in-one
+  {{- if .Values.allInOne.annotations }}
+  annotations:
+    {{- toYaml .Values.allInOne.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.allInOne.data.size }}
+  {{- if .Values.allInOne.data.storageClass }}
+  storageClassName: {{ .Values.allInOne.data.storageClass }}
+  {{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/all-in-one-service.yml

@@ -0,0 +1,83 @@
+{{- if .Values.allInOne.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "seaweedfs.name" . }}-all-in-one
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: seaweedfs-all-in-one
+  {{- if .Values.allInOne.service.annotations }}
+  annotations:
+    {{- toYaml .Values.allInOne.service.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  internalTrafficPolicy: {{ .Values.allInOne.service.internalTrafficPolicy | default "Cluster" }}
+  ports:
+  # Master ports
+  - name: "swfs-master"
+    port: {{ .Values.master.port }}
+    targetPort: {{ .Values.master.port }}
+    protocol: TCP
+  - name: "swfs-master-grpc"
+    port: {{ .Values.master.grpcPort }}
+    targetPort: {{ .Values.master.grpcPort }}
+    protocol: TCP
+  
+  # Volume ports
+  - name: "swfs-volume"
+    port: {{ .Values.volume.port }}
+    targetPort: {{ .Values.volume.port }}
+    protocol: TCP
+  - name: "swfs-volume-grpc"
+    port: {{ .Values.volume.grpcPort }}
+    targetPort: {{ .Values.volume.grpcPort }}
+    protocol: TCP
+  
+  # Filer ports
+  - name: "swfs-filer"
+    port: {{ .Values.filer.port }}
+    targetPort: {{ .Values.filer.port }}
+    protocol: TCP
+  - name: "swfs-filer-grpc"
+    port: {{ .Values.filer.grpcPort }}
+    targetPort: {{ .Values.filer.grpcPort }}
+    protocol: TCP
+  
+  # S3 ports (if enabled)
+  {{- if .Values.allInOne.s3.enabled }}
+  - name: "swfs-s3"
+    port: {{ if .Values.allInOne.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
+    targetPort: {{ if .Values.allInOne.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
+    protocol: TCP
+  {{- if and .Values.allInOne.s3.enabled .Values.s3.httpsPort }}
+  - name: "swfs-s3-tls"
+    port: {{ .Values.s3.httpsPort }}
+    targetPort: {{ .Values.s3.httpsPort }}
+    protocol: TCP
+  {{- end }}
+  {{- end }}
+  
+  # SFTP ports (if enabled)
+  {{- if .Values.allInOne.sftp.enabled }}
+  - name: "swfs-sftp"
+    port: {{ .Values.sftp.port }}
+    targetPort: {{ .Values.sftp.port }}
+    protocol: TCP
+  {{- end }}
+  
+  # Server metrics port (single metrics endpoint for all services)
+  {{- if .Values.allInOne.metricsPort }}
+  - name: "server-metrics"
+    port: {{ .Values.allInOne.metricsPort }}
+    targetPort: {{ .Values.allInOne.metricsPort }}
+    protocol: TCP
+  {{- end }}
+  
+  selector:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    app.kubernetes.io/component: seaweedfs-all-in-one
+{{- end }}

k8s/charts/seaweedfs/templates/all-in-one-servicemonitor.yaml

@@ -0,0 +1,29 @@
+{{- if .Values.allInOne.enabled }}
+{{- if .Values.global.monitoring.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "seaweedfs.name" . }}-all-in-one
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: all-in-one
+    {{- with .Values.global.monitoring.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+    {{- if .Values.allInOne.metricsPort }}
+    - interval: 30s
+      port: server-metrics
+      scrapeTimeout: 5s
+    {{- end }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+      app.kubernetes.io/component: seaweedfs-all-in-one
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/cosi-deployment.yaml

@@ -9,6 +9,7 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: objectstorage-provisioner
 spec:
   replicas: {{ .Values.cosi.replicas }}
   selector:
@@ -39,6 +40,13 @@ spec:
       {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.cosi.restartPolicy }}
+      {{- if .Values.cosi.affinity }}
+      affinity:
+        {{ tpl .Values.cosi.affinity . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.cosi.topologySpreadConstraints }}
+      {{- include "seaweedfs.topologySpreadConstraints" (dict "Values" .Values "component" "objectstorage-provisioner") | nindent 6 }}
+      {{- end }}
       {{- if .Values.cosi.tolerations }}
       tolerations:
         {{ tpl .Values.cosi.tolerations . | nindent 8 | trim }}

k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@@ -61,9 +61,9 @@ spec:
       affinity:
         {{ tpl .Values.filer.affinity . | nindent 8 | trim }}
       {{- end }}
-      {{- with .Values.filer.topologySpreadConstraints }}
+      {{- if .Values.filer.topologySpreadConstraints }}
       topologySpreadConstraints:
-        {{- toYaml . | nindent 8 }}
+      {{- include "seaweedfs.topologySpreadConstraints" (dict "Values" .Values "component" "filer") | nindent 6 }}
       {{- end }}
       {{- if .Values.filer.tolerations }}
       tolerations:
@@ -162,6 +162,9 @@ spec:
               {{- if .Values.filer.metricsPort }}
               -metricsPort={{ .Values.filer.metricsPort }} \
               {{- end }}
+              {{- if .Values.filer.metricsIp }}
+              -metricsIp={{ .Values.filer.metricsIp }} \
+              {{- end }}
               {{- if .Values.filer.redirectOnRead }}
               -redirectOnRead \
               {{- end }}
@@ -187,6 +190,7 @@ spec:
               -encryptVolumeData \
               {{- end }}
               -ip=${POD_IP} \
+              -ip.bind={{ .Values.filer.ipBind }} \
               {{- if .Values.filer.filerGroup}}
               -filerGroup={{ .Values.filer.filerGroup}} \
               {{- end }}
@@ -219,7 +223,10 @@ spec:
               -s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
               {{- end }}
               {{- end }}
-              -master={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }}
+              -master={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }} \
+              {{- range .Values.filer.extraArgs }}
+              {{ . }} \
+              {{- end }}
           volumeMounts:
             {{- if (or (eq .Values.filer.logs.type "hostPath") (eq .Values.filer.logs.type "persistentVolumeClaim") (eq .Values.filer.logs.type "emptyDir")) }}
             - name: seaweedfs-filer-log-volume

k8s/charts/seaweedfs/templates/master-configmap.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.master.enabled }}
+{{- if or .Values.master.enabled .Values.allInOne.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

k8s/charts/seaweedfs/templates/master-statefulset.yaml

@@ -9,6 +9,7 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: master
 {{- if .Values.master.annotations }}
   annotations:
     {{- toYaml .Values.master.annotations | nindent 4 }}
@@ -54,9 +55,8 @@ spec:
       affinity:
         {{ tpl .Values.master.affinity . | nindent 8 | trim }}
       {{- end }}
-      {{- with .Values.master.topologySpreadConstraints }}
-      topologySpreadConstraints:
-        {{- toYaml . | nindent 8 }}
+      {{- if .Values.master.topologySpreadConstraints }}
+      {{- include "seaweedfs.topologySpreadConstraints" (dict "Values" .Values "component" "master") | nindent 6 }}
       {{- end }}
       {{- if .Values.master.tolerations }}
       tolerations:
@@ -157,18 +157,36 @@ spec:
               {{- if .Values.master.metricsPort }}
               -metricsPort={{ .Values.master.metricsPort }} \
               {{- end }}
+              {{- if .Values.master.metricsIp }}
+              -metricsIp={{ .Values.master.metricsIp }} \
+              {{- end }}
               -volumeSizeLimitMB={{ .Values.master.volumeSizeLimitMB }} \
               {{- if .Values.master.disableHttp }}
               -disableHttp \
               {{- end }}
-              {{- if .Values.master.pulseSeconds }}
-              -pulseSeconds={{ .Values.master.pulseSeconds }} \
+              {{- if .Values.master.resumeState }}
+              -resumeState \
+              {{- end }}
+              {{- if .Values.master.raftHashicorp }}
+              -raftHashicorp \
+              {{- end }}
+              {{- if .Values.master.raftBootstrap }}
+              -raftBootstrap \
+              {{- end }}
+              {{- if .Values.master.electionTimeout }}
+              -electionTimeout={{ .Values.master.electionTimeout }} \
+              {{- end }}
+              {{- if .Values.master.heartbeatInterval }}
+              -heartbeatInterval={{ .Values.master.heartbeatInterval }} \
               {{- end }}
               {{- if .Values.master.garbageThreshold }}
               -garbageThreshold={{ .Values.master.garbageThreshold }} \
               {{- end }}
               -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master.{{ .Release.Namespace }} \
-              -peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
+              -peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} \
+              {{- range .Values.master.extraArgs }}
+              {{ . }} \
+              {{- end }}
           volumeMounts:
             - name : data-{{ .Release.Namespace }}
               mountPath: /data

k8s/charts/seaweedfs/templates/s3-deployment.yaml

@@ -9,6 +9,7 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: s3
 {{- if .Values.s3.annotations }}
   annotations:
     {{- toYaml .Values.s3.annotations | nindent 4 }}
@@ -42,6 +43,13 @@ spec:
       {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.s3.restartPolicy }}
+      {{- if .Values.s3.affinity }}
+      affinity:
+        {{ tpl .Values.s3.affinity . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.s3.topologySpreadConstraints }}
+      {{- include "seaweedfs.topologySpreadConstraints" (dict "Values" .Values "component" "s3") | nindent 6 }}
+      {{- end }}
       {{- if .Values.s3.tolerations }}
       tolerations:
         {{ tpl .Values.s3.tolerations . | nindent 8 | trim }}

k8s/charts/seaweedfs/templates/s3-secret.yaml

@@ -1,8 +1,8 @@
-{{- if or (and .Values.filer.s3.enabled .Values.filer.s3.enableAuth (not .Values.filer.s3.existingConfigSecret)) (and .Values.s3.enabled .Values.s3.enableAuth (not .Values.s3.existingConfigSecret)) }}
-{{- $access_key_admin := randAlphaNum 16 -}}
-{{- $secret_key_admin := randAlphaNum 32 -}}
-{{- $access_key_read := randAlphaNum 16 -}}
-{{- $secret_key_read := randAlphaNum 32 -}}
+{{- if or (and (or .Values.s3.enabled .Values.allInOne.enabled) .Values.s3.enableAuth (not .Values.s3.existingConfigSecret)) (and .Values.filer.s3.enabled .Values.filer.s3.enableAuth (not .Values.filer.s3.existingConfigSecret)) }}
+{{- $access_key_admin := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-s3-secret" "key" "admin_access_key_id" "length" 20) -}}
+{{- $secret_key_admin := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-s3-secret" "key" "admin_secret_access_key" "length" 40) -}}
+{{- $access_key_read := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-s3-secret" "key" "read_access_key_id" "length" 20) -}}
+{{- $secret_key_read := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-s3-secret" "key" "read_secret_access_key" "length" 40) -}}
 apiVersion: v1
 kind: Secret
 type: Opaque
@@ -11,7 +11,7 @@ metadata:
   namespace: {{ .Release.Namespace }}
   annotations:
     "helm.sh/resource-policy": keep
-    "helm.sh/hook": "pre-install"
+    "helm.sh/hook": "pre-install,pre-upgrade"
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
@@ -32,4 +32,4 @@ stringData:
   s3_auditLogConfig.json: |
     {{ toJson .Values.s3.auditLogConfig | nindent 4 }}
   {{- end }}
-{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/seaweedfs-grafana-dashboard.yaml

@@ -1,20 +1,19 @@
 {{- if .Values.global.monitoring.enabled }}
 {{- $files := .Files.Glob "dashboards/*.json" }}
 {{- if $files }}
-apiVersion: v1
-kind: ConfigMapList
-items:
-{{- range $path, $fileContents := $files }}
+{{- range $path, $file := $files }}
 {{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }}
-- apiVersion: v1
-  kind: ConfigMap
-  metadata:
-    name: {{ printf "%s" $dashboardName | lower | replace "_" "-" }}
-    namespace: {{ $.Release.Namespace }}
-    labels:
-     grafana_dashboard: "1"
-  data:
-    {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%s" $dashboardName | lower | replace "_" "-" }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    grafana_dashboard: "1"
+data:
+  {{ $dashboardName }}.json: |-
+{{ toString $file | indent 4 }}
+{{- end }}
 {{- end }}
 {{- end }}
-{{- end }}

k8s/charts/seaweedfs/templates/sftp-deployment.yaml

@@ -0,0 +1,300 @@
+{{- if .Values.sftp.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "seaweedfs.name" . }}-sftp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: sftp
+{{- if .Values.sftp.annotations }}
+  annotations:
+    {{- toYaml .Values.sftp.annotations | nindent 4 }}
+{{- end }}
+spec:
+  replicas: {{ .Values.sftp.replicas }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: sftp
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+        helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: sftp
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.sftp.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.sftp.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      restartPolicy: {{ default .Values.global.restartPolicy .Values.sftp.restartPolicy }}
+      {{- if .Values.sftp.affinity }}
+      affinity:
+        {{ tpl .Values.sftp.affinity . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.sftp.topologySpreadConstraints }}
+      {{- include "seaweedfs.topologySpreadConstraints" (dict "Values" .Values "component" "sftp") | nindent 6 }}
+      {{- end }}
+      {{- if .Values.sftp.tolerations }}
+      tolerations:
+        {{ tpl .Values.sftp.tolerations . | nindent 8 | trim }}
+      {{- end }}
+      {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
+      terminationGracePeriodSeconds: 10
+      {{- if .Values.sftp.priorityClassName }}
+      priorityClassName: {{ .Values.sftp.priorityClassName | quote }}
+      {{- end }}
+      enableServiceLinks: false
+      {{- if .Values.sftp.serviceAccountName }}
+      serviceAccountName: {{ .Values.sftp.serviceAccountName | quote }}
+      {{- end }}
+      {{- if .Values.sftp.initContainers }}
+      initContainers:
+        {{ tpl .Values.sftp.initContainers . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.sftp.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.sftp.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: seaweedfs
+          image: {{ template "sftp.image" . }}
+          imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: SEAWEEDFS_FULLNAME
+              value: "{{ template "seaweedfs.name" . }}"
+            {{- if .Values.sftp.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.sftp.extraEnvironmentVars }}
+            - name: {{ $key }}
+            {{- if kindIs "string" $value }}
+              value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
+            {{- end }}
+            {{- end }}
+            {{- if .Values.global.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
+            - name: {{ $key }}
+            {{- if kindIs "string" $value }}
+              value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
+            {{- end }}
+            {{- end }}
+          command:
+            - "/bin/sh"
+            - "-ec"
+            - |
+              exec /usr/bin/weed \
+              {{- if or (eq .Values.sftp.logs.type "hostPath") (eq .Values.sftp.logs.type "emptyDir") }}
+              -logdir=/logs \
+              {{- else }}
+              -logtostderr=true \
+              {{- end }}
+              {{- if .Values.sftp.loggingOverrideLevel }}
+              -v={{ .Values.sftp.loggingOverrideLevel }} \
+              {{- else }}
+              -v={{ .Values.global.loggingLevel }} \
+              {{- end }}
+              sftp \
+              -ip.bind={{ .Values.sftp.bindAddress }} \
+              -port={{ .Values.sftp.port }} \
+              {{- if .Values.sftp.metricsPort }}
+              -metricsPort={{ .Values.sftp.metricsPort }} \
+              {{- end }}
+              {{- if .Values.sftp.metricsIp }}
+              -metricsIp={{ .Values.sftp.metricsIp }} \
+              {{- end }}
+              {{- if .Values.sftp.sshPrivateKey }}
+              -sshPrivateKey={{ .Values.sftp.sshPrivateKey }} \
+              {{- end }}
+              {{- if .Values.sftp.hostKeysFolder }}
+              -hostKeysFolder={{ .Values.sftp.hostKeysFolder }} \
+              {{- end }}
+              {{- if .Values.sftp.authMethods }}
+              -authMethods={{ .Values.sftp.authMethods }} \
+              {{- end }}
+              {{- if .Values.sftp.maxAuthTries }}
+              -maxAuthTries={{ .Values.sftp.maxAuthTries }} \
+              {{- end }}
+              {{- if .Values.sftp.bannerMessage }}
+              -bannerMessage="{{ .Values.sftp.bannerMessage }}" \
+              {{- end }}
+              {{- if .Values.sftp.loginGraceTime }}
+              -loginGraceTime={{ .Values.sftp.loginGraceTime }} \
+              {{- end }}
+              {{- if .Values.sftp.clientAliveInterval }}
+              -clientAliveInterval={{ .Values.sftp.clientAliveInterval }} \
+              {{- end }}
+              {{- if .Values.sftp.clientAliveCountMax }}
+              -clientAliveCountMax={{ .Values.sftp.clientAliveCountMax }} \
+              {{- end }}
+              {{- if .Values.sftp.dataCenter }}
+              -dataCenter={{ .Values.sftp.dataCenter }} \
+              {{- end }}
+              {{- if .Values.sftp.localSocket }}
+              -localSocket={{ .Values.sftp.localSocket }} \
+              {{- end }}
+              {{- if .Values.global.enableSecurity }}
+              -cert.file=/usr/local/share/ca-certificates/client/tls.crt \
+              -key.file=/usr/local/share/ca-certificates/client/tls.key \
+              {{- end }}
+              -userStoreFile=/etc/sw/seaweedfs_sftp_config \
+              -filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
+          volumeMounts:
+            {{- if or (eq .Values.sftp.logs.type "hostPath") (eq .Values.sftp.logs.type "emptyDir") }}
+            - name: logs
+              mountPath: "/logs/"
+            {{- end }}
+            {{- if .Values.sftp.enableAuth }}
+            - mountPath: /etc/sw
+              name: config-users
+              readOnly: true
+            {{- end }}
+            - mountPath: /etc/sw/ssh
+              name: config-ssh
+              readOnly: true
+            {{- if .Values.global.enableSecurity }}
+            - name: security-config
+              readOnly: true
+              mountPath: /etc/seaweedfs/security.toml
+              subPath: security.toml
+            - name: ca-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/ca/
+            - name: master-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/master/
+            - name: volume-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/volume/
+            - name: filer-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/filer/
+            - name: client-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/client/
+            {{- end }}
+            {{ tpl .Values.sftp.extraVolumeMounts . | nindent 12 | trim }}
+          ports:
+            - containerPort: {{ .Values.sftp.port }}
+              name: swfs-sftp
+            {{- if .Values.sftp.metricsPort }}
+            - containerPort: {{ .Values.sftp.metricsPort }}
+              name: metrics
+            {{- end }}
+          {{- if .Values.sftp.readinessProbe.enabled }}
+          readinessProbe:
+            tcpSocket:
+              port: {{ .Values.sftp.port }}
+            initialDelaySeconds: {{ .Values.sftp.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.sftp.readinessProbe.periodSeconds }}
+            successThreshold: {{ .Values.sftp.readinessProbe.successThreshold }}
+            failureThreshold: {{ .Values.sftp.readinessProbe.failureThreshold }}
+            timeoutSeconds: {{ .Values.sftp.readinessProbe.timeoutSeconds }}
+          {{- end }}
+          {{- if .Values.sftp.livenessProbe.enabled }}
+          livenessProbe:
+            tcpSocket:
+              port: {{ .Values.sftp.port }}
+            initialDelaySeconds: {{ .Values.sftp.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.sftp.livenessProbe.periodSeconds }}
+            successThreshold: {{ .Values.sftp.livenessProbe.successThreshold }}
+            failureThreshold: {{ .Values.sftp.livenessProbe.failureThreshold }}
+            timeoutSeconds: {{ .Values.sftp.livenessProbe.timeoutSeconds }}
+          {{- end }}
+          {{- with .Values.sftp.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- if .Values.sftp.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.sftp.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
+      {{- if .Values.sftp.sidecars }}
+      {{- include "common.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
+      {{- end }}
+      volumes:
+        {{- if .Values.sftp.enableAuth }}
+        - name: config-users
+          secret:
+            defaultMode: 420
+            {{- if .Values.sftp.existingConfigSecret }}
+            secretName: {{ .Values.sftp.existingConfigSecret }}
+            {{- else }}
+            secretName: seaweedfs-sftp-secret
+            {{- end }}
+        {{- end }}
+        - name: config-ssh
+          secret:
+            defaultMode: 420
+            {{- if .Values.sftp.existingSshConfigSecret }}
+            secretName: {{ .Values.sftp.existingSshConfigSecret }}
+            {{- else }}
+            secretName: seaweedfs-sftp-ssh-secret
+            {{- end }}
+        {{- if eq .Values.sftp.logs.type "hostPath" }}
+        - name: logs
+          hostPath:
+            path: {{ .Values.sftp.logs.hostPathPrefix }}/logs/seaweedfs/sftp
+            type: DirectoryOrCreate
+        {{- end }}
+        {{- if eq .Values.sftp.logs.type "emptyDir" }}
+        - name: logs
+          emptyDir: {}
+        {{- end }}
+        {{- if .Values.global.enableSecurity }}
+        - name: security-config
+          configMap:
+            name: {{ template "seaweedfs.name" . }}-security-config
+        - name: ca-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-ca-cert
+        - name: master-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-master-cert
+        - name: volume-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-volume-cert
+        - name: filer-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-filer-cert
+        - name: client-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-client-cert
+        {{- end }}
+        {{ tpl .Values.sftp.extraVolumes . | indent 8 | trim }}
+      {{- if .Values.sftp.nodeSelector }}
+      nodeSelector:
+        {{ tpl .Values.sftp.nodeSelector . | indent 8 | trim }}
+      {{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/sftp-secret.yaml

@@ -0,0 +1,33 @@
+{{- if or .Values.sftp.enabled .Values.allInOne.enabled }}
+{{- $admin_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-sftp-secret" "key" "admin_password" 20) -}}
+{{- $read_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-sftp-secret" "key" "readonly_password" 20) -}}
+{{- $public_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-sftp-secret" "key" "public_user_password" 20) -}}
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: seaweedfs-sftp-secret
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/resource-policy": keep
+    "helm.sh/hook": "pre-install,pre-upgrade"
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: sftp
+stringData:
+  admin_password: {{ $admin_pwd }}
+  readonly_password: {{ $read_user_pwd }}
+  public_user_password: {{ $public_user_pwd }}
+  seaweedfs_sftp_config: '[{"Username":"admin","Password":"{{ $admin_pwd }}","PublicKeys":[],"HomeDir":"/","Permissions":{"/":["read","write","list"]},"Uid":0,"Gid":0},{"Username":"readonly_user","Password":"{{ $read_user_pwd }}","PublicKeys":[],"HomeDir":"/","Permissions":{"/":["read","list"]},"Uid":1112,"Gid":1112},{"Username":"public_user","Password":"{{ $public_user_pwd }}","PublicKeys":[],"HomeDir":"/public","Permissions":{"/public":["write","read","list"]},"Uid":1113,"Gid":1113}]'
+  seaweedfs_sftp_ssh_private_key: |
+    -----BEGIN OPENSSH PRIVATE KEY-----
+    b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+    QyNTUxOQAAACDH4McwcDphteXVullu6q7ephEN1N60z+w0qZw0UVW8OwAAAJDjxkmk48ZJ
+    pAAAAAtzc2gtZWQyNTUxOQAAACDH4McwcDphteXVullu6q7ephEN1N60z+w0qZw0UVW8Ow
+    AAAEAeVy/4+gf6rjj2jla/AHqJpC1LcS5hn04IUs4q+iVq/MfgxzBwOmG15dW6WW7qrt6m
+    EQ3U3rTP7DSpnDRRVbw7AAAADHNla291ckAwMDY2NwE=
+    -----END OPENSSH PRIVATE KEY-----
+{{- end }}

k8s/charts/seaweedfs/templates/sftp-service.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.sftp.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "seaweedfs.name" . }}-sftp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    app.kubernetes.io/component: sftp
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- if .Values.sftp.annotations }}
+  annotations:
+    {{- toYaml .Values.sftp.annotations | nindent 4 }}
+{{- end }}
+spec:
+  internalTrafficPolicy: {{ .Values.sftp.internalTrafficPolicy | default "Cluster" }}
+  ports:
+  - name: "swfs-sftp"
+    port: {{ .Values.sftp.port }}
+    targetPort: {{ .Values.sftp.port }}
+    protocol: TCP
+{{- if .Values.sftp.metricsPort }}
+  - name: "metrics"
+    port: {{ .Values.sftp.metricsPort }}
+    targetPort: {{ .Values.sftp.metricsPort }}
+    protocol: TCP
+{{- end }}
+  selector:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    app.kubernetes.io/component: sftp
+{{- end }}

k8s/charts/seaweedfs/templates/sftp-servicemonitor.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.sftp.enabled }}
+{{- if .Values.sftp.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "seaweedfs.name" . }}-sftp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: sftp
+    {{- with .Values.global.monitoring.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+{{- if .Values.sftp.annotations }}
+  annotations:
+    {{- toYaml .Values.sftp.annotations | nindent 4 }}
+{{- end }}
+spec:
+  endpoints:
+    - interval: 30s
+      port: metrics
+      scrapeTimeout: 5s
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+      app.kubernetes.io/component: sftp
+{{- end }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/volume-statefulset.yaml

@@ -9,6 +9,7 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: volume
 {{- if .Values.volume.annotations }}
   annotations:
     {{- toYaml .Values.volume.annotations | nindent 4 }}
@@ -47,9 +48,8 @@ spec:
       affinity:
         {{ tpl .Values.volume.affinity . | nindent 8 | trim }}
       {{- end }}
-      {{- with .Values.volume.topologySpreadConstraints }}
-      topologySpreadConstraints:
-        {{- toYaml . | nindent 8 }}
+      {{- if .Values.volume.topologySpreadConstraints }}
+      {{- include "seaweedfs.topologySpreadConstraints" (dict "Values" .Values "component" "volume") | nindent 6 }}
       {{- end }}
       restartPolicy: {{ default .Values.global.restartPolicy .Values.volume.restartPolicy }}
       {{- if .Values.volume.tolerations }}
@@ -150,6 +150,9 @@ spec:
                 {{- if .Values.volume.metricsPort }}
                 -metricsPort={{ .Values.volume.metricsPort }} \
                 {{- end }}
+                {{- if .Values.volume.metricsIp }}
+                -metricsIp={{ .Values.volume.metricsIp }} \
+                {{- end }}
                 -dir {{range $index, $dir :=  .Values.volume.dataDirs }}{{if ne $index 0}},{{end}}/{{$dir.name}}{{end}} \
                 {{- if .Values.volume.idx }}
                 -dir.idx=/idx \
@@ -183,7 +186,10 @@ spec:
                 -minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \
                 -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume.{{ .Release.Namespace }} \
                 -compactionMBps={{ .Values.volume.compactionMBps }} \
-                -mserver={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }}
+                -mserver={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }} \
+                {{- range .Values.volume.extraArgs }}
+                {{ . }} \
+                {{- end }}
           volumeMounts:
             {{- range $dir := .Values.volume.dataDirs }}
             {{- if not ( eq $dir.type "custom" ) }}
@@ -351,7 +357,9 @@ spec:
   volumeClaimTemplates:
     {{- range $dir := .Values.volume.dataDirs }}
     {{- if eq $dir.type "persistentVolumeClaim" }}
-    - metadata:
+    - apiVersion: v1
+      kind: PersistentVolumeClaim
+      metadata:
         name: {{ $dir.name }}
         {{- with $dir.annotations }}
         annotations:
@@ -367,7 +375,9 @@ spec:
     {{- end }}
 
     {{- if and .Values.volume.idx (eq .Values.volume.idx.type "persistentVolumeClaim") }}
-    - metadata:
+    - apiVersion: v1
+      kind: PersistentVolumeClaim
+      metadata:
         name: idx
         {{- with .Values.volume.idx.annotations }}
         annotations:
@@ -381,7 +391,9 @@ spec:
             storage: {{ .Values.volume.idx.size }}
     {{- end }}
     {{- if and .Values.volume.logs (eq .Values.volume.logs.type "persistentVolumeClaim") }}
-    - metadata:
+    - apiVersion: v1
+      kind: PersistentVolumeClaim
+      metadata:
         name: logs
         {{- with .Values.volume.logs.annotations }}
         annotations:

k8s/charts/seaweedfs/values.yaml

@@ -46,6 +46,7 @@ global:
 image:
   registry: ""
   repository: ""
+  tag: ""
 
 master:
   enabled: true
@@ -55,12 +56,11 @@ master:
   port: 9333
   grpcPort: 19333
   metricsPort: 9327
+  metricsIp: ""  # Metrics listen IP. If empty, defaults to ipBind
   ipBind: "0.0.0.0"
   volumePreallocate: false
   volumeSizeLimitMB: 1000
   loggingOverrideLevel: null
-  # number of seconds between heartbeats, default 5
-  pulseSeconds: null
   # threshold to vacuum and reclaim spaces, default 0.3 (30%)
   garbageThreshold: null
   # Prometheus push interval in seconds, default 15
@@ -74,6 +74,25 @@ master:
   # Disable http request, only gRpc operations are allowed
   disableHttp: false
 
+  # Resume previous state on start master server
+  resumeState: false
+  # Use Hashicorp Raft
+  raftHashicorp: false
+  # Whether to bootstrap the Raft cluster. Only use it when use Hashicorp Raft
+  raftBootstrap: false
+
+  # election timeout of master servers
+  electionTimeout: "10s"
+  # heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)
+  heartbeatInterval: "300ms"
+
+  # Custom command line arguments to add to the master command
+  # Example to fix IPv6 metrics connectivity issues:
+  # extraArgs: ["-metricsIp", "0.0.0.0"]
+  # Example with multiple args:
+  # extraArgs: ["-customFlag", "value", "-anotherFlag"]
+  extraArgs: []
+
   config: |-
     # Enter any extra configuration for master.toml here.
     # It may be a multi-line string.
@@ -172,7 +191,7 @@ master:
   # Topology Spread Constraints Settings
   # This should map directly to the value of the topologySpreadConstraints
   # for a PodSpec. By Default no constraints are set.
-  topologySpreadConstraints: {}
+  topologySpreadConstraints: null
 
   # Toleration Settings for master pods
   # This should be a multi-line string matching the Toleration array
@@ -276,6 +295,7 @@ volume:
   port: 8080
   grpcPort: 18080
   metricsPort: 9327
+  metricsIp: ""  # Metrics listen IP. If empty, defaults to ipBind
   ipBind: "0.0.0.0"
   replicas: 1
   loggingOverrideLevel: null
@@ -288,6 +308,13 @@ volume:
   # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly
   minFreeSpacePercent: 7
 
+  # Custom command line arguments to add to the volume command
+  # Example to fix IPv6 metrics connectivity issues:
+  # extraArgs: ["-metricsIp", "0.0.0.0"]
+  # Example with multiple args:
+  # extraArgs: ["-customFlag", "value", "-anotherFlag"]
+  extraArgs: []
+
   # For each data disk you may use ANY storage-class, example with local-path-provisioner
   # Annotations are optional.
   #  dataDirs:
@@ -435,7 +462,7 @@ volume:
   # Topology Spread Constraints Settings
   # This should map directly to the value of the topologySpreadConstraints
   # for a PodSpec. By Default no constraints are set.
-  topologySpreadConstraints: {}
+  topologySpreadConstraints: null
 
   # Resource requests, limits, etc. for the server cluster placement. This
   # should map directly to the value of the resources field for a PodSpec,
@@ -519,6 +546,8 @@ filer:
   port: 8888
   grpcPort: 18888
   metricsPort: 9327
+  metricsIp: ""  # Metrics listen IP. If empty, defaults to ipBind
+  ipBind: "0.0.0.0"  # IP address to bind to. Set to 0.0.0.0 to allow external traffic
   loggingOverrideLevel: null
   filerGroup: ""
   # prefer to read and write to volumes in this data center (not set by default)
@@ -546,6 +575,13 @@ filer:
   # Disable http request, only gRpc operations are allowed
   disableHttp: false
 
+  # Custom command line arguments to add to the filer command
+  # Example to fix IPv6 metrics connectivity issues:
+  # extraArgs: ["-metricsIp", "0.0.0.0"]
+  # Example with multiple args:
+  # extraArgs: ["-customFlag", "value", "-anotherFlag"]
+  extraArgs: []
+
   # Add a custom notification.toml to configure filer notifications
   # Example:
   # notificationConfig: |-
@@ -654,7 +690,7 @@ filer:
   # Topology Spread Constraints Settings
   # This should map directly to the value of the topologySpreadConstraints
   # for a PodSpec. By Default no constraints are set.
-  topologySpreadConstraints: {}
+  topologySpreadConstraints: null
 
   # updatePartition is used to control a careful rolling update of SeaweedFS
   # masters.
@@ -949,6 +985,215 @@ s3:
     annotations: {}
     tls: []
 
+sftp:
+  enabled: false
+  imageOverride: null
+  restartPolicy: null
+  replicas: 1
+  bindAddress: 0.0.0.0
+  port: 2022  # Default SFTP port
+  metricsPort: 9327
+  metricsIp: ""  # If empty, defaults to bindAddress
+  loggingOverrideLevel: null
+
+  # SSH server configuration
+  sshPrivateKey: "/etc/sw/seaweedfs_sftp_ssh_private_key"  # Path to the SSH private key file for host authentication
+  hostKeysFolder: "/etc/sw/ssh"  # path to folder containing SSH private key files for host authentication
+  authMethods: "password,publickey"  # Comma-separated list of allowed auth methods: password, publickey, keyboard-interactive
+  maxAuthTries: 6  # Maximum number of authentication attempts per connection
+  bannerMessage: "SeaweedFS SFTP Server"  # Message displayed before authentication
+  loginGraceTime: "2m"  # Timeout for authentication
+  clientAliveInterval: "5s"  # Interval for sending keep-alive messages
+  clientAliveCountMax: 3  # Maximum number of missed keep-alive messages before disconnecting
+  dataCenter: ""  # Prefer to read and write to volumes in this data center
+  localSocket: ""  # Default to /tmp/seaweedfs-sftp-<port>.sock
+
+  # User authentication
+  enableAuth: false
+  # Set to the name of an existing kubernetes Secret with the sftp json config file
+  # Should have a secret key called seaweedfs_sftp_config with an inline json config
+  existingConfigSecret: null
+  # Set to the name of an existing kubernetes Secret with the list of ssh private keys for sftp
+  existingSshConfigSecret: null
+
+  # Additional resources
+  sidecars: []
+  initContainers: ""
+  extraVolumes: ""
+  extraVolumeMounts: ""
+  podLabels: {}
+  podAnnotations: {}
+  annotations: {}
+  resources: {}
+  tolerations: ""
+  nodeSelector: |
+    kubernetes.io/arch: amd64
+  priorityClassName: ""
+  serviceAccountName: ""
+  podSecurityContext: {}
+  containerSecurityContext: {}
+
+  logs:
+    type: "hostPath"
+    hostPathPrefix: /storage
+
+  extraEnvironmentVars: {}
+
+  # Health checks
+  # Health checks for SFTP - using tcpSocket instead of httpGet
+  livenessProbe:
+    enabled: true
+    initialDelaySeconds: 20
+    periodSeconds: 60
+    successThreshold: 1
+    failureThreshold: 20
+    timeoutSeconds: 10
+
+  # Health checks for SFTP - using tcpSocket instead of httpGet
+  readinessProbe:
+    enabled: true
+    initialDelaySeconds: 15
+    periodSeconds: 15
+    successThreshold: 1
+    failureThreshold: 100
+    timeoutSeconds: 10
+
+# All-in-one deployment configuration
+allInOne:
+  enabled: false
+  imageOverride: null
+  restartPolicy: Always
+  replicas: 1
+
+  # Core configuration
+  idleTimeout: 30  # Connection idle seconds
+  dataCenter: ""  # Current volume server's data center name
+  rack: ""  # Current volume server's rack name
+  whiteList: ""  # Comma separated IP addresses having write permission
+  disableHttp: false  # Disable HTTP requests, only gRPC operations are allowed
+  metricsPort: 9324  # Prometheus metrics listen port
+  metricsIp: ""  # Metrics listen IP. If empty, defaults to bindAddress
+  loggingOverrideLevel: null  # Override logging level
+
+  # Service configuration
+  s3:
+    enabled: false  # Whether to enable S3 gateway
+  sftp:
+    enabled: false  # Whether to enable SFTP server
+
+  # Service settings
+  service:
+    annotations: {}  # Annotations for the service
+    type: ClusterIP  # Service type (ClusterIP, NodePort, LoadBalancer)
+
+  # Storage configuration
+  data:
+    type: "emptyDir"  # Options: "hostPath", "persistentVolumeClaim", "emptyDir"
+    hostPathPrefix: /mnt/data  # Path prefix for hostPath volumes
+    claimName: seaweedfs-data-pvc  # Name of the PVC to use
+    size: ""  # Size of the PVC
+    storageClass: ""  # Storage class for the PVC
+
+  # Health checks
+  readinessProbe:
+    enabled: true
+    httpGet:
+      path: /cluster/status
+      port: 9333
+    scheme: HTTP
+    initialDelaySeconds: 10
+    periodSeconds: 15
+    successThreshold: 1
+    failureThreshold: 3
+    timeoutSeconds: 5
+
+  livenessProbe:
+    enabled: true
+    httpGet:
+      path: /cluster/status
+      port: 9333
+    scheme: HTTP
+    initialDelaySeconds: 20
+    periodSeconds: 30
+    successThreshold: 1
+    failureThreshold: 5
+    timeoutSeconds: 5
+
+  # Additional resources
+  extraEnvironmentVars: {}  # Additional environment variables
+  extraVolumeMounts: ""  # Additional volume mounts
+  extraVolumes: ""  # Additional volumes
+  initContainers: ""  # Init containers
+  sidecars: ""  # Sidecar containers
+  annotations: {}  # Annotations for the deployment
+  podAnnotations: {}  # Annotations for the pods
+  podLabels: {}  # Labels for the pods
+
+  # Scheduling configuration
+  # Affinity Settings
+  # Commenting out or setting as empty the affinity variable, will allow
+  # deployment to single node services such as Minikube
+  affinity: |
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        - labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+              app.kubernetes.io/instance: {{ .Release.Name }}
+              app.kubernetes.io/component: master
+          topologyKey: kubernetes.io/hostname
+
+  # Topology Spread Constraints Settings
+  # This should map directly to the value of the topologySpreadConstraints
+  # for a PodSpec. By Default no constraints are set.
+  topologySpreadConstraints: null
+
+  # Toleration Settings for master pods
+  # This should be a multi-line string matching the Toleration array
+  # in a PodSpec.
+  tolerations: ""
+
+  # nodeSelector labels for master pod assignment, formatted as a muli-line string.
+  # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
+  nodeSelector: |
+    kubernetes.io/arch: amd64
+
+  # Used to assign priority to master pods
+  # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+  priorityClassName: ""
+
+  # Used to assign a service account.
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
+  serviceAccountName: ""
+
+  # Configure security context for Pod
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # podSecurityContext:
+  #   enabled: true
+  #   runAsUser: 1000
+  #   runAsGroup: 3000
+  #   fsGroup: 2000
+  podSecurityContext: {}
+
+  # Configure security context for Container
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # containerSecurityContext:
+  #   enabled: true
+  #   runAsUser: 2000
+  #   allowPrivilegeEscalation: false
+  containerSecurityContext: {}
+
+  # Resource management
+  resources:
+    limits:
+      cpu: "2"
+      memory: "2Gi"
+    requests:
+      cpu: "500m"
+      memory: "1Gi"
+
 # Deploy Kubernetes COSI Driver for SeaweedFS
 # Requires COSI CRDs and controller to be installed in the cluster
 # For more information, visit: https://container-object-storage-interface.github.io/docs/deployment-guide

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -14,11 +14,13 @@ public class FilerClient extends FilerGrpcClient {
     private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
 
     public FilerClient(String filerHost, int filerGrpcPort) {
-        super(filerHost, filerGrpcPort-10000, filerGrpcPort);
+        this(filerHost, filerGrpcPort-10000, filerGrpcPort, "");
     }
+    public FilerClient(String filerHost, int filerGrpcPort, String cn) { this(filerHost, filerGrpcPort-10000, filerGrpcPort, cn); }
+    public FilerClient(String filerHost, int filerPort, int filerGrpcPort) { this(filerHost, filerPort, filerGrpcPort, ""); }
 
-    public FilerClient(String filerHost, int filerPort, int filerGrpcPort) {
-        super(filerHost, filerPort, filerGrpcPort);
+    public FilerClient(String filerHost, int filerPort, int filerGrpcPort, String cn) {
+        super(filerHost, filerPort, filerGrpcPort, cn);
     }
 
     public static String toFileId(FilerProto.FileId fid) {

other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java

@@ -8,7 +8,6 @@ import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.net.ssl.SSLException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Random;
@@ -17,14 +16,12 @@ import java.util.concurrent.TimeUnit;
 public class FilerGrpcClient {
 
     private static final Logger logger = LoggerFactory.getLogger(FilerGrpcClient.class);
-    static SslContext sslContext;
+    private static final SslContext sslContext;
+    private static final String protocol;
 
     static {
-        try {
-            sslContext = FilerSslContext.loadSslContext();
-        } catch (SSLException e) {
-            logger.warn("failed to load ssl context", e);
-        }
+        sslContext = FilerSecurityContext.getGrpcSslContext();
+        protocol = FilerSecurityContext.isHttpSecurityEnabled() ? "https" : "http";
     }
 
     public final int VOLUME_SERVER_ACCESS_DIRECT = 0;
@@ -42,19 +39,27 @@ public class FilerGrpcClient {
     private int volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT;
     private String filerAddress;
 
-    public FilerGrpcClient(String host, int port, int grpcPort) {
-        this(host, port, grpcPort, sslContext);
+    public FilerGrpcClient(String host, int port, int grpcPort, String cn) {
+        this(host, port, grpcPort, cn, sslContext);
     }
 
-    public FilerGrpcClient(String host, int port, int grpcPort, SslContext sslContext) {
+    public FilerGrpcClient(String host, int port, int grpcPort, String cn, SslContext sslContext) {
 
         this(sslContext == null ?
-                ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext()
+                ManagedChannelBuilder.forAddress(host, grpcPort)
+                        .usePlaintext()
                         .maxInboundMessageSize(1024 * 1024 * 1024) :
-                NettyChannelBuilder.forAddress(host, grpcPort)
-                        .maxInboundMessageSize(1024 * 1024 * 1024)
-                        .negotiationType(NegotiationType.TLS)
-                        .sslContext(sslContext));
+                cn.isEmpty() ?
+                    NettyChannelBuilder.forAddress(host, grpcPort)
+                            .maxInboundMessageSize(1024 * 1024 * 1024)
+                            .negotiationType(NegotiationType.TLS)
+                            .sslContext(sslContext) :
+                    NettyChannelBuilder.forAddress(host, grpcPort)
+                            .maxInboundMessageSize(1024 * 1024 * 1024)
+                            .negotiationType(NegotiationType.TLS)
+                            .overrideAuthority(cn) //will not check hostname of the filer server
+                            .sslContext(sslContext)
+                );
 
         filerAddress = SeaweedUtil.joinHostPort(host, port);
 
@@ -130,12 +135,11 @@ public class FilerGrpcClient {
     public String getChunkUrl(String chunkId, String url, String publicUrl) {
         switch (this.volumeServerAccess) {
             case VOLUME_SERVER_ACCESS_PUBLIC_URL:
-                return String.format("http://%s/%s", publicUrl, chunkId);
+                return String.format("%s://%s/%s", protocol, publicUrl, chunkId);
             case VOLUME_SERVER_ACCESS_FILER_PROXY:
-                return String.format("http://%s/?proxyChunkId=%s", this.filerAddress, chunkId);
+                return String.format("%s://%s/?proxyChunkId=%s", protocol, this.filerAddress, chunkId);
             default:
-                return String.format("http://%s/%s", url, chunkId);
+                return String.format("%s://%s/%s", protocol, url, chunkId);
         }
     }
-
 }

other/java/client/src/main/java/seaweedfs/client/FilerSecurityContext.java

@@ -0,0 +1,164 @@
+package seaweedfs.client;
+
+import com.google.common.base.Strings;
+import com.moandjiezana.toml.Toml;
+import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
+import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.http.ssl.SSLContexts;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.*;
+import java.io.File;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+
+public abstract class FilerSecurityContext extends SslContext {
+//extends Netty SslContext to access its protected static utility methods in
+//buildHttpSslContext()
+
+    private static final Logger logger = LoggerFactory.getLogger(FilerSecurityContext.class);
+    private static boolean grpcSecurityEnabled;
+    private static boolean httpSecurityEnabled;
+    private static SslContext grpcSslContext;
+    private static SSLContext httpSslContext;
+
+    private static String grpcTrustCertCollectionFilePath;
+    private static String grpcClientCertChainFilePath;
+    private static String grpcClientPrivateKeyFilePath;
+
+    private static String httpTrustCertCollectionFilePath;
+    private static String httpClientCertChainFilePath;
+    private static String httpClientPrivateKeyFilePath;
+
+
+    static {
+        String securityFileName = "security.toml";
+        String home = System.getProperty("user.home");
+        File f1 = new File("./"+securityFileName);
+        File f2 = new File(home + "/.seaweedfs/"+securityFileName);
+        File f3 = new File("/etc/seaweedfs/"+securityFileName);
+
+        File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null;
+
+        if (securityFile==null){
+            logger.debug("Security file not found");
+            grpcSecurityEnabled = false;
+            httpSecurityEnabled = false;
+        } else {
+
+            Toml toml = new Toml().read(securityFile);
+            logger.debug("reading ssl setup from {}", securityFile);
+
+            grpcTrustCertCollectionFilePath = toml.getString("grpc.ca");
+            logger.debug("loading gRPC ca from {}", grpcTrustCertCollectionFilePath);
+            grpcClientCertChainFilePath = toml.getString("grpc.client.cert");
+            logger.debug("loading gRPC client ca from {}", grpcClientCertChainFilePath);
+            grpcClientPrivateKeyFilePath = toml.getString("grpc.client.key");
+            logger.debug("loading gRPC client key from {}", grpcClientPrivateKeyFilePath);
+
+            if (Strings.isNullOrEmpty(grpcClientCertChainFilePath) && Strings.isNullOrEmpty(grpcClientPrivateKeyFilePath)) {
+                logger.debug("gRPC private key file locations not set");
+                grpcSecurityEnabled = false;
+            } else {
+                try {
+                    grpcSslContext = buildGrpcSslContext();
+                    grpcSecurityEnabled = true;
+                } catch (Exception e) {
+                    logger.warn("Couldn't initialize gRPC security context, filer operations are likely to fail!", e);
+                    grpcSslContext = null;
+                    grpcSecurityEnabled = false;
+                }
+            }
+
+            if (toml.getBoolean("https.client.enabled")) {
+                httpTrustCertCollectionFilePath = toml.getString("https.client.ca");
+                logger.debug("loading HTTP ca from {}", httpTrustCertCollectionFilePath);
+                httpClientCertChainFilePath = toml.getString("https.client.cert");
+                logger.debug("loading HTTP client ca from {}", httpClientCertChainFilePath);
+                httpClientPrivateKeyFilePath = toml.getString("https.client.key");
+                logger.debug("loading HTTP client key from {}", httpClientPrivateKeyFilePath);
+
+                if (Strings.isNullOrEmpty(httpClientCertChainFilePath) && Strings.isNullOrEmpty(httpClientPrivateKeyFilePath)) {
+                    logger.debug("HTTP private key file locations not set");
+                    httpSecurityEnabled = false;
+                } else {
+                    try {
+                        httpSslContext = buildHttpSslContext();
+                        httpSecurityEnabled = true;
+                    } catch (Exception e) {
+                        logger.warn("Couldn't initialize HTTP security context, volume operations are likely to fail!", e);
+                        httpSslContext = null;
+                        httpSecurityEnabled = false;
+                    }
+                }
+            } else {
+                httpSecurityEnabled = false;
+            }
+        }
+        // possibly fix the format https://netty.io/wiki/sslcontextbuilder-and-private-key.html
+    }
+
+    public static boolean isGrpcSecurityEnabled() {
+        return grpcSecurityEnabled;
+    }
+
+    public static boolean isHttpSecurityEnabled() {
+        return httpSecurityEnabled;
+    }
+
+    public static SslContext getGrpcSslContext() {
+        return grpcSslContext;
+    }
+
+    public static SSLContext getHttpSslContext() {
+        return httpSslContext;
+    }
+
+    private static SslContext buildGrpcSslContext() throws SSLException {
+        SslContextBuilder builder = GrpcSslContexts.forClient();
+        if (grpcTrustCertCollectionFilePath != null) {
+            builder.trustManager(new File(grpcTrustCertCollectionFilePath));
+        }
+        if (grpcClientCertChainFilePath != null && grpcClientPrivateKeyFilePath != null) {
+            builder.keyManager(new File(grpcClientCertChainFilePath), new File(grpcClientPrivateKeyFilePath));
+        }
+
+        return builder.build();
+    }
+
+    private static SSLContext buildHttpSslContext() throws GeneralSecurityException, IOException {
+        SSLContextBuilder builder = SSLContexts.custom();
+
+        if (httpTrustCertCollectionFilePath != null) {
+            final X509Certificate[] trustCerts = toX509Certificates(new File(httpTrustCertCollectionFilePath));
+            final KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
+            ks.load(null, null);
+
+            int i = 0;
+            for (X509Certificate cert: trustCerts) {
+                String alias = Integer.toString(++i);
+                ks.setCertificateEntry(alias, cert);
+            }
+
+            builder.loadTrustMaterial(ks, null);
+        }
+
+        if (httpClientCertChainFilePath != null && httpClientPrivateKeyFilePath != null) {
+            final X509Certificate[] keyCerts = toX509Certificates(new File(httpClientCertChainFilePath));
+            final PrivateKey key = toPrivateKey(new File(httpClientPrivateKeyFilePath), null);
+            char[] emptyPassword = new char[0];
+            final KeyStore ks = buildKeyStore(keyCerts, key, emptyPassword, null);
+            logger.debug("Loaded {} key certificates", ks.size());
+            builder.loadKeyMaterial(ks, emptyPassword);
+        }
+
+        return builder.build();
+    }
+}

other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java

@@ -1,64 +0,0 @@
-package seaweedfs.client;
-
-import com.google.common.base.Strings;
-import com.moandjiezana.toml.Toml;
-import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
-import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
-import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
-import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.SSLException;
-import java.io.File;
-
-public class FilerSslContext {
-
-    private static final Logger logger = LoggerFactory.getLogger(FilerSslContext.class);
-
-    public static SslContext loadSslContext() throws SSLException {
-        String securityFileName = "security.toml";
-        String home = System.getProperty("user.home");
-        File f1 = new File("./"+securityFileName);
-        File f2 = new File(home + "/.seaweedfs/"+securityFileName);
-        File f3 = new File("/etc/seaweedfs/"+securityFileName);
-
-        File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null;
-
-        if (securityFile==null){
-            return null;
-        }
-
-        Toml toml = new Toml().read(securityFile);
-        logger.debug("reading ssl setup from {}", securityFile);
-
-        String trustCertCollectionFilePath = toml.getString("grpc.ca");
-        logger.debug("loading ca from {}", trustCertCollectionFilePath);
-        String clientCertChainFilePath = toml.getString("grpc.client.cert");
-        logger.debug("loading client ca from {}", clientCertChainFilePath);
-        String clientPrivateKeyFilePath = toml.getString("grpc.client.key");
-        logger.debug("loading client key from {}", clientPrivateKeyFilePath);
-
-        if (Strings.isNullOrEmpty(clientPrivateKeyFilePath) && Strings.isNullOrEmpty(clientPrivateKeyFilePath)){
-            return null;
-        }
-
-        // possibly fix the format https://netty.io/wiki/sslcontextbuilder-and-private-key.html
-
-        return buildSslContext(trustCertCollectionFilePath, clientCertChainFilePath, clientPrivateKeyFilePath);
-    }
-
-
-    private static SslContext buildSslContext(String trustCertCollectionFilePath,
-                                             String clientCertChainFilePath,
-                                             String clientPrivateKeyFilePath) throws SSLException {
-        SslContextBuilder builder = GrpcSslContexts.forClient();
-        if (trustCertCollectionFilePath != null) {
-            builder.trustManager(new File(trustCertCollectionFilePath));
-        }
-        if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
-            builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
-        }
-        return builder.trustManager(InsecureTrustManagerFactory.INSTANCE).build();
-    }
-}

other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java

@@ -1,27 +1,64 @@
 package seaweedfs.client;
 
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.impl.DefaultConnectionReuseStrategy;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLContext;
 
 public class SeaweedUtil {
 
-    static PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
+    private static final Logger logger = LoggerFactory.getLogger(SeaweedUtil.class);
+    static PoolingHttpClientConnectionManager cm;
     static CloseableHttpClient httpClient;
 
     static {
+        //Apache HTTP client has a terrible API that makes you configure everything twice
+        //NoopHostnameVerifier is required because SeaweedFS doesn't verify hostnames
+        //and the servers are likely to have TLS certificates that do not match their hosts
+        if (FilerSecurityContext.isHttpSecurityEnabled()) {
+            SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(
+                    FilerSecurityContext.getHttpSslContext(),
+                    NoopHostnameVerifier.INSTANCE);
+
+            Registry<ConnectionSocketFactory> socketFactoryRegistry =
+                    RegistryBuilder.<ConnectionSocketFactory>create()
+                            .register("https", sslSocketFactory)
+                            .register("http", new PlainConnectionSocketFactory())
+                            .build();
+            cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+        } else {
+            cm = new PoolingHttpClientConnectionManager();
+        }
+
         // Increase max total connection to 200
         cm.setMaxTotal(200);
         // Increase default max connection per route to 20
         cm.setDefaultMaxPerRoute(20);
 
-        httpClient = HttpClientBuilder.create()
+        HttpClientBuilder builder = HttpClientBuilder.create()
                 .setConnectionManager(cm)
                 .setConnectionReuseStrategy(DefaultConnectionReuseStrategy.INSTANCE)
-                .setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE)
-                .build();
+                .setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE);
+
+        if (FilerSecurityContext.isHttpSecurityEnabled()) {
+            builder.setSSLContext(FilerSecurityContext.getHttpSslContext());
+            builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
+        }
+
+        httpClient = builder.build();
     }
 
     public static CloseableHttpClient getClosableHttpClient() {

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -29,6 +29,7 @@ public class SeaweedFileSystem extends FileSystem {
     public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
     public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access";
     public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
+    public static final String FS_SEAWEED_FILER_CN = "fs.seaweed.filer.cn";
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
 
@@ -63,8 +64,9 @@ public class SeaweedFileSystem extends FileSystem {
         setConf(conf);
         this.uri = uri;
 
-        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf);
+        String cn = conf.get(FS_SEAWEED_FILER_CN, "");
 
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, cn, conf);
     }
 
     @Override

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -27,8 +27,8 @@ public class SeaweedFileSystemStore {
     private FilerClient filerClient;
     private Configuration conf;
 
-    public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) {
-        filerClient = new FilerClient(host, port, grpcPort);
+    public SeaweedFileSystemStore(String host, int port, int grpcPort, String cn, Configuration conf) {
+        filerClient = new FilerClient(host, port, grpcPort, cn);
         this.conf = conf;
         String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
         if (volumeServerAccessMode.equals("publicUrl")) {
@@ -36,7 +36,6 @@ public class SeaweedFileSystemStore {
         } else if (volumeServerAccessMode.equals("filerProxy")) {
             filerClient.setAccessVolumeServerByFilerProxy();
         }
-
     }
 
     public void close() {

other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -29,6 +29,7 @@ public class SeaweedFileSystem extends FileSystem {
     public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
     public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access";
     public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
+    public static final String FS_SEAWEED_FILER_CN = "fs.seaweed.filer.cn";
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
 
@@ -63,8 +64,9 @@ public class SeaweedFileSystem extends FileSystem {
         setConf(conf);
         this.uri = uri;
 
-        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf);
+        String cn = conf.get(FS_SEAWEED_FILER_CN, "");
 
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, cn, conf);
     }
 
     @Override

other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -27,8 +27,8 @@ public class SeaweedFileSystemStore {
     private FilerClient filerClient;
     private Configuration conf;
 
-    public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) {
-        filerClient = new FilerClient(host, port, grpcPort);
+    public SeaweedFileSystemStore(String host, int port, int grpcPort, String cn, Configuration conf) {
+        filerClient = new FilerClient(host, port, grpcPort, cn);
         this.conf = conf;
         String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
         if (volumeServerAccessMode.equals("publicUrl")) {
@@ -36,7 +36,6 @@ public class SeaweedFileSystemStore {
         } else if (volumeServerAccessMode.equals("filerProxy")) {
             filerClient.setAccessVolumeServerByFilerProxy();
         }
-
     }
 
     public void close() {

test/mq/Dockerfile.test

@@ -0,0 +1,38 @@
+FROM golang:1.24-alpine
+
+# Install necessary tools
+RUN apk add --no-cache \
+    curl \
+    netcat-openbsd \
+    bash \
+    git \
+    build-base \
+    ca-certificates
+
+# Set working directory
+WORKDIR /workspace
+
+# Copy go mod files first for better caching
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy only the necessary source code for building and testing
+COPY weed/ ./weed/
+COPY test/mq/integration/ ./test/mq/integration/
+
+# Build the weed binary for testing (optional - tests can use external cluster)
+RUN cd weed && CGO_ENABLED=1 GOOS=linux go build -o ../weed .
+
+# Create test results directory
+RUN mkdir -p /test-results
+
+# Set up environment
+ENV CGO_ENABLED=1
+ENV GOOS=linux
+ENV GO111MODULE=on
+
+# Default working directory for tests
+WORKDIR /workspace
+
+# Entry point for running tests - use explicit bash
+ENTRYPOINT ["/bin/bash", "-c"] 

test/mq/Makefile

@@ -0,0 +1,223 @@
+.PHONY: help build test test-basic test-performance test-failover test-agent clean up down logs
+
+# Detect architecture and Docker platform compatibility
+ARCH := $(shell uname -m)
+OS := $(shell uname -s)
+ifeq ($(ARCH),arm64)
+    ifeq ($(OS),Darwin)
+        # On Apple Silicon macOS, use native arm64 for better performance
+        DOCKER_PLATFORM := linux/arm64
+    else
+        DOCKER_PLATFORM := linux/arm64
+    endif
+else
+    DOCKER_PLATFORM := linux/amd64
+endif
+
+# Default target
+help:
+	@echo "SeaweedMQ Integration Test Suite"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  build          - Build SeaweedFS Docker images"
+	@echo "  test           - Run all integration tests (in Docker)"
+	@echo "  test-basic     - Run basic pub/sub tests (in Docker)"
+	@echo "  test-native    - Run all tests natively (no Docker test container)"
+	@echo "  test-basic-native - Run basic tests natively (recommended for Apple Silicon)"
+	@echo "  test-performance - Run performance tests"
+	@echo "  test-failover  - Run failover tests"
+	@echo "  test-agent     - Run agent tests"
+	@echo "  up             - Start test environment (local build)"
+	@echo "  up-prod        - Start test environment (production images)"
+	@echo "  up-cluster     - Start cluster only (no test runner)"
+	@echo "  down           - Stop test environment"
+	@echo "  clean          - Clean up test environment and results"
+	@echo "  logs           - Show container logs"
+
+# Build SeaweedFS Docker images
+build:
+	@echo "Building SeaweedFS Docker image for $(DOCKER_PLATFORM)..."
+	cd ../.. && docker build --platform $(DOCKER_PLATFORM) -f docker/Dockerfile.go_build -t chrislusf/seaweedfs:local .
+	@echo "Building test runner image for $(DOCKER_PLATFORM)..."
+	cd ../.. && docker build --platform $(DOCKER_PLATFORM) -f test/mq/Dockerfile.test -t seaweedfs-test-runner .
+
+# Start the test environment
+up: build
+	@echo "Starting SeaweedMQ test environment..."
+	docker-compose -f docker-compose.test.yml up -d master0 master1 master2
+	@echo "Waiting for masters to be ready..."
+	sleep 10
+	docker-compose -f docker-compose.test.yml up -d volume1 volume2 volume3
+	@echo "Waiting for volumes to be ready..."
+	sleep 10
+	docker-compose -f docker-compose.test.yml up -d filer1 filer2
+	@echo "Waiting for filers to be ready..."
+	sleep 15
+	docker-compose -f docker-compose.test.yml up -d broker1 broker2 broker3
+	@echo "Waiting for brokers to be ready..."
+	sleep 20
+	@echo "Test environment is ready!"
+
+# Start the test environment with production images (no build required)
+up-prod: build-test-runner
+	@echo "Starting SeaweedMQ test environment with production images..."
+	docker-compose -f docker-compose.production.yml up -d master0 master1 master2
+	@echo "Waiting for masters to be ready..."
+	sleep 10
+	docker-compose -f docker-compose.production.yml up -d volume1 volume2 volume3
+	@echo "Waiting for volumes to be ready..."
+	sleep 10
+	docker-compose -f docker-compose.production.yml up -d filer1 filer2
+	@echo "Waiting for filers to be ready..."
+	sleep 15
+	docker-compose -f docker-compose.production.yml up -d broker1 broker2 broker3
+	@echo "Waiting for brokers to be ready..."
+	sleep 20
+	@echo "Test environment is ready!"
+
+# Build only the test runner image (for production setup)
+build-test-runner:
+	@echo "Building test runner image for $(DOCKER_PLATFORM)..."
+	cd ../.. && docker build --platform $(DOCKER_PLATFORM) -f test/mq/Dockerfile.test -t seaweedfs-test-runner .
+
+# Start cluster only (no test runner, no build required)
+up-cluster:
+	@echo "Starting SeaweedMQ cluster only..."
+	docker-compose -f docker-compose.cluster.yml up -d master0 master1 master2
+	@echo "Waiting for masters to be ready..."
+	sleep 10
+	docker-compose -f docker-compose.cluster.yml up -d volume1 volume2 volume3
+	@echo "Waiting for volumes to be ready..."
+	sleep 10
+	docker-compose -f docker-compose.cluster.yml up -d filer1 filer2
+	@echo "Waiting for filers to be ready..."
+	sleep 15
+	docker-compose -f docker-compose.cluster.yml up -d broker1 broker2 broker3
+	@echo "Waiting for brokers to be ready..."
+	sleep 20
+	@echo "SeaweedMQ cluster is ready!"
+	@echo "Masters: http://localhost:19333, http://localhost:19334, http://localhost:19335"
+	@echo "Filers: http://localhost:18888, http://localhost:18889"
+	@echo "Brokers: localhost:17777, localhost:17778, localhost:17779"
+
+# Stop the test environment
+down:
+	@echo "Stopping SeaweedMQ test environment..."
+	docker-compose -f docker-compose.test.yml down
+	docker-compose -f docker-compose.production.yml down
+	docker-compose -f docker-compose.cluster.yml down
+
+# Clean up everything
+clean:
+	@echo "Cleaning up test environment..."
+	docker-compose -f docker-compose.test.yml down -v
+	docker system prune -f
+	sudo rm -rf /tmp/test-results/*
+
+# Show container logs
+logs:
+	docker-compose -f docker-compose.test.yml logs -f
+
+# Run all integration tests
+test:
+	@echo "Running all integration tests..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=30m ./test/mq/integration/... -args -test.parallel=4"
+
+# Run basic pub/sub tests
+test-basic:
+	@echo "Running basic pub/sub tests natively (no container restart)..."
+	cd ../.. && SEAWEED_MASTERS="localhost:19333,localhost:19334,localhost:19335" \
+	SEAWEED_BROKERS="localhost:17777,localhost:17778,localhost:17779" \
+	SEAWEED_FILERS="localhost:18888,localhost:18889" \
+	go test -v -timeout=10m ./test/mq/integration/ -run TestBasic
+
+# Run performance tests
+test-performance:
+	@echo "Running performance tests..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=20m ./test/mq/integration/ -run TestPerformance"
+
+# Run failover tests
+test-failover:
+	@echo "Running failover tests..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=15m ./test/mq/integration/ -run TestFailover"
+
+# Run agent tests
+test-agent:
+	@echo "Running agent tests..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=10m ./test/mq/integration/ -run TestAgent"
+
+# Development targets (run tests natively without Docker container)
+test-dev:
+	@echo "Running tests in development mode (using local binaries)..."
+	SEAWEED_MASTERS="localhost:19333,localhost:19334,localhost:19335" \
+	SEAWEED_BROKERS="localhost:17777,localhost:17778,localhost:17779" \
+	SEAWEED_FILERS="localhost:18888,localhost:18889" \
+	go test -v -timeout=10m ./integration/...
+
+# Native test running (no Docker container for tests)
+test-native:
+	@echo "Running tests natively (without Docker container for tests)..."
+	cd ../.. && SEAWEED_MASTERS="localhost:19333,localhost:19334,localhost:19335" \
+	SEAWEED_BROKERS="localhost:17777,localhost:17778,localhost:17779" \
+	SEAWEED_FILERS="localhost:18888,localhost:18889" \
+	go test -v -timeout=10m ./test/mq/integration/...
+
+# Basic native tests
+test-basic-native:
+	@echo "Running basic tests natively..."
+	cd ../.. && SEAWEED_MASTERS="localhost:19333,localhost:19334,localhost:19335" \
+	SEAWEED_BROKERS="localhost:17777,localhost:17778,localhost:17779" \
+	SEAWEED_FILERS="localhost:18888,localhost:18889" \
+	go test -v -timeout=10m ./test/mq/integration/ -run TestBasic
+
+# Quick smoke test
+smoke-test:
+	@echo "Running smoke test..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=5m ./test/mq/integration/ -run TestBasicPublishSubscribe"
+
+# Performance benchmarks
+benchmark:
+	@echo "Running performance benchmarks..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=30m -bench=. ./test/mq/integration/..."
+
+# Check test environment health
+health:
+	@echo "Checking test environment health..."
+	@echo "Masters:"
+	@curl -s http://localhost:19333/cluster/status || echo "Master 0 not accessible"
+	@curl -s http://localhost:19334/cluster/status || echo "Master 1 not accessible"
+	@curl -s http://localhost:19335/cluster/status || echo "Master 2 not accessible"
+	@echo ""
+	@echo "Filers:"
+	@curl -s http://localhost:18888/ || echo "Filer 1 not accessible"
+	@curl -s http://localhost:18889/ || echo "Filer 2 not accessible"
+	@echo ""
+	@echo "Brokers:"
+	@nc -z localhost 17777 && echo "Broker 1 accessible" || echo "Broker 1 not accessible"
+	@nc -z localhost 17778 && echo "Broker 2 accessible" || echo "Broker 2 not accessible"  
+	@nc -z localhost 17779 && echo "Broker 3 accessible" || echo "Broker 3 not accessible"
+
+# Generate test reports
+report:
+	@echo "Generating test reports..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=30m ./test/mq/integration/... -json > /test-results/test-report.json"
+
+# Load testing
+load-test:
+	@echo "Running load tests..."
+	docker-compose -f docker-compose.test.yml run --rm test-runner \
+		sh -c "go test -v -timeout=45m ./test/mq/integration/ -run TestLoad"
+
+# View monitoring dashboards
+monitoring:
+	@echo "Starting monitoring stack..."
+	docker-compose -f docker-compose.test.yml up -d prometheus grafana
+	@echo "Prometheus: http://localhost:19090"
+	@echo "Grafana: http://localhost:13000 (admin/admin)" 

test/mq/README.md

@@ -0,0 +1,414 @@
+# SeaweedMQ Integration Test Suite
+
+This directory contains a comprehensive integration test suite for SeaweedMQ, designed to validate all critical functionalities from basic pub/sub operations to advanced features like auto-scaling, failover, and performance testing.
+
+## Overview
+
+The integration test suite provides:
+
+- **Automated Environment Setup**: Docker Compose based test clusters
+- **Comprehensive Test Coverage**: Basic pub/sub, scaling, failover, performance
+- **Monitoring & Metrics**: Prometheus and Grafana integration
+- **CI/CD Ready**: Configurable for continuous integration pipelines
+- **Load Testing**: Performance benchmarks and stress tests
+
+## Architecture
+
+The test environment consists of:
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│  Master Cluster │    │ Volume Servers  │    │  Filer Cluster  │
+│   (3 nodes)     │    │   (3 nodes)     │    │   (2 nodes)     │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+         │                       │                       │
+         └───────────────────────┼───────────────────────┘
+                                 │
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│ Broker Cluster  │    │ Test Framework  │    │   Monitoring    │
+│   (3 nodes)     │    │   (Go Tests)    │    │ (Prometheus +   │
+└─────────────────┘    └─────────────────┘    │   Grafana)      │
+                                              └─────────────────┘
+```
+
+## Quick Start
+
+### Prerequisites
+
+- Docker and Docker Compose
+- Go 1.21+
+- Make
+- 8GB+ RAM recommended
+- 20GB+ disk space for test data
+
+### Basic Usage
+
+Choose one of three approaches:
+
+#### Option 1: Quick Cluster Setup (Fastest)
+Just starts a SeaweedMQ cluster - no test runner, no build required.
+
+1. **Start Cluster**:
+   ```bash
+   cd test/mq
+   make up-cluster
+   ```
+
+2. **Test manually** or with your own code against:
+   - Masters: http://localhost:19333, http://localhost:19334, http://localhost:19335
+   - Brokers: localhost:17777, localhost:17778, localhost:17779
+   - Filers: http://localhost:18888, http://localhost:18889
+
+#### Option 2: Use Production Images (Recommended for testing)
+Uses official SeaweedFS images from Docker Hub with test runner.
+
+1. **Start Test Environment**:
+   ```bash
+   cd test/mq
+   make up-prod
+   ```
+
+2. **Run Tests**:
+   ```bash
+   make test-basic      # Basic pub/sub tests
+   ```
+
+#### Option 3: Build from Source (Development)
+Builds SeaweedFS from your local source code to test latest changes.
+
+1. **Start Test Environment**:
+   ```bash
+   cd test/mq
+   make up  # This will build and start
+   ```
+
+2. **Run Tests**:
+   ```bash
+   make test-basic      # Basic pub/sub tests
+   ```
+
+#### Common Commands
+
+3. **Run All Tests**:
+   ```bash
+   make test
+   ```
+
+4. **Run Specific Test Categories**:
+   ```bash
+   make test-performance # Performance tests
+   make test-failover   # Failover tests
+   make test-agent      # Agent tests
+   ```
+
+5. **Quick Smoke Test**:
+   ```bash
+   make smoke-test
+   ```
+
+6. **Check Health**:
+   ```bash
+   make health
+   ```
+
+7. **Clean Up**:
+   ```bash
+   make down
+   ```
+
+## Test Categories
+
+### 1. Basic Functionality Tests
+
+**File**: `integration/basic_pubsub_test.go`
+
+- **TestBasicPublishSubscribe**: Basic message publishing and consumption
+- **TestMultipleConsumers**: Load balancing across multiple consumers
+- **TestMessageOrdering**: FIFO ordering within partitions
+- **TestSchemaValidation**: Schema validation and complex nested structures
+
+### 2. Partitioning and Scaling Tests
+
+**File**: `integration/scaling_test.go` (to be implemented)
+
+- **TestPartitionDistribution**: Message distribution across partitions
+- **TestAutoSplitMerge**: Automatic partition split/merge based on load
+- **TestBrokerScaling**: Adding/removing brokers during operation
+- **TestLoadBalancing**: Even load distribution verification
+
+### 3. Failover and Reliability Tests
+
+**File**: `integration/failover_test.go` (to be implemented)
+
+- **TestBrokerFailover**: Leader failover scenarios
+- **TestBrokerRecovery**: Recovery from broker failures
+- **TestMessagePersistence**: Data durability across restarts
+- **TestFollowerReplication**: Leader-follower consistency
+
+### 4. Performance Tests
+
+**File**: `integration/performance_test.go` (to be implemented)
+
+- **TestHighThroughputPublish**: High-volume message publishing
+- **TestHighThroughputSubscribe**: High-volume message consumption
+- **TestLatencyMeasurement**: End-to-end latency analysis
+- **TestResourceUtilization**: CPU, memory, and disk usage
+
+### 5. Agent Tests
+
+**File**: `integration/agent_test.go` (to be implemented)
+
+- **TestAgentPublishSessions**: Session management for publishers
+- **TestAgentSubscribeSessions**: Session management for subscribers
+- **TestAgentFailover**: Agent reconnection and failover
+- **TestAgentConcurrency**: Concurrent session handling
+
+## Configuration
+
+### Environment Variables
+
+The test framework supports configuration via environment variables:
+
+```bash
+# Cluster endpoints
+SEAWEED_MASTERS="master0:9333,master1:9334,master2:9335"
+SEAWEED_BROKERS="broker1:17777,broker2:17778,broker3:17779"
+SEAWEED_FILERS="filer1:8888,filer2:8889"
+
+# Test configuration
+GO_TEST_TIMEOUT="30m"
+TEST_RESULTS_DIR="/test-results"
+```
+
+### Docker Compose Override
+
+Create `docker-compose.override.yml` to customize the test environment:
+
+```yaml
+version: '3.9'
+services:
+  broker1:
+    environment:
+      - CUSTOM_ENV_VAR=value
+  test-runner:
+    volumes:
+      - ./custom-config:/config
+```
+
+## Monitoring and Metrics
+
+### Prometheus Metrics
+
+Access Prometheus at: http://localhost:19090
+
+Key metrics to monitor:
+- Message throughput: `seaweedmq_messages_published_total`
+- Consumer lag: `seaweedmq_consumer_lag_seconds`
+- Broker health: `seaweedmq_broker_health`
+- Resource usage: `seaweedfs_disk_usage_bytes`
+
+### Grafana Dashboards
+
+Access Grafana at: http://localhost:13000 (admin/admin)
+
+Pre-configured dashboards:
+- **SeaweedMQ Overview**: System health and throughput
+- **Performance Metrics**: Latency and resource usage
+- **Error Analysis**: Error rates and failure patterns
+
+## Development
+
+### Writing New Tests
+
+1. **Create Test File**:
+   ```bash
+   touch integration/my_new_test.go
+   ```
+
+2. **Use Test Framework**:
+   ```go
+   func TestMyFeature(t *testing.T) {
+       suite := NewIntegrationTestSuite(t)
+       require.NoError(t, suite.Setup())
+       
+       // Your test logic here
+   }
+   ```
+
+3. **Run Specific Test**:
+   ```bash
+   go test -v ./integration/ -run TestMyFeature
+   ```
+
+### Test Framework Components
+
+**IntegrationTestSuite**: Base test framework with cluster management
+**MessageCollector**: Utility for collecting and verifying received messages
+**TestMessage**: Standard message structure for testing
+**Schema Builders**: Helpers for creating test schemas
+
+### Local Development
+
+Run tests against a local SeaweedMQ cluster:
+
+```bash
+make test-dev
+```
+
+This uses local binaries instead of Docker containers.
+
+## Continuous Integration
+
+### GitHub Actions Example
+
+```yaml
+name: Integration Tests
+on: [push, pull_request]
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.21
+      - name: Run Integration Tests
+        run: |
+          cd test/mq
+          make test
+```
+
+### Jenkins Pipeline
+
+```groovy
+pipeline {
+    agent any
+    stages {
+        stage('Setup') {
+            steps {
+                sh 'cd test/mq && make up'
+            }
+        }
+        stage('Test') {
+            steps {
+                sh 'cd test/mq && make test'
+            }
+            post {
+                always {
+                    sh 'cd test/mq && make down'
+                }
+            }
+        }
+    }
+}
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Port Conflicts**:
+   ```bash
+   # Check port usage
+   netstat -tulpn | grep :19333
+   
+   # Kill conflicting processes
+   sudo kill -9 $(lsof -t -i:19333)
+   ```
+
+2. **Docker Resource Issues**:
+   ```bash
+   # Increase Docker memory (8GB+)
+   # Clean up Docker resources
+   docker system prune -a
+   ```
+
+3. **Test Timeouts**:
+   ```bash
+   # Increase timeout
+   GO_TEST_TIMEOUT=60m make test
+   ```
+
+### Debug Mode
+
+Run tests with verbose logging:
+
+```bash
+docker-compose -f docker-compose.test.yml run --rm test-runner \
+  sh -c "go test -v -race ./test/mq/integration/... -args -test.v"
+```
+
+### Container Logs
+
+View real-time logs:
+
+```bash
+make logs
+
+# Or specific service
+docker-compose -f docker-compose.test.yml logs -f broker1
+```
+
+## Performance Benchmarks
+
+### Throughput Benchmarks
+
+```bash
+make benchmark
+```
+
+Expected performance (on 8-core, 16GB RAM):
+- **Publish Throughput**: 50K+ messages/second/broker
+- **Subscribe Throughput**: 100K+ messages/second/broker
+- **End-to-End Latency**: P95 < 100ms
+- **Storage Efficiency**: < 20% overhead
+
+### Load Testing
+
+```bash
+make load-test
+```
+
+Stress tests with:
+- 1M+ messages
+- 100+ concurrent producers
+- 50+ concurrent consumers
+- Multiple topic scenarios
+
+## Contributing
+
+### Test Guidelines
+
+1. **Test Isolation**: Each test should be independent
+2. **Resource Cleanup**: Always clean up resources in test teardown
+3. **Timeouts**: Set appropriate timeouts for operations
+4. **Error Handling**: Test both success and failure scenarios
+5. **Documentation**: Document test purpose and expected behavior
+
+### Code Style
+
+- Follow Go testing conventions
+- Use testify for assertions
+- Include setup/teardown in test functions
+- Use descriptive test names
+
+## Future Enhancements
+
+- [ ] Chaos engineering tests (network partitions, node failures)
+- [ ] Multi-datacenter deployment testing
+- [ ] Schema evolution compatibility tests
+- [ ] Security and authentication tests
+- [ ] Performance regression detection
+- [ ] Automated load pattern generation
+
+## Support
+
+For issues and questions:
+- Check existing GitHub issues
+- Review SeaweedMQ documentation
+- Join SeaweedFS community discussions
+
+---
+
+*This integration test suite ensures SeaweedMQ's reliability, performance, and functionality across all critical use cases and failure scenarios.* 

test/mq/docker-compose.cluster.yml

@@ -0,0 +1,165 @@
+services:
+  # Masters
+  master0:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "19333:9333"
+    volumes:
+      - /tmp/seaweedfs-test/master0:/data
+    command: "master -port=9333 -mdir=/data -peers=master0:9333,master1:9334,master2:9335 -ip=master0 -defaultReplication=001"
+    networks:
+      - seaweedfs-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master0:9333/cluster/status"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+
+  master1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "19334:9334"
+    volumes:
+      - /tmp/seaweedfs-test/master1:/data
+    command: "master -port=9334 -mdir=/data -peers=master0:9333,master1:9334,master2:9335 -ip=master1 -defaultReplication=001"
+    networks:
+      - seaweedfs-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master1:9334/cluster/status"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+
+  master2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "19335:9335"
+    volumes:
+      - /tmp/seaweedfs-test/master2:/data
+    command: "master -port=9335 -mdir=/data -peers=master0:9333,master1:9334,master2:9335 -ip=master2 -defaultReplication=001"
+    networks:
+      - seaweedfs-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master2:9335/cluster/status"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+
+  # Volume Servers
+  volume1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18080:8080"
+    volumes:
+      - /tmp/seaweedfs-test/volume1:/data
+    command: "volume -port=8080 -mserver=master0:9333,master1:9334,master2:9335 -dir=/data"
+    depends_on:
+      - master0
+      - master1
+      - master2
+    networks:
+      - seaweedfs-test
+
+  volume2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18081:8081"
+    volumes:
+      - /tmp/seaweedfs-test/volume2:/data
+    command: "volume -port=8081 -mserver=master0:9333,master1:9334,master2:9335 -dir=/data"
+    depends_on:
+      - master0
+      - master1
+      - master2
+    networks:
+      - seaweedfs-test
+
+  volume3:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18082:8082"
+    volumes:
+      - /tmp/seaweedfs-test/volume3:/data
+    command: "volume -port=8082 -mserver=master0:9333,master1:9334,master2:9335 -dir=/data"
+    depends_on:
+      - master0
+      - master1
+      - master2
+    networks:
+      - seaweedfs-test
+
+  # Filers
+  filer1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18888:8888"
+    volumes:
+      - /tmp/seaweedfs-test/filer1:/data
+    command: "filer -port=8888 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - volume1
+      - volume2
+      - volume3
+    networks:
+      - seaweedfs-test
+
+  filer2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18889:8889"
+    volumes:
+      - /tmp/seaweedfs-test/filer2:/data
+    command: "filer -port=8889 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - volume1
+      - volume2
+      - volume3
+    networks:
+      - seaweedfs-test
+
+  # Message Queue Brokers
+  broker1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "17777:17777"
+    volumes:
+      - /tmp/seaweedfs-test/broker1:/data
+    command: "mq.broker -port=17777 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - filer1
+      - filer2
+    networks:
+      - seaweedfs-test
+
+  broker2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "17778:17778"
+    volumes:
+      - /tmp/seaweedfs-test/broker2:/data
+    command: "mq.broker -port=17778 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - filer1
+      - filer2
+    networks:
+      - seaweedfs-test
+
+  broker3:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "17779:17779"
+    volumes:
+      - /tmp/seaweedfs-test/broker3:/data
+    command: "mq.broker -port=17779 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - filer1
+      - filer2
+    networks:
+      - seaweedfs-test
+
+networks:
+  seaweedfs-test:
+    driver: bridge 

test/mq/docker-compose.production.yml

@@ -0,0 +1,260 @@
+services:
+  # Masters
+  master0:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "19333:19333"
+      - "9333:9333"
+    volumes:
+      - /tmp/seaweedfs-test/master0:/data
+    command: "master -port=9333 -mdir=/data -peers=master0:9333,master1:9334,master2:9335 -defaultReplication=001"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 6
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 3
+    networks:
+      - seaweedfs-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master0:9333/cluster/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  master1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "19334:9334"
+      - "9334:9334"
+    volumes:
+      - /tmp/seaweedfs-test/master1:/data
+    command: "master -port=9334 -mdir=/data -peers=master0:9333,master1:9334,master2:9335 -defaultReplication=001"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 6
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 3
+    networks:
+      - seaweedfs-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master1:9334/cluster/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  master2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "19335:9335"
+      - "9335:9335"
+    volumes:
+      - /tmp/seaweedfs-test/master2:/data
+    command: "master -port=9335 -mdir=/data -peers=master0:9333,master1:9334,master2:9335 -defaultReplication=001"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 6
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 3
+    networks:
+      - seaweedfs-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master2:9335/cluster/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  # Volume Servers
+  volume1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18080:8080"
+      - "8080:8080"
+    volumes:
+      - /tmp/seaweedfs-test/volume1:/data
+    command: "volume -port=8080 -mserver=master0:9333,master1:9334,master2:9335 -dir=/data"
+    depends_on:
+      master0:
+        condition: service_healthy
+      master1:
+        condition: service_healthy
+      master2:
+        condition: service_healthy
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_VOLUME_MAX: 100
+
+  volume2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18081:8081"
+      - "8081:8081"
+    volumes:
+      - /tmp/seaweedfs-test/volume2:/data
+    command: "volume -port=8081 -mserver=master0:9333,master1:9334,master2:9335 -dir=/data"
+    depends_on:
+      master0:
+        condition: service_healthy
+      master1:
+        condition: service_healthy
+      master2:
+        condition: service_healthy
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_VOLUME_MAX: 100
+
+  volume3:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18082:8082"
+      - "8082:8082"
+    volumes:
+      - /tmp/seaweedfs-test/volume3:/data
+    command: "volume -port=8082 -mserver=master0:9333,master1:9334,master2:9335 -dir=/data"
+    depends_on:
+      master0:
+        condition: service_healthy
+      master1:
+        condition: service_healthy
+      master2:
+        condition: service_healthy
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_VOLUME_MAX: 100
+
+  # Filers
+  filer1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18888:8888"
+      - "8888:8888"
+    volumes:
+      - /tmp/seaweedfs-test/filer1:/data
+    command: "filer -port=8888 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - volume1
+      - volume2
+      - volume3
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_LEVELDB2_DIR: "/data/filerldb2"
+
+  filer2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "18889:8889"
+      - "8889:8889"
+    volumes:
+      - /tmp/seaweedfs-test/filer2:/data
+    command: "filer -port=8889 -master=master0:9333,master1:9334,master2:9335"
+    depends_on:
+      - volume1
+      - volume2
+      - volume3
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_LEVELDB2_DIR: "/data/filerldb2"
+
+  # Message Queue Brokers
+  broker1:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "17777:17777"
+    volumes:
+      - /tmp/seaweedfs-test/broker1:/data
+    command: "mq.broker -port=17777 -filer=filer1:8888,filer2:8889"
+    depends_on:
+      - filer1
+      - filer2
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_MQ_BROKER_CPU_PERCENT: 80
+
+  broker2:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "17778:17778"
+    volumes:
+      - /tmp/seaweedfs-test/broker2:/data
+    command: "mq.broker -port=17778 -filer=filer1:8888,filer2:8889"
+    depends_on:
+      - filer1
+      - filer2
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_MQ_BROKER_CPU_PERCENT: 80
+
+  broker3:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "17779:17779"
+    volumes:
+      - /tmp/seaweedfs-test/broker3:/data
+    command: "mq.broker -port=17779 -filer=filer1:8888,filer2:8889"
+    depends_on:
+      - filer1
+      - filer2
+    networks:
+      - seaweedfs-test
+    environment:
+      WEED_MQ_BROKER_CPU_PERCENT: 80
+
+  # Test Runner
+  test-runner:
+    image: seaweedfs-test-runner
+    volumes:
+      - ../..:/workspace
+      - /tmp/test-results:/test-results
+    working_dir: /workspace
+    networks:
+      - seaweedfs-test
+    environment:
+      - SEAWEED_MASTERS=master0:9333,master1:9334,master2:9335
+      - SEAWEED_BROKERS=broker1:17777,broker2:17778,broker3:17779
+      - SEAWEED_FILERS=filer1:8888,filer2:8889
+      - GO111MODULE=on
+      - CGO_ENABLED=0
+    depends_on:
+      - broker1
+      - broker2
+      - broker3
+
+  # Monitoring
+  prometheus:
+    image: prom/prometheus:latest
+    ports:
+      - "19090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    networks:
+      - seaweedfs-test
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume1
+      - volume2
+      - volume3
+      - filer1
+      - filer2
+      - broker1
+      - broker2
+      - broker3
+
+  grafana:
+    image: grafana/grafana:latest
+    ports:
+      - "13000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    networks:
+      - seaweedfs-test
+    depends_on:
+      - prometheus
+
+networks:
+  seaweedfs-test:
+    driver: bridge 

test/mq/docker-compose.test.yml

@@ -0,0 +1,341 @@
+services:
+  # Master cluster for coordination and metadata
+  master0:
+    image: chrislusf/seaweedfs:local
+    container_name: test-master0
+    ports:
+      - "19333:9333"
+      - "29333:19333"
+    command: >
+      -v=1 
+      master 
+      -volumeSizeLimitMB=100 
+      -resumeState=false 
+      -ip=master0 
+      -port=9333 
+      -peers=master0:9333,master1:9334,master2:9335 
+      -mdir=/tmp/master0
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+    networks:
+      - seaweedmq-test
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://master0:9333/cluster/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  master1:
+    image: chrislusf/seaweedfs:local
+    container_name: test-master1
+    ports:
+      - "19334:9334"
+      - "29334:19334"
+    command: >
+      -v=1 
+      master 
+      -volumeSizeLimitMB=100 
+      -resumeState=false 
+      -ip=master1 
+      -port=9334 
+      -peers=master0:9333,master1:9334,master2:9335 
+      -mdir=/tmp/master1
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+    networks:
+      - seaweedmq-test
+    depends_on:
+      - master0
+
+  master2:
+    image: chrislusf/seaweedfs:local
+    container_name: test-master2
+    ports:
+      - "19335:9335"
+      - "29335:19335"
+    command: >
+      -v=1 
+      master 
+      -volumeSizeLimitMB=100 
+      -resumeState=false 
+      -ip=master2 
+      -port=9335 
+      -peers=master0:9333,master1:9334,master2:9335 
+      -mdir=/tmp/master2
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+    networks:
+      - seaweedmq-test
+    depends_on:
+      - master0
+
+  # Volume servers for data storage
+  volume1:
+    image: chrislusf/seaweedfs:local
+    container_name: test-volume1
+    ports:
+      - "18080:8080"
+      - "28080:18080"
+    volumes:
+      - volume1-data:/data/volume1
+    entrypoint: ["/bin/sh", "-c"]
+    command: >
+      "mkdir -p /data/volume1 && exec weed -v=1
+      volume 
+      -dataCenter=dc1 
+      -rack=rack1 
+      -mserver=master0:9333,master1:9334,master2:9335 
+      -port=8080 
+      -ip=volume1 
+      -publicUrl=localhost:18080 
+      -preStopSeconds=1
+      -dir=/data/volume1"
+    networks:
+      - seaweedmq-test
+    depends_on:
+      master0:
+        condition: service_healthy
+
+  volume2:
+    image: chrislusf/seaweedfs:local
+    container_name: test-volume2
+    ports:
+      - "18081:8081"
+      - "28081:18081"
+    volumes:
+      - volume2-data:/data/volume2
+    entrypoint: ["/bin/sh", "-c"]
+    command: >
+      "mkdir -p /data/volume2 && exec weed -v=1
+      volume 
+      -dataCenter=dc1 
+      -rack=rack2 
+      -mserver=master0:9333,master1:9334,master2:9335 
+      -port=8081 
+      -ip=volume2 
+      -publicUrl=localhost:18081 
+      -preStopSeconds=1
+      -dir=/data/volume2"
+    networks:
+      - seaweedmq-test
+    depends_on:
+      master0:
+        condition: service_healthy
+
+  volume3:
+    image: chrislusf/seaweedfs:local
+    container_name: test-volume3
+    ports:
+      - "18082:8082"
+      - "28082:18082"
+    volumes:
+      - volume3-data:/data/volume3
+    entrypoint: ["/bin/sh", "-c"]
+    command: >
+      "mkdir -p /data/volume3 && exec weed -v=1
+      volume 
+      -dataCenter=dc2 
+      -rack=rack1 
+      -mserver=master0:9333,master1:9334,master2:9335 
+      -port=8082 
+      -ip=volume3 
+      -publicUrl=localhost:18082 
+      -preStopSeconds=1
+      -dir=/data/volume3"
+    networks:
+      - seaweedmq-test
+    depends_on:
+      master0:
+        condition: service_healthy
+
+  # Filer servers for metadata
+  filer1:
+    image: chrislusf/seaweedfs:local
+    container_name: test-filer1
+    ports:
+      - "18888:8888"
+      - "28888:18888"
+    command: >
+      -v=1
+      filer 
+      -defaultReplicaPlacement=100 
+      -master=master0:9333,master1:9334,master2:9335 
+      -port=8888 
+      -ip=filer1
+      -dataCenter=dc1
+    networks:
+      - seaweedmq-test
+    depends_on:
+      master0:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://filer1:8888/"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  filer2:
+    image: chrislusf/seaweedfs:local
+    container_name: test-filer2
+    ports:
+      - "18889:8889"
+      - "28889:18889"
+    command: >
+      -v=1
+      filer 
+      -defaultReplicaPlacement=100 
+      -master=master0:9333,master1:9334,master2:9335 
+      -port=8889 
+      -ip=filer2
+      -dataCenter=dc2
+    networks:
+      - seaweedmq-test
+    depends_on:
+      filer1:
+        condition: service_healthy
+
+  # Message Queue Brokers
+  broker1:
+    image: chrislusf/seaweedfs:local
+    container_name: test-broker1
+    ports:
+      - "17777:17777"
+    command: >
+      -v=1
+      mq.broker 
+      -master=master0:9333,master1:9334,master2:9335 
+      -port=17777 
+      -ip=127.0.0.1
+      -dataCenter=dc1
+      -rack=rack1
+    networks:
+      - seaweedmq-test
+    depends_on:
+      filer1:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "nc", "-z", "localhost", "17777"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  broker2:
+    image: chrislusf/seaweedfs:local
+    container_name: test-broker2
+    ports:
+      - "17778:17778"
+    command: >
+      -v=1
+      mq.broker 
+      -master=master0:9333,master1:9334,master2:9335 
+      -port=17778 
+      -ip=127.0.0.1
+      -dataCenter=dc1
+      -rack=rack2
+    networks:
+      - seaweedmq-test
+    depends_on:
+      broker1:
+        condition: service_healthy
+
+  broker3:
+    image: chrislusf/seaweedfs:local
+    container_name: test-broker3
+    ports:
+      - "17779:17779"
+    command: >
+      -v=1
+      mq.broker 
+      -master=master0:9333,master1:9334,master2:9335 
+      -port=17779 
+      -ip=127.0.0.1
+      -dataCenter=dc2
+      -rack=rack1
+    networks:
+      - seaweedmq-test
+    depends_on:
+      broker1:
+        condition: service_healthy
+
+  # Test runner container
+  test-runner:
+    build:
+      context: ../../
+      dockerfile: test/mq/Dockerfile.test
+    container_name: test-runner
+    volumes:
+      - ../../:/app
+      - /tmp/test-results:/test-results
+    working_dir: /app
+    environment:
+      - SEAWEED_MASTERS=master0:9333,master1:9334,master2:9335
+      - SEAWEED_BROKERS=broker1:17777,broker2:17778,broker3:17779
+      - SEAWEED_FILERS=filer1:8888,filer2:8889
+      - TEST_RESULTS_DIR=/test-results
+      - GO_TEST_TIMEOUT=30m
+    networks:
+      - seaweedmq-test
+    depends_on:
+      broker1:
+        condition: service_healthy
+      broker2:
+        condition: service_started
+      broker3:
+        condition: service_started
+    command: >
+      sh -c "
+        echo 'Waiting for cluster to be ready...' &&
+        sleep 30 &&
+        echo 'Running integration tests...' &&
+        go test -v -timeout=30m ./test/mq/integration/... -args -test.parallel=4
+      "
+
+  # Monitoring and metrics
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: test-prometheus
+    ports:
+      - "19090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    networks:
+      - seaweedmq-test
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.enable-lifecycle'
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: test-grafana
+    ports:
+      - "13000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    volumes:
+      - grafana-storage:/var/lib/grafana
+      - ./grafana/dashboards:/etc/grafana/provisioning/dashboards
+      - ./grafana/datasources:/etc/grafana/provisioning/datasources
+    networks:
+      - seaweedmq-test
+
+networks:
+  seaweedmq-test:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.20.0.0/16
+
+volumes:
+  grafana-storage:
+  volume1-data:
+  volume2-data:
+  volume3-data: 

test/mq/integration/basic_pubsub_test.go

@@ -0,0 +1,339 @@
+package integration
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBasicPublishSubscribe(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	namespace := "test"
+	topicName := fmt.Sprintf("basic-pubsub-%d", time.Now().UnixNano()) // Unique topic name per run
+	testSchema := CreateTestSchema()
+	messageCount := 10
+
+	// Create publisher
+	pubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName,
+		PartitionCount: 1,
+		PublisherName:  "basic-publisher",
+		RecordType:     testSchema,
+	}
+
+	publisher, err := suite.CreatePublisher(pubConfig)
+	require.NoError(t, err)
+
+	// Create subscriber
+	subConfig := &SubscriberTestConfig{
+		Namespace:          namespace,
+		TopicName:          topicName,
+		ConsumerGroup:      "test-group",
+		ConsumerInstanceId: "consumer-1",
+		MaxPartitionCount:  1,
+		SlidingWindowSize:  10,
+		OffsetType:         schema_pb.OffsetType_RESET_TO_EARLIEST,
+	}
+
+	subscriber, err := suite.CreateSubscriber(subConfig)
+	require.NoError(t, err, "Failed to create subscriber")
+
+	// Set up message collector
+	collector := NewMessageCollector(messageCount)
+	subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
+		t.Logf("[Subscriber] Received message with key: %s, ts: %d", string(m.Data.Key), m.Data.TsNs)
+		collector.AddMessage(TestMessage{
+			ID:        fmt.Sprintf("msg-%d", len(collector.GetMessages())),
+			Content:   m.Data.Value,
+			Timestamp: time.Unix(0, m.Data.TsNs),
+			Key:       m.Data.Key,
+		})
+	})
+
+	// Start subscriber
+	go func() {
+		err := subscriber.Subscribe()
+		if err != nil {
+			t.Logf("Subscriber error: %v", err)
+		}
+	}()
+
+	// Wait for subscriber to be ready
+	t.Logf("[Test] Waiting for subscriber to be ready...")
+	time.Sleep(2 * time.Second)
+
+	// Publish test messages
+	for i := 0; i < messageCount; i++ {
+		record := schema.RecordBegin().
+			SetString("id", fmt.Sprintf("msg-%d", i)).
+			SetInt64("timestamp", time.Now().UnixNano()).
+			SetString("content", fmt.Sprintf("Test message %d", i)).
+			SetInt32("sequence", int32(i)).
+			RecordEnd()
+
+		key := []byte(fmt.Sprintf("key-%d", i))
+		t.Logf("[Publisher] Publishing message %d with key: %s", i, string(key))
+		err := publisher.PublishRecord(key, record)
+		require.NoError(t, err, "Failed to publish message %d", i)
+	}
+
+	t.Logf("[Test] Waiting for messages to be received...")
+	messages := collector.WaitForMessages(30 * time.Second)
+	t.Logf("[Test] WaitForMessages returned. Received %d messages.", len(messages))
+
+	// Verify all messages were received
+	assert.Len(t, messages, messageCount, "Expected %d messages, got %d", messageCount, len(messages))
+
+	// Verify message content
+	for i, msg := range messages {
+		assert.NotEmpty(t, msg.Content, "Message %d should have content", i)
+		assert.NotEmpty(t, msg.Key, "Message %d should have key", i)
+	}
+
+	t.Logf("[Test] TestBasicPublishSubscribe completed.")
+}
+
+func TestMultipleConsumers(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	namespace := "test"
+	topicName := "multi-consumer"
+	testSchema := CreateTestSchema()
+	messageCount := 20
+	consumerCount := 3
+
+	// Create publisher
+	pubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName,
+		PartitionCount: 3, // Multiple partitions for load distribution
+		PublisherName:  "multi-publisher",
+		RecordType:     testSchema,
+	}
+
+	publisher, err := suite.CreatePublisher(pubConfig)
+	require.NoError(t, err)
+
+	// Create multiple consumers
+	collectors := make([]*MessageCollector, consumerCount)
+	for i := 0; i < consumerCount; i++ {
+		collectors[i] = NewMessageCollector(messageCount / consumerCount) // Expect roughly equal distribution
+
+		subConfig := &SubscriberTestConfig{
+			Namespace:          namespace,
+			TopicName:          topicName,
+			ConsumerGroup:      "multi-consumer-group", // Same group for load balancing
+			ConsumerInstanceId: fmt.Sprintf("consumer-%d", i),
+			MaxPartitionCount:  1,
+			SlidingWindowSize:  10,
+			OffsetType:         schema_pb.OffsetType_RESET_TO_EARLIEST,
+		}
+
+		subscriber, err := suite.CreateSubscriber(subConfig)
+		require.NoError(t, err)
+
+		// Set up message collection for this consumer
+		collectorIndex := i
+		subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
+			collectors[collectorIndex].AddMessage(TestMessage{
+				ID:        fmt.Sprintf("consumer-%d-msg-%d", collectorIndex, len(collectors[collectorIndex].GetMessages())),
+				Content:   m.Data.Value,
+				Timestamp: time.Unix(0, m.Data.TsNs),
+				Key:       m.Data.Key,
+			})
+		})
+
+		// Start subscriber
+		go func() {
+			subscriber.Subscribe()
+		}()
+	}
+
+	// Wait for subscribers to be ready
+	time.Sleep(3 * time.Second)
+
+	// Publish messages with different keys to distribute across partitions
+	for i := 0; i < messageCount; i++ {
+		record := schema.RecordBegin().
+			SetString("id", fmt.Sprintf("multi-msg-%d", i)).
+			SetInt64("timestamp", time.Now().UnixNano()).
+			SetString("content", fmt.Sprintf("Multi consumer test message %d", i)).
+			SetInt32("sequence", int32(i)).
+			RecordEnd()
+
+		key := []byte(fmt.Sprintf("partition-key-%d", i%3)) // Distribute across 3 partitions
+		err := publisher.PublishRecord(key, record)
+		require.NoError(t, err)
+	}
+
+	// Wait for all messages to be consumed
+	time.Sleep(10 * time.Second)
+
+	// Verify message distribution
+	totalReceived := 0
+	for i, collector := range collectors {
+		messages := collector.GetMessages()
+		t.Logf("Consumer %d received %d messages", i, len(messages))
+		totalReceived += len(messages)
+	}
+
+	// All messages should be consumed across all consumers
+	assert.Equal(t, messageCount, totalReceived, "Total messages received should equal messages sent")
+}
+
+func TestMessageOrdering(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	namespace := "test"
+	topicName := "ordering-test"
+	testSchema := CreateTestSchema()
+	messageCount := 15
+
+	// Create publisher
+	pubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName,
+		PartitionCount: 1, // Single partition to guarantee ordering
+		PublisherName:  "ordering-publisher",
+		RecordType:     testSchema,
+	}
+
+	publisher, err := suite.CreatePublisher(pubConfig)
+	require.NoError(t, err)
+
+	// Create subscriber
+	subConfig := &SubscriberTestConfig{
+		Namespace:          namespace,
+		TopicName:          topicName,
+		ConsumerGroup:      "ordering-group",
+		ConsumerInstanceId: "ordering-consumer",
+		MaxPartitionCount:  1,
+		SlidingWindowSize:  5,
+		OffsetType:         schema_pb.OffsetType_RESET_TO_EARLIEST,
+	}
+
+	subscriber, err := suite.CreateSubscriber(subConfig)
+	require.NoError(t, err)
+
+	// Set up message collector
+	collector := NewMessageCollector(messageCount)
+	subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
+		collector.AddMessage(TestMessage{
+			ID:        fmt.Sprintf("ordered-msg"),
+			Content:   m.Data.Value,
+			Timestamp: time.Unix(0, m.Data.TsNs),
+			Key:       m.Data.Key,
+		})
+	})
+
+	// Start subscriber
+	go func() {
+		subscriber.Subscribe()
+	}()
+
+	// Wait for consumer to be ready
+	time.Sleep(2 * time.Second)
+
+	// Publish messages with same key to ensure they go to same partition
+	publishTimes := make([]time.Time, messageCount)
+	for i := 0; i < messageCount; i++ {
+		publishTimes[i] = time.Now()
+
+		record := schema.RecordBegin().
+			SetString("id", fmt.Sprintf("ordered-%d", i)).
+			SetInt64("timestamp", publishTimes[i].UnixNano()).
+			SetString("content", fmt.Sprintf("Ordered message %d", i)).
+			SetInt32("sequence", int32(i)).
+			RecordEnd()
+
+		key := []byte("same-partition-key") // Same key ensures same partition
+		err := publisher.PublishRecord(key, record)
+		require.NoError(t, err)
+
+		// Small delay to ensure different timestamps
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	// Wait for all messages
+	messages := collector.WaitForMessages(30 * time.Second)
+	require.Len(t, messages, messageCount)
+
+	// Verify ordering within the partition
+	suite.AssertMessageOrdering(t, messages)
+}
+
+func TestSchemaValidation(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	namespace := "test"
+	topicName := "schema-validation"
+
+	// Test with simple schema
+	simpleSchema := CreateTestSchema()
+
+	pubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName,
+		PartitionCount: 1,
+		PublisherName:  "schema-publisher",
+		RecordType:     simpleSchema,
+	}
+
+	publisher, err := suite.CreatePublisher(pubConfig)
+	require.NoError(t, err)
+
+	// Test valid record
+	validRecord := schema.RecordBegin().
+		SetString("id", "valid-msg").
+		SetInt64("timestamp", time.Now().UnixNano()).
+		SetString("content", "Valid message").
+		SetInt32("sequence", 1).
+		RecordEnd()
+
+	err = publisher.PublishRecord([]byte("test-key"), validRecord)
+	assert.NoError(t, err, "Valid record should be published successfully")
+
+	// Test with complex nested schema
+	complexSchema := CreateComplexTestSchema()
+
+	complexPubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName + "-complex",
+		PartitionCount: 1,
+		PublisherName:  "complex-publisher",
+		RecordType:     complexSchema,
+	}
+
+	complexPublisher, err := suite.CreatePublisher(complexPubConfig)
+	require.NoError(t, err)
+
+	// Test complex nested record
+	complexRecord := schema.RecordBegin().
+		SetString("user_id", "user123").
+		SetString("name", "John Doe").
+		SetInt32("age", 30).
+		SetStringList("emails", "john@example.com", "john.doe@company.com").
+		SetRecord("address",
+			schema.RecordBegin().
+				SetString("street", "123 Main St").
+				SetString("city", "New York").
+				SetString("zipcode", "10001").
+				RecordEnd()).
+		SetInt64("created_at", time.Now().UnixNano()).
+		RecordEnd()
+
+	err = complexPublisher.PublishRecord([]byte("complex-key"), complexRecord)
+	assert.NoError(t, err, "Complex nested record should be published successfully")
+}

test/mq/integration/framework.go

@@ -0,0 +1,379 @@
+package integration
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/agent"
+	"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/client/sub_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// TestEnvironment holds the configuration for the test environment
+type TestEnvironment struct {
+	Masters      []string
+	Brokers      []string
+	Filers       []string
+	TestTimeout  time.Duration
+	CleanupFuncs []func()
+	mutex        sync.Mutex
+}
+
+// IntegrationTestSuite provides the base test framework
+type IntegrationTestSuite struct {
+	env         *TestEnvironment
+	agents      map[string]*agent.MessageQueueAgent
+	publishers  map[string]*pub_client.TopicPublisher
+	subscribers map[string]*sub_client.TopicSubscriber
+	subCancels  map[string]context.CancelFunc
+	cleanupOnce sync.Once
+	t           *testing.T
+}
+
+// NewIntegrationTestSuite creates a new test suite instance
+func NewIntegrationTestSuite(t *testing.T) *IntegrationTestSuite {
+	env := &TestEnvironment{
+		Masters:     getEnvList("SEAWEED_MASTERS", []string{"localhost:19333"}),
+		Brokers:     getEnvList("SEAWEED_BROKERS", []string{"localhost:17777"}),
+		Filers:      getEnvList("SEAWEED_FILERS", []string{"localhost:18888"}),
+		TestTimeout: getEnvDuration("GO_TEST_TIMEOUT", 30*time.Minute),
+	}
+
+	return &IntegrationTestSuite{
+		env:         env,
+		agents:      make(map[string]*agent.MessageQueueAgent),
+		publishers:  make(map[string]*pub_client.TopicPublisher),
+		subscribers: make(map[string]*sub_client.TopicSubscriber),
+		subCancels:  make(map[string]context.CancelFunc),
+		t:           t,
+	}
+}
+
+// Setup initializes the test environment
+func (its *IntegrationTestSuite) Setup() error {
+	// Wait for cluster to be ready
+	if err := its.waitForClusterReady(); err != nil {
+		return fmt.Errorf("cluster not ready: %v", err)
+	}
+
+	// Register cleanup
+	its.t.Cleanup(its.Cleanup)
+
+	return nil
+}
+
+// Cleanup performs cleanup operations
+func (its *IntegrationTestSuite) Cleanup() {
+	its.cleanupOnce.Do(func() {
+		// Close all subscribers first (they use context cancellation)
+		for name := range its.subscribers {
+			if cancel, ok := its.subCancels[name]; ok && cancel != nil {
+				cancel()
+				its.t.Logf("Cancelled subscriber context: %s", name)
+			}
+			its.t.Logf("Cleaned up subscriber: %s", name)
+		}
+
+		// Wait a moment for gRPC connections to close gracefully
+		time.Sleep(1 * time.Second)
+
+		// Close all publishers
+		for name, publisher := range its.publishers {
+			if publisher != nil {
+				// Add timeout to prevent deadlock during shutdown
+				done := make(chan bool, 1)
+				go func(p *pub_client.TopicPublisher, n string) {
+					p.Shutdown()
+					done <- true
+				}(publisher, name)
+
+				select {
+				case <-done:
+					its.t.Logf("Cleaned up publisher: %s", name)
+				case <-time.After(5 * time.Second):
+					its.t.Logf("Publisher shutdown timed out: %s", name)
+				}
+			}
+		}
+
+		// Execute additional cleanup functions
+		its.env.mutex.Lock()
+		for _, cleanup := range its.env.CleanupFuncs {
+			cleanup()
+		}
+		its.env.mutex.Unlock()
+	})
+}
+
+// CreatePublisher creates a new topic publisher
+func (its *IntegrationTestSuite) CreatePublisher(config *PublisherTestConfig) (*pub_client.TopicPublisher, error) {
+	publisherConfig := &pub_client.PublisherConfiguration{
+		Topic:          topic.NewTopic(config.Namespace, config.TopicName),
+		PartitionCount: config.PartitionCount,
+		Brokers:        its.env.Brokers,
+		PublisherName:  config.PublisherName,
+		RecordType:     config.RecordType,
+	}
+
+	publisher, err := pub_client.NewTopicPublisher(publisherConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create publisher: %v", err)
+	}
+
+	its.publishers[config.PublisherName] = publisher
+	return publisher, nil
+}
+
+// CreateSubscriber creates a new topic subscriber
+func (its *IntegrationTestSuite) CreateSubscriber(config *SubscriberTestConfig) (*sub_client.TopicSubscriber, error) {
+	subscriberConfig := &sub_client.SubscriberConfiguration{
+		ConsumerGroup:           config.ConsumerGroup,
+		ConsumerGroupInstanceId: config.ConsumerInstanceId,
+		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
+		MaxPartitionCount:       config.MaxPartitionCount,
+		SlidingWindowSize:       config.SlidingWindowSize,
+	}
+
+	contentConfig := &sub_client.ContentConfiguration{
+		Topic:            topic.NewTopic(config.Namespace, config.TopicName),
+		Filter:           config.Filter,
+		PartitionOffsets: config.PartitionOffsets,
+		OffsetType:       config.OffsetType,
+		OffsetTsNs:       config.OffsetTsNs,
+	}
+
+	offsetChan := make(chan sub_client.KeyedOffset, 1024)
+	ctx, cancel := context.WithCancel(context.Background())
+	subscriber := sub_client.NewTopicSubscriber(
+		ctx,
+		its.env.Brokers,
+		subscriberConfig,
+		contentConfig,
+		offsetChan,
+	)
+
+	its.subscribers[config.ConsumerInstanceId] = subscriber
+	its.subCancels[config.ConsumerInstanceId] = cancel
+	return subscriber, nil
+}
+
+// CreateAgent creates a new message queue agent
+func (its *IntegrationTestSuite) CreateAgent(name string) (*agent.MessageQueueAgent, error) {
+	var brokerAddresses []pb.ServerAddress
+	for _, broker := range its.env.Brokers {
+		brokerAddresses = append(brokerAddresses, pb.ServerAddress(broker))
+	}
+
+	agentOptions := &agent.MessageQueueAgentOptions{
+		SeedBrokers: brokerAddresses,
+	}
+
+	mqAgent := agent.NewMessageQueueAgent(
+		agentOptions,
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+	)
+
+	its.agents[name] = mqAgent
+	return mqAgent, nil
+}
+
+// PublisherTestConfig holds configuration for creating test publishers
+type PublisherTestConfig struct {
+	Namespace      string
+	TopicName      string
+	PartitionCount int32
+	PublisherName  string
+	RecordType     *schema_pb.RecordType
+}
+
+// SubscriberTestConfig holds configuration for creating test subscribers
+type SubscriberTestConfig struct {
+	Namespace          string
+	TopicName          string
+	ConsumerGroup      string
+	ConsumerInstanceId string
+	MaxPartitionCount  int32
+	SlidingWindowSize  int32
+	Filter             string
+	PartitionOffsets   []*schema_pb.PartitionOffset
+	OffsetType         schema_pb.OffsetType
+	OffsetTsNs         int64
+}
+
+// TestMessage represents a test message with metadata
+type TestMessage struct {
+	ID        string
+	Content   []byte
+	Timestamp time.Time
+	Key       []byte
+}
+
+// MessageCollector collects received messages for verification
+type MessageCollector struct {
+	messages []TestMessage
+	mutex    sync.RWMutex
+	waitCh   chan struct{}
+	expected int
+	closed   bool // protect against closing waitCh multiple times
+}
+
+// NewMessageCollector creates a new message collector
+func NewMessageCollector(expectedCount int) *MessageCollector {
+	return &MessageCollector{
+		messages: make([]TestMessage, 0),
+		waitCh:   make(chan struct{}),
+		expected: expectedCount,
+	}
+}
+
+// AddMessage adds a received message to the collector
+func (mc *MessageCollector) AddMessage(msg TestMessage) {
+	mc.mutex.Lock()
+	defer mc.mutex.Unlock()
+
+	mc.messages = append(mc.messages, msg)
+	if len(mc.messages) >= mc.expected && !mc.closed {
+		close(mc.waitCh)
+		mc.closed = true
+	}
+}
+
+// WaitForMessages waits for the expected number of messages or timeout
+func (mc *MessageCollector) WaitForMessages(timeout time.Duration) []TestMessage {
+	select {
+	case <-mc.waitCh:
+	case <-time.After(timeout):
+	}
+
+	mc.mutex.RLock()
+	defer mc.mutex.RUnlock()
+
+	result := make([]TestMessage, len(mc.messages))
+	copy(result, mc.messages)
+	return result
+}
+
+// GetMessages returns all collected messages
+func (mc *MessageCollector) GetMessages() []TestMessage {
+	mc.mutex.RLock()
+	defer mc.mutex.RUnlock()
+
+	result := make([]TestMessage, len(mc.messages))
+	copy(result, mc.messages)
+	return result
+}
+
+// CreateTestSchema creates a simple test schema
+func CreateTestSchema() *schema_pb.RecordType {
+	return schema.RecordTypeBegin().
+		WithField("id", schema.TypeString).
+		WithField("timestamp", schema.TypeInt64).
+		WithField("content", schema.TypeString).
+		WithField("sequence", schema.TypeInt32).
+		RecordTypeEnd()
+}
+
+// CreateComplexTestSchema creates a complex test schema with nested structures
+func CreateComplexTestSchema() *schema_pb.RecordType {
+	addressType := schema.RecordTypeBegin().
+		WithField("street", schema.TypeString).
+		WithField("city", schema.TypeString).
+		WithField("zipcode", schema.TypeString).
+		RecordTypeEnd()
+
+	return schema.RecordTypeBegin().
+		WithField("user_id", schema.TypeString).
+		WithField("name", schema.TypeString).
+		WithField("age", schema.TypeInt32).
+		WithField("emails", schema.ListOf(schema.TypeString)).
+		WithRecordField("address", addressType).
+		WithField("created_at", schema.TypeInt64).
+		RecordTypeEnd()
+}
+
+// Helper functions
+
+func getEnvList(key string, defaultValue []string) []string {
+	value := os.Getenv(key)
+	if value == "" {
+		return defaultValue
+	}
+	return strings.Split(value, ",")
+}
+
+func getEnvDuration(key string, defaultValue time.Duration) time.Duration {
+	value := os.Getenv(key)
+	if value == "" {
+		return defaultValue
+	}
+
+	duration, err := time.ParseDuration(value)
+	if err != nil {
+		return defaultValue
+	}
+	return duration
+}
+
+func (its *IntegrationTestSuite) waitForClusterReady() error {
+	maxRetries := 30
+	retryInterval := 2 * time.Second
+
+	for i := 0; i < maxRetries; i++ {
+		if its.isClusterReady() {
+			return nil
+		}
+		its.t.Logf("Waiting for cluster to be ready... attempt %d/%d", i+1, maxRetries)
+		time.Sleep(retryInterval)
+	}
+
+	return fmt.Errorf("cluster not ready after %d attempts", maxRetries)
+}
+
+func (its *IntegrationTestSuite) isClusterReady() bool {
+	// Check if at least one broker is accessible
+	for _, broker := range its.env.Brokers {
+		if its.isBrokerReady(broker) {
+			return true
+		}
+	}
+	return false
+}
+
+func (its *IntegrationTestSuite) isBrokerReady(broker string) bool {
+	// Simple connection test
+	conn, err := grpc.NewClient(broker, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false
+	}
+	defer conn.Close()
+
+	// TODO: Add actual health check call here
+	return true
+}
+
+// AssertMessagesReceived verifies that expected messages were received
+func (its *IntegrationTestSuite) AssertMessagesReceived(t *testing.T, collector *MessageCollector, expectedCount int, timeout time.Duration) {
+	messages := collector.WaitForMessages(timeout)
+	require.Len(t, messages, expectedCount, "Expected %d messages, got %d", expectedCount, len(messages))
+}
+
+// AssertMessageOrdering verifies that messages are received in the expected order
+func (its *IntegrationTestSuite) AssertMessageOrdering(t *testing.T, messages []TestMessage) {
+	for i := 1; i < len(messages); i++ {
+		require.True(t, messages[i].Timestamp.After(messages[i-1].Timestamp) || messages[i].Timestamp.Equal(messages[i-1].Timestamp),
+			"Messages not in chronological order: message %d timestamp %v should be >= message %d timestamp %v",
+			i, messages[i].Timestamp, i-1, messages[i-1].Timestamp)
+	}
+}

test/mq/integration/performance_test.go

@@ -0,0 +1,666 @@
+package integration
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// PerformanceMetrics holds test metrics
+type PerformanceMetrics struct {
+	MessagesPublished int64
+	MessagesConsumed  int64
+	PublishLatencies  []time.Duration
+	ConsumeLatencies  []time.Duration
+	StartTime         time.Time
+	EndTime           time.Time
+	ErrorCount        int64
+	mu                sync.RWMutex
+}
+
+func (m *PerformanceMetrics) AddPublishLatency(d time.Duration) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.PublishLatencies = append(m.PublishLatencies, d)
+}
+
+func (m *PerformanceMetrics) AddConsumeLatency(d time.Duration) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.ConsumeLatencies = append(m.ConsumeLatencies, d)
+}
+
+func (m *PerformanceMetrics) GetThroughput() float64 {
+	duration := m.EndTime.Sub(m.StartTime).Seconds()
+	if duration == 0 {
+		return 0
+	}
+	return float64(atomic.LoadInt64(&m.MessagesPublished)) / duration
+}
+
+func (m *PerformanceMetrics) GetP95Latency(latencies []time.Duration) time.Duration {
+	if len(latencies) == 0 {
+		return 0
+	}
+
+	// Simple P95 calculation - in production use proper percentile library
+	index := int(float64(len(latencies)) * 0.95)
+	if index >= len(latencies) {
+		index = len(latencies) - 1
+	}
+
+	// Sort latencies (simplified)
+	for i := 0; i < len(latencies)-1; i++ {
+		for j := 0; j < len(latencies)-i-1; j++ {
+			if latencies[j] > latencies[j+1] {
+				latencies[j], latencies[j+1] = latencies[j+1], latencies[j]
+			}
+		}
+	}
+
+	return latencies[index]
+}
+
+// Enhanced performance metrics with connection error tracking
+type EnhancedPerformanceMetrics struct {
+	MessagesPublished int64
+	MessagesConsumed  int64
+	PublishLatencies  []time.Duration
+	ConsumeLatencies  []time.Duration
+	StartTime         time.Time
+	EndTime           time.Time
+	ErrorCount        int64
+	ConnectionErrors  int64
+	ApplicationErrors int64
+	RetryAttempts     int64
+	mu                sync.RWMutex
+}
+
+func (m *EnhancedPerformanceMetrics) AddPublishLatency(d time.Duration) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.PublishLatencies = append(m.PublishLatencies, d)
+}
+
+func (m *EnhancedPerformanceMetrics) AddConsumeLatency(d time.Duration) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.ConsumeLatencies = append(m.ConsumeLatencies, d)
+}
+
+func (m *EnhancedPerformanceMetrics) GetThroughput() float64 {
+	duration := m.EndTime.Sub(m.StartTime).Seconds()
+	if duration == 0 {
+		return 0
+	}
+	return float64(atomic.LoadInt64(&m.MessagesPublished)) / duration
+}
+
+func (m *EnhancedPerformanceMetrics) GetP95Latency(latencies []time.Duration) time.Duration {
+	if len(latencies) == 0 {
+		return 0
+	}
+
+	// Simple P95 calculation
+	index := int(float64(len(latencies)) * 0.95)
+	if index >= len(latencies) {
+		index = len(latencies) - 1
+	}
+
+	// Sort latencies (simplified bubble sort for small datasets)
+	sorted := make([]time.Duration, len(latencies))
+	copy(sorted, latencies)
+	for i := 0; i < len(sorted)-1; i++ {
+		for j := 0; j < len(sorted)-i-1; j++ {
+			if sorted[j] > sorted[j+1] {
+				sorted[j], sorted[j+1] = sorted[j+1], sorted[j]
+			}
+		}
+	}
+
+	return sorted[index]
+}
+
+// isConnectionError determines if an error is a connection-level error
+func isConnectionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errStr := err.Error()
+
+	// Check for gRPC status codes
+	if st, ok := status.FromError(err); ok {
+		switch st.Code() {
+		case codes.Unavailable, codes.DeadlineExceeded, codes.Canceled, codes.Unknown:
+			return true
+		}
+	}
+
+	// Check for common connection error strings
+	connectionErrorPatterns := []string{
+		"EOF",
+		"error reading server preface",
+		"connection refused",
+		"connection reset",
+		"broken pipe",
+		"network is unreachable",
+		"no route to host",
+		"transport is closing",
+		"connection error",
+		"dial tcp",
+		"context deadline exceeded",
+	}
+
+	for _, pattern := range connectionErrorPatterns {
+		if containsString(errStr, pattern) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func containsString(s, substr string) bool {
+	for i := 0; i <= len(s)-len(substr); i++ {
+		if s[i:i+len(substr)] == substr {
+			return true
+		}
+	}
+	return false
+}
+
+func TestPerformanceThroughput(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	topicName := "performance-throughput-test"
+	namespace := "perf-test"
+
+	metrics := &PerformanceMetrics{
+		StartTime: time.Now(),
+	}
+
+	// Test parameters
+	numMessages := 50000
+	numPublishers := 5
+	messageSize := 1024 // 1KB messages
+
+	// Create message payload
+	payload := make([]byte, messageSize)
+	for i := range payload {
+		payload[i] = byte(i % 256)
+	}
+
+	t.Logf("Starting throughput test: %d messages, %d publishers, %d bytes per message",
+		numMessages, numPublishers, messageSize)
+
+	// Start publishers
+	var publishWg sync.WaitGroup
+	messagesPerPublisher := numMessages / numPublishers
+
+	for i := 0; i < numPublishers; i++ {
+		publishWg.Add(1)
+		go func(publisherID int) {
+			defer publishWg.Done()
+
+			// Create publisher for this goroutine
+			pubConfig := &PublisherTestConfig{
+				Namespace:      namespace,
+				TopicName:      topicName,
+				PartitionCount: 4, // Multiple partitions for better throughput
+				PublisherName:  fmt.Sprintf("perf-publisher-%d", publisherID),
+				RecordType:     nil, // Use raw publish
+			}
+
+			publisher, err := suite.CreatePublisher(pubConfig)
+			if err != nil {
+				atomic.AddInt64(&metrics.ErrorCount, 1)
+				t.Errorf("Failed to create publisher %d: %v", publisherID, err)
+				return
+			}
+
+			for j := 0; j < messagesPerPublisher; j++ {
+				messageKey := fmt.Sprintf("publisher-%d-msg-%d", publisherID, j)
+
+				start := time.Now()
+				err := publisher.Publish([]byte(messageKey), payload)
+				latency := time.Since(start)
+
+				if err != nil {
+					atomic.AddInt64(&metrics.ErrorCount, 1)
+					continue
+				}
+
+				atomic.AddInt64(&metrics.MessagesPublished, 1)
+				metrics.AddPublishLatency(latency)
+
+				// Small delay to prevent overwhelming the system
+				if j%1000 == 0 {
+					time.Sleep(1 * time.Millisecond)
+				}
+			}
+		}(i)
+	}
+
+	// Wait for publishing to complete
+	publishWg.Wait()
+	metrics.EndTime = time.Now()
+
+	// Verify results
+	publishedCount := atomic.LoadInt64(&metrics.MessagesPublished)
+	errorCount := atomic.LoadInt64(&metrics.ErrorCount)
+	throughput := metrics.GetThroughput()
+
+	t.Logf("Performance Results:")
+	t.Logf("  Messages Published: %d", publishedCount)
+	t.Logf("  Errors: %d", errorCount)
+	t.Logf("  Throughput: %.2f messages/second", throughput)
+	t.Logf("  Duration: %v", metrics.EndTime.Sub(metrics.StartTime))
+
+	if len(metrics.PublishLatencies) > 0 {
+		p95Latency := metrics.GetP95Latency(metrics.PublishLatencies)
+		t.Logf("  P95 Publish Latency: %v", p95Latency)
+
+		// Performance assertions
+		assert.Less(t, p95Latency, 100*time.Millisecond, "P95 publish latency should be under 100ms")
+	}
+
+	// Throughput requirements
+	expectedMinThroughput := 5000.0 // 5K messages/sec minimum (relaxed from 10K for initial testing)
+	assert.Greater(t, throughput, expectedMinThroughput,
+		"Throughput should exceed %.0f messages/second", expectedMinThroughput)
+
+	// Error rate should be low
+	errorRate := float64(errorCount) / float64(publishedCount+errorCount)
+	assert.Less(t, errorRate, 0.05, "Error rate should be less than 5%")
+}
+
+func TestPerformanceLatency(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	topicName := "performance-latency-test"
+	namespace := "perf-test"
+
+	// Create publisher
+	pubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName,
+		PartitionCount: 1, // Single partition for latency testing
+		PublisherName:  "latency-publisher",
+		RecordType:     nil,
+	}
+
+	publisher, err := suite.CreatePublisher(pubConfig)
+	require.NoError(t, err)
+
+	// Start consumer first
+	subConfig := &SubscriberTestConfig{
+		Namespace:          namespace,
+		TopicName:          topicName,
+		ConsumerGroup:      "latency-test-group",
+		ConsumerInstanceId: "latency-consumer-1",
+		MaxPartitionCount:  1,
+		SlidingWindowSize:  10,
+		OffsetType:         schema_pb.OffsetType_RESET_TO_EARLIEST,
+	}
+
+	subscriber, err := suite.CreateSubscriber(subConfig)
+	require.NoError(t, err)
+
+	numMessages := 5000
+	collector := NewMessageCollector(numMessages)
+
+	// Set up message handler
+	subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
+		collector.AddMessage(TestMessage{
+			ID:        string(m.Data.Key),
+			Content:   m.Data.Value,
+			Timestamp: time.Unix(0, m.Data.TsNs),
+			Key:       m.Data.Key,
+		})
+	})
+
+	// Start subscriber
+	go func() {
+		err := subscriber.Subscribe()
+		if err != nil {
+			t.Logf("Subscriber error: %v", err)
+		}
+	}()
+
+	// Wait for consumer to be ready
+	time.Sleep(2 * time.Second)
+
+	metrics := &PerformanceMetrics{
+		StartTime: time.Now(),
+	}
+
+	t.Logf("Starting latency test with %d messages", numMessages)
+
+	// Publish messages with controlled timing
+	for i := 0; i < numMessages; i++ {
+		messageKey := fmt.Sprintf("latency-msg-%d", i)
+		payload := fmt.Sprintf("test-payload-data-%d", i)
+
+		start := time.Now()
+		err := publisher.Publish([]byte(messageKey), []byte(payload))
+		publishLatency := time.Since(start)
+
+		require.NoError(t, err)
+		metrics.AddPublishLatency(publishLatency)
+
+		// Controlled rate for latency measurement
+		if i%100 == 0 {
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+	metrics.EndTime = time.Now()
+
+	// Wait for messages to be consumed
+	messages := collector.WaitForMessages(30 * time.Second)
+
+	// Analyze latency results
+	t.Logf("Latency Test Results:")
+	t.Logf("  Messages Published: %d", numMessages)
+	t.Logf("  Messages Consumed: %d", len(messages))
+
+	if len(metrics.PublishLatencies) > 0 {
+		p95PublishLatency := metrics.GetP95Latency(metrics.PublishLatencies)
+		t.Logf("  P95 Publish Latency: %v", p95PublishLatency)
+
+		// Latency assertions
+		assert.Less(t, p95PublishLatency, 50*time.Millisecond,
+			"P95 publish latency should be under 50ms")
+	}
+
+	// Verify message delivery
+	deliveryRate := float64(len(messages)) / float64(numMessages)
+	assert.Greater(t, deliveryRate, 0.80, "Should deliver at least 80% of messages")
+}
+
+func TestPerformanceConcurrentConsumers(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	topicName := "performance-concurrent-test"
+	namespace := "perf-test"
+	numPartitions := int32(4)
+
+	// Create publisher first
+	pubConfig := &PublisherTestConfig{
+		Namespace:      namespace,
+		TopicName:      topicName,
+		PartitionCount: numPartitions,
+		PublisherName:  "concurrent-publisher",
+		RecordType:     nil,
+	}
+
+	publisher, err := suite.CreatePublisher(pubConfig)
+	require.NoError(t, err)
+
+	// Test parameters
+	numConsumers := 8
+	numMessages := 20000
+	consumerGroup := "concurrent-perf-group"
+
+	t.Logf("Starting concurrent consumer test: %d consumers, %d messages, %d partitions",
+		numConsumers, numMessages, numPartitions)
+
+	// Start multiple consumers
+	var collectors []*MessageCollector
+
+	for i := 0; i < numConsumers; i++ {
+		collector := NewMessageCollector(numMessages / numConsumers) // Expected per consumer
+		collectors = append(collectors, collector)
+
+		subConfig := &SubscriberTestConfig{
+			Namespace:          namespace,
+			TopicName:          topicName,
+			ConsumerGroup:      consumerGroup,
+			ConsumerInstanceId: fmt.Sprintf("consumer-%d", i),
+			MaxPartitionCount:  numPartitions,
+			SlidingWindowSize:  10,
+			OffsetType:         schema_pb.OffsetType_RESET_TO_EARLIEST,
+		}
+
+		subscriber, err := suite.CreateSubscriber(subConfig)
+		require.NoError(t, err)
+
+		// Set up message handler for this consumer
+		func(consumerID int, c *MessageCollector) {
+			subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
+				c.AddMessage(TestMessage{
+					ID:        fmt.Sprintf("%d-%s", consumerID, string(m.Data.Key)),
+					Content:   m.Data.Value,
+					Timestamp: time.Unix(0, m.Data.TsNs),
+					Key:       m.Data.Key,
+				})
+			})
+		}(i, collector)
+
+		// Start subscriber
+		go func(consumerID int, s *SubscriberTestConfig) {
+			sub, _ := suite.CreateSubscriber(s)
+			err := sub.Subscribe()
+			if err != nil {
+				t.Logf("Consumer %d error: %v", consumerID, err)
+			}
+		}(i, subConfig)
+	}
+
+	// Wait for consumers to initialize
+	time.Sleep(3 * time.Second)
+
+	// Start publishing
+	startTime := time.Now()
+
+	var publishWg sync.WaitGroup
+	numPublishers := 3
+	messagesPerPublisher := numMessages / numPublishers
+
+	for i := 0; i < numPublishers; i++ {
+		publishWg.Add(1)
+		go func(publisherID int) {
+			defer publishWg.Done()
+
+			for j := 0; j < messagesPerPublisher; j++ {
+				messageKey := fmt.Sprintf("concurrent-msg-%d-%d", publisherID, j)
+				payload := fmt.Sprintf("publisher-%d-message-%d-data", publisherID, j)
+
+				err := publisher.Publish([]byte(messageKey), []byte(payload))
+
+				if err != nil {
+					t.Logf("Publish error: %v", err)
+				}
+
+				// Rate limiting
+				if j%1000 == 0 {
+					time.Sleep(2 * time.Millisecond)
+				}
+			}
+		}(i)
+	}
+
+	publishWg.Wait()
+	publishDuration := time.Since(startTime)
+
+	// Allow time for message consumption
+	time.Sleep(10 * time.Second)
+
+	// Analyze results
+	totalConsumed := int64(0)
+	for i, collector := range collectors {
+		messages := collector.GetMessages()
+		consumed := int64(len(messages))
+		totalConsumed += consumed
+		t.Logf("Consumer %d consumed %d messages", i, consumed)
+	}
+
+	publishThroughput := float64(numMessages) / publishDuration.Seconds()
+	consumeThroughput := float64(totalConsumed) / publishDuration.Seconds()
+
+	t.Logf("Concurrent Consumer Test Results:")
+	t.Logf("  Total Published: %d", numMessages)
+	t.Logf("  Total Consumed: %d", totalConsumed)
+	t.Logf("  Publish Throughput: %.2f msg/sec", publishThroughput)
+	t.Logf("  Consume Throughput: %.2f msg/sec", consumeThroughput)
+	t.Logf("  Test Duration: %v", publishDuration)
+
+	// Performance assertions (relaxed for initial testing)
+	deliveryRate := float64(totalConsumed) / float64(numMessages)
+	assert.Greater(t, deliveryRate, 0.70, "Should consume at least 70% of messages")
+
+	expectedMinThroughput := 2000.0 // 2K messages/sec minimum for concurrent consumption
+	assert.Greater(t, consumeThroughput, expectedMinThroughput,
+		"Consume throughput should exceed %.0f messages/second", expectedMinThroughput)
+}
+
+func TestPerformanceWithErrorHandling(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	require.NoError(t, suite.Setup())
+
+	topicName := "performance-error-handling-test"
+	namespace := "perf-test"
+
+	metrics := &EnhancedPerformanceMetrics{
+		StartTime: time.Now(),
+	}
+
+	// Test parameters
+	numMessages := 50000
+	numPublishers := 5
+	messageSize := 1024 // 1KB messages
+
+	// Create message payload
+	payload := make([]byte, messageSize)
+	for i := range payload {
+		payload[i] = byte(i % 256)
+	}
+
+	t.Logf("Starting performance test with enhanced error handling: %d messages, %d publishers, %d bytes per message",
+		numMessages, numPublishers, messageSize)
+
+	// Start publishers
+	var publishWg sync.WaitGroup
+	messagesPerPublisher := numMessages / numPublishers
+
+	for i := 0; i < numPublishers; i++ {
+		publishWg.Add(1)
+		go func(publisherID int) {
+			defer publishWg.Done()
+
+			// Create publisher for this goroutine
+			pubConfig := &PublisherTestConfig{
+				Namespace:      namespace,
+				TopicName:      topicName,
+				PartitionCount: 4, // Multiple partitions for better throughput
+				PublisherName:  fmt.Sprintf("error-aware-publisher-%d", publisherID),
+				RecordType:     nil, // Use raw publish
+			}
+
+			publisher, err := suite.CreatePublisher(pubConfig)
+			if err != nil {
+				atomic.AddInt64(&metrics.ErrorCount, 1)
+				t.Errorf("Failed to create publisher %d: %v", publisherID, err)
+				return
+			}
+
+			for j := 0; j < messagesPerPublisher; j++ {
+				messageKey := fmt.Sprintf("publisher-%d-msg-%d", publisherID, j)
+
+				start := time.Now()
+				err := publisher.Publish([]byte(messageKey), payload)
+				latency := time.Since(start)
+
+				if err != nil {
+					atomic.AddInt64(&metrics.ErrorCount, 1)
+
+					// Classify the error type
+					if isConnectionError(err) {
+						atomic.AddInt64(&metrics.ConnectionErrors, 1)
+						t.Logf("Connection error (publisher %d, msg %d): %v", publisherID, j, err)
+					} else {
+						atomic.AddInt64(&metrics.ApplicationErrors, 1)
+						t.Logf("Application error (publisher %d, msg %d): %v", publisherID, j, err)
+					}
+					continue
+				}
+
+				atomic.AddInt64(&metrics.MessagesPublished, 1)
+				metrics.AddPublishLatency(latency)
+
+				// Small delay to prevent overwhelming the system
+				if j%1000 == 0 {
+					time.Sleep(1 * time.Millisecond)
+				}
+			}
+		}(i)
+	}
+
+	// Wait for publishing to complete
+	publishWg.Wait()
+	metrics.EndTime = time.Now()
+
+	// Analyze results with enhanced error reporting
+	publishedCount := atomic.LoadInt64(&metrics.MessagesPublished)
+	totalErrors := atomic.LoadInt64(&metrics.ErrorCount)
+	connectionErrors := atomic.LoadInt64(&metrics.ConnectionErrors)
+	applicationErrors := atomic.LoadInt64(&metrics.ApplicationErrors)
+	throughput := metrics.GetThroughput()
+
+	t.Logf("Enhanced Performance Results:")
+	t.Logf("  Messages Successfully Published: %d", publishedCount)
+	t.Logf("  Total Errors: %d", totalErrors)
+	t.Logf("  Connection-Level Errors: %d", connectionErrors)
+	t.Logf("  Application-Level Errors: %d", applicationErrors)
+	t.Logf("  Throughput: %.2f messages/second", throughput)
+	t.Logf("  Duration: %v", metrics.EndTime.Sub(metrics.StartTime))
+
+	if len(metrics.PublishLatencies) > 0 {
+		p95Latency := metrics.GetP95Latency(metrics.PublishLatencies)
+		t.Logf("  P95 Publish Latency: %v", p95Latency)
+
+		// Performance assertions (adjusted for error handling overhead)
+		assert.Less(t, p95Latency, 100*time.Millisecond, "P95 publish latency should be under 100ms")
+	}
+
+	// Enhanced error analysis
+	totalAttempts := publishedCount + totalErrors
+	if totalAttempts > 0 {
+		successRate := float64(publishedCount) / float64(totalAttempts)
+		connectionErrorRate := float64(connectionErrors) / float64(totalAttempts)
+		applicationErrorRate := float64(applicationErrors) / float64(totalAttempts)
+
+		t.Logf("Error Analysis:")
+		t.Logf("  Success Rate: %.2f%%", successRate*100)
+		t.Logf("  Connection Error Rate: %.2f%%", connectionErrorRate*100)
+		t.Logf("  Application Error Rate: %.2f%%", applicationErrorRate*100)
+
+		// Assertions based on error types
+		assert.Greater(t, successRate, 0.80, "Success rate should be greater than 80%")
+		assert.Less(t, applicationErrorRate, 0.01, "Application error rate should be less than 1%")
+
+		// Connection errors are expected under high load but should be handled
+		if connectionErrors > 0 {
+			t.Logf("Note: %d connection errors detected - this indicates the test is successfully stressing the system", connectionErrors)
+			t.Logf("Recommendation: Implement retry logic for production applications to handle these connection errors")
+		}
+	}
+
+	// Throughput requirements (adjusted for error handling)
+	expectedMinThroughput := 5000.0 // 5K messages/sec minimum
+	assert.Greater(t, throughput, expectedMinThroughput,
+		"Throughput should exceed %.0f messages/second", expectedMinThroughput)
+}

test/mq/integration/resilient_publisher.go

@@ -0,0 +1,336 @@
+package integration
+
+import (
+	"fmt"
+	"math"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// ResilientPublisher wraps TopicPublisher with enhanced error handling
+type ResilientPublisher struct {
+	publisher *pub_client.TopicPublisher
+	config    *PublisherTestConfig
+	suite     *IntegrationTestSuite
+
+	// Error tracking
+	connectionErrors  int64
+	applicationErrors int64
+	retryAttempts     int64
+	totalPublishes    int64
+
+	// Retry configuration
+	maxRetries    int
+	baseDelay     time.Duration
+	maxDelay      time.Duration
+	backoffFactor float64
+
+	// Circuit breaker
+	circuitOpen     bool
+	circuitOpenTime time.Time
+	circuitTimeout  time.Duration
+
+	mu sync.RWMutex
+}
+
+// RetryConfig holds retry configuration
+type RetryConfig struct {
+	MaxRetries     int
+	BaseDelay      time.Duration
+	MaxDelay       time.Duration
+	BackoffFactor  float64
+	CircuitTimeout time.Duration
+}
+
+// DefaultRetryConfig returns sensible defaults for retry configuration
+func DefaultRetryConfig() *RetryConfig {
+	return &RetryConfig{
+		MaxRetries:     5,
+		BaseDelay:      10 * time.Millisecond,
+		MaxDelay:       5 * time.Second,
+		BackoffFactor:  2.0,
+		CircuitTimeout: 30 * time.Second,
+	}
+}
+
+// NewResilientPublisher creates a new resilient publisher
+func (its *IntegrationTestSuite) CreateResilientPublisher(config *PublisherTestConfig, retryConfig *RetryConfig) (*ResilientPublisher, error) {
+	if retryConfig == nil {
+		retryConfig = DefaultRetryConfig()
+	}
+
+	publisher, err := its.CreatePublisher(config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create base publisher: %v", err)
+	}
+
+	return &ResilientPublisher{
+		publisher:      publisher,
+		config:         config,
+		suite:          its,
+		maxRetries:     retryConfig.MaxRetries,
+		baseDelay:      retryConfig.BaseDelay,
+		maxDelay:       retryConfig.MaxDelay,
+		backoffFactor:  retryConfig.BackoffFactor,
+		circuitTimeout: retryConfig.CircuitTimeout,
+	}, nil
+}
+
+// PublishWithRetry publishes a message with retry logic and error handling
+func (rp *ResilientPublisher) PublishWithRetry(key, value []byte) error {
+	atomic.AddInt64(&rp.totalPublishes, 1)
+
+	// Check circuit breaker
+	if rp.isCircuitOpen() {
+		atomic.AddInt64(&rp.applicationErrors, 1)
+		return fmt.Errorf("circuit breaker is open")
+	}
+
+	var lastErr error
+	for attempt := 0; attempt <= rp.maxRetries; attempt++ {
+		if attempt > 0 {
+			atomic.AddInt64(&rp.retryAttempts, 1)
+			delay := rp.calculateDelay(attempt)
+			glog.V(1).Infof("Retrying publish after %v (attempt %d/%d)", delay, attempt, rp.maxRetries)
+			time.Sleep(delay)
+		}
+
+		err := rp.publisher.Publish(key, value)
+		if err == nil {
+			// Success - reset circuit breaker if it was open
+			rp.resetCircuitBreaker()
+			return nil
+		}
+
+		lastErr = err
+
+		// Classify error type
+		if rp.isConnectionError(err) {
+			atomic.AddInt64(&rp.connectionErrors, 1)
+			glog.V(1).Infof("Connection error on attempt %d: %v", attempt+1, err)
+
+			// For connection errors, try to recreate the publisher
+			if attempt < rp.maxRetries {
+				if recreateErr := rp.recreatePublisher(); recreateErr != nil {
+					glog.Warningf("Failed to recreate publisher: %v", recreateErr)
+				}
+			}
+			continue
+		} else {
+			// Application error - don't retry
+			atomic.AddInt64(&rp.applicationErrors, 1)
+			glog.Warningf("Application error (not retrying): %v", err)
+			break
+		}
+	}
+
+	// All retries exhausted or non-retryable error
+	rp.openCircuitBreaker()
+	return fmt.Errorf("publish failed after %d attempts, last error: %v", rp.maxRetries+1, lastErr)
+}
+
+// PublishRecord publishes a record with retry logic
+func (rp *ResilientPublisher) PublishRecord(key []byte, record *schema_pb.RecordValue) error {
+	atomic.AddInt64(&rp.totalPublishes, 1)
+
+	if rp.isCircuitOpen() {
+		atomic.AddInt64(&rp.applicationErrors, 1)
+		return fmt.Errorf("circuit breaker is open")
+	}
+
+	var lastErr error
+	for attempt := 0; attempt <= rp.maxRetries; attempt++ {
+		if attempt > 0 {
+			atomic.AddInt64(&rp.retryAttempts, 1)
+			delay := rp.calculateDelay(attempt)
+			time.Sleep(delay)
+		}
+
+		err := rp.publisher.PublishRecord(key, record)
+		if err == nil {
+			rp.resetCircuitBreaker()
+			return nil
+		}
+
+		lastErr = err
+
+		if rp.isConnectionError(err) {
+			atomic.AddInt64(&rp.connectionErrors, 1)
+			if attempt < rp.maxRetries {
+				if recreateErr := rp.recreatePublisher(); recreateErr != nil {
+					glog.Warningf("Failed to recreate publisher: %v", recreateErr)
+				}
+			}
+			continue
+		} else {
+			atomic.AddInt64(&rp.applicationErrors, 1)
+			break
+		}
+	}
+
+	rp.openCircuitBreaker()
+	return fmt.Errorf("publish record failed after %d attempts, last error: %v", rp.maxRetries+1, lastErr)
+}
+
+// recreatePublisher attempts to recreate the underlying publisher
+func (rp *ResilientPublisher) recreatePublisher() error {
+	rp.mu.Lock()
+	defer rp.mu.Unlock()
+
+	// Shutdown old publisher
+	if rp.publisher != nil {
+		rp.publisher.Shutdown()
+	}
+
+	// Create new publisher
+	newPublisher, err := rp.suite.CreatePublisher(rp.config)
+	if err != nil {
+		return fmt.Errorf("failed to recreate publisher: %v", err)
+	}
+
+	rp.publisher = newPublisher
+	glog.V(1).Infof("Successfully recreated publisher")
+	return nil
+}
+
+// isConnectionError determines if an error is a connection-level error that should be retried
+func (rp *ResilientPublisher) isConnectionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errStr := err.Error()
+
+	// Check for gRPC status codes
+	if st, ok := status.FromError(err); ok {
+		switch st.Code() {
+		case codes.Unavailable, codes.DeadlineExceeded, codes.Canceled, codes.Unknown:
+			return true
+		}
+	}
+
+	// Check for common connection error strings
+	connectionErrorPatterns := []string{
+		"EOF",
+		"error reading server preface",
+		"connection refused",
+		"connection reset",
+		"broken pipe",
+		"network is unreachable",
+		"no route to host",
+		"transport is closing",
+		"connection error",
+		"dial tcp",
+		"context deadline exceeded",
+	}
+
+	for _, pattern := range connectionErrorPatterns {
+		if contains(errStr, pattern) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// contains checks if a string contains a substring (case-insensitive)
+func contains(s, substr string) bool {
+	return len(s) >= len(substr) &&
+		(s == substr ||
+			len(s) > len(substr) &&
+				(s[:len(substr)] == substr ||
+					s[len(s)-len(substr):] == substr ||
+					containsSubstring(s, substr)))
+}
+
+func containsSubstring(s, substr string) bool {
+	for i := 0; i <= len(s)-len(substr); i++ {
+		if s[i:i+len(substr)] == substr {
+			return true
+		}
+	}
+	return false
+}
+
+// calculateDelay calculates exponential backoff delay
+func (rp *ResilientPublisher) calculateDelay(attempt int) time.Duration {
+	delay := float64(rp.baseDelay) * math.Pow(rp.backoffFactor, float64(attempt-1))
+	if delay > float64(rp.maxDelay) {
+		delay = float64(rp.maxDelay)
+	}
+	return time.Duration(delay)
+}
+
+// Circuit breaker methods
+func (rp *ResilientPublisher) isCircuitOpen() bool {
+	rp.mu.RLock()
+	defer rp.mu.RUnlock()
+
+	if !rp.circuitOpen {
+		return false
+	}
+
+	// Check if circuit should be reset
+	if time.Since(rp.circuitOpenTime) > rp.circuitTimeout {
+		return false
+	}
+
+	return true
+}
+
+func (rp *ResilientPublisher) openCircuitBreaker() {
+	rp.mu.Lock()
+	defer rp.mu.Unlock()
+
+	rp.circuitOpen = true
+	rp.circuitOpenTime = time.Now()
+	glog.Warningf("Circuit breaker opened due to repeated failures")
+}
+
+func (rp *ResilientPublisher) resetCircuitBreaker() {
+	rp.mu.Lock()
+	defer rp.mu.Unlock()
+
+	if rp.circuitOpen {
+		rp.circuitOpen = false
+		glog.V(1).Infof("Circuit breaker reset")
+	}
+}
+
+// GetErrorStats returns error statistics
+func (rp *ResilientPublisher) GetErrorStats() ErrorStats {
+	return ErrorStats{
+		ConnectionErrors:  atomic.LoadInt64(&rp.connectionErrors),
+		ApplicationErrors: atomic.LoadInt64(&rp.applicationErrors),
+		RetryAttempts:     atomic.LoadInt64(&rp.retryAttempts),
+		TotalPublishes:    atomic.LoadInt64(&rp.totalPublishes),
+		CircuitOpen:       rp.isCircuitOpen(),
+	}
+}
+
+// ErrorStats holds error statistics
+type ErrorStats struct {
+	ConnectionErrors  int64
+	ApplicationErrors int64
+	RetryAttempts     int64
+	TotalPublishes    int64
+	CircuitOpen       bool
+}
+
+// Shutdown gracefully shuts down the resilient publisher
+func (rp *ResilientPublisher) Shutdown() error {
+	rp.mu.Lock()
+	defer rp.mu.Unlock()
+
+	if rp.publisher != nil {
+		return rp.publisher.Shutdown()
+	}
+	return nil
+}

test/mq/integration_test_design.md

@@ -0,0 +1,286 @@
+# SeaweedMQ Integration Test Design
+
+## Overview
+
+This document outlines the comprehensive integration test strategy for SeaweedMQ, covering all critical functionalities from basic pub/sub operations to advanced features like auto-scaling, failover, and performance testing.
+
+## Architecture Under Test
+
+SeaweedMQ consists of:
+- **Masters**: Cluster coordination and metadata management
+- **Volume Servers**: Storage layer for persistent messages
+- **Filers**: File system interface for metadata storage
+- **Brokers**: Message processing and routing (stateless)
+- **Agents**: Client interface for pub/sub operations
+- **Schema System**: Protobuf-based message schema management
+
+## Test Categories
+
+### 1. Basic Functionality Tests
+
+#### 1.1 Basic Pub/Sub Operations
+- **Test**: `TestBasicPublishSubscribe`
+  - Publish messages to a topic
+  - Subscribe and receive messages
+  - Verify message content and ordering
+  - Test with different data types (string, int, bytes, records)
+
+- **Test**: `TestMultipleConsumers`
+  - Multiple subscribers on same topic
+  - Verify message distribution
+  - Test consumer group functionality
+
+- **Test**: `TestMessageOrdering`
+  - Publish messages in sequence
+  - Verify FIFO ordering within partitions
+  - Test with different partition keys
+
+#### 1.2 Schema Management
+- **Test**: `TestSchemaValidation`
+  - Publish with valid schemas
+  - Reject invalid schema messages
+  - Test schema evolution scenarios
+
+- **Test**: `TestRecordTypes`
+  - Nested record structures
+  - List types and complex schemas
+  - Schema-to-Parquet conversion
+
+### 2. Partitioning and Scaling Tests
+
+#### 2.1 Partition Management
+- **Test**: `TestPartitionDistribution`
+  - Messages distributed across partitions based on keys
+  - Verify partition assignment logic
+  - Test partition rebalancing
+
+- **Test**: `TestAutoSplitMerge`
+  - Simulate high load to trigger auto-split
+  - Simulate low load to trigger auto-merge
+  - Verify data consistency during splits/merges
+
+#### 2.2 Broker Scaling
+- **Test**: `TestBrokerAddRemove`
+  - Add brokers during operation
+  - Remove brokers gracefully
+  - Verify partition reassignment
+
+- **Test**: `TestLoadBalancing`
+  - Verify even load distribution across brokers
+  - Test with varying message sizes and rates
+  - Monitor broker resource utilization
+
+### 3. Failover and Reliability Tests
+
+#### 3.1 Broker Failover
+- **Test**: `TestBrokerFailover`
+  - Kill leader broker during publishing
+  - Verify seamless failover to follower
+  - Test data consistency after failover
+
+- **Test**: `TestBrokerRecovery`
+  - Broker restart scenarios
+  - State recovery from storage
+  - Partition reassignment after recovery
+
+#### 3.2 Data Durability
+- **Test**: `TestMessagePersistence`
+  - Publish messages and restart cluster
+  - Verify all messages are recovered
+  - Test with different replication settings
+
+- **Test**: `TestFollowerReplication`
+  - Leader-follower message replication
+  - Verify consistency between replicas
+  - Test follower promotion scenarios
+
+### 4. Agent Functionality Tests
+
+#### 4.1 Session Management
+- **Test**: `TestPublishSessions`
+  - Create/close publish sessions
+  - Concurrent session management
+  - Session cleanup after failures
+
+- **Test**: `TestSubscribeSessions`
+  - Subscribe session lifecycle
+  - Consumer group management
+  - Offset tracking and acknowledgments
+
+#### 4.2 Error Handling
+- **Test**: `TestConnectionFailures`
+  - Network partitions between agent and broker
+  - Automatic reconnection logic
+  - Message buffering during outages
+
+### 5. Performance and Load Tests
+
+#### 5.1 Throughput Tests
+- **Test**: `TestHighThroughputPublish`
+  - Publish 100K+ messages/second
+  - Monitor system resources
+  - Verify no message loss
+
+- **Test**: `TestHighThroughputSubscribe`
+  - Multiple consumers processing high volume
+  - Monitor processing latency
+  - Test backpressure handling
+
+#### 5.2 Spike Traffic Tests
+- **Test**: `TestTrafficSpikes`
+  - Sudden increase in message volume
+  - Auto-scaling behavior verification
+  - Resource utilization patterns
+
+- **Test**: `TestLargeMessages`
+  - Messages with large payloads (MB size)
+  - Memory usage monitoring
+  - Storage efficiency testing
+
+### 6. End-to-End Scenarios
+
+#### 6.1 Complete Workflow Tests
+- **Test**: `TestProducerConsumerWorkflow`
+  - Multi-stage data processing pipeline
+  - Producer → Topic → Multiple Consumers
+  - Data transformation and aggregation
+
+- **Test**: `TestMultiTopicOperations`
+  - Multiple topics with different schemas
+  - Cross-topic message routing
+  - Topic management operations
+
+## Test Infrastructure
+
+### Environment Setup
+
+#### Docker Compose Configuration
+```yaml
+# test-environment.yml
+version: '3.9'
+services:
+  master-cluster:
+    # 3 master nodes for HA
+  volume-cluster:
+    # 3 volume servers for data storage
+  filer-cluster:
+    # 2 filers for metadata
+  broker-cluster:
+    # 3 brokers for message processing
+  test-runner:
+    # Container to run integration tests
+```
+
+#### Test Data Management
+- Pre-defined test schemas
+- Sample message datasets
+- Performance benchmarking data
+
+### Test Framework Structure
+
+```go
+// Base test framework
+type IntegrationTestSuite struct {
+    masters     []string
+    brokers     []string
+    filers      []string
+    testClient  *TestClient
+    cleanup     []func()
+}
+
+// Test utilities
+type TestClient struct {
+    publishers  map[string]*pub_client.TopicPublisher
+    subscribers map[string]*sub_client.TopicSubscriber
+    agents      []*agent.MessageQueueAgent
+}
+```
+
+### Monitoring and Metrics
+
+#### Health Checks
+- Broker connectivity status
+- Master cluster health
+- Storage system availability
+- Network connectivity between components
+
+#### Performance Metrics
+- Message throughput (msgs/sec)
+- End-to-end latency
+- Resource utilization (CPU, Memory, Disk)
+- Network bandwidth usage
+
+## Test Execution Strategy
+
+### Parallel Test Execution
+- Categorize tests by resource requirements
+- Run independent tests in parallel
+- Serialize tests that modify cluster state
+
+### Continuous Integration
+- Automated test runs on PR submissions
+- Performance regression detection
+- Multi-platform testing (Linux, macOS, Windows)
+
+### Test Environment Management
+- Docker-based isolated environments
+- Automatic cleanup after test completion
+- Resource monitoring and alerts
+
+## Success Criteria
+
+### Functional Requirements
+- ✅ All messages published are received by subscribers
+- ✅ Message ordering preserved within partitions
+- ✅ Schema validation works correctly
+- ✅ Auto-scaling triggers at expected thresholds
+- ✅ Failover completes within 30 seconds
+- ✅ No data loss during normal operations
+
+### Performance Requirements
+- ✅ Throughput: 50K+ messages/second/broker
+- ✅ Latency: P95 < 100ms end-to-end
+- ✅ Memory usage: < 1GB per broker under normal load
+- ✅ Storage efficiency: < 20% overhead vs raw message size
+
+### Reliability Requirements
+- ✅ 99.9% uptime during normal operations
+- ✅ Automatic recovery from single component failures
+- ✅ Data consistency maintained across all scenarios
+- ✅ Graceful degradation under resource constraints
+
+## Implementation Timeline
+
+### Phase 1: Core Functionality (Week 1-2)
+- Basic pub/sub tests
+- Schema validation tests
+- Simple failover scenarios
+
+### Phase 2: Advanced Features (Week 3-4)
+- Auto-scaling tests
+- Complex failover scenarios
+- Agent functionality tests
+
+### Phase 3: Performance & Load (Week 5-6)
+- Throughput and latency tests
+- Spike traffic handling
+- Resource utilization monitoring
+
+### Phase 4: End-to-End (Week 7-8)
+- Complete workflow tests
+- Multi-component integration
+- Performance regression testing
+
+## Maintenance and Updates
+
+### Regular Updates
+- Add tests for new features
+- Update performance baselines
+- Enhance error scenarios coverage
+
+### Test Data Refresh
+- Generate new test datasets quarterly
+- Update schema examples
+- Refresh performance benchmarks
+
+This comprehensive test design ensures SeaweedMQ's reliability, performance, and functionality across all critical use cases and failure scenarios. 

test/mq/prometheus.yml

@@ -0,0 +1,54 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+scrape_configs:
+  # SeaweedFS Masters
+  - job_name: 'seaweedfs-master'
+    static_configs:
+      - targets: 
+        - 'master0:9333'
+        - 'master1:9334'
+        - 'master2:9335'
+    metrics_path: '/metrics'
+    scrape_interval: 10s
+
+  # SeaweedFS Volume Servers
+  - job_name: 'seaweedfs-volume'
+    static_configs:
+      - targets:
+        - 'volume1:8080'
+        - 'volume2:8081'
+        - 'volume3:8082'
+    metrics_path: '/metrics'
+    scrape_interval: 10s
+
+  # SeaweedFS Filers
+  - job_name: 'seaweedfs-filer'
+    static_configs:
+      - targets:
+        - 'filer1:8888'
+        - 'filer2:8889'
+    metrics_path: '/metrics'
+    scrape_interval: 10s
+
+  # SeaweedMQ Brokers
+  - job_name: 'seaweedmq-broker'
+    static_configs:
+      - targets:
+        - 'broker1:17777'
+        - 'broker2:17778'
+        - 'broker3:17779'
+    metrics_path: '/metrics'
+    scrape_interval: 5s
+
+  # Docker containers
+  - job_name: 'docker'
+    static_configs:
+      - targets: ['localhost:9323']
+    metrics_path: '/metrics'
+    scrape_interval: 30s 

test/mq/simple_demo.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+)
+
+func main() {
+	log.Println("Starting SeaweedMQ logging test...")
+
+	// Create publisher configuration
+	config := &pub_client.PublisherConfiguration{
+		Topic:          topic.NewTopic("test", "logging-demo"),
+		PartitionCount: 3,
+		Brokers:        []string{"127.0.0.1:17777"},
+		PublisherName:  "logging-test-client",
+	}
+
+	log.Println("Creating topic publisher...")
+	publisher, err := pub_client.NewTopicPublisher(config)
+	if err != nil {
+		log.Printf("Failed to create publisher: %v", err)
+		return
+	}
+	defer publisher.Shutdown()
+
+	log.Println("Publishing test messages...")
+
+	// Publish some test messages
+	for i := 0; i < 100; i++ {
+		key := fmt.Sprintf("key-%d", i)
+		value := fmt.Sprintf("message-%d-timestamp-%d", i, time.Now().Unix())
+
+		err := publisher.Publish([]byte(key), []byte(value))
+		if err != nil {
+			log.Printf("Failed to publish message %d: %v", i, err)
+		}
+
+		// Small delay to create some connection stress
+		if i%10 == 0 {
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+	log.Println("Finishing publish...")
+	err = publisher.FinishPublish()
+	if err != nil {
+		log.Printf("Failed to finish publish: %v", err)
+	}
+
+	log.Println("Test completed successfully!")
+}

unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go

@@ -1,18 +1,19 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"strconv"
 	"strings"
 	"time"
-	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
 var (
@@ -60,7 +61,7 @@ func startGenerateMetadata() {
 		for i := 0; i < *n; i++ {
 			name := fmt.Sprintf("file%d", i)
 			glog.V(0).Infof("write %s/%s", *dir, name)
-			if err := filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{
+			if err := filer_pb.CreateEntry(context.Background(), client, &filer_pb.CreateEntryRequest{
 				Directory: *dir,
 				Entry: &filer_pb.Entry{
 					Name: name,

unmaintained/repeated_vacuum/repeated_vacuum.go

@@ -1,13 +1,13 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"log"
 	"math/rand"
 	"time"
-	"context"
 
 	"google.golang.org/grpc"
 
@@ -56,7 +56,7 @@ func main() {
 }
 
 func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, string) {
-	assignResult, err := operation.Assign(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*master) }, grpcDialOption, &operation.VolumeAssignRequest{
+	assignResult, err := operation.Assign(context.Background(), func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*master) }, grpcDialOption, &operation.VolumeAssignRequest{
 		Count:       1,
 		Replication: *replication,
 	})
@@ -84,7 +84,7 @@ func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, st
 		log.Fatalf("upload: %v", err)
 	}
 
-	_, err = uploader.UploadData(data, uploadOption)
+	_, err = uploader.UploadData(context.Background(), data, uploadOption)
 	if err != nil {
 		log.Fatalf("upload: %v", err)
 	}

weed/cluster/lock_manager/lock_ring.go

@@ -1,12 +1,13 @@
 package lock_manager
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"sort"
 	"sync"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 type LockRingSnapshot struct {
@@ -22,6 +23,7 @@ type LockRing struct {
 	lastCompactTime  time.Time
 	snapshotInterval time.Duration
 	onTakeSnapshot   func(snapshot []pb.ServerAddress)
+	cleanupWg        sync.WaitGroup
 }
 
 func NewLockRing(snapshotInterval time.Duration) *LockRing {
@@ -87,7 +89,9 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
 
 	r.addOneSnapshot(servers)
 
+	r.cleanupWg.Add(1)
 	go func() {
+		defer r.cleanupWg.Done()
 		<-time.After(r.snapshotInterval)
 		r.compactSnapshots()
 	}()
@@ -96,7 +100,9 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
 func (r *LockRing) takeSnapshotWithDelayedCompaction() {
 	r.doTakeSnapshot()
 
+	r.cleanupWg.Add(1)
 	go func() {
+		defer r.cleanupWg.Done()
 		<-time.After(r.snapshotInterval)
 		r.compactSnapshots()
 	}()
@@ -172,6 +178,12 @@ func (r *LockRing) GetSnapshot() (servers []pb.ServerAddress) {
 	return r.snapshots[0].servers
 }
 
+// WaitForCleanup waits for all pending cleanup operations to complete
+// This is useful for testing to ensure deterministic behavior
+func (r *LockRing) WaitForCleanup() {
+	r.cleanupWg.Wait()
+}
+
 func hashKeyToServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
 	if len(servers) == 0 {
 		return ""

weed/cluster/lock_manager/lock_ring_test.go

@@ -1,10 +1,11 @@
 package lock_manager
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/stretchr/testify/assert"
 	"testing"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestAddServer(t *testing.T) {
@@ -21,7 +22,9 @@ func TestAddServer(t *testing.T) {
 
 	assert.Equal(t, 8, len(r.snapshots))
 
-	time.Sleep(110 * time.Millisecond)
+	// Wait for all cleanup operations to complete instead of using time.Sleep
+	time.Sleep(110 * time.Millisecond) // Still need to wait for the cleanup interval
+	r.WaitForCleanup()                 // Ensure all cleanup goroutines have finished
 
 	assert.Equal(t, 2, len(r.snapshots))
 

weed/command/backup.go

@@ -115,7 +115,10 @@ func runBackup(cmd *Command, args []string) bool {
 			return true
 		}
 	}
-	v, err := storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0)
+
+	ver := needle.Version(stats.Version)
+
+	v, err := storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, ver, 0, 0)
 	if err != nil {
 		fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
 		return true
@@ -142,7 +145,7 @@ func runBackup(cmd *Command, args []string) bool {
 			fmt.Printf("Error destroying volume: %v\n", err)
 		}
 		// recreate an empty volume
-		v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0)
+		v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, ver, 0, 0)
 		if err != nil {
 			fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
 			return true

weed/command/benchmark.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"io"
 	"math"
 	"math/rand"
@@ -115,7 +116,7 @@ func runBenchmark(cmd *Command, args []string) bool {
 	util.LoadSecurityConfiguration()
 	b.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
 
-	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)
+	fmt.Printf("This is SeaweedFS version %s %s %s\n", version.Version(), runtime.GOOS, runtime.GOARCH)
 	if *b.maxCpu < 1 {
 		*b.maxCpu = runtime.NumCPU()
 	}
@@ -241,7 +242,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 			Replication: *b.replication,
 			DiskType:    *b.diskType,
 		}
-		if assignResult, err := operation.Assign(b.masterClient.GetMaster, b.grpcDialOption, ar); err == nil {
+		if assignResult, err := operation.Assign(context.Background(), b.masterClient.GetMaster, b.grpcDialOption, ar); err == nil {
 			fp.Server, fp.Fid, fp.Pref.Collection = assignResult.Url, assignResult.Fid, *b.collection
 			if !isSecure && assignResult.Auth != "" {
 				isSecure = true
@@ -288,7 +289,7 @@ func readFiles(fileIdLineChan chan string, s *stat) {
 		start := time.Now()
 		var bytesRead int
 		var err error
-		urls, err := b.masterClient.LookupFileId(fid)
+		urls, err := b.masterClient.LookupFileId(context.Background(), fid)
 		if err != nil {
 			s.failed++
 			println("!!!! ", fid, " location not found!!!!!")

weed/command/command.go

@@ -43,6 +43,7 @@ var Commands = []*Command{
 	cmdVersion,
 	cmdVolume,
 	cmdWebDav,
+	cmdSftp,
 }
 
 type Command struct {

weed/command/compact.go

@@ -41,7 +41,7 @@ func runCompact(cmd *Command, args []string) bool {
 	preallocate := *compactVolumePreallocate * (1 << 20)
 
 	vid := needle.VolumeId(*compactVolumeId)
-	v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, 0, 0)
+	v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, needle.GetCurrentVersion(), 0, 0)
 	if err != nil {
 		glog.Fatalf("Load Volume [ERROR] %s\n", err)
 	}

weed/command/filer.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net"
 	"net/http"
 	"os"
@@ -35,6 +36,8 @@ var (
 	filerWebDavOptions WebDavOption
 	filerStartIam      *bool
 	filerIamOptions    IamOptions
+	filerStartSftp     *bool
+	filerSftpOptions   SftpOptions
 )
 
 type FilerOptions struct {
@@ -121,6 +124,7 @@ func init() {
 	filerS3Options.localSocket = cmdFiler.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
 	filerS3Options.tlsCACertificate = cmdFiler.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")
 	filerS3Options.tlsVerifyClientCert = cmdFiler.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate")
+	filerS3Options.bindIp = cmdFiler.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")
 	filerS3Options.idleTimeout = cmdFiler.Flag.Int("s3.idleTimeout", 10, "connection idle seconds")
 
 	// start webdav on filer
@@ -140,6 +144,19 @@ func init() {
 	filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
 	filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
 	filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
+
+	filerStartSftp = cmdFiler.Flag.Bool("sftp", false, "whether to start the SFTP server")
+	filerSftpOptions.port = cmdFiler.Flag.Int("sftp.port", 2022, "SFTP server listen port")
+	filerSftpOptions.sshPrivateKey = cmdFiler.Flag.String("sftp.sshPrivateKey", "", "path to the SSH private key file for host authentication")
+	filerSftpOptions.hostKeysFolder = cmdFiler.Flag.String("sftp.hostKeysFolder", "", "path to folder containing SSH private key files for host authentication")
+	filerSftpOptions.authMethods = cmdFiler.Flag.String("sftp.authMethods", "password,publickey", "comma-separated list of allowed auth methods: password, publickey, keyboard-interactive")
+	filerSftpOptions.maxAuthTries = cmdFiler.Flag.Int("sftp.maxAuthTries", 6, "maximum number of authentication attempts per connection")
+	filerSftpOptions.bannerMessage = cmdFiler.Flag.String("sftp.bannerMessage", "SeaweedFS SFTP Server - Unauthorized access is prohibited", "message displayed before authentication")
+	filerSftpOptions.loginGraceTime = cmdFiler.Flag.Duration("sftp.loginGraceTime", 2*time.Minute, "timeout for authentication")
+	filerSftpOptions.clientAliveInterval = cmdFiler.Flag.Duration("sftp.clientAliveInterval", 5*time.Second, "interval for sending keep-alive messages")
+	filerSftpOptions.clientAliveCountMax = cmdFiler.Flag.Int("sftp.clientAliveCountMax", 3, "maximum number of missed keep-alive messages before disconnecting")
+	filerSftpOptions.userStoreFile = cmdFiler.Flag.String("sftp.userStoreFile", "", "path to JSON file containing user credentials and permissions")
+	filerSftpOptions.localSocket = cmdFiler.Flag.String("sftp.localSocket", "", "default to /tmp/seaweedfs-sftp-<port>.sock")
 }
 
 func filerLongDesc() string {
@@ -198,7 +215,9 @@ func runFiler(cmd *Command, args []string) bool {
 	startDelay := time.Duration(2)
 	if *filerStartS3 {
 		filerS3Options.filer = &filerAddress
-		filerS3Options.bindIp = f.bindIp
+		if *filerS3Options.bindIp == "" {
+			filerS3Options.bindIp = f.bindIp
+		}
 		filerS3Options.localFilerSocket = f.localSocket
 		if *f.dataCenter != "" && *filerS3Options.dataCenter == "" {
 			filerS3Options.dataCenter = f.dataCenter
@@ -232,6 +251,18 @@ func runFiler(cmd *Command, args []string) bool {
 			time.Sleep(delay * time.Second)
 			filerIamOptions.startIamServer()
 		}(startDelay)
+		startDelay++
+	}
+
+	if *filerStartSftp {
+		sftpOptions.filer = &filerAddress
+		if *f.dataCenter != "" && *filerSftpOptions.dataCenter == "" {
+			filerSftpOptions.dataCenter = f.dataCenter
+		}
+		go func(delay time.Duration) {
+			time.Sleep(delay * time.Second)
+			sftpOptions.startSftpServer()
+		}(startDelay)
 	}
 
 	f.masters = pb.ServerAddresses(*f.mastersString).ToServiceDiscovery()
@@ -299,7 +330,7 @@ func (fo *FilerOptions) startFiler() {
 
 	if *fo.publicPort != 0 {
 		publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
-		glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
+		glog.V(0).Infoln("Start Seaweed filer server", version.Version(), "public at", publicListeningAddress)
 		publicListener, localPublicListener, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
 		if e != nil {
 			glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
@@ -318,7 +349,7 @@ func (fo *FilerOptions) startFiler() {
 		}
 	}
 
-	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
+	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", version.Version(), *fo.ip, *fo.port)
 	filerListener, filerLocalListener, e := util.NewIpAndLocalListeners(
 		*fo.bindIp, *fo.port,
 		time.Duration(10)*time.Second,

weed/command/filer_cat.go

@@ -28,9 +28,9 @@ type FilerCatOptions struct {
 }
 
 func (fco *FilerCatOptions) GetLookupFileIdFunction() wdclient.LookupFileIdFunctionType {
-	return func(fileId string) (targetUrls []string, err error) {
+	return func(ctx context.Context, fileId string) (targetUrls []string, err error) {
 		vid := filer.VolumeId(fileId)
-		resp, err := fco.filerClient.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{
+		resp, err := fco.filerClient.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
 			VolumeIds: []string{vid},
 		})
 		if err != nil {
@@ -102,7 +102,7 @@ func runFilerCat(cmd *Command, args []string) bool {
 			Name:      name,
 			Directory: dir,
 		}
-		respLookupEntry, err := filer_pb.LookupEntry(client, request)
+		respLookupEntry, err := filer_pb.LookupEntry(context.Background(), client, request)
 		if err != nil {
 			return err
 		}

weed/command/filer_copy.go

@@ -394,7 +394,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 			},
 		}
 
-		if err := filer_pb.CreateEntry(client, request); err != nil {
+		if err := filer_pb.CreateEntry(context.Background(), client, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
 		}
 		return nil
@@ -511,7 +511,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 			},
 		}
 
-		if err := filer_pb.CreateEntry(client, request); err != nil {
+		if err := filer_pb.CreateEntry(context.Background(), client, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
 		}
 		return nil

weed/command/filer_remote_gateway_buckets.go

@@ -1,6 +1,7 @@
 package command
 
 import (
+	"context"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -408,7 +409,7 @@ func (option *RemoteGatewayOptions) collectRemoteStorageConf() (err error) {
 
 	option.remoteConfs = make(map[string]*remote_pb.RemoteConf)
 	var lastConfName string
-	err = filer_pb.List(option, filer.DirectoryEtcRemote, "", func(entry *filer_pb.Entry, isLast bool) error {
+	err = filer_pb.List(context.Background(), option, filer.DirectoryEtcRemote, "", func(entry *filer_pb.Entry, isLast bool) error {
 		if !strings.HasSuffix(entry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
 			return nil
 		}

weed/command/filer_remote_sync_dir.go

@@ -230,7 +230,7 @@ func collectLastSyncOffset(filerClient filer_pb.FilerClient, grpcDialOption grpc
 	// 3. directory creation time
 	var lastOffsetTs time.Time
 	if timeAgo == 0 {
-		mountedDirEntry, err := filer_pb.GetEntry(filerClient, util.FullPath(mountedDir))
+		mountedDirEntry, err := filer_pb.GetEntry(context.Background(), filerClient, util.FullPath(mountedDir))
 		if err != nil {
 			glog.V(0).Infof("get mounted directory %s: %v", mountedDir, err)
 			return time.Now()

weed/command/fuse.go

@@ -1,250 +1,9 @@
 package command
 
-import (
-	"fmt"
-	"os"
-	"strconv"
-	"strings"
-	"time"
-)
-
 func init() {
 	cmdFuse.Run = runFuse // break init cycle
 }
 
-type parameter struct {
-	name  string
-	value string
-}
-
-func runFuse(cmd *Command, args []string) bool {
-	rawArgs := strings.Join(args, " ")
-	rawArgsLen := len(rawArgs)
-	option := strings.Builder{}
-	options := []parameter{}
-	masterProcess := true
-	fusermountPath := ""
-
-	// first parameter
-	i := 0
-	for i = 0; i < rawArgsLen && rawArgs[i] != ' '; i++ {
-		option.WriteByte(rawArgs[i])
-	}
-	options = append(options, parameter{"arg0", option.String()})
-	option.Reset()
-
-	for i++; i < rawArgsLen; i++ {
-
-		// space separator check for filled option
-		if rawArgs[i] == ' ' {
-			if option.Len() > 0 {
-				options = append(options, parameter{option.String(), "true"})
-				option.Reset()
-			}
-
-			// dash separator read option until next space
-		} else if rawArgs[i] == '-' {
-			for i++; i < rawArgsLen && rawArgs[i] != ' '; i++ {
-				option.WriteByte(rawArgs[i])
-			}
-			// ignore "-o"
-			if option.String() != "o" {
-				options = append(options, parameter{option.String(), "true"})
-			}
-			option.Reset()
-
-			// equal separator start option with pending value
-		} else if rawArgs[i] == '=' {
-			name := option.String()
-			option.Reset()
-
-			for i++; i < rawArgsLen && rawArgs[i] != ',' && rawArgs[i] != ' '; i++ {
-				// double quote separator read option until next double quote
-				if rawArgs[i] == '"' {
-					for i++; i < rawArgsLen && rawArgs[i] != '"'; i++ {
-						option.WriteByte(rawArgs[i])
-					}
-
-					// single quote separator read option until next single quote
-				} else if rawArgs[i] == '\'' {
-					for i++; i < rawArgsLen && rawArgs[i] != '\''; i++ {
-						option.WriteByte(rawArgs[i])
-					}
-
-					// add chars before comma
-				} else if rawArgs[i] != ' ' {
-					option.WriteByte(rawArgs[i])
-				}
-			}
-
-			options = append(options, parameter{name, option.String()})
-			option.Reset()
-
-			// comma separator just read current option
-		} else if rawArgs[i] == ',' {
-			options = append(options, parameter{option.String(), "true"})
-			option.Reset()
-
-			// what is not a separator fill option buffer
-		} else {
-			option.WriteByte(rawArgs[i])
-		}
-	}
-
-	// get residual option data
-	if option.Len() > 0 {
-		// add value to pending option
-		options = append(options, parameter{option.String(), "true"})
-		option.Reset()
-	}
-
-	// scan each parameter
-	for i := 0; i < len(options); i++ {
-		parameter := options[i]
-
-		switch parameter.name {
-		case "child":
-			masterProcess = false
-		case "arg0":
-			mountOptions.dir = &parameter.value
-		case "filer":
-			mountOptions.filer = &parameter.value
-		case "filer.path":
-			mountOptions.filerMountRootPath = &parameter.value
-		case "dirAutoCreate":
-			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
-				mountOptions.dirAutoCreate = &parsed
-			} else {
-				panic(fmt.Errorf("dirAutoCreate: %s", err))
-			}
-		case "collection":
-			mountOptions.collection = &parameter.value
-		case "replication":
-			mountOptions.replication = &parameter.value
-		case "disk":
-			mountOptions.diskType = &parameter.value
-		case "ttl":
-			if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err == nil {
-				intValue := int(parsed)
-				mountOptions.ttlSec = &intValue
-			} else {
-				panic(fmt.Errorf("ttl: %s", err))
-			}
-		case "chunkSizeLimitMB":
-			if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err == nil {
-				intValue := int(parsed)
-				mountOptions.chunkSizeLimitMB = &intValue
-			} else {
-				panic(fmt.Errorf("chunkSizeLimitMB: %s", err))
-			}
-		case "concurrentWriters":
-			i++
-			if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err == nil {
-				intValue := int(parsed)
-				mountOptions.concurrentWriters = &intValue
-			} else {
-				panic(fmt.Errorf("concurrentWriters: %s", err))
-			}
-		case "cacheDir":
-			mountOptions.cacheDirForRead = &parameter.value
-		case "cacheCapacityMB":
-			if parsed, err := strconv.ParseInt(parameter.value, 0, 64); err == nil {
-				mountOptions.cacheSizeMBForRead = &parsed
-			} else {
-				panic(fmt.Errorf("cacheCapacityMB: %s", err))
-			}
-		case "cacheDirWrite":
-			mountOptions.cacheDirForWrite = &parameter.value
-		case "dataCenter":
-			mountOptions.dataCenter = &parameter.value
-		case "allowOthers":
-			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
-				mountOptions.allowOthers = &parsed
-			} else {
-				panic(fmt.Errorf("allowOthers: %s", err))
-			}
-		case "umask":
-			mountOptions.umaskString = &parameter.value
-		case "nonempty":
-			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
-				mountOptions.nonempty = &parsed
-			} else {
-				panic(fmt.Errorf("nonempty: %s", err))
-			}
-		case "volumeServerAccess":
-			mountOptions.volumeServerAccess = &parameter.value
-		case "map.uid":
-			mountOptions.uidMap = &parameter.value
-		case "map.gid":
-			mountOptions.gidMap = &parameter.value
-		case "readOnly":
-			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
-				mountOptions.readOnly = &parsed
-			} else {
-				panic(fmt.Errorf("readOnly: %s", err))
-			}
-		case "cpuprofile":
-			mountCpuProfile = &parameter.value
-		case "memprofile":
-			mountMemProfile = &parameter.value
-		case "readRetryTime":
-			if parsed, err := time.ParseDuration(parameter.value); err == nil {
-				mountReadRetryTime = &parsed
-			} else {
-				panic(fmt.Errorf("readRetryTime: %s", err))
-			}
-		case "fusermount.path":
-			fusermountPath = parameter.value
-		default:
-			t := parameter.name
-			if parameter.value != "true" {
-				t = fmt.Sprintf("%s=%s", parameter.name, parameter.value)
-			}
-			mountOptions.extraOptions = append(mountOptions.extraOptions, t)
-		}
-	}
-
-	// the master start the child, release it then finish himself
-	if masterProcess {
-		arg0, err := os.Executable()
-		if err != nil {
-			panic(err)
-		}
-
-		argv := append(os.Args, "-o", "child")
-
-		attr := os.ProcAttr{}
-		attr.Env = os.Environ()
-
-		child, err := os.StartProcess(arg0, argv, &attr)
-
-		if err != nil {
-			panic(fmt.Errorf("master process can not start child process: %s", err))
-		}
-
-		err = child.Release()
-
-		if err != nil {
-			panic(fmt.Errorf("master process can not release child process: %s", err))
-		}
-
-		return true
-	}
-
-	if fusermountPath != "" {
-		if err := os.Setenv("PATH", fusermountPath); err != nil {
-			panic(fmt.Errorf("setenv: %s", err))
-		}
-	} else if os.Getenv("PATH") == "" {
-		if err := os.Setenv("PATH", "/bin:/sbin:/usr/bin:/usr/sbin"); err != nil {
-			panic(fmt.Errorf("setenv: %s", err))
-		}
-	}
-
-	// just call "weed mount" command
-	return runMount(cmdMount, []string{})
-}
-
 var cmdFuse = &Command{
 	UsageLine: "fuse /mnt/mount/point -o \"filer=localhost:8888,filer.path=/\"",
 	Short:     "Allow use weed with linux's mount command",

weed/command/fuse_notsupported.go

@@ -0,0 +1,15 @@
+//go:build !linux && !darwin
+// +build !linux,!darwin
+
+package command
+
+import (
+	"fmt"
+	"runtime"
+)
+
+func runFuse(cmd *Command, args []string) bool {
+	fmt.Printf("Fuse is not supported on %s %s\n", runtime.GOOS, runtime.GOARCH)
+
+	return true
+}

weed/command/fuse_std.go

@@ -0,0 +1,271 @@
+//go:build linux || darwin
+// +build linux darwin
+
+package command
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"os/signal"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+)
+
+type parameter struct {
+	name  string
+	value string
+}
+
+func runFuse(cmd *Command, args []string) bool {
+	rawArgs := strings.Join(args, " ")
+	rawArgsLen := len(rawArgs)
+	option := strings.Builder{}
+	options := []parameter{}
+	masterProcess := true
+	fusermountPath := ""
+
+	// first parameter
+	i := 0
+	for i = 0; i < rawArgsLen && rawArgs[i] != ' '; i++ {
+		option.WriteByte(rawArgs[i])
+	}
+	options = append(options, parameter{"arg0", option.String()})
+	option.Reset()
+
+	for i++; i < rawArgsLen; i++ {
+
+		// space separator check for filled option
+		if rawArgs[i] == ' ' {
+			if option.Len() > 0 {
+				options = append(options, parameter{option.String(), "true"})
+				option.Reset()
+			}
+
+			// dash separator read option until next space
+		} else if rawArgs[i] == '-' {
+			for i++; i < rawArgsLen && rawArgs[i] != ' '; i++ {
+				option.WriteByte(rawArgs[i])
+			}
+			// ignore "-o"
+			if option.String() != "o" {
+				options = append(options, parameter{option.String(), "true"})
+			}
+			option.Reset()
+
+			// equal separator start option with pending value
+		} else if rawArgs[i] == '=' {
+			name := option.String()
+			option.Reset()
+
+			for i++; i < rawArgsLen && rawArgs[i] != ',' && rawArgs[i] != ' '; i++ {
+				// double quote separator read option until next double quote
+				if rawArgs[i] == '"' {
+					for i++; i < rawArgsLen && rawArgs[i] != '"'; i++ {
+						option.WriteByte(rawArgs[i])
+					}
+
+					// single quote separator read option until next single quote
+				} else if rawArgs[i] == '\'' {
+					for i++; i < rawArgsLen && rawArgs[i] != '\''; i++ {
+						option.WriteByte(rawArgs[i])
+					}
+
+					// add chars before comma
+				} else if rawArgs[i] != ' ' {
+					option.WriteByte(rawArgs[i])
+				}
+			}
+
+			options = append(options, parameter{name, option.String()})
+			option.Reset()
+
+			// comma separator just read current option
+		} else if rawArgs[i] == ',' {
+			options = append(options, parameter{option.String(), "true"})
+			option.Reset()
+
+			// what is not a separator fill option buffer
+		} else {
+			option.WriteByte(rawArgs[i])
+		}
+	}
+
+	// get residual option data
+	if option.Len() > 0 {
+		// add value to pending option
+		options = append(options, parameter{option.String(), "true"})
+		option.Reset()
+	}
+
+	// scan each parameter
+	for i := 0; i < len(options); i++ {
+		parameter := options[i]
+
+		switch parameter.name {
+		case "child":
+			masterProcess = false
+			if parsed, err := strconv.ParseInt(parameter.value, 10, 64); err == nil {
+				if parsed > math.MaxInt || parsed <= 0 {
+					panic(fmt.Errorf("parent PID %d is invalid", parsed))
+				}
+				mountOptions.fuseCommandPid = int(parsed)
+			} else {
+				panic(fmt.Errorf("parent PID %s is invalid: %w", parameter.value, err))
+			}
+		case "arg0":
+			mountOptions.dir = &parameter.value
+		case "filer":
+			mountOptions.filer = &parameter.value
+		case "filer.path":
+			mountOptions.filerMountRootPath = &parameter.value
+		case "dirAutoCreate":
+			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
+				mountOptions.dirAutoCreate = &parsed
+			} else {
+				panic(fmt.Errorf("dirAutoCreate: %s", err))
+			}
+		case "collection":
+			mountOptions.collection = &parameter.value
+		case "replication":
+			mountOptions.replication = &parameter.value
+		case "disk":
+			mountOptions.diskType = &parameter.value
+		case "ttl":
+			if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err == nil {
+				intValue := int(parsed)
+				mountOptions.ttlSec = &intValue
+			} else {
+				panic(fmt.Errorf("ttl: %s", err))
+			}
+		case "chunkSizeLimitMB":
+			if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err == nil {
+				intValue := int(parsed)
+				mountOptions.chunkSizeLimitMB = &intValue
+			} else {
+				panic(fmt.Errorf("chunkSizeLimitMB: %s", err))
+			}
+		case "concurrentWriters":
+			i++
+			if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err == nil {
+				intValue := int(parsed)
+				mountOptions.concurrentWriters = &intValue
+			} else {
+				panic(fmt.Errorf("concurrentWriters: %s", err))
+			}
+		case "cacheDir":
+			mountOptions.cacheDirForRead = &parameter.value
+		case "cacheCapacityMB":
+			if parsed, err := strconv.ParseInt(parameter.value, 0, 64); err == nil {
+				mountOptions.cacheSizeMBForRead = &parsed
+			} else {
+				panic(fmt.Errorf("cacheCapacityMB: %s", err))
+			}
+		case "cacheDirWrite":
+			mountOptions.cacheDirForWrite = &parameter.value
+		case "dataCenter":
+			mountOptions.dataCenter = &parameter.value
+		case "allowOthers":
+			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
+				mountOptions.allowOthers = &parsed
+			} else {
+				panic(fmt.Errorf("allowOthers: %s", err))
+			}
+		case "umask":
+			mountOptions.umaskString = &parameter.value
+		case "nonempty":
+			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
+				mountOptions.nonempty = &parsed
+			} else {
+				panic(fmt.Errorf("nonempty: %s", err))
+			}
+		case "volumeServerAccess":
+			mountOptions.volumeServerAccess = &parameter.value
+		case "map.uid":
+			mountOptions.uidMap = &parameter.value
+		case "map.gid":
+			mountOptions.gidMap = &parameter.value
+		case "readOnly":
+			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
+				mountOptions.readOnly = &parsed
+			} else {
+				panic(fmt.Errorf("readOnly: %s", err))
+			}
+		case "disableXAttr":
+			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
+
+				mountOptions.disableXAttr = &parsed
+			} else {
+				panic(fmt.Errorf("disableXAttr: %s", err))
+			}
+		case "cpuprofile":
+			mountCpuProfile = &parameter.value
+		case "memprofile":
+			mountMemProfile = &parameter.value
+		case "readRetryTime":
+			if parsed, err := time.ParseDuration(parameter.value); err == nil {
+				mountReadRetryTime = &parsed
+			} else {
+				panic(fmt.Errorf("readRetryTime: %s", err))
+			}
+		case "fusermount.path":
+			fusermountPath = parameter.value
+		default:
+			t := parameter.name
+			if parameter.value != "true" {
+				t = fmt.Sprintf("%s=%s", parameter.name, parameter.value)
+			}
+			mountOptions.extraOptions = append(mountOptions.extraOptions, t)
+		}
+	}
+
+	// the master start the child, release it then finish himself
+	if masterProcess {
+		arg0, err := os.Executable()
+		if err != nil {
+			panic(err)
+		}
+
+		// pass our PID to the child process
+		pid := os.Getpid()
+		argv := append(os.Args, "-o", "child="+strconv.Itoa(pid))
+
+		c := make(chan os.Signal, 1)
+		signal.Notify(c, syscall.SIGTERM)
+
+		attr := os.ProcAttr{}
+		attr.Env = os.Environ()
+
+		child, err := os.StartProcess(arg0, argv, &attr)
+
+		if err != nil {
+			panic(fmt.Errorf("master process can not start child process: %s", err))
+		}
+
+		err = child.Release()
+
+		if err != nil {
+			panic(fmt.Errorf("master process can not release child process: %s", err))
+		}
+
+		select {
+		case <-c:
+			return true
+		}
+	}
+
+	if fusermountPath != "" {
+		if err := os.Setenv("PATH", fusermountPath); err != nil {
+			panic(fmt.Errorf("setenv: %s", err))
+		}
+	} else if os.Getenv("PATH") == "" {
+		if err := os.Setenv("PATH", "/bin:/sbin:/usr/bin:/usr/sbin"); err != nil {
+			panic(fmt.Errorf("setenv: %s", err))
+		}
+	}
+
+	// just call "weed mount" command
+	return runMount(cmdMount, []string{})
+}

weed/command/iam.go

@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 
 	"time"
@@ -89,7 +90,7 @@ func (iamopt *IamOptions) startIamServer() bool {
 		glog.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err)
 	}
 
-	glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", util.Version(), *iamopt.port)
+	glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", version.Version(), *iamopt.port)
 	if iamApiLocalListener != nil {
 		go func() {
 			if err = httpS.Serve(iamApiLocalListener); err != nil {

weed/command/master.go

@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"os"
 	"path"
@@ -160,7 +161,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers)
 	listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
-	glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
+	glog.V(0).Infof("Start Seaweed Master %s at %s", version.Version(), listeningAddress)
 	masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
@@ -211,7 +212,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		protobuf.RegisterRaftServer(grpcS, raftServer)
 	}
 	reflection.Register(grpcS)
-	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOption.ipBind, grpcPort)
+	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", version.Version(), *masterOption.ipBind, grpcPort)
 	if grpcLocalL != nil {
 		go grpcS.Serve(grpcLocalL)
 	}

weed/command/master_follower.go

@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"time"
 
@@ -119,7 +120,7 @@ func startMasterFollower(masterOptions MasterOptions) {
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, option, masters)
 	listeningAddress := util.JoinHostPort(*masterOptions.ipBind, *masterOptions.port)
-	glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
+	glog.V(0).Infof("Start Seaweed Master %s at %s", version.Version(), listeningAddress)
 	masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOptions.ipBind, *masterOptions.port, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
@@ -134,7 +135,7 @@ func startMasterFollower(masterOptions MasterOptions) {
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
 	master_pb.RegisterSeaweedServer(grpcS, ms)
 	reflection.Register(grpcS)
-	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOptions.ip, grpcPort)
+	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", version.Version(), *masterOptions.ip, grpcPort)
 	if grpcLocalL != nil {
 		go grpcS.Serve(grpcLocalL)
 	}

weed/command/mount.go

@@ -34,6 +34,7 @@ type MountOptions struct {
 	localSocket        *string
 	disableXAttr       *bool
 	extraOptions       []string
+	fuseCommandPid     int
 }
 
 var (
@@ -72,6 +73,7 @@ func init() {
 	mountOptions.debugPort = cmdMount.Flag.Int("debug.port", 6061, "http port for debugging")
 	mountOptions.localSocket = cmdMount.Flag.String("localSocket", "", "default to /tmp/seaweedfs-mount-<mount_dir_hash>.sock")
 	mountOptions.disableXAttr = cmdMount.Flag.Bool("disableXAttr", false, "disable xattr")
+	mountOptions.fuseCommandPid = 0
 
 	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
 	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")

weed/command/mount_freebsd.go

@@ -0,0 +1,5 @@
+package command
+
+func checkMountPointAvailable(dir string) bool {
+	return true
+}

weed/command/mount_notsupported.go

@@ -1,5 +1,5 @@
-//go:build !linux && !darwin
-// +build !linux,!darwin
+//go:build !linux && !darwin && !freebsd
+// +build !linux,!darwin,!freebsd
 
 package command
 

weed/command/mount_std.go

@@ -1,5 +1,5 @@
-//go:build linux || darwin
-// +build linux darwin
+//go:build linux || darwin || freebsd
+// +build linux darwin freebsd
 
 package command
 
@@ -13,8 +13,11 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+
 	"github.com/hanwen/go-fuse/v2/fuse"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mount"
@@ -255,7 +258,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	// create mount root
 	mountRootPath := util.FullPath(mountRoot)
 	mountRootParent, mountDir := mountRootPath.DirAndName()
-	if err = filer_pb.Mkdir(seaweedFileSystem, mountRootParent, mountDir, nil); err != nil {
+	if err = filer_pb.Mkdir(context.Background(), seaweedFileSystem, mountRootParent, mountDir, nil); err != nil {
 		fmt.Printf("failed to create dir %s on filer %s: %v\n", mountRoot, filerAddresses, err)
 		return false
 	}
@@ -268,6 +271,15 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		unmount.Unmount(dir)
 	})
 
+	if mountOptions.fuseCommandPid != 0 {
+		// send a signal to the parent process to notify that the mount is ready
+		err = syscall.Kill(mountOptions.fuseCommandPid, syscall.SIGTERM)
+		if err != nil {
+			fmt.Printf("failed to notify parent process: %v\n", err)
+			return false
+		}
+	}
+
 	grpcS := pb.NewGrpcServer()
 	mount_pb.RegisterSeaweedMountServer(grpcS, seaweedFileSystem)
 	reflection.Register(grpcS)
@@ -280,7 +292,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	}
 
 	glog.V(0).Infof("mounted %s%s to %v", *option.filer, mountRoot, dir)
-	glog.V(0).Infof("This is SeaweedFS version %s %s %s", util.Version(), runtime.GOOS, runtime.GOARCH)
+	glog.V(0).Infof("This is SeaweedFS version %s %s %s", version.Version(), runtime.GOOS, runtime.GOARCH)
 
 	server.Serve()
 

weed/command/mq_agent.go

@@ -32,7 +32,7 @@ func init() {
 
 var cmdMqAgent = &Command{
 	UsageLine: "mq.agent [-port=16777] [-broker=<ip:port>]",
-	Short:     "<WIP> start a message queue agent",
+	Short:     "start a message queue agent",
 	Long: `start a message queue agent
 
 	The agent runs on local server to accept gRPC calls to write or read messages. 

weed/command/mq_broker.go

@@ -43,7 +43,7 @@ func init() {
 
 var cmdMqBroker = &Command{
 	UsageLine: "mq.broker [-port=17777] [-master=<ip:port>]",
-	Short:     "<WIP> start a message queue broker",
+	Short:     "start a message queue broker",
 	Long: `start a message queue broker
 
 	The broker can accept gRPC calls to write or read messages. The messages are stored via filer.

weed/command/s3.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -339,7 +340,7 @@ func (s3opt *S3Options) startS3Server() bool {
 			glog.Fatalf("error with tls config: %v", err)
 		}
 		if *s3opt.portHttps == 0 {
-			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
+			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", version.Version(), *s3opt.port)
 			if s3ApiLocalListener != nil {
 				go func() {
 					if err = httpS.ServeTLS(s3ApiLocalListener, "", ""); err != nil {
@@ -351,9 +352,9 @@ func (s3opt *S3Options) startS3Server() bool {
 				glog.Fatalf("S3 API Server Fail to serve: %v", err)
 			}
 		} else {
-			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.portHttps)
+			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", version.Version(), *s3opt.portHttps)
 			s3ApiListenerHttps, s3ApiLocalListenerHttps, _ := util.NewIpAndLocalListeners(
-                *s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second)
+				*s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second)
 			if s3ApiLocalListenerHttps != nil {
 				go func() {
 					if err = httpS.ServeTLS(s3ApiLocalListenerHttps, "", ""); err != nil {
@@ -369,7 +370,7 @@ func (s3opt *S3Options) startS3Server() bool {
 		}
 	}
 	if *s3opt.tlsPrivateKey == "" || *s3opt.portHttps > 0 {
-		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.Version(), *s3opt.port)
+		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", version.Version(), *s3opt.port)
 		if s3ApiLocalListener != nil {
 			go func() {
 				if err = httpS.Serve(s3ApiLocalListener); err != nil {

weed/command/scaffold/filer.toml

@@ -54,6 +54,10 @@ enabled = false
 # dsn will take priority over "hostname, port, username, password, database".
 # [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
 dsn = "root@tcp(localhost:3306)/seaweedfs?collation=utf8mb4_bin"
+enable_tls = false
+ca_crt = ""                # ca.crt dir when enable_tls set true
+client_crt = ""            # mysql client.crt dir when enable_tls set true
+client_key = ""            # mysql client.key dir when enable_tls set true
 hostname = "localhost"
 port = 3306
 username = "root"
@@ -112,7 +116,13 @@ connection_max_open = 100
 connection_max_lifetime_seconds = 0
 # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
 enableUpsert = true
-upsertQuery = """UPSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)"""
+upsertQuery = """
+  INSERT INTO "%[1]s" (dirhash, name, directory, meta)
+    VALUES($1, $2, $3, $4)
+    ON CONFLICT (dirhash, name) DO UPDATE SET
+      directory=EXCLUDED.directory,
+      meta=EXCLUDED.meta
+"""
 
 [postgres2]
 enabled = false
@@ -137,7 +147,13 @@ connection_max_open = 100
 connection_max_lifetime_seconds = 0
 # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
 enableUpsert = true
-upsertQuery = """UPSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)"""
+upsertQuery = """
+  INSERT INTO "%[1]s" (dirhash, name, directory, meta)
+    VALUES($1, $2, $3, $4)
+    ON CONFLICT (dirhash, name) DO UPDATE SET
+      directory=EXCLUDED.directory,
+      meta=EXCLUDED.meta
+"""
 
 [cassandra2]
 # CREATE TABLE filemeta (

weed/command/server.go

@@ -28,6 +28,7 @@ var (
 	masterOptions   MasterOptions
 	filerOptions    FilerOptions
 	s3Options       S3Options
+	sftpOptions     SftpOptions
 	iamOptions      IamOptions
 	webdavOptions   WebDavOption
 	mqBrokerOptions MessageQueueBrokerOptions
@@ -73,6 +74,7 @@ var (
 	isStartingVolumeServer = cmdServer.Flag.Bool("volume", true, "whether to start volume server")
 	isStartingFiler        = cmdServer.Flag.Bool("filer", false, "whether to start filer")
 	isStartingS3           = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
+	isStartingSftp         = cmdServer.Flag.Bool("sftp", false, "whether to start Sftp server")
 	isStartingIam          = cmdServer.Flag.Bool("iam", false, "whether to start IAM service")
 	isStartingWebDav       = cmdServer.Flag.Bool("webdav", false, "whether to start WebDAV gateway")
 	isStartingMqBroker     = cmdServer.Flag.Bool("mq.broker", false, "whether to start message queue broker")
@@ -159,6 +161,17 @@ func init() {
 	s3Options.bindIp = cmdServer.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")
 	s3Options.idleTimeout = cmdServer.Flag.Int("s3.idleTimeout", 10, "connection idle seconds")
 
+	sftpOptions.port = cmdServer.Flag.Int("sftp.port", 2022, "SFTP server listen port")
+	sftpOptions.sshPrivateKey = cmdServer.Flag.String("sftp.sshPrivateKey", "", "path to the SSH private key file for host authentication")
+	sftpOptions.hostKeysFolder = cmdServer.Flag.String("sftp.hostKeysFolder", "", "path to folder containing SSH private key files for host authentication")
+	sftpOptions.authMethods = cmdServer.Flag.String("sftp.authMethods", "password,publickey", "comma-separated list of allowed auth methods: password, publickey, keyboard-interactive")
+	sftpOptions.maxAuthTries = cmdServer.Flag.Int("sftp.maxAuthTries", 6, "maximum number of authentication attempts per connection")
+	sftpOptions.bannerMessage = cmdServer.Flag.String("sftp.bannerMessage", "SeaweedFS SFTP Server - Unauthorized access is prohibited", "message displayed before authentication")
+	sftpOptions.loginGraceTime = cmdServer.Flag.Duration("sftp.loginGraceTime", 2*time.Minute, "timeout for authentication")
+	sftpOptions.clientAliveInterval = cmdServer.Flag.Duration("sftp.clientAliveInterval", 5*time.Second, "interval for sending keep-alive messages")
+	sftpOptions.clientAliveCountMax = cmdServer.Flag.Int("sftp.clientAliveCountMax", 3, "maximum number of missed keep-alive messages before disconnecting")
+	sftpOptions.userStoreFile = cmdServer.Flag.String("sftp.userStoreFile", "", "path to JSON file containing user credentials and permissions")
+	sftpOptions.localSocket = cmdServer.Flag.String("sftp.localSocket", "", "default to /tmp/seaweedfs-sftp-<port>.sock")
 	iamOptions.port = cmdServer.Flag.Int("iam.port", 8111, "iam server http listen port")
 
 	webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port")
@@ -190,6 +203,9 @@ func runServer(cmd *Command, args []string) bool {
 	if *isStartingS3 {
 		*isStartingFiler = true
 	}
+	if *isStartingSftp {
+		*isStartingFiler = true
+	}
 	if *isStartingIam {
 		*isStartingFiler = true
 	}
@@ -223,6 +239,9 @@ func runServer(cmd *Command, args []string) bool {
 	if *s3Options.bindIp == "" {
 		s3Options.bindIp = serverBindIp
 	}
+	if sftpOptions.bindIp == nil || *sftpOptions.bindIp == "" {
+		sftpOptions.bindIp = serverBindIp
+	}
 	iamOptions.ip = serverBindIp
 	iamOptions.masters = masterOptions.peers
 	webdavOptions.ipBind = serverBindIp
@@ -246,11 +265,13 @@ func runServer(cmd *Command, args []string) bool {
 	mqBrokerOptions.dataCenter = serverDataCenter
 	mqBrokerOptions.rack = serverRack
 	s3Options.dataCenter = serverDataCenter
+	sftpOptions.dataCenter = serverDataCenter
 	filerOptions.disableHttp = serverDisableHttp
 	masterOptions.disableHttp = serverDisableHttp
 
 	filerAddress := string(pb.NewServerAddress(*serverIp, *filerOptions.port, *filerOptions.portGrpc))
 	s3Options.filer = &filerAddress
+	sftpOptions.filer = &filerAddress
 	iamOptions.filer = &filerAddress
 	webdavOptions.filer = &filerAddress
 	mqBrokerOptions.filerGroup = filerOptions.filerGroup
@@ -291,6 +312,14 @@ func runServer(cmd *Command, args []string) bool {
 		}()
 	}
 
+	if *isStartingSftp {
+		go func() {
+			time.Sleep(2 * time.Second)
+			sftpOptions.localSocket = filerOptions.localSocket
+			sftpOptions.startSftpServer()
+		}()
+	}
+
 	if *isStartingIam {
 		go func() {
 			time.Sleep(2 * time.Second)

weed/command/sftp.go

@@ -0,0 +1,194 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+	"net"
+	"os"
+	"runtime"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	filer_pb "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/sftpd"
+	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var (
+	sftpOptionsStandalone SftpOptions
+)
+
+// SftpOptions holds configuration options for the SFTP server.
+type SftpOptions struct {
+	filer               *string
+	bindIp              *string
+	port                *int
+	sshPrivateKey       *string
+	hostKeysFolder      *string
+	authMethods         *string
+	maxAuthTries        *int
+	bannerMessage       *string
+	loginGraceTime      *time.Duration
+	clientAliveInterval *time.Duration
+	clientAliveCountMax *int
+	userStoreFile       *string
+	dataCenter          *string
+	metricsHttpPort     *int
+	metricsHttpIp       *string
+	localSocket         *string
+}
+
+// cmdSftp defines the SFTP command similar to the S3 command.
+var cmdSftp = &Command{
+	UsageLine: "sftp [-port=2022] [-filer=<ip:port>] [-sshPrivateKey=</path/to/private_key>]",
+	Short:     "start an SFTP server that is backed by a SeaweedFS filer",
+	Long: `Start an SFTP server that leverages the SeaweedFS filer service to handle file operations.
+
+Instead of reading from or writing to a local filesystem, all file operations
+are routed through the filer (filer_pb) gRPC API. This allows you to centralize
+your file management in SeaweedFS.
+	`,
+}
+
+func init() {
+	// Register the command to avoid cyclic dependencies.
+	cmdSftp.Run = runSftp
+
+	sftpOptionsStandalone.filer = cmdSftp.Flag.String("filer", "localhost:8888", "filer server address (ip:port)")
+	sftpOptionsStandalone.bindIp = cmdSftp.Flag.String("ip.bind", "0.0.0.0", "ip address to bind SFTP server")
+	sftpOptionsStandalone.port = cmdSftp.Flag.Int("port", 2022, "SFTP server listen port")
+	sftpOptionsStandalone.sshPrivateKey = cmdSftp.Flag.String("sshPrivateKey", "", "path to the SSH private key file for host authentication")
+	sftpOptionsStandalone.hostKeysFolder = cmdSftp.Flag.String("hostKeysFolder", "", "path to folder containing SSH private key files for host authentication")
+	sftpOptionsStandalone.authMethods = cmdSftp.Flag.String("authMethods", "password,publickey", "comma-separated list of allowed auth methods: password, publickey, keyboard-interactive")
+	sftpOptionsStandalone.maxAuthTries = cmdSftp.Flag.Int("maxAuthTries", 6, "maximum number of authentication attempts per connection")
+	sftpOptionsStandalone.bannerMessage = cmdSftp.Flag.String("bannerMessage", "SeaweedFS SFTP Server - Unauthorized access is prohibited", "message displayed before authentication")
+	sftpOptionsStandalone.loginGraceTime = cmdSftp.Flag.Duration("loginGraceTime", 2*time.Minute, "timeout for authentication")
+	sftpOptionsStandalone.clientAliveInterval = cmdSftp.Flag.Duration("clientAliveInterval", 5*time.Second, "interval for sending keep-alive messages")
+	sftpOptionsStandalone.clientAliveCountMax = cmdSftp.Flag.Int("clientAliveCountMax", 3, "maximum number of missed keep-alive messages before disconnecting")
+	sftpOptionsStandalone.userStoreFile = cmdSftp.Flag.String("userStoreFile", "", "path to JSON file containing user credentials and permissions")
+	sftpOptionsStandalone.dataCenter = cmdSftp.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
+	sftpOptionsStandalone.metricsHttpPort = cmdSftp.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
+	sftpOptionsStandalone.metricsHttpIp = cmdSftp.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
+	sftpOptionsStandalone.localSocket = cmdSftp.Flag.String("localSocket", "", "default to /tmp/seaweedfs-sftp-<port>.sock")
+}
+
+// runSftp is the command entry point.
+func runSftp(cmd *Command, args []string) bool {
+	// Load security configuration as done in other SeaweedFS services.
+	util.LoadSecurityConfiguration()
+
+	// Configure metrics
+	switch {
+	case *sftpOptionsStandalone.metricsHttpIp != "":
+		// nothing to do, use sftpOptionsStandalone.metricsHttpIp
+	case *sftpOptionsStandalone.bindIp != "":
+		*sftpOptionsStandalone.metricsHttpIp = *sftpOptionsStandalone.bindIp
+	}
+	go stats_collect.StartMetricsServer(*sftpOptionsStandalone.metricsHttpIp, *sftpOptionsStandalone.metricsHttpPort)
+
+	return sftpOptionsStandalone.startSftpServer()
+}
+
+func (sftpOpt *SftpOptions) startSftpServer() bool {
+	filerAddress := pb.ServerAddress(*sftpOpt.filer)
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	// metrics read from the filer
+	var metricsAddress string
+	var metricsIntervalSec int
+	var filerGroup string
+
+	// Connect to the filer service and try to retrieve basic configuration.
+	for {
+		err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			if err != nil {
+				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
+			}
+			metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec)
+			filerGroup = resp.FilerGroup
+			glog.V(0).Infof("SFTP read filer configuration, using filer at: %s", filerAddress)
+			return nil
+		})
+		if err != nil {
+			glog.V(0).Infof("Waiting to connect to filer %s grpc address %s...", *sftpOpt.filer, filerAddress.ToGrpcAddress())
+			time.Sleep(time.Second)
+		} else {
+			glog.V(0).Infof("Connected to filer %s grpc address %s", *sftpOpt.filer, filerAddress.ToGrpcAddress())
+			break
+		}
+	}
+
+	go stats_collect.LoopPushingMetric("sftp", stats_collect.SourceName(uint32(*sftpOpt.port)), metricsAddress, metricsIntervalSec)
+
+	// Parse auth methods
+	var authMethods []string
+	if *sftpOpt.authMethods != "" {
+		authMethods = util.StringSplit(*sftpOpt.authMethods, ",")
+	}
+
+	// Create a new SFTP service instance with all options
+	service := sftpd.NewSFTPService(&sftpd.SFTPServiceOptions{
+		GrpcDialOption:      grpcDialOption,
+		DataCenter:          *sftpOpt.dataCenter,
+		FilerGroup:          filerGroup,
+		Filer:               filerAddress,
+		SshPrivateKey:       *sftpOpt.sshPrivateKey,
+		HostKeysFolder:      *sftpOpt.hostKeysFolder,
+		AuthMethods:         authMethods,
+		MaxAuthTries:        *sftpOpt.maxAuthTries,
+		BannerMessage:       *sftpOpt.bannerMessage,
+		LoginGraceTime:      *sftpOpt.loginGraceTime,
+		ClientAliveInterval: *sftpOpt.clientAliveInterval,
+		ClientAliveCountMax: *sftpOpt.clientAliveCountMax,
+		UserStoreFile:       *sftpOpt.userStoreFile,
+	})
+
+	// Set up Unix socket if on non-Windows platforms
+	if runtime.GOOS != "windows" {
+		localSocket := *sftpOpt.localSocket
+		if localSocket == "" {
+			localSocket = fmt.Sprintf("/tmp/seaweedfs-sftp-%d.sock", *sftpOpt.port)
+		}
+		if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
+			glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
+		}
+		go func() {
+			// start on local unix socket
+			sftpSocketListener, err := net.Listen("unix", localSocket)
+			if err != nil {
+				glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
+			}
+			if err := service.Serve(sftpSocketListener); err != nil {
+				glog.Fatalf("Failed to serve SFTP on socket %s: %v", localSocket, err)
+			}
+		}()
+	}
+
+	// Start the SFTP service on TCP
+	listenAddress := fmt.Sprintf("%s:%d", *sftpOpt.bindIp, *sftpOpt.port)
+	sftpListener, sftpLocalListener, err := util.NewIpAndLocalListeners(*sftpOpt.bindIp, *sftpOpt.port, time.Duration(10)*time.Second)
+	if err != nil {
+		glog.Fatalf("SFTP server listener on %s error: %v", listenAddress, err)
+	}
+
+	glog.V(0).Infof("Start Seaweed SFTP Server %s at %s", version.Version(), listenAddress)
+
+	if sftpLocalListener != nil {
+		go func() {
+			if err := service.Serve(sftpLocalListener); err != nil {
+				glog.Fatalf("SFTP Server failed to serve on local listener: %v", err)
+			}
+		}()
+	}
+
+	if err := service.Serve(sftpListener); err != nil {
+		glog.Fatalf("SFTP Server failed to serve: %v", err)
+	}
+
+	return true
+}

weed/command/update.go

@@ -10,6 +10,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	swv "github.com/seaweedfs/seaweedfs/weed/util/version"
 	"io"
 	"net/http"
 	"os"
@@ -117,7 +118,7 @@ func runUpdate(cmd *Command, args []string) bool {
 }
 
 func downloadRelease(ctx context.Context, target string, ver string) (version string, err error) {
-	currentVersion := util.VERSION_NUMBER
+	currentVersion := swv.VERSION_NUMBER
 	rel, err := GitHubLatestRelease(ctx, ver, "seaweedfs", "seaweedfs")
 	if err != nil {
 		return "", err

weed/command/version.go

@@ -2,9 +2,8 @@ package command
 
 import (
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"runtime"
-
-	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 var cmdVersion = &Command{
@@ -19,6 +18,6 @@ func runVersion(cmd *Command, args []string) bool {
 		cmd.Usage()
 	}
 
-	fmt.Printf("version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)
+	fmt.Printf("version %s %s %s\n", version.Version(), runtime.GOOS, runtime.GOARCH)
 	return true
 }

weed/command/volume.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	httppprof "net/http/pprof"
 	"os"
@@ -351,7 +352,7 @@ func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerSe
 
 func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server {
 	publicListeningAddress := util.JoinHostPort(*v.bindIp, *v.publicPort)
-	glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "public at", publicListeningAddress)
+	glog.V(0).Infoln("Start Seaweed volume server", version.Version(), "public at", publicListeningAddress)
 	publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
 	if e != nil {
 		glog.Fatalf("Volume server listener error:%v", e)
@@ -378,7 +379,7 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpd
 	}
 
 	listeningAddress := util.JoinHostPort(*v.bindIp, *v.port)
-	glog.V(0).Infof("Start Seaweed volume server %s at %s", util.Version(), listeningAddress)
+	glog.V(0).Infof("Start Seaweed volume server %s at %s", version.Version(), listeningAddress)
 	listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
 	if e != nil {
 		glog.Fatalf("Volume server listener error:%v", e)

weed/command/webdav.go

@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"os"
 	"os/user"
@@ -65,7 +66,7 @@ func runWebDav(cmd *Command, args []string) bool {
 	util.LoadSecurityConfiguration()
 
 	listenAddress := fmt.Sprintf("%s:%d", *webDavStandaloneOptions.ipBind, *webDavStandaloneOptions.port)
-	glog.V(0).Infof("Starting Seaweed WebDav Server %s at %s", util.Version(), listenAddress)
+	glog.V(0).Infof("Starting Seaweed WebDav Server %s at %s", version.Version(), listenAddress)
 
 	return webDavStandaloneOptions.startWebDav()
 
@@ -136,12 +137,12 @@ func (wo *WebDavOption) startWebDav() bool {
 	}
 
 	if *wo.tlsPrivateKey != "" {
-		glog.V(0).Infof("Start Seaweed WebDav Server %s at https %s", util.Version(), listenAddress)
+		glog.V(0).Infof("Start Seaweed WebDav Server %s at https %s", version.Version(), listenAddress)
 		if err = httpS.ServeTLS(webDavListener, *wo.tlsCertificate, *wo.tlsPrivateKey); err != nil {
 			glog.Fatalf("WebDav Server Fail to serve: %v", err)
 		}
 	} else {
-		glog.V(0).Infof("Start Seaweed WebDav Server %s at http %s", util.Version(), listenAddress)
+		glog.V(0).Infof("Start Seaweed WebDav Server %s at http %s", version.Version(), listenAddress)
 		if err = httpS.Serve(webDavListener); err != nil {
 			glog.Fatalf("WebDav Server Fail to serve: %v", err)
 		}

weed/filer/filechunk_group.go

@@ -1,6 +1,7 @@
 package filer
 
 import (
+	"context"
 	"io"
 	"sync"
 
@@ -89,7 +90,7 @@ func (group *ChunkGroup) SetChunks(chunks []*filer_pb.FileChunk) error {
 			continue
 		}
 
-		resolvedChunks, err := ResolveOneChunkManifest(group.lookupFn, chunk)
+		resolvedChunks, err := ResolveOneChunkManifest(context.Background(), group.lookupFn, chunk)
 		if err != nil {
 			return err
 		}

weed/filer/filechunk_manifest.go

@@ -2,6 +2,7 @@ package filer
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"math"
@@ -48,7 +49,7 @@ func SeparateManifestChunks(chunks []*filer_pb.FileChunk) (manifestChunks, nonMa
 	return
 }
 
-func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset, stopOffset int64) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {
+func ResolveChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset, stopOffset int64) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {
 	// TODO maybe parallel this
 	for _, chunk := range chunks {
 
@@ -61,14 +62,14 @@ func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chun
 			continue
 		}
 
-		resolvedChunks, err := ResolveOneChunkManifest(lookupFileIdFn, chunk)
+		resolvedChunks, err := ResolveOneChunkManifest(ctx, lookupFileIdFn, chunk)
 		if err != nil {
 			return dataChunks, nil, err
 		}
 
 		manifestChunks = append(manifestChunks, chunk)
 		// recursive
-		subDataChunks, subManifestChunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset)
+		subDataChunks, subManifestChunks, subErr := ResolveChunkManifest(ctx, lookupFileIdFn, resolvedChunks, startOffset, stopOffset)
 		if subErr != nil {
 			return dataChunks, nil, subErr
 		}
@@ -78,7 +79,7 @@ func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chun
 	return
 }
 
-func ResolveOneChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunk *filer_pb.FileChunk) (dataChunks []*filer_pb.FileChunk, manifestResolveErr error) {
+func ResolveOneChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunk *filer_pb.FileChunk) (dataChunks []*filer_pb.FileChunk, manifestResolveErr error) {
 	if !chunk.IsChunkManifest {
 		return
 	}
@@ -87,7 +88,7 @@ func ResolveOneChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, c
 	bytesBuffer := bytesBufferPool.Get().(*bytes.Buffer)
 	bytesBuffer.Reset()
 	defer bytesBufferPool.Put(bytesBuffer)
-	err := fetchWholeChunk(bytesBuffer, lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed)
+	err := fetchWholeChunk(ctx, bytesBuffer, lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed)
 	if err != nil {
 		return nil, fmt.Errorf("fail to read manifest %s: %v", chunk.GetFileIdString(), err)
 	}
@@ -102,13 +103,13 @@ func ResolveOneChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, c
 }
 
 // TODO fetch from cache for weed mount?
-func fetchWholeChunk(bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool) error {
-	urlStrings, err := lookupFileIdFn(fileId)
+func fetchWholeChunk(ctx context.Context, bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool) error {
+	urlStrings, err := lookupFileIdFn(ctx, fileId)
 	if err != nil {
 		glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err)
 		return err
 	}
-	err = retriedStreamFetchChunkData(bytesBuffer, urlStrings, "", cipherKey, isGzipped, true, 0, 0)
+	err = retriedStreamFetchChunkData(ctx, bytesBuffer, urlStrings, "", cipherKey, isGzipped, true, 0, 0)
 	if err != nil {
 		return err
 	}
@@ -116,15 +117,15 @@ func fetchWholeChunk(bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFi
 }
 
 func fetchChunkRange(buffer []byte, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool, offset int64) (int, error) {
-	urlStrings, err := lookupFileIdFn(fileId)
+	urlStrings, err := lookupFileIdFn(context.Background(), fileId)
 	if err != nil {
 		glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err)
 		return 0, err
 	}
-	return util_http.RetriedFetchChunkData(buffer, urlStrings, cipherKey, isGzipped, false, offset)
+	return util_http.RetriedFetchChunkData(context.Background(), buffer, urlStrings, cipherKey, isGzipped, false, offset)
 }
 
-func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, jwt string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) {
+func retriedStreamFetchChunkData(ctx context.Context, writer io.Writer, urlStrings []string, jwt string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) {
 
 	var shouldRetry bool
 	var totalWritten int
@@ -135,7 +136,7 @@ func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, jwt stri
 			retriedCnt++
 			var localProcessed int
 			var writeErr error
-			shouldRetry, err = util_http.ReadUrlAsStreamAuthenticated(urlString+"?readDeleted=true", jwt, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
+			shouldRetry, err = util_http.ReadUrlAsStreamAuthenticated(ctx, urlString+"?readDeleted=true", jwt, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
 				if totalWritten > localProcessed {
 					toBeSkipped := totalWritten - localProcessed
 					if len(data) <= toBeSkipped {