3.87

related to https://github.com/seaweedfs/seaweedfs/pull/6738

.github/workflows/binaries_release4.yml

@@ -36,7 +36,7 @@ jobs:
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}
           overwrite: true
-          build_flags: -tags elastic,gocdk,rclone,sqlite,tikv,ydb
+          build_flags: -tags elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
           ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
@@ -52,7 +52,7 @@ jobs:
           goarch: ${{ matrix.goarch }}
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
-          build_flags: -tags 5BytesOffset,elastic,gocdk,rclone,sqlite,tikv,ydb
+          build_flags: -tags 5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
           ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed

.github/workflows/container_dev.yml

@@ -20,7 +20,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -33,30 +33,30 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -21,7 +21,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -34,30 +34,30 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -20,7 +20,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -34,20 +34,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -21,7 +21,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -35,20 +35,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -21,7 +21,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -35,20 +35,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release4.yml

@@ -20,7 +20,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -34,25 +34,25 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build
-          build-args: TAGS=elastic,gocdk,rclone,sqlite,tikv,ydb
+          build-args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
           platforms: linux/amd64
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release5.yml

@@ -20,7 +20,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -34,25 +34,25 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build
-          build-args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tikv,ydb
+          build-args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
           platforms: linux/amd64
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/depsreview.yml

@@ -11,4 +11,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@0659a74c94536054bfa5aeb92241f70d680cc78e
+        uses: actions/dependency-review-action@0c155c5e8556a497adf53f2c18edabf945ed8e70

.github/workflows/go.yml

@@ -34,7 +34,7 @@ jobs:
         cd weed; go get -v -t -d ./...
 
     - name: Build
-      run: cd weed; go build -tags "elastic gocdk sqlite ydb tikv rclone" -v .
+      run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
 
     - name: Test
-      run: cd weed; go test -tags "elastic gocdk sqlite ydb tikv rclone" -v ./...
+      run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...

.github/workflows/test-s3-over-https-using-awscli.yml

@@ -0,0 +1,79 @@
+name: "test s3 over https using aws-cli"
+
+on:
+  push:
+    branches: [master, test-https-s3-awscli]
+  pull_request:
+    branches: [master, test-https-s3-awscli]
+
+env:
+  AWS_ACCESS_KEY_ID: some_access_key1
+  AWS_SECRET_ACCESS_KEY: some_secret_key1
+  AWS_ENDPOINT_URL: https://localhost:8443
+
+defaults:
+  run:
+    working-directory: weed
+
+jobs:
+  awscli-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ^1.24
+
+      - name: Build SeaweedFS
+        run: |
+          go build
+
+      - name: Start SeaweedFS
+        run: |
+          set -e
+          mkdir -p /tmp/data
+          ./weed server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json &
+          until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
+
+      - name: Setup Caddy
+        run: |
+          curl -fsSL "https://caddyserver.com/api/download?os=linux&arch=amd64" -o caddy
+          chmod +x caddy
+          ./caddy version
+          echo "{
+            auto_https disable_redirects
+            local_certs
+          }
+          localhost:8443 {
+            tls internal
+            reverse_proxy localhost:8333
+          }" > Caddyfile
+
+      - name: Start Caddy
+        run: |
+          ./caddy start
+          until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
+
+      - name: Create Bucket
+        run: |
+          aws --no-verify-ssl s3api create-bucket --bucket bucket
+
+      - name: Test PutObject
+        run: |
+          set -e
+          dd if=/dev/urandom of=generated bs=1M count=2
+          aws --no-verify-ssl s3api put-object --bucket bucket --key test-putobject --body generated
+          aws --no-verify-ssl s3api get-object --bucket bucket --key test-putobject downloaded
+          diff -q generated downloaded
+          rm -f generated downloaded
+
+      - name: Test Multi-part Upload
+        run: |
+          set -e
+          dd if=/dev/urandom of=generated bs=1M count=32
+          aws --no-verify-ssl s3 cp --no-progress generated s3://bucket/test-multipart
+          aws --no-verify-ssl s3 cp --no-progress s3://bucket/test-multipart downloaded
+          diff -q generated downloaded
+          rm -f generated downloaded

.gitignore

@@ -87,7 +87,6 @@ other/java/hdfs/dependency-reduced-pom.xml
 
 # binary file
 weed/weed
-weed/mq/client/cmd/weed_pub_kv/weed_pub
 docker/weed
 
 # test generated files
@@ -95,7 +94,4 @@ weed/*/*.jpg
 docker/weed_sub
 docker/weed_pub
 weed/mq/schema/example.parquet
-docker/weed_pub_kv
-docker/weed_pub_record
-docker/weed_sub_kv
-docker/weed_sub_record
+docker/agent_sub_record

Makefile

@@ -14,7 +14,7 @@ warp_install:
 	go install github.com/minio/warp@v0.7.6
 
 full_install:
-	cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone"
+	cd weed; go install -tags "elastic gocdk sqlite ydb tarantool tikv rclone"
 
 server: install
 	weed -v 0 server -s3 -filer -filer.maxMB=64 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324
@@ -34,4 +34,4 @@ benchmark_with_pprof: debug = 1
 benchmark_with_pprof: benchmark
 
 test:
-	cd weed; go test -tags "elastic gocdk sqlite ydb tikv rclone" -v ./...
+	cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...

docker/Dockerfile.go_build

@@ -1,4 +1,4 @@
-FROM golang:1.23-alpine as builder
+FROM golang:1.24-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/seaweedfs/
 RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs

docker/Dockerfile.rocksdb_dev_env

@@ -1,9 +1,9 @@
-FROM golang:1.23 as builder
+FROM golang:1.24 as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v8.10.0
+ENV ROCKSDB_VERSION v9.10.0
 
 # build RocksDB
 RUN cd /tmp && \

docker/Dockerfile.rocksdb_large

@@ -1,9 +1,9 @@
-FROM golang:1.23 as builder
+FROM golang:1.24 as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v8.10.0
+ENV ROCKSDB_VERSION v9.10.0
 
 # build RocksDB
 RUN cd /tmp && \

docker/Dockerfile.tarantool.dev_env

@@ -0,0 +1,17 @@
+FROM tarantool/tarantool:3.3.1 AS builder
+
+# install dependencies
+RUN apt update && \
+  apt install -y git unzip cmake tt=2.7.0
+
+# init tt dir structure, create dir for app, create symlink
+RUN tt init && \
+  mkdir app && \
+  ln -sfn ${PWD}/app/ ${PWD}/instances.enabled/app
+
+# copy cluster configs
+COPY tarantool /opt/tarantool/app
+
+# build app
+RUN tt build app
+

docker/Makefile

@@ -9,10 +9,8 @@ binary:
 	export SWCOMMIT=$(shell git rev-parse --short HEAD)
 	export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
 	cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w -extldflags -static $(SWLDFLAGS)" && mv weed ../docker/
-	cd ../weed/mq/client/cmd/weed_pub_kv && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_pub_kv ../../../../../docker/
-	cd ../weed/mq/client/cmd/weed_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_pub_record ../../../../../docker/
-	cd ../weed/mq/client/cmd/weed_sub_kv && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_sub_kv ../../../../../docker/
-	cd ../weed/mq/client/cmd/weed_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_sub_record ../../../../../docker/
+	cd ../other/mq_client_example/agent_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_pub_record ../../../docker/
+	cd ../other/mq_client_example/agent_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_sub_record ../../../docker/
 
 binary_race: options = -race
 binary_race: cgo = 1
@@ -24,7 +22,7 @@ build: binary
 build_e2e: binary_race
 	docker build --no-cache -t chrislusf/seaweedfs:e2e -f Dockerfile.e2e .
 
-go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset
+go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset,tarantool
 	docker build --build-arg TAGS=$(tags) --no-cache -t chrislusf/seaweedfs:go_build -f Dockerfile.go_build .
 
 go_build_large_disk:
@@ -39,6 +37,9 @@ build_rocksdb_local: build_rocksdb_dev_env
 build_rocksdb:
 	docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large .
 
+build_tarantool_dev_env:
+	docker build --no-cache -t chrislusf/tarantool_dev_env -f Dockerfile.tarantool.dev_env .
+
 s3tests_build:
 	docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
 
@@ -97,6 +98,9 @@ s3tests: build s3tests_build
 brokers: build
 	docker compose -f compose/local-brokers-compose.yml -p seaweedfs up
 
+agent: build
+	docker compose -f compose/local-mq-test.yml -p seaweedfs up
+
 filer_etcd: build
 	docker stack deploy -c compose/swarm-etcd.yml fs
 
@@ -105,9 +109,12 @@ test_etcd: build
 
 test_ydb: tags = ydb
 test_ydb: build
-	export
 	docker compose -f compose/test-ydb-filer.yml -p seaweedfs up
 
+test_tarantool: tags = tarantool
+test_tarantool: build_tarantool_dev_env build
+	docker compose -f compose/test-tarantool-filer.yml -p seaweedfs up
+
 clean:
 	rm ./weed
 

docker/compose/fluent.conf

@@ -0,0 +1,8 @@
+<source>
+  @type forward
+  port 24224
+</source>
+
+<match **>
+  @type stdout  # Output logs to container's stdout (visible via `docker logs`)
+</match>

docker/compose/local-auditlog-compose.yml

@@ -19,7 +19,9 @@ services:
     depends_on:
       - fluent
   fluent:
-    image: fluent/fluentd:v1.14
+    image: fluent/fluentd:v1.17
+    volumes:
+      - ./fluent.conf:/fluentd/etc/fluent.conf
     ports:
       - 24224:24224
   #s3tests:

docker/compose/local-mq-test.yml

@@ -1,5 +1,3 @@
-version: '3.9'
-
 services:
   server:
     image: chrislusf/seaweedfs:local
@@ -19,9 +17,16 @@ services:
     depends_on:
       server:
         condition: service_healthy
+  mq_agent:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 16777:16777
+    command: "mq.agent -broker=mq_broker:17777 -port=16777"
+    depends_on:
+      - mq_broker
   mq_client:
     image: chrislusf/seaweedfs:local
     # run a custom command instead of entrypoint
     command: "ls -al"
     depends_on:
-      - mq_broker
+      - mq_agent

docker/compose/test-tarantool-filer.yml

@@ -0,0 +1,30 @@
+version: '3.9'
+
+services:
+  tarantool:
+    image: chrislusf/tarantool_dev_env
+    entrypoint: "tt start app -i"
+    environment:
+      APP_USER_PASSWORD: "app"
+      CLIENT_USER_PASSWORD: "client"
+      REPLICATOR_USER_PASSWORD: "replicator"
+      STORAGE_USER_PASSWORD: "storage"
+    network_mode: "host"
+    ports:
+      - "3303:3303"
+
+  s3:
+    image: chrislusf/seaweedfs:local
+    command: "server -ip=127.0.0.1 -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false"
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
+    environment:
+      WEED_LEVELDB2_ENABLED: "false"
+      WEED_TARANTOOL_ENABLED: "true"
+      WEED_TARANTOOL_ADDRESS: "127.0.0.1:3303"
+      WEED_TARANTOOL_USER: "client"
+      WEED_TARANTOOL_PASSWORD: "client"
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+    network_mode: "host"
+    depends_on:
+      - tarantool

docker/tarantool/app-scm-1.rockspec

@@ -0,0 +1,14 @@
+package = 'app'
+version = 'scm-1'
+source  = {
+    url = '/dev/null',
+}
+dependencies = {
+    'crud == 1.5.2-1',
+    'expirationd == 1.6.0-1',
+    'metrics-export-role == 0.3.0-1',
+    'vshard == 0.1.32-1'
+}
+build = {
+    type = 'none';
+}

docker/tarantool/config.yaml

@@ -0,0 +1,145 @@
+config:
+  context:
+    app_user_password:
+      from: env
+      env: APP_USER_PASSWORD
+    client_user_password:
+      from: env
+      env: CLIENT_USER_PASSWORD
+    replicator_user_password:
+      from: env
+      env: REPLICATOR_USER_PASSWORD
+    storage_user_password:
+      from: env
+      env: STORAGE_USER_PASSWORD
+
+credentials:
+  roles:
+    crud-role:
+      privileges:
+        - permissions: [ "execute" ]
+          lua_call: [ "crud.delete", "crud.get", "crud.upsert" ]
+  users:
+    app:
+      password: '{{ context.app_user_password }}'
+      roles: [ public, crud-role ]
+    client:
+      password: '{{ context.client_user_password }}'
+      roles: [ super ]
+    replicator:
+      password: '{{ context.replicator_user_password }}'
+      roles: [ replication ]
+    storage:
+      password: '{{ context.storage_user_password }}'
+      roles: [ sharding ]
+
+iproto:
+  advertise:
+    peer:
+      login: replicator
+    sharding:
+      login: storage
+
+sharding:
+  bucket_count: 10000
+
+metrics:
+  include: [ all ]
+  exclude: [ vinyl ]
+  labels:
+    alias: '{{ instance_name }}'
+
+
+groups:
+  storages:
+    roles:
+      - roles.crud-storage
+      - roles.expirationd
+      - roles.metrics-export
+    roles_cfg:
+      roles.expirationd:
+        cfg:
+          metrics: true
+        filer_metadata_task:
+          space: filer_metadata
+          is_expired: filer_metadata.is_expired
+          options:
+            atomic_iteration: true
+            force: true
+            index: 'expire_at_idx'
+            iterator_type: GT
+            start_key:
+              - 0
+            tuples_per_iteration: 10000
+    app:
+      module: storage
+    sharding:
+      roles: [ storage ]
+    replication:
+      failover: election
+    database:
+      use_mvcc_engine: true
+    replicasets:
+      storage-001:
+        instances:
+          storage-001-a:
+            roles_cfg:
+              roles.metrics-export:
+                http:
+                  - listen: '0.0.0.0:8081'
+                    endpoints:
+                      - path: /metrics/prometheus/
+                        format: prometheus
+                      - path: /metrics/json
+                        format: json
+            iproto:
+              listen:
+                - uri: 127.0.0.1:3301
+              advertise:
+                client: 127.0.0.1:3301
+          storage-001-b:
+            roles_cfg:
+              roles.metrics-export:
+                http:
+                  - listen: '0.0.0.0:8082'
+                    endpoints:
+                      - path: /metrics/prometheus/
+                        format: prometheus
+                      - path: /metrics/json
+                        format: json
+            iproto:
+              listen:
+                - uri: 127.0.0.1:3302
+              advertise:
+                client: 127.0.0.1:3302
+  routers:
+    roles:
+      - roles.crud-router
+      - roles.metrics-export
+    roles_cfg:
+      roles.crud-router:
+        stats: true
+        stats_driver: metrics
+        stats_quantiles: true
+    app:
+      module: router
+    sharding:
+      roles: [ router ]
+    replicasets:
+      router-001:
+        instances:
+          router-001-a:
+            roles_cfg:
+              roles.metrics-export:
+                http:
+                  - listen: '0.0.0.0:8083'
+                    endpoints:
+                      - path: /metrics/prometheus/
+                        format: prometheus
+                      - path: /metrics/json
+                        format: json
+            iproto:
+              listen:
+                - uri: 127.0.0.1:3303
+              advertise:
+                client: 127.0.0.1:3303

docker/tarantool/instances.yaml

@@ -0,0 +1,7 @@
+---
+storage-001-a:
+
+storage-001-b:
+
+router-001-a:
+

docker/tarantool/router.lua

@@ -0,0 +1,77 @@
+local vshard = require('vshard')
+local log = require('log')
+
+-- Bootstrap the vshard router.
+while true do
+    local ok, err = vshard.router.bootstrap({
+        if_not_bootstrapped = true,
+    })
+    if ok then
+        break
+    end
+    log.info(('Router bootstrap error: %s'):format(err))
+end
+
+-- functions for filer_metadata space
+local filer_metadata = {
+    delete_by_directory_idx = function(directory)
+        -- find all storages
+        local storages = require('vshard').router.routeall()
+        -- on each storage
+        for _, storage in pairs(storages) do
+            -- call local function
+            local result, err = storage:callrw('filer_metadata.delete_by_directory_idx', { directory })
+            -- check for error
+            if err then
+                error("Failed to call function on storage: " .. tostring(err))
+            end
+        end
+        -- return
+        return true
+    end,
+    find_by_directory_idx_and_name = function(dirPath, startFileName, includeStartFile, limit)
+        -- init results
+        local results = {}
+        -- find all storages
+        local storages = require('vshard').router.routeall()
+        -- on each storage
+        for _, storage in pairs(storages) do
+            -- call local function
+            local result, err = storage:callro('filer_metadata.find_by_directory_idx_and_name', {
+                dirPath,
+                startFileName,
+                includeStartFile,
+                limit
+            })
+            -- check for error
+            if err then
+                error("Failed to call function on storage: " .. tostring(err))
+            end
+            -- add to results
+            for _, tuple in ipairs(result) do
+                table.insert(results, tuple)
+            end
+        end
+        -- sort
+        table.sort(results, function(a, b) return a[3] < b[3] end)
+        -- apply limit
+        if #results > limit then
+            local limitedResults = {}
+            for i = 1, limit do
+                table.insert(limitedResults, results[i])
+            end
+            results = limitedResults
+        end
+        -- return
+        return results
+    end,
+}
+
+rawset(_G, 'filer_metadata', filer_metadata)
+
+-- register functions for filer_metadata space, set grants
+for name, _ in pairs(filer_metadata) do
+    box.schema.func.create('filer_metadata.' .. name, { if_not_exists = true })
+    box.schema.user.grant('app', 'execute', 'function', 'filer_metadata.' .. name, { if_not_exists = true })
+    box.schema.user.grant('client', 'execute', 'function', 'filer_metadata.' .. name, { if_not_exists = true })
+end

docker/tarantool/storage.lua

@@ -0,0 +1,97 @@
+box.watch('box.status', function()
+    if box.info.ro then
+        return
+    end
+
+    -- ====================================
+    -- key_value space
+    -- ====================================
+    box.schema.create_space('key_value', {
+        format = {
+            { name = 'key', type = 'string' },
+            { name = 'bucket_id', type = 'unsigned' },
+            { name = 'value', type = 'string' }
+        },
+        if_not_exists = true
+    })
+
+    -- create key_value space indexes
+    box.space.key_value:create_index('id', {type = 'tree', parts = { 'key' }, unique = true, if_not_exists = true})
+    box.space.key_value:create_index('bucket_id', { type = 'tree', parts = { 'bucket_id' }, unique = false, if_not_exists = true })
+
+    -- ====================================
+    -- filer_metadata space
+    -- ====================================
+    box.schema.create_space('filer_metadata', {
+        format = {
+            { name = 'directory', type = 'string' },
+            { name = 'bucket_id', type = 'unsigned' },
+            { name = 'name', type = 'string' },
+            { name = 'expire_at', type = 'unsigned' },
+            { name = 'data', type = 'string' }
+        },
+        if_not_exists = true
+    })
+
+    -- create filer_metadata space indexes
+    box.space.filer_metadata:create_index('id', {type = 'tree', parts = { 'directory', 'name' }, unique = true, if_not_exists = true})
+    box.space.filer_metadata:create_index('bucket_id', { type = 'tree', parts = { 'bucket_id' }, unique = false, if_not_exists = true })
+    box.space.filer_metadata:create_index('directory_idx', { type = 'tree', parts = { 'directory' }, unique = false, if_not_exists = true })
+    box.space.filer_metadata:create_index('name_idx', { type = 'tree', parts = { 'name' }, unique = false, if_not_exists = true })
+    box.space.filer_metadata:create_index('expire_at_idx', { type = 'tree', parts = { 'expire_at' }, unique = false, if_not_exists = true})
+end)
+
+-- functions for filer_metadata space
+local filer_metadata = {
+    delete_by_directory_idx = function(directory)
+        local space = box.space.filer_metadata
+        local index = space.index.directory_idx
+        -- for each finded directories
+        for _, tuple in index:pairs({ directory }, { iterator = 'EQ' }) do
+            space:delete({ tuple[1], tuple[3] })
+        end
+        return true
+    end,
+    find_by_directory_idx_and_name = function(dirPath, startFileName, includeStartFile, limit)
+        local space = box.space.filer_metadata
+        local directory_idx = space.index.directory_idx
+        -- choose filter name function
+        local filter_filename_func
+        if includeStartFile then
+            filter_filename_func = function(value) return value >= startFileName end
+        else
+            filter_filename_func = function(value) return value > startFileName end
+        end
+        -- init results
+        local results = {}
+        -- for each finded directories
+        for _, tuple in directory_idx:pairs({ dirPath }, { iterator = 'EQ' }) do
+            -- filter by name
+            if filter_filename_func(tuple[3]) then
+                table.insert(results, tuple)
+            end
+        end
+        -- sort
+        table.sort(results, function(a, b) return a[3] < b[3] end)
+        -- apply limit
+        if #results > limit then
+            local limitedResults = {}
+            for i = 1, limit do
+                table.insert(limitedResults, results[i])
+            end
+            results = limitedResults
+        end
+        -- return
+        return results
+    end,
+    is_expired = function(args, tuple)
+        return (tuple[4] > 0) and (require('fiber').time() > tuple[4])
+    end
+}
+
+-- register functions for filer_metadata space, set grants
+rawset(_G, 'filer_metadata', filer_metadata)
+for name, _ in pairs(filer_metadata) do
+    box.schema.func.create('filer_metadata.' .. name, { setuid = true, if_not_exists = true })
+    box.schema.user.grant('storage', 'execute', 'function', 'filer_metadata.' .. name, { if_not_exists = true })
+end

go.mod

@@ -1,13 +1,13 @@
 module github.com/seaweedfs/seaweedfs
 
-go 1.23
+go 1.24
 
-toolchain go1.23.3
+toolchain go1.24.1
 
 require (
-	cloud.google.com/go v0.118.1 // indirect
-	cloud.google.com/go/pubsub v1.47.0
-	cloud.google.com/go/storage v1.50.0
+	cloud.google.com/go v0.120.0 // indirect
+	cloud.google.com/go/pubsub v1.49.0
+	cloud.google.com/go/storage v1.51.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Shopify/sarama v1.38.1
@@ -30,12 +30,12 @@ require (
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fclairamb/ftpserverlib v0.25.0
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/go-redsync/redsync/v4 v4.13.0
-	github.com/go-sql-driver/mysql v1.8.1
+	github.com/go-sql-driver/mysql v1.9.1
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.7.0
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/golang/protobuf v1.5.4
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.3
@@ -47,23 +47,20 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jcmturner/gofork v1.7.6 // indirect
 	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
 	github.com/jinzhu/copier v0.4.0
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
-	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/reedsolomon v1.12.4
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.9
-	github.com/linxGnu/grocksdb v1.9.8
-	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/linxGnu/grocksdb v1.9.9
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.11 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/olivere/elastic/v7 v7.0.32
@@ -72,18 +69,18 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/posener/complete v1.2.3
 	github.com/pquerna/cachecontrol v0.2.0
-	github.com/prometheus/client_golang v1.20.5
+	github.com/prometheus/client_golang v1.21.1
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
-	github.com/prometheus/procfs v0.15.1
+	github.com/prometheus/common v0.62.0 // indirect
+	github.com/prometheus/procfs v0.16.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
 	github.com/seaweedfs/raft v1.1.3
 	github.com/sirupsen/logrus v1.9.3 // indirect
-	github.com/spf13/afero v1.11.0 // indirect
-	github.com/spf13/cast v1.6.0 // indirect
-	github.com/spf13/viper v1.19.0
+	github.com/spf13/afero v1.12.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/viper v1.20.1
 	github.com/stretchr/testify v1.10.0
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
@@ -99,89 +96,91 @@ require (
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
 	go.etcd.io/etcd/client/v3 v3.5.18
-	go.mongodb.org/mongo-driver v1.17.2
+	go.mongodb.org/mongo-driver v1.17.3
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.40.0
-	gocloud.dev/pubsub/natspubsub v0.40.0
-	gocloud.dev/pubsub/rabbitpubsub v0.40.0
-	golang.org/x/crypto v0.33.0 // indirect
-	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
-	golang.org/x/image v0.23.0
-	golang.org/x/net v0.35.0
-	golang.org/x/oauth2 v0.26.0 // indirect
-	golang.org/x/sys v0.30.0
-	golang.org/x/text v0.22.0 // indirect
-	golang.org/x/tools v0.30.0
-	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
-	google.golang.org/api v0.221.0
-	google.golang.org/genproto v0.0.0-20250122153221-138b5a5a4fd4 // indirect
-	google.golang.org/grpc v1.70.0
-	google.golang.org/protobuf v1.36.5
+	gocloud.dev v0.41.0
+	gocloud.dev/pubsub/natspubsub v0.41.0
+	gocloud.dev/pubsub/rabbitpubsub v0.41.0
+	golang.org/x/crypto v0.37.0 // indirect
+	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394
+	golang.org/x/image v0.24.0
+	golang.org/x/net v0.39.0
+	golang.org/x/oauth2 v0.29.0 // indirect
+	golang.org/x/sys v0.32.0
+	golang.org/x/text v0.24.0 // indirect
+	golang.org/x/tools v0.31.0
+	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
+	google.golang.org/api v0.230.0
+	google.golang.org/genproto v0.0.0-20250324211829-b45e905df463 // indirect
+	google.golang.org/grpc v1.72.0
+	google.golang.org/protobuf v1.36.6
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
 	modernc.org/mathutil v1.7.1
-	modernc.org/memory v1.8.0 // indirect
-	modernc.org/sqlite v1.34.5
+	modernc.org/memory v1.9.1 // indirect
+	modernc.org/sqlite v1.37.0
 	modernc.org/strutil v1.2.1
 )
 
 require (
 	github.com/Jille/raft-grpc-transport v1.6.1
-	github.com/arangodb/go-driver v1.6.5
+	github.com/arangodb/go-driver v1.6.6
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.36.1
-	github.com/aws/aws-sdk-go-v2/config v1.29.1
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.59
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0
+	github.com/aws/aws-sdk-go-v2 v1.36.3
+	github.com/aws/aws-sdk-go-v2/config v1.29.13
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.66
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2
 	github.com/cognusion/imaging v1.0.1
 	github.com/fluent/fluent-logger-golang v1.9.0
 	github.com/getsentry/sentry-go v0.31.1
-	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
 	github.com/hanwen/go-fuse/v2 v2.7.2
-	github.com/hashicorp/raft v1.7.2
+	github.com/hashicorp/raft v1.7.3
 	github.com/hashicorp/raft-boltdb/v2 v2.3.1
+	github.com/minio/crc64nvme v1.0.1
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/parquet-go/parquet-go v0.24.0
 	github.com/rabbitmq/amqp091-go v1.10.0
-	github.com/rclone/rclone v1.68.2
+	github.com/rclone/rclone v1.69.1
 	github.com/rdleal/intervalst v1.4.1
-	github.com/redis/go-redis/v9 v9.7.0
+	github.com/redis/go-redis/v9 v9.7.3
 	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/shirou/gopsutil/v3 v3.24.5
+	github.com/tarantool/go-tarantool/v2 v2.3.0
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.99.10
-	go.etcd.io/etcd/client/pkg/v3 v3.5.18
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.106.1
+	go.etcd.io/etcd/client/pkg/v3 v3.5.21
 	go.uber.org/atomic v1.11.0
-	golang.org/x/sync v0.11.0
+	golang.org/x/sync v0.13.0
 	google.golang.org/grpc/security/advancedtls v1.0.0
 )
 
 require (
-	cel.dev/expr v0.19.0 // indirect
-	cloud.google.com/go/auth v0.14.1 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect
+	cel.dev/expr v0.22.1 // indirect
+	cloud.google.com/go/auth v0.16.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.6.0 // indirect
-	cloud.google.com/go/iam v1.3.1 // indirect
-	cloud.google.com/go/monitoring v1.23.0 // indirect
+	cloud.google.com/go/iam v1.4.2 // indirect
+	cloud.google.com/go/monitoring v1.24.1 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.2.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.4.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
-	github.com/Files-com/files-sdk-go/v3 v3.2.34 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
+	github.com/Files-com/files-sdk-go/v3 v3.2.97 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
-	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
 	github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
-	github.com/ProtonMail/go-crypto v1.0.0 // indirect
+	github.com/ProtonMail/go-crypto v1.1.3 // indirect
 	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
 	github.com/ProtonMail/go-srp v0.0.7 // indirect
 	github.com/ProtonMail/gopenpgp/v2 v2.7.4 // indirect
@@ -191,62 +190,67 @@ require (
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.31.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect
-	github.com/aws/smithy-go v1.22.2 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.69 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.34.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 // indirect
+	github.com/aws/smithy-go v1.22.3 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.2 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
 	github.com/buengese/sgzip v0.1.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
-	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
-	github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891 // indirect
-	github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
+	github.com/cloudinary/cloudinary-go/v2 v2.9.0 // indirect
+	github.com/cloudsoda/go-smb2 v0.0.0-20241223203758-52b943b88fd6 // indirect
+	github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect
 	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
+	github.com/creasty/defaults v1.7.0 // indirect
 	github.com/cronokirby/saferith v0.33.0 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
+	github.com/ebitengine/purego v0.8.1 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/emersion/go-message v0.18.0 // indirect
 	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
 	github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
-	github.com/envoyproxy/go-control-plane v0.13.1 // indirect
-	github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
+	github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/fclairamb/go-log v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/flynn/noise v1.0.1 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.4 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.7 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
 	github.com/go-chi/chi/v5 v5.1.0 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-resty/resty/v2 v2.11.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
+	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
@@ -261,12 +265,12 @@ require (
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
-	github.com/jlaffaye/ftp v0.2.0 // indirect
+	github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 // indirect
 	github.com/jonboulle/clockwork v0.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
@@ -279,18 +283,20 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/nats-io/nats.go v1.37.0 // indirect
-	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/nats-io/nats.go v1.40.1 // indirect
+	github.com/nats-io/nkeys v0.4.10 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/ncw/swift/v2 v2.0.3 // indirect
 	github.com/nxadm/tail v1.4.11 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/onsi/ginkgo/v2 v2.19.0 // indirect
+	github.com/onsi/gomega v1.34.1 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.69.2 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.80.0 // indirect
 	github.com/panjf2000/ants/v2 v2.9.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
@@ -299,8 +305,8 @@ require (
 	github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/pkg/sftp v1.13.6 // indirect
-	github.com/pkg/xattr v0.4.9 // indirect
+	github.com/pkg/sftp v1.13.7 // indirect
+	github.com/pkg/xattr v0.4.10 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
@@ -308,18 +314,20 @@ require (
 	github.com/rfjakob/eme v1.1.2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
-	github.com/sagikazarmark/locafero v0.4.0 // indirect
-	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sagikazarmark/locafero v0.7.0 // indirect
 	github.com/samber/lo v1.47.0 // indirect
+	github.com/shirou/gopsutil/v4 v4.24.12 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
 	github.com/sony/gobreaker v0.5.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 // indirect
+	github.com/t3rm1n4l/go-mega v0.0.0-20241213150454-ec0027fb0002 // indirect
+	github.com/tarantool/go-iproto v1.1.0 // indirect
 	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
 	github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
 	github.com/tinylib/msgp v1.1.8 // indirect
@@ -327,6 +335,8 @@ require (
 	github.com/tklauser/numcpus v0.7.0 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
 	github.com/unknwon/goconfig v1.0.0 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
 	github.com/ydb-platform/ydb-go-genproto v0.0.0-20241112172322-ea1f63298f77 // indirect
@@ -335,31 +345,29 @@ require (
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
-	github.com/zeebo/errs v1.3.0 // indirect
+	github.com/zeebo/errs v1.4.0 // indirect
 	go.etcd.io/bbolt v1.3.10 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.18 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/contrib/detectors/gcp v1.33.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
-	go.opentelemetry.io/otel v1.34.0 // indirect
-	go.opentelemetry.io/otel/metric v1.34.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.34.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect
-	go.opentelemetry.io/otel/trace v1.34.0 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
+	go.opentelemetry.io/otel v1.35.0 // indirect
+	go.opentelemetry.io/otel/metric v1.35.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.35.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
+	go.opentelemetry.io/otel/trace v1.35.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/mod v0.23.0 // indirect
-	golang.org/x/term v0.29.0 // indirect
-	golang.org/x/time v0.10.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250124145028-65684f501c47 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
+	golang.org/x/term v0.31.0 // indirect
+	golang.org/x/time v0.11.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250414145226-207652e42e2e // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	modernc.org/libc v1.55.3 // indirect
+	modernc.org/libc v1.62.1 // indirect
 	moul.io/http2curl/v2 v2.3.0 // indirect
 	storj.io/common v0.0.0-20240812101423-26b53789c348 // indirect
 	storj.io/drpc v0.0.35-0.20240709171858-0075ac871661 // indirect
@@ -370,8 +378,3 @@ require (
 )
 
 // replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
-
-replace (
-	github.com/boltdb/bolt => go.etcd.io/bbolt v1.3.10
-	go.etcd.io/bbolt => github.com/etcd-io/bbolt v1.3.10
-)

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.85"
+appVersion: "3.87"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.385
+version: 4.0.387

k8s/charts/seaweedfs/templates/cosi-deployment.yaml

@@ -140,6 +140,10 @@ spec:
               mountPath: /usr/local/share/ca-certificates/client/
             {{- end }}
             {{ tpl .Values.cosi.extraVolumeMounts . | nindent 12 | trim }}
+          {{- with .Values.cosi.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
         - name: seaweedfs-cosi-sidecar
           image: "{{ .Values.cosi.sidecar.image }}"
           imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
@@ -153,7 +157,7 @@ spec:
           volumeMounts:
             - mountPath: /var/lib/cosi
               name: socket
-          {{- with .Values.cosi.resources }}
+          {{- with .Values.cosi.sidecar.resources }}
           resources:
             {{- toYaml . | nindent 12 }}
           {{- end }}
@@ -173,7 +177,7 @@ spec:
             {{- if .Values.cosi.existingConfigSecret }}
             secretName: {{ .Values.cosi.existingConfigSecret }}
             {{- else }}
-            secretName: seaweedfs-client-cert
+            secretName: seaweedfs-s3-secret
             {{- end }}
         {{- end }}
         {{- if .Values.global.enableSecurity }}

k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@@ -173,7 +173,7 @@ spec:
               {{- end }}
               -dirListLimit={{ .Values.filer.dirListLimit }} \
               {{- if .Values.global.enableReplication }}
-              -defaultReplicaPlacement={{ .Values.global.replicationPlacment }} \
+              -defaultReplicaPlacement={{ .Values.global.replicationPlacement }} \
               {{- else }}
               -defaultReplicaPlacement={{ .Values.filer.defaultReplicaPlacement }} \
               {{- end }}

k8s/charts/seaweedfs/templates/master-statefulset.yaml

@@ -139,7 +139,7 @@ spec:
               -mdir=/data \
               -ip.bind={{ .Values.master.ipBind }} \
               {{- if .Values.global.enableReplication }}
-              -defaultReplication={{ .Values.global.replicationPlacment }} \
+              -defaultReplication={{ .Values.global.replicationPlacement }} \
               {{- else }}
               -defaultReplication={{ .Values.master.defaultReplication }} \
               {{- end }}

k8s/charts/seaweedfs/values.yaml

@@ -27,13 +27,13 @@ global:
     gatewayHost: null
     gatewayPort: null
     additionalLabels: {}
-  # if enabled will use global.replicationPlacment and override master & filer defaultReplicaPlacement config
+  # if enabled will use global.replicationPlacement and override master & filer defaultReplicaPlacement config
   enableReplication: false
   #  replication type is XYZ:
   # X number of replica in other data centers
   # Y number of replica in other racks in the same data center
   # Z number of replica in other servers in the same rack
-  replicationPlacment: "001"
+  replicationPlacement: "001"
   extraEnvironmentVars:
     WEED_CLUSTER_DEFAULT: "sw"
     WEED_CLUSTER_SW_MASTER: "seaweedfs-master.seaweedfs:9333"
@@ -291,7 +291,7 @@ volume:
   # For each data disk you may use ANY storage-class, example with local-path-provisioner
   # Annotations are optional.
   #  dataDirs:
-  #   - name: data:
+  #   - name: data
   #     type: "persistentVolumeClaim"
   #     size: "24Ti"
   #     storageClass: "local-path-provisioner"
@@ -962,6 +962,11 @@ cosi:
 
   sidecar:
     image: gcr.io/k8s-staging-sig-storage/objectstorage-sidecar/objectstorage-sidecar:v20230130-v0.1.0-24-gc0cf995
+    # Resource requests, limits, etc. for the server cluster placement. This
+    # should map directly to the value of the resources field for a PodSpec,
+    # formatted as a multi-line string. By default no direct resource request
+    # is made.
+    resources: {}
 
   # enable user & permission to s3 (need to inject to all services)
   enableAuth: false
@@ -975,6 +980,12 @@ cosi:
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  # Resource requests, limits, etc. for the server cluster placement. This
+  # should map directly to the value of the resources field for a PodSpec,
+  # formatted as a multi-line string. By default no direct resource request
+  # is made.
+  resources: {}
+
 certificates:
   commonName: "SeaweedFS CA"
   ipAddresses: []

other/mq_client_example/agent_pub_record/agent_pub_record.go

@@ -3,9 +3,9 @@ package main
 import (
 	"flag"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/other/mq_client_example/example"
 	"github.com/seaweedfs/seaweedfs/weed/mq/client/agent_client"
 	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
-	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
 	"log"
 	"sync"
 	"sync/atomic"
@@ -20,13 +20,26 @@ var (
 
 	clientName = flag.String("client", "c1", "client name")
 
-	namespace   = flag.String("ns", "test", "namespace")
-	t           = flag.String("t", "test", "t")
-	seedBrokers = flag.String("brokers", "localhost:17777", "seed brokers")
+	namespace = flag.String("ns", "test", "namespace")
+	t         = flag.String("t", "test", "t")
+	agent     = flag.String("agent", "localhost:16777", "mq agent address")
 
 	counter int32
 )
 
+func genMyRecord(id int32) *example.MyRecord {
+	return &example.MyRecord{
+		Key:    []byte(fmt.Sprintf("key-%s-%d", *clientName, id)),
+		Field1: []byte(fmt.Sprintf("field1-%s-%d", *clientName, id)),
+		Field2: fmt.Sprintf("field2-%s-%d", *clientName, id),
+		Field3: id,
+		Field4: int64(id),
+		Field5: float32(id),
+		Field6: float64(id),
+		Field7: id%2 == 0,
+	}
+}
+
 func doPublish(publisher *agent_client.PublishSession, id int) {
 	startTime := time.Now()
 	for {
@@ -49,58 +62,12 @@ func doPublish(publisher *agent_client.PublishSession, id int) {
 	log.Printf("Publisher %s-%d finished in %s", *clientName, id, elapsed)
 }
 
-type MyRecord struct {
-	Key    []byte
-	Field1 []byte
-	Field2 string
-	Field3 int32
-	Field4 int64
-	Field5 float32
-	Field6 float64
-	Field7 bool
-}
-
-func genMyRecord(id int32) *MyRecord {
-	return &MyRecord{
-		Key:    []byte(fmt.Sprintf("key-%s-%d", *clientName, id)),
-		Field1: []byte(fmt.Sprintf("field1-%s-%d", *clientName, id)),
-		Field2: fmt.Sprintf("field2-%s-%d", *clientName, id),
-		Field3: id,
-		Field4: int64(id),
-		Field5: float32(id),
-		Field6: float64(id),
-		Field7: id%2 == 0,
-	}
-}
-
-func (r *MyRecord) ToRecordValue() *schema_pb.RecordValue {
-	return schema.RecordBegin().
-		SetBytes("key", r.Key).
-		SetBytes("field1", r.Field1).
-		SetString("field2", r.Field2).
-		SetInt32("field3", r.Field3).
-		SetInt64("field4", r.Field4).
-		SetFloat("field5", r.Field5).
-		SetDouble("field6", r.Field6).
-		SetBool("field7", r.Field7).
-		RecordEnd()
-}
-
 func main() {
 	flag.Parse()
 
-	recordType := schema.RecordTypeBegin().
-		WithField("key", schema.TypeBytes).
-		WithField("field1", schema.TypeBytes).
-		WithField("field2", schema.TypeString).
-		WithField("field3", schema.TypeInt32).
-		WithField("field4", schema.TypeInt64).
-		WithField("field5", schema.TypeFloat).
-		WithField("field6", schema.TypeDouble).
-		WithField("field7", schema.TypeBoolean).
-		RecordTypeEnd()
+	recordType := example.MyRecordType()
 
-	session, err := agent_client.NewPublishSession("localhost:16777", schema.NewSchema(*namespace, *t, recordType), *partitionCount, *clientName)
+	session, err := agent_client.NewPublishSession(*agent, schema.NewSchema(*namespace, *t, recordType), *partitionCount, *clientName)
 	if err != nil {
 		log.Printf("failed to create session: %v", err)
 		return

other/mq_client_example/agent_sub_record/agent_sub_record.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/other/mq_client_example/example"
+	"github.com/seaweedfs/seaweedfs/weed/mq/client/agent_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"log"
+	"time"
+)
+
+var (
+	namespace         = flag.String("ns", "test", "namespace")
+	t                 = flag.String("topic", "test", "topic")
+	agent             = flag.String("agent", "localhost:16777", "mq agent address")
+	maxPartitionCount = flag.Int("maxPartitionCount", 3, "max partition count")
+	slidingWindowSize = flag.Int("slidingWindowSize", 1, "per partition concurrency")
+	timeAgo           = flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
+
+	clientId = flag.Uint("client_id", uint(util.RandomInt32()), "client id")
+)
+
+func main() {
+	flag.Parse()
+
+	// determine the start of the messages
+	var startTsNs int64
+	startType := schema_pb.OffsetType_RESUME_OR_EARLIEST
+	if *timeAgo > 0 {
+		startTsNs = time.Now().Add(-*timeAgo).UnixNano()
+		startType = schema_pb.OffsetType_EXACT_TS_NS
+	}
+
+	session, err := agent_client.NewSubscribeSession(*agent, &agent_client.SubscribeOption{
+		ConsumerGroup:           "test",
+		ConsumerGroupInstanceId: fmt.Sprintf("client-%d", *clientId),
+		Topic:                   topic.NewTopic(*namespace, *t),
+		OffsetType:              startType,
+		OffsetTsNs:              startTsNs,
+		Filter:                  "",
+		MaxSubscribedPartitions: int32(*maxPartitionCount),
+		SlidingWindowSize:       int32(*slidingWindowSize),
+	})
+	if err != nil {
+		log.Printf("new subscribe session: %v", err)
+		return
+	}
+	defer session.CloseSession()
+
+	counter := 0
+	session.SubscribeMessageRecord(func(key []byte, recordValue *schema_pb.RecordValue) {
+		counter++
+		record := example.FromRecordValue(recordValue)
+		fmt.Printf("%d %s %v\n", counter, string(key), record.Field2)
+	}, func() {
+		log.Printf("done received %d messages", counter)
+	})
+
+}

other/mq_client_example/example/my_record.go

@@ -0,0 +1,56 @@
+package example
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+type MyRecord struct {
+	Key    []byte
+	Field1 []byte
+	Field2 string
+	Field3 int32
+	Field4 int64
+	Field5 float32
+	Field6 float64
+	Field7 bool
+}
+
+func MyRecordType() *schema_pb.RecordType {
+	return schema.RecordTypeBegin().
+		WithField("key", schema.TypeBytes).
+		WithField("field1", schema.TypeBytes).
+		WithField("field2", schema.TypeString).
+		WithField("field3", schema.TypeInt32).
+		WithField("field4", schema.TypeInt64).
+		WithField("field5", schema.TypeFloat).
+		WithField("field6", schema.TypeDouble).
+		WithField("field7", schema.TypeBoolean).
+		RecordTypeEnd()
+}
+
+func (r *MyRecord) ToRecordValue() *schema_pb.RecordValue {
+	return schema.RecordBegin().
+		SetBytes("key", r.Key).
+		SetBytes("field1", r.Field1).
+		SetString("field2", r.Field2).
+		SetInt32("field3", r.Field3).
+		SetInt64("field4", r.Field4).
+		SetFloat("field5", r.Field5).
+		SetDouble("field6", r.Field6).
+		SetBool("field7", r.Field7).
+		RecordEnd()
+}
+
+func FromRecordValue(recordValue *schema_pb.RecordValue) *MyRecord {
+	return &MyRecord{
+		Key:    recordValue.Fields["key"].GetBytesValue(),
+		Field1: recordValue.Fields["field1"].GetBytesValue(),
+		Field2: recordValue.Fields["field2"].GetStringValue(),
+		Field3: recordValue.Fields["field3"].GetInt32Value(),
+		Field4: recordValue.Fields["field4"].GetInt64Value(),
+		Field5: recordValue.Fields["field5"].GetFloatValue(),
+		Field6: recordValue.Fields["field6"].GetDoubleValue(),
+		Field7: recordValue.Fields["field7"].GetBoolValue(),
+	}
+}

weed/Makefile

@@ -41,6 +41,10 @@ debug_mq:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 mq.broker
 
+debug_mq_agent:
+	go build -gcflags="all=-N -l"
+	dlv --listen=:2346 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 mq.agent -broker=localhost:17777
+
 debug_filer_copy:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 filer.backup -filer=localhost:8888 -filerProxy -timeAgo=10h

weed/command/command.go

@@ -32,6 +32,7 @@ var Commands = []*Command{
 	cmdMaster,
 	cmdMasterFollower,
 	cmdMount,
+	cmdMqAgent,
 	cmdMqBroker,
 	cmdS3,
 	cmdScaffold,

weed/command/filer.go

@@ -119,6 +119,9 @@ func init() {
 	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
 	filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 	filerS3Options.localSocket = cmdFiler.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
+	filerS3Options.tlsCACertificate = cmdFiler.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")
+	filerS3Options.tlsVerifyClientCert = cmdFiler.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate")
+	filerS3Options.idleTimeout = cmdFiler.Flag.Int("s3.idleTimeout", 10, "connection idle seconds")
 
 	// start webdav on filer
 	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
@@ -209,6 +212,7 @@ func runFiler(cmd *Command, args []string) bool {
 
 	if *filerStartWebDav {
 		filerWebDavOptions.filer = &filerAddress
+		filerWebDavOptions.ipBind = f.bindIp
 
 		if *filerWebDavOptions.disk == "" {
 			filerWebDavOptions.disk = f.diskType

weed/command/imports.go

@@ -31,6 +31,7 @@ import (
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/redis2"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/redis3"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/sqlite"
+	_ "github.com/seaweedfs/seaweedfs/weed/filer/tarantool"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/tikv"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/ydb"
 )

weed/command/mount_std.go

@@ -284,5 +284,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 
 	server.Serve()
 
+	seaweedFileSystem.ClearCacheDir()
+
 	return true
 }

weed/command/mq_agent.go

@@ -26,12 +26,12 @@ type MessageQueueAgentOptions struct {
 func init() {
 	cmdMqAgent.Run = runMqAgent // break init cycle
 	mqAgentOptions.brokersString = cmdMqAgent.Flag.String("broker", "localhost:17777", "comma-separated message queue brokers")
-	mqAgentOptions.ip = cmdMqAgent.Flag.String("ip", "localhost", "message queue agent host address")
+	mqAgentOptions.ip = cmdMqAgent.Flag.String("ip", "", "message queue agent host address")
 	mqAgentOptions.port = cmdMqAgent.Flag.Int("port", 16777, "message queue agent gRPC server port")
 }
 
 var cmdMqAgent = &Command{
-	UsageLine: "mq.agent [-port=6377] [-master=<ip:port>]",
+	UsageLine: "mq.agent [-port=16777] [-broker=<ip:port>]",
 	Short:     "<WIP> start a message queue agent",
 	Long: `start a message queue agent
 
@@ -64,6 +64,7 @@ func (mqAgentOpt *MessageQueueAgentOptions) startQueueAgent() bool {
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", *mqAgentOpt.port, err)
 	}
+	glog.Infof("Start Seaweed Message Queue Agent on %s:%d", *mqAgentOpt.ip, *mqAgentOpt.port)
 	grpcS := pb.NewGrpcServer()
 	mq_agent_pb.RegisterSeaweedMessagingAgentServer(grpcS, agentServer)
 	reflection.Register(grpcS)

weed/command/s3.go

@@ -57,6 +57,7 @@ type S3Options struct {
 	dataCenter                *string
 	localSocket               *string
 	certProvider              certprovider.Provider
+	idleTimeout               *int
 }
 
 func init() {
@@ -81,6 +82,7 @@ func init() {
 	s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 	s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path")
 	s3StandaloneOptions.localSocket = cmdS3.Flag.String("localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
+	s3StandaloneOptions.idleTimeout = cmdS3.Flag.Int("idleTimeout", 10, "connection idle seconds")
 }
 
 var cmdS3 = &Command{
@@ -275,7 +277,8 @@ func (s3opt *S3Options) startS3Server() bool {
 	}
 
 	listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port)
-	s3ApiListener, s3ApiLocalListener, err := util.NewIpAndLocalListeners(*s3opt.bindIp, *s3opt.port, time.Duration(10)*time.Second)
+	s3ApiListener, s3ApiLocalListener, err := util.NewIpAndLocalListeners(
+		*s3opt.bindIp, *s3opt.port, time.Duration(*s3opt.idleTimeout)*time.Second)
 	if err != nil {
 		glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
 	}
@@ -350,7 +353,7 @@ func (s3opt *S3Options) startS3Server() bool {
 		} else {
 			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.portHttps)
 			s3ApiListenerHttps, s3ApiLocalListenerHttps, _ := util.NewIpAndLocalListeners(
-				*s3opt.bindIp, *s3opt.portHttps, time.Duration(10)*time.Second)
+                *s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second)
 			if s3ApiLocalListenerHttps != nil {
 				go func() {
 					if err = httpS.ServeTLS(s3ApiLocalListenerHttps, "", ""); err != nil {

weed/command/scaffold/filer.toml

@@ -139,12 +139,13 @@ connection_max_lifetime_seconds = 0
 enableUpsert = true
 upsertQuery = """UPSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)"""
 
-[cassandra]
+[cassandra2]
 # CREATE TABLE filemeta (
+#    dirhash bigint,
 #    directory varchar,
 #    name varchar,
 #    meta blob,
-#    PRIMARY KEY (directory, name)
+#    PRIMARY KEY ((dirhash, directory), name)
 # ) WITH CLUSTERING ORDER BY (name ASC);
 enabled = false
 keyspace = "seaweedfs"
@@ -170,6 +171,10 @@ enabled = false
 address = "localhost:6379"
 password = ""
 database = 0
+enable_tls = false
+ca_cert_path = ""
+client_cert_path = ""
+client_key_path = ""
 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
 superLargeDirectories = []
 
@@ -180,6 +185,10 @@ masterName = "master"
 username = ""
 password = ""
 database = 0
+enable_tls = false
+ca_cert_path = ""
+client_cert_path = ""
+client_key_path = ""
 
 [redis_cluster2]
 enabled = false
@@ -192,6 +201,10 @@ addresses = [
     "localhost:30006",
 ]
 password = ""
+enable_tls = false
+ca_cert_path = ""
+client_cert_path = ""
+client_key_path = ""
 # allows reads from slave servers or the master, but all writes still go to the master
 readOnly = false
 # automatically use the closest Redis server for reads
@@ -199,11 +212,16 @@ routeByLatency = false
 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
 superLargeDirectories = []
 
+# The following lua redis stores uses lua to ensure atomicity
 [redis_lua]
 enabled = false
 address = "localhost:6379"
 password = ""
 database = 0
+enable_tls = false
+ca_cert_path = ""
+client_cert_path = ""
+client_key_path = ""
 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
 superLargeDirectories = []
 
@@ -214,6 +232,10 @@ masterName = "master"
 username = ""
 password = ""
 database = 0
+enable_tls = false
+ca_cert_path = ""
+client_cert_path = ""
+client_key_path = ""
 
 [redis_lua_cluster]
 enabled = false
@@ -226,6 +248,10 @@ addresses = [
     "localhost:30006",
 ]
 password = ""
+enable_tls = false
+ca_cert_path = ""
+client_cert_path = ""
+client_key_path = ""
 # allows reads from slave servers or the master, but all writes still go to the master
 readOnly = false
 # automatically use the closest Redis server for reads
@@ -233,36 +259,6 @@ routeByLatency = false
 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
 superLargeDirectories = []
 
-[redis3] # beta
-enabled = false
-address = "localhost:6379"
-password = ""
-database = 0
-
-[redis3_sentinel]
-enabled = false
-addresses = ["172.22.12.7:26379","172.22.12.8:26379","172.22.12.9:26379"]
-masterName = "master"
-username = ""
-password = ""
-database = 0
-
-[redis_cluster3] # beta
-enabled = false
-addresses = [
-    "localhost:30001",
-    "localhost:30002",
-    "localhost:30003",
-    "localhost:30004",
-    "localhost:30005",
-    "localhost:30006",
-]
-password = ""
-# allows reads from slave servers or the master, but all writes still go to the master
-readOnly = false
-# automatically use the closest Redis server for reads
-routeByLatency = false
-
 [etcd]
 enabled = false
 servers = "localhost:2379"
@@ -333,7 +329,7 @@ dialTimeOut = 10
 ##########################
 # To add path-specific filer store:
 #
-# 1. Add a name following the store type separated by a dot ".". E.g., cassandra.tmp
+# 1. Add a name following the store type separated by a dot ".". E.g., cassandra2.tmp
 # 2. Add a location configuration. E.g., location = "/tmp/"
 # 3. Copy and customize all other configurations.
 #     Make sure they are not the same if using the same store type!
@@ -365,3 +361,10 @@ cert_path=""
 key_path=""
 # The name list used to verify the cn name
 verify_cn=""
+
+[tarantool]
+address = "localhost:3301"
+user = "guest"
+password = ""
+timeout = "5s"
+maxReconnects = 1000

weed/command/server.go

@@ -156,6 +156,8 @@ func init() {
 	s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
 	s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 	s3Options.localSocket = cmdServer.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
+	s3Options.bindIp = cmdServer.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")
+	s3Options.idleTimeout = cmdServer.Flag.Int("s3.idleTimeout", 10, "connection idle seconds")
 
 	iamOptions.port = cmdServer.Flag.Int("iam.port", 8111, "iam server http listen port")
 
@@ -218,9 +220,12 @@ func runServer(cmd *Command, args []string) bool {
 	filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToServiceDiscovery()
 	filerOptions.ip = serverIp
 	filerOptions.bindIp = serverBindIp
-	s3Options.bindIp = serverBindIp
+	if *s3Options.bindIp == "" {
+		s3Options.bindIp = serverBindIp
+	}
 	iamOptions.ip = serverBindIp
 	iamOptions.masters = masterOptions.peers
+	webdavOptions.ipBind = serverBindIp
 	serverOptions.v.ip = serverIp
 	serverOptions.v.bindIp = serverBindIp
 	serverOptions.v.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()

weed/command/update_full.go

@@ -1,5 +1,5 @@
-//go:build elastic && gocdk && rclone && sqlite && tikv && ydb
-// +build elastic,gocdk,rclone,sqlite,tikv,ydb
+//go:build elastic && gocdk && rclone && sqlite && tarantool && tikv && ydb
+// +build elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
 
 package command
 

weed/command/webdav.go

@@ -23,6 +23,7 @@ var (
 
 type WebDavOption struct {
 	filer          *string
+	ipBind         *string
 	filerRootPath  *string
 	port           *int
 	collection     *string
@@ -38,6 +39,7 @@ type WebDavOption struct {
 func init() {
 	cmdWebDav.Run = runWebDav // break init cycle
 	webDavStandaloneOptions.filer = cmdWebDav.Flag.String("filer", "localhost:8888", "filer server address")
+	webDavStandaloneOptions.ipBind = cmdWebDav.Flag.String("ip.bind", "", "ip address to bind to. Default listen to all.")
 	webDavStandaloneOptions.port = cmdWebDav.Flag.Int("port", 7333, "webdav server http listen port")
 	webDavStandaloneOptions.collection = cmdWebDav.Flag.String("collection", "", "collection to create the files")
 	webDavStandaloneOptions.replication = cmdWebDav.Flag.String("replication", "", "replication to create the files")
@@ -62,7 +64,8 @@ func runWebDav(cmd *Command, args []string) bool {
 
 	util.LoadSecurityConfiguration()
 
-	glog.V(0).Infof("Starting Seaweed WebDav Server %s at https port %d", util.Version(), *webDavStandaloneOptions.port)
+	listenAddress := fmt.Sprintf("%s:%d", *webDavStandaloneOptions.ipBind, *webDavStandaloneOptions.port)
+	glog.V(0).Infof("Starting Seaweed WebDav Server %s at %s", util.Version(), listenAddress)
 
 	return webDavStandaloneOptions.startWebDav()
 
@@ -126,19 +129,19 @@ func (wo *WebDavOption) startWebDav() bool {
 
 	httpS := &http.Server{Handler: ws.Handler}
 
-	listenAddress := fmt.Sprintf(":%d", *wo.port)
+	listenAddress := fmt.Sprintf("%s:%d", *wo.ipBind, *wo.port)
 	webDavListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
 	if err != nil {
 		glog.Fatalf("WebDav Server listener on %s error: %v", listenAddress, err)
 	}
 
 	if *wo.tlsPrivateKey != "" {
-		glog.V(0).Infof("Start Seaweed WebDav Server %s at https port %d", util.Version(), *wo.port)
+		glog.V(0).Infof("Start Seaweed WebDav Server %s at https %s", util.Version(), listenAddress)
 		if err = httpS.ServeTLS(webDavListener, *wo.tlsCertificate, *wo.tlsPrivateKey); err != nil {
 			glog.Fatalf("WebDav Server Fail to serve: %v", err)
 		}
 	} else {
-		glog.V(0).Infof("Start Seaweed WebDav Server %s at http port %d", util.Version(), *wo.port)
+		glog.V(0).Infof("Start Seaweed WebDav Server %s at http %s", util.Version(), listenAddress)
 		if err = httpS.Serve(webDavListener); err != nil {
 			glog.Fatalf("WebDav Server Fail to serve: %v", err)
 		}

weed/filer/cassandra/README.txt

@@ -1,14 +1 @@
-1. create a keyspace
-
-CREATE KEYSPACE seaweedfs WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 1};
-
-2. create filemeta table
-
- USE seaweedfs;
-
- CREATE TABLE filemeta (
-    directory varchar,
-    name varchar,
-    meta blob,
-    PRIMARY KEY (directory, name)
- ) WITH CLUSTERING ORDER BY (name ASC);
+Deprecated by cassandra2

weed/filer/cassandra/cassandra_store.go

@@ -209,7 +209,7 @@ func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath u
 			break
 		}
 	}
-	if err := iter.Close(); err != nil {
+	if err = iter.Close(); err != nil {
 		glog.V(0).Infof("list iterator close: %v", err)
 	}
 

weed/filer/cassandra2/README.txt

@@ -0,0 +1,15 @@
+1. create a keyspace
+
+CREATE KEYSPACE seaweedfs WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 1};
+
+2. create filemeta table
+
+ USE seaweedfs;
+
+ CREATE TABLE filemeta (
+    dirhash bigint,
+    directory varchar,
+    name varchar,
+    meta blob,
+    PRIMARY KEY ((dirhash, directory), name)
+ ) WITH CLUSTERING ORDER BY (name ASC);

weed/filer/cassandra2/cassandra_store.go

@@ -0,0 +1,221 @@
+package cassandra2
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/gocql/gocql"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &Cassandra2Store{})
+}
+
+type Cassandra2Store struct {
+	cluster                 *gocql.ClusterConfig
+	session                 *gocql.Session
+	superLargeDirectoryHash map[string]string
+}
+
+func (store *Cassandra2Store) GetName() string {
+	return "cassandra2"
+}
+
+func (store *Cassandra2Store) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetString(prefix+"keyspace"),
+		configuration.GetStringSlice(prefix+"hosts"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetStringSlice(prefix+"superLargeDirectories"),
+		configuration.GetString(prefix+"localDC"),
+		configuration.GetInt(prefix+"connection_timeout_millisecond"),
+	)
+}
+
+func (store *Cassandra2Store) isSuperLargeDirectory(dir string) (dirHash string, isSuperLargeDirectory bool) {
+	dirHash, isSuperLargeDirectory = store.superLargeDirectoryHash[dir]
+	return
+}
+
+func (store *Cassandra2Store) initialize(keyspace string, hosts []string, username string, password string, superLargeDirectories []string, localDC string, timeout int) (err error) {
+	store.cluster = gocql.NewCluster(hosts...)
+	if username != "" && password != "" {
+		store.cluster.Authenticator = gocql.PasswordAuthenticator{Username: username, Password: password}
+	}
+	store.cluster.Keyspace = keyspace
+	store.cluster.Timeout = time.Duration(timeout) * time.Millisecond
+	glog.V(0).Infof("timeout = %d", timeout)
+	fallback := gocql.RoundRobinHostPolicy()
+	if localDC != "" {
+		fallback = gocql.DCAwareRoundRobinPolicy(localDC)
+	}
+	store.cluster.PoolConfig.HostSelectionPolicy = gocql.TokenAwareHostPolicy(fallback)
+	store.cluster.Consistency = gocql.LocalQuorum
+
+	store.session, err = store.cluster.CreateSession()
+	if err != nil {
+		glog.V(0).Infof("Failed to open cassandra2 store, hosts %v, keyspace %s", hosts, keyspace)
+	}
+
+	// set directory hash
+	store.superLargeDirectoryHash = make(map[string]string)
+	existingHash := make(map[string]string)
+	for _, dir := range superLargeDirectories {
+		// adding dir hash to avoid duplicated names
+		dirHash := util.Md5String([]byte(dir))[:4]
+		store.superLargeDirectoryHash[dir] = dirHash
+		if existingDir, found := existingHash[dirHash]; found {
+			glog.Fatalf("directory %s has the same hash as %s", dir, existingDir)
+		}
+		existingHash[dirHash] = dir
+	}
+	return
+}
+
+func (store *Cassandra2Store) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *Cassandra2Store) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *Cassandra2Store) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *Cassandra2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	dir, name := entry.FullPath.DirAndName()
+	if dirHash, ok := store.isSuperLargeDirectory(dir); ok {
+		dir, name = dirHash+name, ""
+	}
+
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
+	}
+
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		meta = util.MaybeGzipData(meta)
+	}
+
+	if err := store.session.Query(
+		"INSERT INTO filemeta (dirhash,directory,name,meta) VALUES(?,?,?,?) USING TTL ? ",
+		util.HashStringToLong(dir), dir, name, meta, entry.TtlSec).Exec(); err != nil {
+		return fmt.Errorf("insert %s: %s", entry.FullPath, err)
+	}
+
+	return nil
+}
+
+func (store *Cassandra2Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *Cassandra2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+
+	dir, name := fullpath.DirAndName()
+	if dirHash, ok := store.isSuperLargeDirectory(dir); ok {
+		dir, name = dirHash+name, ""
+	}
+
+	var data []byte
+	if err := store.session.Query(
+		"SELECT meta FROM filemeta WHERE dirhash=? AND directory=? AND name=?",
+		util.HashStringToLong(dir), dir, name).Scan(&data); err != nil {
+		if errors.Is(err, gocql.ErrNotFound) {
+			return nil, filer_pb.ErrNotFound
+		}
+		return nil, err
+	}
+
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *Cassandra2Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
+
+	dir, name := fullpath.DirAndName()
+	if dirHash, ok := store.isSuperLargeDirectory(dir); ok {
+		dir, name = dirHash+name, ""
+	}
+
+	if err := store.session.Query(
+		"DELETE FROM filemeta WHERE dirhash=? AND directory=? AND name=?",
+		util.HashStringToLong(dir), dir, name).Exec(); err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *Cassandra2Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
+	if _, ok := store.isSuperLargeDirectory(string(fullpath)); ok {
+		return nil // filer.ErrUnsupportedSuperLargeDirectoryListing
+	}
+
+	if err := store.session.Query(
+		"DELETE FROM filemeta WHERE dirhash=? AND directory=?",
+		util.HashStringToLong(string(fullpath)), fullpath).Exec(); err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *Cassandra2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
+}
+
+func (store *Cassandra2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+
+	if _, ok := store.isSuperLargeDirectory(string(dirPath)); ok {
+		return // nil, filer.ErrUnsupportedSuperLargeDirectoryListing
+	}
+
+	cqlStr := "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND directory=? AND name>? ORDER BY NAME ASC LIMIT ?"
+	if includeStartFile {
+		cqlStr = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND directory=? AND name>=? ORDER BY NAME ASC LIMIT ?"
+	}
+
+	var data []byte
+	var name string
+	iter := store.session.Query(cqlStr, util.HashStringToLong(string(dirPath)), string(dirPath), startFileName, limit+1).Iter()
+	for iter.Scan(&name, &data) {
+		entry := &filer.Entry{
+			FullPath: util.NewFullPath(string(dirPath), name),
+		}
+		lastFileName = name
+		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
+			break
+		}
+		if !eachEntryFunc(entry) {
+			break
+		}
+	}
+	if err = iter.Close(); err != nil {
+		glog.V(0).Infof("list iterator close: %v", err)
+	}
+
+	return lastFileName, err
+}
+
+func (store *Cassandra2Store) Shutdown() {
+	store.session.Close()
+}

weed/filer/cassandra2/cassandra_store_kv.go

@@ -0,0 +1,63 @@
+package cassandra2
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"github.com/gocql/gocql"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func (store *Cassandra2Store) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+	dir, name := genDirAndName(key)
+
+	if err := store.session.Query(
+		"INSERT INTO filemeta (dirhash,directory,name,meta) VALUES(?,?,?,?) USING TTL ? ",
+		util.HashStringToLong(dir), dir, name, value, 0).Exec(); err != nil {
+		return fmt.Errorf("kv insert: %s", err)
+	}
+
+	return nil
+}
+
+func (store *Cassandra2Store) KvGet(ctx context.Context, key []byte) (data []byte, err error) {
+	dir, name := genDirAndName(key)
+
+	if err := store.session.Query(
+		"SELECT meta FROM filemeta WHERE dirhash=? AND directory=? AND name=?",
+		util.HashStringToLong(dir), dir, name).Scan(&data); err != nil {
+		if err != gocql.ErrNotFound {
+			return nil, filer.ErrKvNotFound
+		}
+	}
+
+	if len(data) == 0 {
+		return nil, filer.ErrKvNotFound
+	}
+
+	return data, nil
+}
+
+func (store *Cassandra2Store) KvDelete(ctx context.Context, key []byte) (err error) {
+	dir, name := genDirAndName(key)
+
+	if err := store.session.Query(
+		"DELETE FROM filemeta WHERE dirhash=? AND directory=? AND name=?",
+		util.HashStringToLong(dir), dir, name).Exec(); err != nil {
+		return fmt.Errorf("kv delete: %v", err)
+	}
+
+	return nil
+}
+
+func genDirAndName(key []byte) (dir string, name string) {
+	for len(key) < 8 {
+		key = append(key, 0)
+	}
+
+	dir = base64.StdEncoding.EncodeToString(key[:8])
+	name = base64.StdEncoding.EncodeToString(key[8:])
+
+	return
+}

weed/filer/filechunk_group.go

@@ -56,6 +56,9 @@ func (group *ChunkGroup) ReadDataAt(fileSize int64, buff []byte, offset int64) (
 	for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
 		section, found := group.sections[si]
 		rangeStart, rangeStop := max(offset, int64(si*SectionSize)), min(offset+int64(len(buff)), int64((si+1)*SectionSize))
+		if rangeStart >= rangeStop {
+			continue
+		}
 		if !found {
 			rangeStop = min(rangeStop, fileSize)
 			for i := rangeStart; i < rangeStop; i++ {

weed/filer/filer.go

@@ -3,12 +3,13 @@ package filer
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
 	"os"
 	"sort"
 	"strings"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
+
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
@@ -289,7 +290,7 @@ func (f *Filer) ensureParentDirectoryEntry(ctx context.Context, entry *Entry, di
 		glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
 		mkdirErr := f.Store.InsertEntry(ctx, dirEntry)
 		if mkdirErr != nil {
-			if _, err := f.FindEntry(ctx, util.FullPath(dirPath)); err == filer_pb.ErrNotFound {
+			if fEntry, err := f.FindEntry(ctx, util.FullPath(dirPath)); err == filer_pb.ErrNotFound || fEntry == nil {
 				glog.V(3).Infof("mkdir %s: %v", dirPath, mkdirErr)
 				return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
 			}

weed/filer/redis/README.md

@@ -0,0 +1,5 @@
+Deprecated by redis2.
+
+This implementaiton uses unsorted set. For example, add a directory child via SAdd.
+
+Redis2 moves to sorted set. Adding a child uses ZAddNX.

weed/filer/redis2/redis_store.go

@@ -1,8 +1,14 @@
 package redis2
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"net"
+	"os"
+
 	"github.com/redis/go-redis/v9"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
@@ -24,15 +30,54 @@ func (store *Redis2Store) Initialize(configuration util.Configuration, prefix st
 		configuration.GetString(prefix+"password"),
 		configuration.GetInt(prefix+"database"),
 		configuration.GetStringSlice(prefix+"superLargeDirectories"),
+		configuration.GetBool(prefix+"enable_mtls"),
+		configuration.GetString(prefix+"ca_cert_path"),
+		configuration.GetString(prefix+"client_cert_path"),
+		configuration.GetString(prefix+"client_key_path"),
 	)
 }
 
-func (store *Redis2Store) initialize(hostPort string, password string, database int, superLargeDirectories []string) (err error) {
-	store.Client = redis.NewClient(&redis.Options{
-		Addr:     hostPort,
-		Password: password,
-		DB:       database,
-	})
+func (store *Redis2Store) initialize(hostPort string, password string, database int, superLargeDirectories []string, enableMtls bool, caCertPath string, clientCertPath string, clientKeyPath string) (err error) {
+	if enableMtls {
+		clientCert, err := tls.LoadX509KeyPair(clientCertPath, clientKeyPath)
+		if err != nil {
+			glog.Fatalf("Error loading client certificate and key pair: %v", err)
+		}
+
+		caCertBytes, err := os.ReadFile(caCertPath)
+		if err != nil {
+			glog.Fatalf("Error reading CA certificate file: %v", err)
+		}
+
+		caCertPool := x509.NewCertPool()
+		if ok := caCertPool.AppendCertsFromPEM(caCertBytes); !ok {
+			glog.Fatalf("Error appending CA certificate to pool")
+		}
+
+		redisHost, _, err := net.SplitHostPort(hostPort)
+		if err != nil {
+			glog.Fatalf("Error parsing redis host and port from %s: %v", hostPort, err)
+		}
+
+		tlsConfig := &tls.Config{
+			Certificates: []tls.Certificate{clientCert},
+			RootCAs: caCertPool,
+			ServerName: redisHost,
+			MinVersion: tls.VersionTLS12,
+		}
+		store.Client = redis.NewClient(&redis.Options{
+			Addr:     hostPort,
+			Password: password,
+			DB:       database,
+			TLSConfig: tlsConfig,
+		})
+	} else {
+		store.Client = redis.NewClient(&redis.Options{
+			Addr:     hostPort,
+			Password: password,
+			DB:       database,
+		})
+	}
 	store.loadSuperLargeDirectories(superLargeDirectories)
 	return
 }

weed/filer/redis3/README.md

@@ -0,0 +1,4 @@
+Desuppported.
+
+This implementation attempts to use skip list.
+Did not get any report on actual benefits.

weed/filer/redis3/redis_store.go

@@ -1,10 +1,16 @@
 package redis3
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"net"
+	"os"
+
 	"github.com/go-redsync/redsync/v4"
 	"github.com/go-redsync/redsync/v4/redis/goredis/v9"
 	"github.com/redis/go-redis/v9"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
@@ -25,15 +31,54 @@ func (store *Redis3Store) Initialize(configuration util.Configuration, prefix st
 		configuration.GetString(prefix+"address"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetInt(prefix+"database"),
+		configuration.GetBool(prefix+"enable_mtls"),
+		configuration.GetString(prefix+"ca_cert_path"),
+		configuration.GetString(prefix+"client_cert_path"),
+		configuration.GetString(prefix+"client_key_path"),
 	)
 }
 
-func (store *Redis3Store) initialize(hostPort string, password string, database int) (err error) {
-	store.Client = redis.NewClient(&redis.Options{
-		Addr:     hostPort,
-		Password: password,
-		DB:       database,
-	})
+func (store *Redis3Store) initialize(hostPort string, password string, database int, enableMtls bool, caCertPath string, clientCertPath string, clientKeyPath string) (err error) {
+	if enableMtls {
+		clientCert, err := tls.LoadX509KeyPair(clientCertPath, clientKeyPath)
+		if err != nil {
+			glog.Fatalf("Error loading client certificate and key pair: %v", err)
+		}
+
+		caCertBytes, err := os.ReadFile(caCertPath)
+		if err != nil {
+			glog.Fatalf("Error reading CA certificate file: %v", err)
+		}
+
+		caCertPool := x509.NewCertPool()
+		if ok := caCertPool.AppendCertsFromPEM(caCertBytes); !ok {
+			glog.Fatalf("Error appending CA certificate to pool")
+		}
+
+		redisHost, _, err := net.SplitHostPort(hostPort)
+		if err != nil {
+			glog.Fatalf("Error parsing redis host and port from %s: %v", hostPort, err)
+		}
+
+		tlsConfig := &tls.Config{
+			Certificates: []tls.Certificate{clientCert},
+			RootCAs:      caCertPool,
+			ServerName:   redisHost,
+			MinVersion:   tls.VersionTLS12,
+		}
+		store.Client = redis.NewClient(&redis.Options{
+			Addr:      hostPort,
+			Password:  password,
+			DB:        database,
+			TLSConfig: tlsConfig,
+		})
+	} else {
+		store.Client = redis.NewClient(&redis.Options{
+			Addr:     hostPort,
+			Password: password,
+			DB:       database,
+		})
+	}
 	store.redsync = redsync.New(goredis.NewPool(store.Client))
 	return
 }

weed/filer/store_test/test_suite.go

@@ -29,16 +29,29 @@ func TestFilerStore(t *testing.T, store filer.FilerStore) {
 		})
 		assert.Nil(t, err, "list directory")
 		assert.Equal(t, 3, counter, "directory list counter")
-		assert.Equal(t, "f00003", lastFileName, "directory list last file")
+		assert.Equal(t, "f00002", lastFileName, "directory list last file")
 		lastFileName, err = store.ListDirectoryEntries(ctx, util.FullPath("/a/b/c"), lastFileName, false, 1024, func(entry *filer.Entry) bool {
 			counter++
 			return true
 		})
 		assert.Nil(t, err, "list directory")
 		assert.Equal(t, 1027, counter, "directory list counter")
-		assert.Equal(t, "f01027", lastFileName, "directory list last file")
+		assert.Equal(t, "f01026", lastFileName, "directory list last file")
 	}
 
+	testKey := []byte("test_key")
+	testValue1 := []byte("test_value1")
+	testValue2 := []byte("test_value2")
+
+	err := store.KvPut(ctx, testKey, testValue1)
+	assert.Nil(t, err, "KV put")
+	value, err := store.KvGet(ctx, testKey)
+	assert.Equal(t, value, testValue1, "KV get")
+
+	err = store.KvPut(ctx, testKey, testValue2)
+	assert.Nil(t, err, "KV update")
+	value, err = store.KvGet(ctx, testKey)
+	assert.Equal(t, value, testValue2, "KV get after update")
 }
 
 func makeEntry(fullPath util.FullPath, isDirectory bool) *filer.Entry {

weed/filer/tarantool/doc.go

@@ -0,0 +1,7 @@
+/*
+Package tarantool is for Tarantool filer store.
+
+The referenced "github.com/tarantool/go-tarantool/v2" library is too big when compiled.
+So this is only compiled in "make full_install".
+*/
+package tarantool

weed/filer/tarantool/readme.md

@@ -0,0 +1,11 @@
+## Tarantool
+
+database: https://www.tarantool.io/
+
+go driver: https://github.com/tarantool/go-tarantool/
+
+To set up local env:
+`make -C docker test_tarantool`
+
+Run tests:
+`RUN_TARANTOOL_TESTS=1 go test -tags=tarantool ./weed/filer/tarantool`

weed/filer/tarantool/tarantool_store.go

@@ -0,0 +1,318 @@
+//go:build tarantool
+// +build tarantool
+
+package tarantool
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	weed_util "github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/tarantool/go-tarantool/v2"
+	"github.com/tarantool/go-tarantool/v2/crud"
+	"github.com/tarantool/go-tarantool/v2/pool"
+)
+
+const (
+	tarantoolSpaceName = "filer_metadata"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &TarantoolStore{})
+}
+
+type TarantoolStore struct {
+	pool *pool.ConnectionPool
+}
+
+func (store *TarantoolStore) GetName() string {
+	return "tarantool"
+}
+
+func (store *TarantoolStore) Initialize(configuration weed_util.Configuration, prefix string) error {
+
+	configuration.SetDefault(prefix+"address", "localhost:3301")
+	configuration.SetDefault(prefix+"user", "guest")
+	configuration.SetDefault(prefix+"password", "")
+	configuration.SetDefault(prefix+"timeout", "5s")
+	configuration.SetDefault(prefix+"maxReconnects", "1000")
+
+	address := configuration.GetString(prefix + "address")
+	user := configuration.GetString(prefix + "user")
+	password := configuration.GetString(prefix + "password")
+
+	timeoutStr := configuration.GetString(prefix + "timeout")
+	timeout, err := time.ParseDuration(timeoutStr)
+	if err != nil {
+		return fmt.Errorf("parse tarantool store timeout: %v", err)
+	}
+
+	maxReconnects := configuration.GetInt(prefix + "maxReconnects")
+	if maxReconnects < 0 {
+		return fmt.Errorf("maxReconnects is negative")
+	}
+
+	addresses := strings.Split(address, ",")
+
+	return store.initialize(addresses, user, password, timeout, uint(maxReconnects))
+}
+
+func (store *TarantoolStore) initialize(addresses []string, user string, password string, timeout time.Duration, maxReconnects uint) error {
+
+	opts := tarantool.Opts{
+		Timeout:       timeout,
+		Reconnect:     time.Second,
+		MaxReconnects: maxReconnects,
+	}
+
+	poolInstances := makePoolInstances(addresses, user, password, opts)
+	poolOpts := pool.Opts{
+		CheckTimeout: time.Second,
+	}
+
+	ctx := context.Background()
+	p, err := pool.ConnectWithOpts(ctx, poolInstances, poolOpts)
+	if err != nil {
+		return fmt.Errorf("Can't create connection pool: %v", err)
+	}
+
+	_, err = p.Do(tarantool.NewPingRequest(), pool.ANY).Get()
+	if err != nil {
+		return err
+	}
+
+	store.pool = p
+
+	return nil
+}
+
+func makePoolInstances(addresses []string, user string, password string, opts tarantool.Opts) []pool.Instance {
+	poolInstances := make([]pool.Instance, 0, len(addresses))
+	for i, address := range addresses {
+		poolInstances = append(poolInstances, makePoolInstance(address, user, password, opts, i))
+	}
+	return poolInstances
+}
+
+func makePoolInstance(address string, user string, password string, opts tarantool.Opts, serial int) pool.Instance {
+	return pool.Instance{
+		Name: fmt.Sprintf("instance%d", serial),
+		Dialer: tarantool.NetDialer{
+			Address:  address,
+			User:     user,
+			Password: password,
+		},
+		Opts: opts,
+	}
+}
+
+func (store *TarantoolStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+
+func (store *TarantoolStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *TarantoolStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *TarantoolStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	dir, name := entry.FullPath.DirAndName()
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
+	}
+
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		meta = util.MaybeGzipData(meta)
+	}
+
+	var ttl int64
+	if entry.TtlSec > 0 {
+		ttl = time.Now().Unix() + int64(entry.TtlSec)
+	} else {
+		ttl = 0
+	}
+
+	var operations = []crud.Operation{
+		{
+			Operator: crud.Insert,
+			Field:    "data",
+			Value:    string(meta),
+		},
+	}
+
+	req := crud.MakeUpsertRequest(tarantoolSpaceName).
+		Tuple([]interface{}{dir, nil, name, ttl, string(meta)}).
+		Operations(operations)
+
+	ret := crud.Result{}
+
+	if err := store.pool.Do(req, pool.RW).GetTyped(&ret); err != nil {
+		return fmt.Errorf("insert %s: %s", entry.FullPath, err)
+	}
+
+	return nil
+}
+
+func (store *TarantoolStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *TarantoolStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {
+	dir, name := fullpath.DirAndName()
+
+	findEntryGetOpts := crud.GetOpts{
+		Fields:        crud.MakeOptTuple([]interface{}{"data"}),
+		Mode:          crud.MakeOptString("read"),
+		PreferReplica: crud.MakeOptBool(true),
+		Balance:       crud.MakeOptBool(true),
+	}
+
+	req := crud.MakeGetRequest(tarantoolSpaceName).
+		Key(crud.Tuple([]interface{}{dir, name})).
+		Opts(findEntryGetOpts)
+
+	resp := crud.Result{}
+
+	err = store.pool.Do(req, pool.PreferRO).GetTyped(&resp)
+	if err != nil {
+		return nil, err
+	}
+
+	results, ok := resp.Rows.([]interface{})
+	if !ok || len(results) != 1 {
+		return nil, filer_pb.ErrNotFound
+	}
+
+	rows, ok := results[0].([]interface{})
+	if !ok || len(rows) != 1 {
+		return nil, filer_pb.ErrNotFound
+	}
+
+	row, ok := rows[0].(string)
+	if !ok {
+		return nil, fmt.Errorf("Can't convert rows[0] field to string. Actual type: %v, value: %v", reflect.TypeOf(rows[0]), rows[0])
+	}
+
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+
+	err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData([]byte(row)))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *TarantoolStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+	dir, name := fullpath.DirAndName()
+
+	delOpts := crud.DeleteOpts{
+		Noreturn: crud.MakeOptBool(true),
+	}
+
+	req := crud.MakeDeleteRequest(tarantoolSpaceName).
+		Key(crud.Tuple([]interface{}{dir, name})).
+		Opts(delOpts)
+
+	if _, err := store.pool.Do(req, pool.RW).Get(); err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *TarantoolStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+	req := tarantool.NewCallRequest("filer_metadata.delete_by_directory_idx").
+		Args([]interface{}{fullpath})
+
+	if _, err := store.pool.Do(req, pool.RW).Get(); err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *TarantoolStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
+}
+
+func (store *TarantoolStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+
+	req := tarantool.NewCallRequest("filer_metadata.find_by_directory_idx_and_name").
+		Args([]interface{}{string(dirPath), startFileName, includeStartFile, limit})
+
+	results, err := store.pool.Do(req, pool.PreferRO).Get()
+	if err != nil {
+		return
+	}
+
+	if len(results) < 1 {
+		glog.Errorf("Can't find results, data is empty")
+		return
+	}
+
+	rows, ok := results[0].([]interface{})
+	if !ok {
+		glog.Errorf("Can't convert results[0] to list")
+		return
+	}
+
+	for _, result := range rows {
+		row, ok := result.([]interface{})
+		if !ok {
+			glog.Errorf("Can't convert result to list")
+			return
+		}
+
+		if len(row) < 5 {
+			glog.Errorf("Length of result is less than needed: %v", len(row))
+			return
+		}
+
+		nameRaw := row[2]
+		name, ok := nameRaw.(string)
+		if !ok {
+			glog.Errorf("Can't convert name field to string. Actual type: %v, value: %v", reflect.TypeOf(nameRaw), nameRaw)
+			return
+		}
+
+		dataRaw := row[4]
+		data, ok := dataRaw.(string)
+		if !ok {
+			glog.Errorf("Can't convert data field to string. Actual type: %v, value: %v", reflect.TypeOf(dataRaw), dataRaw)
+			return
+		}
+
+		entry := &filer.Entry{
+			FullPath: util.NewFullPath(string(dirPath), name),
+		}
+		lastFileName = name
+		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data))); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
+			break
+		}
+		if !eachEntryFunc(entry) {
+			break
+		}
+	}
+
+	return lastFileName, err
+}
+
+func (store *TarantoolStore) Shutdown() {
+	store.pool.Close()
+}

weed/filer/tarantool/tarantool_store_kv.go

@@ -0,0 +1,95 @@
+//go:build tarantool
+// +build tarantool
+
+package tarantool
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/tarantool/go-tarantool/v2/crud"
+	"github.com/tarantool/go-tarantool/v2/pool"
+)
+
+const (
+	tarantoolKVSpaceName = "key_value"
+)
+
+func (store *TarantoolStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+
+	var operations = []crud.Operation{
+		{
+			Operator: crud.Insert,
+			Field:    "value",
+			Value:    string(value),
+		},
+	}
+
+	req := crud.MakeUpsertRequest(tarantoolKVSpaceName).
+		Tuple([]interface{}{string(key), nil, string(value)}).
+		Operations(operations)
+
+	ret := crud.Result{}
+	if err := store.pool.Do(req, pool.RW).GetTyped(&ret); err != nil {
+		return fmt.Errorf("kv put: %v", err)
+	}
+
+	return nil
+}
+
+func (store *TarantoolStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+
+	getOpts := crud.GetOpts{
+		Fields:        crud.MakeOptTuple([]interface{}{"value"}),
+		Mode:          crud.MakeOptString("read"),
+		PreferReplica: crud.MakeOptBool(true),
+		Balance:       crud.MakeOptBool(true),
+	}
+
+	req := crud.MakeGetRequest(tarantoolKVSpaceName).
+		Key(crud.Tuple([]interface{}{string(key)})).
+		Opts(getOpts)
+
+	resp := crud.Result{}
+
+	err = store.pool.Do(req, pool.PreferRO).GetTyped(&resp)
+	if err != nil {
+		return nil, err
+	}
+
+	results, ok := resp.Rows.([]interface{})
+	if !ok || len(results) != 1 {
+		return nil, filer.ErrKvNotFound
+	}
+
+	rows, ok := results[0].([]interface{})
+	if !ok || len(rows) != 1 {
+		return nil, filer.ErrKvNotFound
+	}
+
+	row, ok := rows[0].(string)
+	if !ok {
+		return nil, fmt.Errorf("Can't convert rows[0] field to string. Actual type: %v, value: %v", reflect.TypeOf(rows[0]), rows[0])
+	}
+
+	return []byte(row), nil
+}
+
+func (store *TarantoolStore) KvDelete(ctx context.Context, key []byte) (err error) {
+
+	delOpts := crud.DeleteOpts{
+		Noreturn: crud.MakeOptBool(true),
+	}
+
+	req := crud.MakeDeleteRequest(tarantoolKVSpaceName).
+		Key(crud.Tuple([]interface{}{string(key)})).
+		Opts(delOpts)
+
+	if _, err := store.pool.Do(req, pool.RW).Get(); err != nil {
+		return fmt.Errorf("kv delete: %v", err)
+	}
+
+	return nil
+}

weed/filer/tarantool/tarantool_store_test.go

@@ -0,0 +1,24 @@
+//go:build tarantool
+// +build tarantool
+
+package tarantool
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer/store_test"
+)
+
+func TestStore(t *testing.T) {
+	// run "make test_tarantool" under docker folder.
+	// to set up local env
+	if os.Getenv("RUN_TARANTOOL_TESTS") != "1" {
+		t.Skip("Tarantool tests are disabled. Set RUN_TARANTOOL_TESTS=1 to enable.")
+	}
+	store := &TarantoolStore{}
+	addresses := []string{"127.0.1:3303"}
+	store.initialize(addresses, "client", "client", 5*time.Second, 1000)
+	store_test.TestFilerStore(t, store)
+}

weed/mount/filehandle_map.go

@@ -1,9 +1,10 @@
 package mount
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"sync"
 
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 )
 
@@ -67,22 +68,26 @@ func (i *FileHandleToInode) ReleaseByInode(inode uint64) {
 		}
 	}
 }
+
 func (i *FileHandleToInode) ReleaseByHandle(fh FileHandleId) {
 	i.Lock()
 	defer i.Unlock()
-	inode, found := i.fh2inode[fh]
-	if found {
-		fhHandle, fhFound := i.inode2fh[inode]
-		if !fhFound {
-			delete(i.fh2inode, fh)
-		} else {
-			fhHandle.counter--
-			if fhHandle.counter <= 0 {
-				delete(i.inode2fh, inode)
-				delete(i.fh2inode, fhHandle.fh)
-				fhHandle.ReleaseHandle()
-			}
-		}
 
+	inode, found := i.fh2inode[fh]
+	if !found {
+		return // Handle already released or invalid
+	}
+
+	fhHandle, fhFound := i.inode2fh[inode]
+	if !fhFound {
+		delete(i.fh2inode, fh)
+		return
+	}
+
+	fhHandle.counter--
+	if fhHandle.counter <= 0 {
+		delete(i.inode2fh, inode)
+		delete(i.fh2inode, fhHandle.fh)
+		fhHandle.ReleaseHandle()
 	}
 }

weed/mount/filehandle_read.go

@@ -89,7 +89,7 @@ func (fh *FileHandle) downloadRemoteEntry(entry *LockedEntry) error {
 			return fmt.Errorf("CacheRemoteObjectToLocalCluster file %s: %v", fileFullPath, err)
 		}
 
-		entry.SetEntry(resp.Entry)
+		fh.SetEntry(resp.Entry)
 
 		fh.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, resp.Entry))
 

weed/mount/meta_cache/meta_cache.go

@@ -119,7 +119,7 @@ func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *fi
 	if err != nil {
 		return nil, err
 	}
-	if entry.TtlSec > 0 && entry.Crtime.Add(time.Duration(entry.TtlSec)).Before(time.Now()) {
+	if entry.TtlSec > 0 && entry.Crtime.Add(time.Duration(entry.TtlSec)*time.Second).Before(time.Now()) {
 		return nil, filer_pb.ErrNotFound
 	}
 	mc.mapIdFromFilerToLocal(entry)
@@ -147,7 +147,7 @@ func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.Full
 	}
 
 	_, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *filer.Entry) bool {
-		if entry.TtlSec > 0 && entry.Crtime.Add(time.Duration(entry.TtlSec)).Before(time.Now()) {
+		if entry.TtlSec > 0 && entry.Crtime.Add(time.Duration(entry.TtlSec)*time.Second).Before(time.Now()) {
 			return true
 		}
 		mc.mapIdFromFilerToLocal(entry)

weed/mount/page_writer/page_chunk_swapfile.go

@@ -140,7 +140,7 @@ func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop in
 			}
 			maxStop = max(maxStop, logicStop)
 
-			if t.TsNs >= tsNs {
+			if t.TsNs > tsNs {
 				println("read new data2", t.TsNs-tsNs, "ns")
 			}
 		}

weed/mount/weedfs.go

@@ -224,6 +224,12 @@ func (wfs *WFS) getCurrentFiler() pb.ServerAddress {
 	return wfs.option.FilerAddresses[i]
 }
 
+func (wfs *WFS) ClearCacheDir() {
+	wfs.metaCache.Shutdown()
+	os.RemoveAll(wfs.option.getUniqueCacheDirForWrite())
+	os.RemoveAll(wfs.option.getUniqueCacheDirForRead())
+}
+
 func (option *Option) setupUniqueCacheDirectory() {
 	cacheUniqueId := util.Md5String([]byte(option.MountDirectory + string(option.FilerAddresses[0]) + option.FilerMountRootPath + util.Version()))[0:8]
 	option.uniqueCacheDirForRead = path.Join(option.CacheDirForRead, cacheUniqueId)

weed/mount/weedfs_file_sync.go

@@ -54,7 +54,9 @@ import (
 func (wfs *WFS) Flush(cancel <-chan struct{}, in *fuse.FlushIn) fuse.Status {
 	fh := wfs.GetHandle(FileHandleId(in.Fh))
 	if fh == nil {
-		return fuse.ENOENT
+		// If handle is not found, it might have been already released
+		// This is not an error condition for FLUSH
+		return fuse.OK
 	}
 
 	return wfs.doFlush(fh, in.Uid, in.Gid)

weed/mq/agent/agent_grpc_pub_session.go

@@ -7,13 +7,12 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pb"
 	"log/slog"
 	"math/rand/v2"
-	"time"
 )
 
 func (a *MessageQueueAgent) StartPublishSession(ctx context.Context, req *mq_agent_pb.StartPublishSessionRequest) (*mq_agent_pb.StartPublishSessionResponse, error) {
 	sessionId := rand.Int64()
 
-	topicPublisher := pub_client.NewTopicPublisher(
+	topicPublisher, err := pub_client.NewTopicPublisher(
 		&pub_client.PublisherConfiguration{
 			Topic:          topic.NewTopic(req.Topic.Namespace, req.Topic.Name),
 			PartitionCount: req.PartitionCount,
@@ -21,18 +20,11 @@ func (a *MessageQueueAgent) StartPublishSession(ctx context.Context, req *mq_age
 			PublisherName:  req.PublisherName,
 			RecordType:     req.RecordType,
 		})
+	if err != nil {
+		return nil, err
+	}
 
 	a.publishersLock.Lock()
-	// remove inactive publishers to avoid memory leak
-	for k, entry := range a.publishers {
-		if entry.lastActiveTsNs == 0 {
-			// this is an active session
-			continue
-		}
-		if time.Unix(0, entry.lastActiveTsNs).Add(10 * time.Hour).Before(time.Now()) {
-			delete(a.publishers, k)
-		}
-	}
 	a.publishers[SessionId(sessionId)] = &SessionEntry[*pub_client.TopicPublisher]{
 		entry: topicPublisher,
 	}

weed/mq/agent/agent_grpc_publish.go

@@ -3,24 +3,25 @@ package agent
 import (
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pb"
-	"time"
 )
 
-func (a *MessageQueueAgent) PublishRecordRequest(stream mq_agent_pb.SeaweedMessagingAgent_PublishRecordServer) error {
+func (a *MessageQueueAgent) PublishRecord(stream mq_agent_pb.SeaweedMessagingAgent_PublishRecordServer) error {
 	m, err := stream.Recv()
 	if err != nil {
 		return err
 	}
+	sessionId := SessionId(m.SessionId)
 	a.publishersLock.RLock()
-	publisherEntry, found := a.publishers[SessionId(m.SessionId)]
+	publisherEntry, found := a.publishers[sessionId]
 	a.publishersLock.RUnlock()
 	if !found {
-		return fmt.Errorf("publish session id %d not found", m.SessionId)
+		return fmt.Errorf("publish session id %d not found", sessionId)
 	}
 	defer func() {
-		publisherEntry.lastActiveTsNs = time.Now().UnixNano()
+		a.publishersLock.Lock()
+		delete(a.publishers, sessionId)
+		a.publishersLock.Unlock()
 	}()
-	publisherEntry.lastActiveTsNs = 0
 
 	if m.Value != nil {
 		if err := publisherEntry.entry.PublishRecord(m.Key, m.Value); err != nil {
@@ -29,7 +30,7 @@ func (a *MessageQueueAgent) PublishRecordRequest(stream mq_agent_pb.SeaweedMessa
 	}
 
 	for {
-		m, err := stream.Recv()
+		m, err = stream.Recv()
 		if err != nil {
 			return err
 		}

weed/mq/agent/agent_grpc_sub_session.go

@@ -1,57 +0,0 @@
-package agent
-
-import (
-	"context"
-	"github.com/seaweedfs/seaweedfs/weed/mq/client/sub_client"
-	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
-	"github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pb"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
-	"math/rand/v2"
-	"time"
-)
-
-func (a *MessageQueueAgent) StartSubscribeSession(ctx context.Context, req *mq_agent_pb.StartSubscribeSessionRequest) (*mq_agent_pb.StartSubscribeSessionResponse, error) {
-	sessionId := rand.Int64()
-
-	subscriberConfig := &sub_client.SubscriberConfiguration{
-		ConsumerGroup:           req.ConsumerGroup,
-		ConsumerGroupInstanceId: req.ConsumerGroupInstanceId,
-		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
-		MaxPartitionCount:       req.MaxSubscribedPartitions,
-		SlidingWindowSize:       req.SlidingWindowSize,
-	}
-
-	contentConfig := &sub_client.ContentConfiguration{
-		Topic:            topic.FromPbTopic(req.Topic),
-		Filter:           req.Filter,
-		PartitionOffsets: req.PartitionOffsets,
-	}
-
-	topicSubscriber := sub_client.NewTopicSubscriber(
-		a.brokersList(),
-		subscriberConfig,
-		contentConfig,
-		make(chan sub_client.KeyedOffset, 1024),
-	)
-
-	a.subscribersLock.Lock()
-	// remove inactive publishers to avoid memory leak
-	for k, entry := range a.subscribers {
-		if entry.lastActiveTsNs == 0 {
-			// this is an active session
-			continue
-		}
-		if time.Unix(0, entry.lastActiveTsNs).Add(10 * time.Hour).Before(time.Now()) {
-			delete(a.subscribers, k)
-		}
-	}
-	a.subscribers[SessionId(sessionId)] = &SessionEntry[*sub_client.TopicSubscriber]{
-		entry: topicSubscriber,
-	}
-	a.subscribersLock.Unlock()
-
-	return &mq_agent_pb.StartSubscribeSessionResponse{
-		SessionId: sessionId,
-	}, nil
-}

weed/mq/agent/agent_grpc_subscribe.go

@@ -1,59 +1,59 @@
 package agent
 
 import (
-	"fmt"
+	"context"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/client/sub_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/protobuf/proto"
-	"time"
 )
 
-func (a *MessageQueueAgent) SubscribeRecordRequest(stream mq_agent_pb.SeaweedMessagingAgent_SubscribeRecordServer) error {
+func (a *MessageQueueAgent) SubscribeRecord(stream mq_agent_pb.SeaweedMessagingAgent_SubscribeRecordServer) error {
 	// the first message is the subscribe request
 	// it should only contain the session id
-	m, err := stream.Recv()
+	initMessage, err := stream.Recv()
 	if err != nil {
 		return err
 	}
-	a.subscribersLock.RLock()
-	subscriberEntry, found := a.subscribers[SessionId(m.SessionId)]
-	a.subscribersLock.RUnlock()
-	if !found {
-		return fmt.Errorf("subscribe session id %d not found", m.SessionId)
-	}
-	defer func() {
-		subscriberEntry.lastActiveTsNs = time.Now().UnixNano()
-	}()
-	subscriberEntry.lastActiveTsNs = 0
+
+	subscriber := a.handleInitSubscribeRecordRequest(stream.Context(), initMessage.Init)
 
 	var lastErr error
-	subscriberEntry.entry.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
-		record := &schema_pb.RecordValue{}
-		err := proto.Unmarshal(m.Data.Value, record)
-		if err != nil {
-			if lastErr == nil {
-				lastErr = err
+	executors := util.NewLimitedConcurrentExecutor(int(subscriber.SubscriberConfig.SlidingWindowSize))
+	subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
+		executors.Execute(func() {
+			record := &schema_pb.RecordValue{}
+			err := proto.Unmarshal(m.Data.Value, record)
+			if err != nil {
+				glog.V(0).Infof("unmarshal record value: %v", err)
+				if lastErr == nil {
+					lastErr = err
+				}
+				return
 			}
-			return
-		}
-		if sendErr := stream.Send(&mq_agent_pb.SubscribeRecordResponse{
-			Key:   m.Data.Key,
-			Value: record,
-			TsNs:  m.Data.TsNs,
-		}); sendErr != nil {
-			if lastErr == nil {
-				lastErr = sendErr
+			if sendErr := stream.Send(&mq_agent_pb.SubscribeRecordResponse{
+				Key:   m.Data.Key,
+				Value: record,
+				TsNs:  m.Data.TsNs,
+			}); sendErr != nil {
+				glog.V(0).Infof("send record: %v", sendErr)
+				if lastErr == nil {
+					lastErr = sendErr
+				}
 			}
-		}
+		})
 	})
 
 	go func() {
-		subErr := subscriberEntry.entry.Subscribe()
+		subErr := subscriber.Subscribe()
 		if subErr != nil {
-			glog.V(0).Infof("subscriber %d subscribe: %v", m.SessionId, subErr)
+			glog.V(0).Infof("subscriber %s subscribe: %v", subscriber.SubscriberConfig.String(), subErr)
 			if lastErr == nil {
 				lastErr = subErr
 			}
@@ -63,13 +63,43 @@ func (a *MessageQueueAgent) SubscribeRecordRequest(stream mq_agent_pb.SeaweedMes
 	for {
 		m, err := stream.Recv()
 		if err != nil {
+			glog.V(0).Infof("subscriber %s receive: %v", subscriber.SubscriberConfig.String(), err)
 			return err
 		}
 		if m != nil {
-			subscriberEntry.entry.PartitionOffsetChan <- sub_client.KeyedOffset{
+			subscriber.PartitionOffsetChan <- sub_client.KeyedOffset{
 				Key:    m.AckKey,
 				Offset: m.AckSequence,
 			}
 		}
 	}
 }
+
+func (a *MessageQueueAgent) handleInitSubscribeRecordRequest(ctx context.Context, req *mq_agent_pb.SubscribeRecordRequest_InitSubscribeRecordRequest) *sub_client.TopicSubscriber {
+
+	subscriberConfig := &sub_client.SubscriberConfiguration{
+		ConsumerGroup:           req.ConsumerGroup,
+		ConsumerGroupInstanceId: req.ConsumerGroupInstanceId,
+		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
+		MaxPartitionCount:       req.MaxSubscribedPartitions,
+		SlidingWindowSize:       req.SlidingWindowSize,
+	}
+
+	contentConfig := &sub_client.ContentConfiguration{
+		Topic:            topic.FromPbTopic(req.Topic),
+		Filter:           req.Filter,
+		PartitionOffsets: req.PartitionOffsets,
+		OffsetType:       req.OffsetType,
+		OffsetTsNs:       req.OffsetTsNs,
+	}
+
+	topicSubscriber := sub_client.NewTopicSubscriber(
+		ctx,
+		a.brokersList(),
+		subscriberConfig,
+		contentConfig,
+		make(chan sub_client.KeyedOffset, 1024),
+	)
+
+	return topicSubscriber
+}

weed/mq/agent/agent_server.go

@@ -11,8 +11,7 @@ import (
 
 type SessionId int64
 type SessionEntry[T any] struct {
-	entry          T
-	lastActiveTsNs int64
+	entry T
 }
 
 type MessageQueueAgentOptions struct {
@@ -32,11 +31,15 @@ type MessageQueueAgent struct {
 
 func NewMessageQueueAgent(option *MessageQueueAgentOptions, grpcDialOption grpc.DialOption) *MessageQueueAgent {
 
-	// check masters to list all brokers
+	// initialize brokers which may change later
+	var brokers []pb.ServerAddress
+	for _, broker := range option.SeedBrokers {
+		brokers = append(brokers, broker)
+	}
 
 	return &MessageQueueAgent{
 		option:         option,
-		brokers:        []pb.ServerAddress{},
+		brokers:        brokers,
 		grpcDialOption: grpcDialOption,
 		publishers:     make(map[SessionId]*SessionEntry[*pub_client.TopicPublisher]),
 		subscribers:    make(map[SessionId]*SessionEntry[*sub_client.TopicSubscriber]),

weed/mq/broker.go

@@ -1,12 +0,0 @@
-package mq
-
-const LAST_MINUTES = 10
-
-type TopicStat struct {
-	MessageCounts [LAST_MINUTES]int64
-	ByteCounts    [LAST_MINUTES]int64
-}
-
-func NewTopicStat() *TopicStat {
-	return &TopicStat{}
-}

weed/mq/broker/broker_grpc_configure.go

@@ -3,6 +3,7 @@ package broker
 import (
 	"context"
 	"fmt"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
@@ -57,7 +58,7 @@ func (b *MessageQueueBroker) ConfigureTopic(ctx context.Context, request *mq_pb.
 	}
 	resp = &mq_pb.ConfigureTopicResponse{}
 	if b.PubBalancer.Brokers.IsEmpty() {
-		return nil, status.Errorf(codes.Unavailable, pub_balancer.ErrNoBroker.Error())
+		return nil, status.Errorf(codes.Unavailable, "no broker available: %v", pub_balancer.ErrNoBroker)
 	}
 	resp.BrokerPartitionAssignments = pub_balancer.AllocateTopicPartitions(b.PubBalancer.Brokers, request.PartitionCount)
 	resp.RecordType = request.RecordType

weed/mq/broker/broker_grpc_sub.go

@@ -2,6 +2,7 @@ package broker
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/sub_coordinator"
@@ -67,7 +68,9 @@ func (b *MessageQueueBroker) SubscribeMessage(stream mq_pb.SeaweedMessaging_Subs
 		} else {
 			defer func() {
 				println("closing SubscribeFollowMe connection", follower)
-				subscribeFollowMeStream.CloseSend()
+				if subscribeFollowMeStream != nil {
+					subscribeFollowMeStream.CloseSend()
+				}
 				// followerGrpcConnection.Close()
 			}()
 			followerClient := mq_pb.NewSeaweedMessagingClient(followerGrpcConnection)
@@ -141,7 +144,9 @@ func (b *MessageQueueBroker) SubscribeMessage(stream mq_pb.SeaweedMessaging_Subs
 					Close: &mq_pb.SubscribeFollowMeRequest_CloseMessage{},
 				},
 			}); err != nil {
-				glog.Errorf("Error sending close to follower: %v", err)
+				if err != io.EOF {
+					glog.Errorf("Error sending close to follower: %v", err)
+				}
 			}
 		}
 	}()
@@ -160,7 +165,7 @@ func (b *MessageQueueBroker) SubscribeMessage(stream mq_pb.SeaweedMessaging_Subs
 		select {
 		case <-ctx.Done():
 			err := ctx.Err()
-			if err == context.Canceled {
+			if errors.Is(err, context.Canceled) {
 				// Client disconnected
 				return false
 			}
@@ -177,6 +182,19 @@ func (b *MessageQueueBroker) SubscribeMessage(stream mq_pb.SeaweedMessaging_Subs
 
 		for imt.IsInflight(logEntry.Key) {
 			time.Sleep(137 * time.Millisecond)
+			// Check if the client has disconnected by monitoring the context
+			select {
+			case <-ctx.Done():
+				err := ctx.Err()
+				if err == context.Canceled {
+					// Client disconnected
+					return false, nil
+				}
+				glog.V(0).Infof("Subscriber %s disconnected: %v", clientName, err)
+				return false, nil
+			default:
+				// Continue processing the request
+			}
 		}
 		if logEntry.Key != nil {
 			imt.EnflightMessage(logEntry.Key, logEntry.TsNs)
@@ -203,20 +221,35 @@ func (b *MessageQueueBroker) getRequestPosition(initMessage *mq_pb.SubscribeMess
 		return
 	}
 	offset := initMessage.GetPartitionOffset()
-	if offset.StartTsNs != 0 {
+	offsetType := initMessage.OffsetType
+
+	// reset to earliest or latest
+	if offsetType == schema_pb.OffsetType_RESET_TO_EARLIEST {
+		startPosition = log_buffer.NewMessagePosition(1, -3)
+		return
+	}
+	if offsetType == schema_pb.OffsetType_RESET_TO_LATEST {
+		startPosition = log_buffer.NewMessagePosition(time.Now().UnixNano(), -4)
+		return
+	}
+
+	// use the exact timestamp
+	if offsetType == schema_pb.OffsetType_EXACT_TS_NS {
 		startPosition = log_buffer.NewMessagePosition(offset.StartTsNs, -2)
 		return
 	}
+
+	// try to resume
 	if storedOffset, err := b.readConsumerGroupOffset(initMessage); err == nil {
 		glog.V(0).Infof("resume from saved offset %v %v %v: %v", initMessage.Topic, initMessage.PartitionOffset.Partition, initMessage.ConsumerGroup, storedOffset)
 		startPosition = log_buffer.NewMessagePosition(storedOffset, -2)
 		return
 	}
 
-	if offset.StartType == schema_pb.PartitionOffsetStartType_EARLIEST {
-		startPosition = log_buffer.NewMessagePosition(1, -3)
-	} else if offset.StartType == schema_pb.PartitionOffsetStartType_LATEST {
-		startPosition = log_buffer.NewMessagePosition(time.Now().UnixNano(), -4)
+	if offsetType == schema_pb.OffsetType_RESUME_OR_EARLIEST {
+		startPosition = log_buffer.NewMessagePosition(1, -5)
+	} else if offsetType == schema_pb.OffsetType_RESUME_OR_LATEST {
+		startPosition = log_buffer.NewMessagePosition(time.Now().UnixNano(), -6)
 	}
 	return
 }

weed/mq/broker/broker_grpc_sub_coordinator.go

@@ -2,6 +2,7 @@ package broker
 
 import (
 	"context"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/sub_coordinator"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
@@ -26,7 +27,7 @@ func (b *MessageQueueBroker) SubscriberToSubCoordinator(stream mq_pb.SeaweedMess
 	if initMessage != nil {
 		cg, cgi, err = b.SubCoordinator.AddSubscriber(initMessage)
 		if err != nil {
-			return status.Errorf(codes.InvalidArgument, err.Error())
+			return status.Errorf(codes.InvalidArgument, "failed to add subscriber: %v", err)
 		}
 		glog.V(0).Infof("subscriber %s/%s/%s connected", initMessage.ConsumerGroup, initMessage.ConsumerGroupInstanceId, initMessage.Topic)
 	} else {

weed/mq/client/agent_client/agent_publish.go

@@ -1,14 +0,0 @@
-package agent_client
-
-import (
-	"github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
-)
-
-func (a *PublishSession) PublishMessageRecord(key []byte, record *schema_pb.RecordValue) error {
-	return a.stream.Send(&mq_agent_pb.PublishRecordRequest{
-		SessionId: a.sessionId,
-		Key:       key,
-		Value:     record,
-	})
-}

weed/mq/client/agent_client/agent_subscribe.go

@@ -1,17 +0,0 @@
-package agent_client
-
-import (
-	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
-)
-
-func (a *SubscribeSession) SubscribeMessageRecord(
-	onEachMessageFn func(key []byte, record *schema_pb.RecordValue),
-	onCompletionFn func()) error {
-	for {
-		resp, err := a.stream.Recv()
-		if err != nil {
-			return err
-		}
-		onEachMessageFn(resp.Key, resp.Value)
-	}
-}

weed/mq/client/agent_client/publish_session.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 type PublishSession struct {
@@ -15,13 +15,12 @@ type PublishSession struct {
 	partitionCount int
 	publisherName  string
 	stream         grpc.BidiStreamingClient[mq_agent_pb.PublishRecordRequest, mq_agent_pb.PublishRecordResponse]
-	sessionId      int64
 }
 
 func NewPublishSession(agentAddress string, topicSchema *schema.Schema, partitionCount int, publisherName string) (*PublishSession, error) {
 
 	// call local agent grpc server to create a new session
-	clientConn, err := pb.GrpcDial(context.Background(), agentAddress, true, grpc.WithInsecure())
+	clientConn, err := grpc.NewClient(agentAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		return nil, fmt.Errorf("dial agent server %s: %v", agentAddress, err)
 	}
@@ -48,12 +47,17 @@ func NewPublishSession(agentAddress string, topicSchema *schema.Schema, partitio
 		return nil, fmt.Errorf("publish record: %v", err)
 	}
 
+	if err = stream.Send(&mq_agent_pb.PublishRecordRequest{
+		SessionId: resp.SessionId,
+	}); err != nil {
+		return nil, fmt.Errorf("send session id: %v", err)
+	}
+
 	return &PublishSession{
 		schema:         topicSchema,
 		partitionCount: partitionCount,
 		publisherName:  publisherName,
 		stream:         stream,
-		sessionId:      resp.SessionId,
 	}, nil
 }
 
@@ -68,3 +72,10 @@ func (a *PublishSession) CloseSession() error {
 	a.schema = nil
 	return err
 }
+
+func (a *PublishSession) PublishMessageRecord(key []byte, record *schema_pb.RecordValue) error {
+	return a.stream.Send(&mq_agent_pb.PublishRecordRequest{
+		Key:   key,
+		Value: record,
+	})
+}

weed/mq/client/agent_client/subscribe_session.go

@@ -4,50 +4,48 @@ import (
 	"context"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 type SubscribeOption struct {
 	ConsumerGroup           string
 	ConsumerGroupInstanceId string
 	Topic                   topic.Topic
+	OffsetType              schema_pb.OffsetType
+	OffsetTsNs              int64
 	Filter                  string
 	MaxSubscribedPartitions int32
-	PerPartitionConcurrency int32
+	SlidingWindowSize       int32
 }
 
 type SubscribeSession struct {
-	Option    *SubscribeOption
-	stream    grpc.BidiStreamingClient[mq_agent_pb.SubscribeRecordRequest, mq_agent_pb.SubscribeRecordResponse]
-	sessionId int64
+	Option *SubscribeOption
+	stream grpc.BidiStreamingClient[mq_agent_pb.SubscribeRecordRequest, mq_agent_pb.SubscribeRecordResponse]
 }
 
 func NewSubscribeSession(agentAddress string, option *SubscribeOption) (*SubscribeSession, error) {
 	// call local agent grpc server to create a new session
-	clientConn, err := pb.GrpcDial(context.Background(), agentAddress, true, grpc.WithInsecure())
+	clientConn, err := grpc.NewClient(agentAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		return nil, fmt.Errorf("dial agent server %s: %v", agentAddress, err)
 	}
 	agentClient := mq_agent_pb.NewSeaweedMessagingAgentClient(clientConn)
 
-	resp, err := agentClient.StartSubscribeSession(context.Background(), &mq_agent_pb.StartSubscribeSessionRequest{
+	initRequest := &mq_agent_pb.SubscribeRecordRequest_InitSubscribeRecordRequest{
 		ConsumerGroup:           option.ConsumerGroup,
 		ConsumerGroupInstanceId: option.ConsumerGroupInstanceId,
 		Topic: &schema_pb.Topic{
 			Namespace: option.Topic.Namespace,
 			Name:      option.Topic.Name,
 		},
+		OffsetType:              option.OffsetType,
+		OffsetTsNs:              option.OffsetTsNs,
 		MaxSubscribedPartitions: option.MaxSubscribedPartitions,
 		Filter:                  option.Filter,
-	})
-	if err != nil {
-		return nil, err
-	}
-	if resp.Error != "" {
-		return nil, fmt.Errorf("start subscribe session: %v", resp.Error)
+		SlidingWindowSize:       option.SlidingWindowSize,
 	}
 
 	stream, err := agentClient.SubscribeRecord(context.Background())
@@ -55,9 +53,35 @@ func NewSubscribeSession(agentAddress string, option *SubscribeOption) (*Subscri
 		return nil, fmt.Errorf("subscribe record: %v", err)
 	}
 
+	if err = stream.Send(&mq_agent_pb.SubscribeRecordRequest{
+		Init: initRequest,
+	}); err != nil {
+		return nil, fmt.Errorf("send session id: %v", err)
+	}
+
 	return &SubscribeSession{
-		Option:    option,
-		stream:    stream,
-		sessionId: resp.SessionId,
+		Option: option,
+		stream: stream,
 	}, nil
 }
+
+func (s *SubscribeSession) CloseSession() error {
+	err := s.stream.CloseSend()
+	return err
+}
+
+func (a *SubscribeSession) SubscribeMessageRecord(
+	onEachMessageFn func(key []byte, record *schema_pb.RecordValue),
+	onCompletionFn func()) error {
+	for {
+		resp, err := a.stream.Recv()
+		if err != nil {
+			return err
+		}
+		onEachMessageFn(resp.Key, resp.Value)
+	}
+	if onCompletionFn != nil {
+		onCompletionFn()
+	}
+	return nil
+}

weed/mq/client/cmd/weed_pub_kv/publisher_kv.go

@@ -1,78 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
-	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
-	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
-	"log"
-	"strings"
-	"sync"
-	"time"
-)
-
-var (
-	messageCount   = flag.Int("n", 1000, "message count")
-	concurrency    = flag.Int("c", 4, "concurrent publishers")
-	partitionCount = flag.Int("p", 6, "partition count")
-
-	clientName = flag.String("client", "c1", "client name")
-
-	namespace   = flag.String("ns", "test", "namespace")
-	t           = flag.String("t", "test", "t")
-	seedBrokers = flag.String("brokers", "localhost:17777", "seed brokers")
-)
-
-func doPublish(publisher *pub_client.TopicPublisher, id int) {
-	startTime := time.Now()
-	for i := 0; i < *messageCount / *concurrency; i++ {
-		// Simulate publishing a message
-		key := []byte(fmt.Sprintf("key-%s-%d-%d", *clientName, id, i))
-		value := []byte(fmt.Sprintf("value-%s-%d-%d", *clientName, id, i))
-		if err := publisher.Publish(key, value); err != nil {
-			fmt.Println(err)
-			break
-		}
-		time.Sleep(time.Second)
-		// println("Published", string(key), string(value))
-	}
-	if err := publisher.FinishPublish(); err != nil {
-		fmt.Println(err)
-	}
-	elapsed := time.Since(startTime)
-	log.Printf("Publisher %s-%d finished in %s", *clientName, id, elapsed)
-}
-
-func main() {
-	flag.Parse()
-	util_http.InitGlobalHttpClient()
-
-	config := &pub_client.PublisherConfiguration{
-		Topic:          topic.NewTopic(*namespace, *t),
-		PartitionCount: int32(*partitionCount),
-		Brokers:        strings.Split(*seedBrokers, ","),
-		PublisherName:  *clientName,
-	}
-	publisher := pub_client.NewTopicPublisher(config)
-
-	startTime := time.Now()
-
-	var wg sync.WaitGroup
-	// Start multiple publishers
-	for i := 0; i < *concurrency; i++ {
-		wg.Add(1)
-		go func(id int) {
-			defer wg.Done()
-			doPublish(publisher, id)
-		}(i)
-	}
-
-	// Wait for all publishers to finish
-	wg.Wait()
-	elapsed := time.Since(startTime)
-	publisher.Shutdown()
-
-	log.Printf("Published %d messages in %s (%.2f msg/s)", *messageCount, elapsed, float64(*messageCount)/elapsed.Seconds())
-
-}

weed/mq/client/cmd/weed_pub_record/publisher_record.go

@@ -1,138 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
-	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
-	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
-	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
-	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
-	"log"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-)
-
-var (
-	messageCount   = flag.Int("n", 1000, "message count")
-	messageDelay   = flag.Duration("d", time.Second, "delay between messages")
-	concurrency    = flag.Int("c", 4, "concurrent publishers")
-	partitionCount = flag.Int("p", 6, "partition count")
-
-	clientName = flag.String("client", "c1", "client name")
-
-	namespace   = flag.String("ns", "test", "namespace")
-	t           = flag.String("t", "test", "t")
-	seedBrokers = flag.String("brokers", "localhost:17777", "seed brokers")
-
-	counter int32
-)
-
-func doPublish(publisher *pub_client.TopicPublisher, id int) {
-	startTime := time.Now()
-	for {
-		i := atomic.AddInt32(&counter, 1)
-		if i > int32(*messageCount) {
-			break
-		}
-		// Simulate publishing a message
-		myRecord := genMyRecord(int32(i))
-		if err := publisher.PublishRecord(myRecord.Key, myRecord.ToRecordValue()); err != nil {
-			fmt.Println(err)
-			break
-		}
-		if *messageDelay > 0 {
-			time.Sleep(*messageDelay)
-			fmt.Printf("sent %+v\n", string(myRecord.Key))
-		}
-	}
-	if err := publisher.FinishPublish(); err != nil {
-		fmt.Println(err)
-	}
-	elapsed := time.Since(startTime)
-	log.Printf("Publisher %s-%d finished in %s", *clientName, id, elapsed)
-}
-
-type MyRecord struct {
-	Key    []byte
-	Field1 []byte
-	Field2 string
-	Field3 int32
-	Field4 int64
-	Field5 float32
-	Field6 float64
-	Field7 bool
-}
-
-func genMyRecord(id int32) *MyRecord {
-	return &MyRecord{
-		Key:    []byte(fmt.Sprintf("key-%s-%d", *clientName, id)),
-		Field1: []byte(fmt.Sprintf("field1-%s-%d", *clientName, id)),
-		Field2: fmt.Sprintf("field2-%s-%d", *clientName, id),
-		Field3: id,
-		Field4: int64(id),
-		Field5: float32(id),
-		Field6: float64(id),
-		Field7: id%2 == 0,
-	}
-}
-
-func (r *MyRecord) ToRecordValue() *schema_pb.RecordValue {
-	return schema.RecordBegin().
-		SetBytes("key", r.Key).
-		SetBytes("field1", r.Field1).
-		SetString("field2", r.Field2).
-		SetInt32("field3", r.Field3).
-		SetInt64("field4", r.Field4).
-		SetFloat("field5", r.Field5).
-		SetDouble("field6", r.Field6).
-		SetBool("field7", r.Field7).
-		RecordEnd()
-}
-
-func main() {
-	flag.Parse()
-	util_http.InitGlobalHttpClient()
-
-	recordType := schema.RecordTypeBegin().
-		WithField("key", schema.TypeBytes).
-		WithField("field1", schema.TypeBytes).
-		WithField("field2", schema.TypeString).
-		WithField("field3", schema.TypeInt32).
-		WithField("field4", schema.TypeInt64).
-		WithField("field5", schema.TypeFloat).
-		WithField("field6", schema.TypeDouble).
-		WithField("field7", schema.TypeBoolean).
-		RecordTypeEnd()
-
-	config := &pub_client.PublisherConfiguration{
-		Topic:          topic.NewTopic(*namespace, *t),
-		PartitionCount: int32(*partitionCount),
-		Brokers:        strings.Split(*seedBrokers, ","),
-		PublisherName:  *clientName,
-		RecordType:     recordType,
-	}
-	publisher := pub_client.NewTopicPublisher(config)
-
-	startTime := time.Now()
-
-	var wg sync.WaitGroup
-	// Start multiple publishers
-	for i := 0; i < *concurrency; i++ {
-		wg.Add(1)
-		go func(id int) {
-			defer wg.Done()
-			doPublish(publisher, id)
-		}(i)
-	}
-
-	// Wait for all publishers to finish
-	wg.Wait()
-	elapsed := time.Since(startTime)
-	publisher.Shutdown()
-
-	log.Printf("Published %d messages in %s (%.2f msg/s)", *messageCount, elapsed, float64(*messageCount)/elapsed.Seconds())
-
-}

weed/mq/client/cmd/weed_sub_kv/subscriber_kv.go

@@ -1,64 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/mq/client/sub_client"
-	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
-	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
-	"strings"
-)
-
-var (
-	namespace               = flag.String("ns", "test", "namespace")
-	t                       = flag.String("topic", "test", "topic")
-	seedBrokers             = flag.String("brokers", "localhost:17777", "seed brokers")
-	maxPartitionCount       = flag.Int("maxPartitionCount", 3, "max partition count")
-	perPartitionConcurrency = flag.Int("perPartitionConcurrency", 1, "per partition concurrency")
-
-	clientId = flag.Uint("client_id", uint(util.RandomInt32()), "client id")
-)
-
-func main() {
-	flag.Parse()
-	util_http.InitGlobalHttpClient()
-
-	subscriberConfig := &sub_client.SubscriberConfiguration{
-		ConsumerGroup:           "test",
-		ConsumerGroupInstanceId: fmt.Sprintf("client-%d", *clientId),
-		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
-		MaxPartitionCount:       int32(*maxPartitionCount),
-		SlidingWindowSize:       int32(*perPartitionConcurrency),
-	}
-
-	contentConfig := &sub_client.ContentConfiguration{
-		Topic:  topic.NewTopic(*namespace, *t),
-		Filter: "",
-	}
-
-	brokers := strings.Split(*seedBrokers, ",")
-	subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, make(chan sub_client.KeyedOffset, 1024))
-
-	counter := 0
-	executors := util.NewLimitedConcurrentExecutor(int(subscriberConfig.SlidingWindowSize))
-	subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
-		executors.Execute(func() {
-			counter++
-			println(string(m.Data.Key), "=>", string(m.Data.Value), counter)
-		})
-	})
-
-	subscriber.SetCompletionFunc(func() {
-		glog.V(0).Infof("done received %d messages", counter)
-	})
-
-	if err := subscriber.Subscribe(); err != nil {
-		fmt.Println(err)
-	}
-
-}

weed/mq/client/cmd/weed_sub_record/subscriber_record.go

@@ -1,99 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/mq/client/sub_client"
-	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
-	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
-	"google.golang.org/protobuf/proto"
-	"strings"
-	"time"
-)
-
-var (
-	namespace               = flag.String("ns", "test", "namespace")
-	t                       = flag.String("topic", "test", "topic")
-	seedBrokers             = flag.String("brokers", "localhost:17777", "seed brokers")
-	maxPartitionCount       = flag.Int("maxPartitionCount", 3, "max partition count")
-	perPartitionConcurrency = flag.Int("perPartitionConcurrency", 1, "per partition concurrency")
-	timeAgo                 = flag.Duration("timeAgo", 1*time.Hour, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
-
-	clientId = flag.Uint("client_id", uint(util.RandomInt32()), "client id")
-)
-
-type MyRecord struct {
-	Key    []byte
-	Field1 []byte
-	Field2 string
-	Field3 int32
-	Field4 int64
-	Field5 float32
-	Field6 float64
-	Field7 bool
-}
-
-func FromSchemaRecordValue(recordValue *schema_pb.RecordValue) *MyRecord {
-	return &MyRecord{
-		Key:    recordValue.Fields["key"].GetBytesValue(),
-		Field1: recordValue.Fields["field1"].GetBytesValue(),
-		Field2: recordValue.Fields["field2"].GetStringValue(),
-		Field3: recordValue.Fields["field3"].GetInt32Value(),
-		Field4: recordValue.Fields["field4"].GetInt64Value(),
-		Field5: recordValue.Fields["field5"].GetFloatValue(),
-		Field6: recordValue.Fields["field6"].GetDoubleValue(),
-		Field7: recordValue.Fields["field7"].GetBoolValue(),
-	}
-}
-
-func main() {
-	flag.Parse()
-	util_http.InitGlobalHttpClient()
-
-	subscriberConfig := &sub_client.SubscriberConfiguration{
-		ConsumerGroup:           "test",
-		ConsumerGroupInstanceId: fmt.Sprintf("client-%d", *clientId),
-		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
-		MaxPartitionCount:       int32(*maxPartitionCount),
-		SlidingWindowSize:       int32(*perPartitionConcurrency),
-	}
-
-	contentConfig := &sub_client.ContentConfiguration{
-		Topic:  topic.NewTopic(*namespace, *t),
-		Filter: "",
-		// StartTime: time.Now().Add(-*timeAgo),
-	}
-
-	brokers := strings.Split(*seedBrokers, ",")
-	subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, make(chan sub_client.KeyedOffset, 1024))
-
-	counter := 0
-	executors := util.NewLimitedConcurrentExecutor(int(subscriberConfig.SlidingWindowSize))
-	subscriber.SetOnDataMessageFn(func(m *mq_pb.SubscribeMessageResponse_Data) {
-		executors.Execute(func() {
-			counter++
-			record := &schema_pb.RecordValue{}
-			err := proto.Unmarshal(m.Data.Value, record)
-			if err != nil {
-				fmt.Printf("unmarshal record value: %v\n", err)
-			} else {
-				fmt.Printf("%s %d: %v\n", string(m.Data.Key), len(m.Data.Value), record)
-			}
-		})
-	})
-
-	subscriber.SetCompletionFunc(func() {
-		glog.V(0).Infof("done received %d messages", counter)
-	})
-
-	if err := subscriber.Subscribe(); err != nil {
-		fmt.Println(err)
-	}
-
-}

weed/mq/client/pub_client/publisher.go

@@ -34,8 +34,8 @@ type TopicPublisher struct {
 	jobs             []*EachPartitionPublishJob
 }
 
-func NewTopicPublisher(config *PublisherConfiguration) *TopicPublisher {
-	tp := &TopicPublisher{
+func NewTopicPublisher(config *PublisherConfiguration) (tp *TopicPublisher, err error) {
+	tp = &TopicPublisher{
 		partition2Buffer: interval.NewSearchTree[*buffered_queue.BufferedQueue[*mq_pb.DataMessage]](func(a, b int32) int {
 			return int(a - b)
 		}),
@@ -46,7 +46,7 @@ func NewTopicPublisher(config *PublisherConfiguration) *TopicPublisher {
 	wg := sync.WaitGroup{}
 	wg.Add(1)
 	go func() {
-		if err := tp.startSchedulerThread(&wg); err != nil {
+		if err = tp.startSchedulerThread(&wg); err != nil {
 			log.Println(err)
 			return
 		}
@@ -54,7 +54,7 @@ func NewTopicPublisher(config *PublisherConfiguration) *TopicPublisher {
 
 	wg.Wait()
 
-	return tp
+	return
 }
 
 func (p *TopicPublisher) Shutdown() error {

weed/mq/client/pub_client/scheduler.go

@@ -7,7 +7,9 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util/buffered_queue"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/status"
 	"log"
 	"sort"
@@ -33,6 +35,7 @@ type EachPartitionPublishJob struct {
 func (p *TopicPublisher) startSchedulerThread(wg *sync.WaitGroup) error {
 
 	if err := p.doConfigureTopic(); err != nil {
+		wg.Done()
 		return fmt.Errorf("configure topic %s: %v", p.config.Topic, err)
 	}
 
@@ -111,6 +114,7 @@ func (p *TopicPublisher) onEachAssignments(generation int, assignments []*mq_pb.
 		go func(job *EachPartitionPublishJob) {
 			defer job.wg.Done()
 			if err := p.doPublishToPartition(job); err != nil {
+				log.Printf("publish to %s partition %v: %v", p.config.Topic, job.Partition, err)
 				errChan <- EachPartitionError{assignment, err, generation}
 			}
 		}(job)
@@ -126,7 +130,7 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
 
 	log.Printf("connecting to %v for topic partition %+v", job.LeaderBroker, job.Partition)
 
-	grpcConnection, err := pb.GrpcDial(context.Background(), job.LeaderBroker, true, p.grpcDialOption)
+	grpcConnection, err := grpc.NewClient(job.LeaderBroker, grpc.WithTransportCredentials(insecure.NewCredentials()), p.grpcDialOption)
 	if err != nil {
 		return fmt.Errorf("dial broker %s: %v", job.LeaderBroker, err)
 	}
@@ -225,7 +229,7 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
 
 func (p *TopicPublisher) doConfigureTopic() (err error) {
 	if len(p.config.Brokers) == 0 {
-		return fmt.Errorf("no bootstrap brokers")
+		return fmt.Errorf("topic configuring found no bootstrap brokers")
 	}
 	var lastErr error
 	for _, brokerAddress := range p.config.Brokers {
@@ -256,7 +260,7 @@ func (p *TopicPublisher) doConfigureTopic() (err error) {
 
 func (p *TopicPublisher) doLookupTopicPartitions() (assignments []*mq_pb.BrokerPartitionAssignment, err error) {
 	if len(p.config.Brokers) == 0 {
-		return nil, fmt.Errorf("no bootstrap brokers")
+		return nil, fmt.Errorf("lookup found no bootstrap brokers")
 	}
 	var lastErr error
 	for _, brokerAddress := range p.config.Brokers {

weed/mq/client/sub_client/connect_to_sub_coordinator.go

@@ -1,7 +1,6 @@
 package sub_client
 
 import (
-	"context"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
@@ -12,10 +11,17 @@ func (sub *TopicSubscriber) doKeepConnectedToSubCoordinator() {
 	waitTime := 1 * time.Second
 	for {
 		for _, broker := range sub.bootstrapBrokers {
+
+			select {
+			case <-sub.ctx.Done():
+				return
+			default:
+			}
+
 			// lookup topic brokers
 			var brokerLeader string
 			err := pb.WithBrokerGrpcClient(false, broker, sub.SubscriberConfig.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
-				resp, err := client.FindBrokerLeader(context.Background(), &mq_pb.FindBrokerLeaderRequest{})
+				resp, err := client.FindBrokerLeader(sub.ctx, &mq_pb.FindBrokerLeaderRequest{})
 				if err != nil {
 					return err
 				}
@@ -30,10 +36,8 @@ func (sub *TopicSubscriber) doKeepConnectedToSubCoordinator() {
 
 			// connect to the balancer
 			pb.WithBrokerGrpcClient(true, brokerLeader, sub.SubscriberConfig.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
-				ctx, cancel := context.WithCancel(context.Background())
-				defer cancel()
 
-				stream, err := client.SubscriberToSubCoordinator(ctx)
+				stream, err := client.SubscriberToSubCoordinator(sub.ctx)
 				if err != nil {
 					glog.V(0).Infof("subscriber %s: %v", sub.ContentConfig.Topic, err)
 					return err
@@ -58,6 +62,13 @@ func (sub *TopicSubscriber) doKeepConnectedToSubCoordinator() {
 
 				go func() {
 					for reply := range sub.brokerPartitionAssignmentAckChan {
+
+						select {
+						case <-sub.ctx.Done():
+							return
+						default:
+						}
+
 						glog.V(0).Infof("subscriber instance %s ack %+v", sub.SubscriberConfig.ConsumerGroupInstanceId, reply)
 						if err := stream.Send(reply); err != nil {
 							glog.V(0).Infof("subscriber %s reply: %v", sub.ContentConfig.Topic, err)
@@ -73,6 +84,13 @@ func (sub *TopicSubscriber) doKeepConnectedToSubCoordinator() {
 						glog.V(0).Infof("subscriber %s receive: %v", sub.ContentConfig.Topic, err)
 						return err
 					}
+
+					select {
+					case <-sub.ctx.Done():
+						return nil
+					default:
+					}
+
 					sub.brokerPartitionAssignmentChan <- resp
 					glog.V(0).Infof("Received assignment: %+v", resp)
 				}

weed/mq/client/sub_client/on_each_partition.go

@@ -2,14 +2,13 @@ package sub_client
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
 	"io"
-	"reflect"
-	"time"
 )
 
 type KeyedOffset struct {
@@ -35,8 +34,7 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
 		if po == nil {
 			po = &schema_pb.PartitionOffset{
 				Partition: assigned.Partition,
-				StartTsNs: time.Now().UnixNano(),
-				StartType: schema_pb.PartitionOffsetStartType_EARLIEST_IN_MEMORY,
+				StartTsNs: sub.ContentConfig.OffsetTsNs,
 			}
 		}
 
@@ -47,6 +45,7 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
 					ConsumerId:        sub.SubscriberConfig.ConsumerGroupInstanceId,
 					Topic:             sub.ContentConfig.Topic.ToPbTopic(),
 					PartitionOffset:   po,
+					OffsetType:        sub.ContentConfig.OffsetType,
 					Filter:            sub.ContentConfig.Filter,
 					FollowerBroker:    assigned.FollowerBroker,
 					SlidingWindowSize: slidingWindowSize,
@@ -65,6 +64,9 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
 		go func() {
 			for {
 				select {
+				case <-sub.ctx.Done():
+					subscribeClient.CloseSend()
+					return
 				case <-stopCh:
 					subscribeClient.CloseSend()
 					return
@@ -86,15 +88,27 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
 		}()
 
 		for {
-			// glog.V(0).Infof("subscriber %s/%s/%s waiting for message", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup)
+			// glog.V(0).Infof("subscriber %s/%s waiting for message", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup)
 			resp, err := subscribeClient.Recv()
 			if err != nil {
+				if errors.Is(err, io.EOF) {
+					return nil
+				}
 				return fmt.Errorf("subscribe recv: %v", err)
 			}
 			if resp.Message == nil {
 				glog.V(0).Infof("subscriber %s/%s received nil message", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup)
 				continue
 			}
+
+			select {
+			case <-sub.ctx.Done():
+				return nil
+			case <-stopCh:
+				return nil
+			default:
+			}
+
 			switch m := resp.Message.(type) {
 			case *mq_pb.SubscribeMessageResponse_Data:
 				if m.Data.Ctrl != nil {
@@ -102,7 +116,7 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
 					continue
 				}
 				if len(m.Data.Key) == 0 {
-					fmt.Printf("empty key %+v, type %v\n", m, reflect.TypeOf(m))
+					// fmt.Printf("empty key %+v, type %v\n", m, reflect.TypeOf(m))
 					continue
 				}
 				onDataMessageFn(m)

weed/mq/client/sub_client/subscribe.go

@@ -72,12 +72,12 @@ func (sub *TopicSubscriber) startProcessors() {
 				executors := util.NewLimitedConcurrentExecutor(int(sub.SubscriberConfig.SlidingWindowSize))
 				onDataMessageFn := func(m *mq_pb.SubscribeMessageResponse_Data) {
 					executors.Execute(func() {
-						processErr := sub.OnEachMessageFunc(m.Data.Key, m.Data.Value)
-						if processErr == nil {
-							sub.PartitionOffsetChan <- KeyedOffset{
-								Key:    m.Data.Key,
-								Offset: m.Data.TsNs,
-							}
+						if sub.OnDataMessageFunc != nil {
+							sub.OnDataMessageFunc(m)
+						}
+						sub.PartitionOffsetChan <- KeyedOffset{
+							Key:    m.Data.Key,
+							Offset: m.Data.TsNs,
 						}
 					})
 				}

weed/mq/client/sub_client/subscriber.go

@@ -1,6 +1,7 @@
 package sub_client
 
 import (
+	"context"
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
@@ -17,50 +18,50 @@ type SubscriberConfiguration struct {
 	SlidingWindowSize       int32 // how many messages to process concurrently per partition
 }
 
+func (s *SubscriberConfiguration) String() string {
+	return "ClientId: " + s.ClientId + ", ConsumerGroup: " + s.ConsumerGroup + ", ConsumerGroupInstanceId: " + s.ConsumerGroupInstanceId
+}
+
 type ContentConfiguration struct {
 	Topic            topic.Topic
 	Filter           string
 	PartitionOffsets []*schema_pb.PartitionOffset
+	OffsetType       schema_pb.OffsetType
+	OffsetTsNs       int64
 }
 
 type OnDataMessageFn func(m *mq_pb.SubscribeMessageResponse_Data)
-type OnEachMessageFunc func(key, value []byte) (err error)
 type OnCompletionFunc func()
 
 type TopicSubscriber struct {
+	ctx                              context.Context
 	SubscriberConfig                 *SubscriberConfiguration
 	ContentConfig                    *ContentConfiguration
 	brokerPartitionAssignmentChan    chan *mq_pb.SubscriberToSubCoordinatorResponse
 	brokerPartitionAssignmentAckChan chan *mq_pb.SubscriberToSubCoordinatorRequest
-	OnDataMessageFnnc                OnDataMessageFn
-	OnEachMessageFunc                OnEachMessageFunc
+	OnDataMessageFunc                OnDataMessageFn
 	OnCompletionFunc                 OnCompletionFunc
 	bootstrapBrokers                 []string
-	waitForMoreMessage               bool
 	activeProcessors                 map[topic.Partition]*ProcessorState
 	activeProcessorsLock             sync.Mutex
 	PartitionOffsetChan              chan KeyedOffset
 }
 
-func NewTopicSubscriber(bootstrapBrokers []string, subscriber *SubscriberConfiguration, content *ContentConfiguration, partitionOffsetChan chan KeyedOffset) *TopicSubscriber {
+func NewTopicSubscriber(ctx context.Context, bootstrapBrokers []string, subscriber *SubscriberConfiguration, content *ContentConfiguration, partitionOffsetChan chan KeyedOffset) *TopicSubscriber {
 	return &TopicSubscriber{
+		ctx:                              ctx,
 		SubscriberConfig:                 subscriber,
 		ContentConfig:                    content,
 		brokerPartitionAssignmentChan:    make(chan *mq_pb.SubscriberToSubCoordinatorResponse, 1024),
 		brokerPartitionAssignmentAckChan: make(chan *mq_pb.SubscriberToSubCoordinatorRequest, 1024),
 		bootstrapBrokers:                 bootstrapBrokers,
-		waitForMoreMessage:               true,
 		activeProcessors:                 make(map[topic.Partition]*ProcessorState),
 		PartitionOffsetChan:              partitionOffsetChan,
 	}
 }
 
-func (sub *TopicSubscriber) SetEachMessageFunc(onEachMessageFn OnEachMessageFunc) {
-	sub.OnEachMessageFunc = onEachMessageFn
-}
-
 func (sub *TopicSubscriber) SetOnDataMessageFn(fn OnDataMessageFn) {
-	sub.OnDataMessageFnnc = fn
+	sub.OnDataMessageFunc = fn
 }
 
 func (sub *TopicSubscriber) SetCompletionFunc(onCompletionFn OnCompletionFunc) {

weed/mq/logstore/read_log_from_disk.go

@@ -36,7 +36,7 @@ func GenLogOnDiskReadFunc(filerClient filer_pb.FilerClient, t topic.Topic, p top
 				err = fmt.Errorf("unexpected unmarshal mq_pb.Message: %v", err)
 				return
 			}
-			if logEntry.TsNs < starTsNs {
+			if logEntry.TsNs <= starTsNs {
 				pos += 4 + int(size)
 				continue
 			}

weed/mq/logstore/read_parquet_to_log.go

@@ -73,7 +73,7 @@ func GenParquetReadFunc(filerClient filer_pb.FilerClient, t topic.Topic, p topic
 					return processedTsNs, fmt.Errorf("ToRecordValue failed: %v", err)
 				}
 				processedTsNs = recordValue.Fields[SW_COLUMN_NAME_TS].GetInt64Value()
-				if processedTsNs < starTsNs {
+				if processedTsNs <= starTsNs {
 					continue
 				}
 				if stopTsNs != 0 && processedTsNs >= stopTsNs {

weed/mq/schema/write_parquet_test.go

@@ -91,13 +91,13 @@ func testWritingParquetFile(t *testing.T, count int, filename string, parquetSch
 		AddRecordValue(rowBuilder, recordType, parquetLevels, recordValue)
 
 		if count < 10 {
-			fmt.Printf("RecordValue: %v\n", recordValue)
+			fmt.Printf("Write RecordValue: %v\n", recordValue)
 		}
 
 		row := rowBuilder.Row()
 
 		if count < 10 {
-			fmt.Printf("Row: %+v\n", row)
+			fmt.Printf("Build Row: %+v\n", row)
 		}
 
 		if err != nil {
@@ -143,7 +143,7 @@ func testReadingParquetFile(t *testing.T, filename string, parquetSchema *parque
 				t.Fatalf("ToRecordValue failed: %v", err)
 			}
 			if rowCount < 10 {
-				fmt.Printf("RecordValue: %v\n", recordValue)
+				fmt.Printf("Read RecordValue: %v\n", recordValue)
 			}
 		}
 		total += rowCount

weed/mq/sub_coordinator/market.go

@@ -20,7 +20,7 @@ Trigger rebalance when:
 * A new consumer group instance is added
 * Some partitions are unassigned from a consumer group instance.
 
-If multiple reblance requests are received, after a certain period, the market will only process the latest request.
+If multiple rebalance requests are received, after a certain period, the market will only process the latest request.
 
 However, if the number of unassigned partition is increased to exactly the total number of partitions,
 and total partitions are less than or equal to the sum of the max partition count of all consumer group instances,

weed/mq/topic/local_partition.go

@@ -89,12 +89,16 @@ func (p *LocalPartition) Subscribe(clientName string, startPosition log_buffer.M
 			return nil
 		}
 
-		startPosition = processedPosition
+		if processedPosition.Time.UnixNano() != 0 {
+			startPosition = processedPosition
+		}
 		processedPosition, isDone, readInMemoryLogErr = p.LogBuffer.LoopProcessLogData(clientName, startPosition, 0, onNoMessageFn, eachMessageFn)
 		if isDone {
 			return nil
 		}
-		startPosition = processedPosition
+		if processedPosition.Time.UnixNano() != 0 {
+			startPosition = processedPosition
+		}
 
 		if readInMemoryLogErr == log_buffer.ResumeFromDiskError {
 			continue

weed/pb/mq_agent.proto

@@ -21,10 +21,6 @@ service SeaweedMessagingAgent {
     }
 
     // Subscribing
-    rpc StartSubscribeSession (StartSubscribeSessionRequest) returns (StartSubscribeSessionResponse) {
-    }
-    rpc CloseSubscribeSession (CloseSubscribeSessionRequest) returns (CloseSubscribeSessionResponse) {
-    }
     rpc SubscribeRecord (stream SubscribeRecordRequest) returns (stream SubscribeRecordResponse) {
     }
 
@@ -59,28 +55,19 @@ message PublishRecordResponse {
     string error = 2;
 }
 //////////////////////////////////////////////////
-message StartSubscribeSessionRequest {
-    string consumer_group = 1;
-    string consumer_group_instance_id = 2;
-    schema_pb.Topic topic = 4;
-    repeated schema_pb.PartitionOffset partition_offsets = 5;
-    string filter = 6;
-    int32 max_subscribed_partitions = 8;
-    int32 sliding_window_size = 9;
-}
-message StartSubscribeSessionResponse {
-    string error = 1;
-    int64 session_id = 2;
-}
-message CloseSubscribeSessionRequest {
-    int64 session_id = 1;
-}
-message CloseSubscribeSessionResponse {
-    string error = 1;
-}
-//////////////////////////////////////////////////
 message SubscribeRecordRequest {
-    int64 session_id = 1; // session_id is required for the first record
+    message InitSubscribeRecordRequest {
+        string consumer_group = 1;
+        string consumer_group_instance_id = 2;
+        schema_pb.Topic topic = 4;
+        repeated schema_pb.PartitionOffset partition_offsets = 5;
+        schema_pb.OffsetType offset_type = 6;
+        int64 offset_ts_ns = 7;
+        string filter = 10;
+        int32 max_subscribed_partitions = 11;
+        int32 sliding_window_size = 12;
+    }
+    InitSubscribeRecordRequest init = 1;
     int64 ack_sequence = 2;
     bytes ack_key = 3;
 }