3.75

s3 add skip bucket encryption handlers

.github/workflows/binaries_release5.yml

@@ -0,0 +1,59 @@
+# This is a basic workflow to help you get started with Actions
+
+name: "go: build versioned binaries for openbsd"
+
+on:
+  push:
+    tags:
+      - '*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+permissions:
+  contents: read
+
+jobs:
+
+  build-release-binaries_openbsd:
+    permissions:
+      contents: write  # for wangyoucao577/go-release-action to upload release assets
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [openbsd]
+        goarch: [amd64, arm, arm64]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
+          # build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
+      - name: Go Release Large Disk Binaries
+        uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"

.github/workflows/container_dev.yml

@@ -33,30 +33,30 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee # v1
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v1
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@a254f8ca60a858f3136a2f1f23a60969f2c402dd # v2
+        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -34,30 +34,30 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee # v1
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v1
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@a254f8ca60a858f3136a2f1f23a60969f2c402dd # v2
+        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -34,20 +34,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee # v1
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v1
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@a254f8ca60a858f3136a2f1f23a60969f2c402dd # v2
+        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -35,20 +35,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee # v1
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v1
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@a254f8ca60a858f3136a2f1f23a60969f2c402dd # v2
+        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -35,20 +35,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee # v1
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v1
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@a254f8ca60a858f3136a2f1f23a60969f2c402dd # v2
+        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release4.yml

@@ -34,20 +34,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee # v1
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v1
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@a254f8ca60a858f3136a2f1f23a60969f2c402dd # v2
+        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release5.yml

@@ -34,20 +34,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee # v1
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v1
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@a254f8ca60a858f3136a2f1f23a60969f2c402dd # v2
+        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

go.mod

@@ -3,17 +3,17 @@ module github.com/seaweedfs/seaweedfs
 go 1.22.0
 
 require (
-	cloud.google.com/go v0.115.0 // indirect
-	cloud.google.com/go/pubsub v1.40.0
-	cloud.google.com/go/storage v1.42.0
+	cloud.google.com/go v0.115.1 // indirect
+	cloud.google.com/go/pubsub v1.43.0
+	cloud.google.com/go/storage v1.43.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Shopify/sarama v1.38.1
-	github.com/aws/aws-sdk-go v1.54.13
+	github.com/aws/aws-sdk-go v1.55.5
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cenkalti/backoff/v4 v4.3.0
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -36,10 +36,10 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/btree v1.1.2
+	github.com/google/btree v1.1.3
 	github.com/google/uuid v1.6.0
 	github.com/google/wire v0.6.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.12.5 // indirect
+	github.com/googleapis/gax-go/v2 v2.13.0 // indirect
 	github.com/gorilla/mux v1.8.1
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -53,10 +53,10 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
 	github.com/klauspost/compress v1.17.9 // indirect
-	github.com/klauspost/reedsolomon v1.12.1
+	github.com/klauspost/reedsolomon v1.12.4
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.9
-	github.com/linxGnu/grocksdb v1.9.2
+	github.com/linxGnu/grocksdb v1.9.3
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.11 // indirect
@@ -70,9 +70,9 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/posener/complete v1.2.3
 	github.com/pquerna/cachecontrol v0.2.0
-	github.com/prometheus/client_golang v1.19.1
-	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/client_golang v1.20.3
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
@@ -85,7 +85,7 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tidwall/gjson v1.17.1
+	github.com/tidwall/gjson v1.17.3
 	github.com/tidwall/match v1.1.1
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
@@ -96,115 +96,118 @@ require (
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	go.etcd.io/etcd/client/v3 v3.5.13
-	go.mongodb.org/mongo-driver v1.15.0
+	go.etcd.io/etcd/client/v3 v3.5.16
+	go.mongodb.org/mongo-driver v1.16.0
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.37.0
-	gocloud.dev/pubsub/natspubsub v0.37.0
-	gocloud.dev/pubsub/rabbitpubsub v0.37.0
-	golang.org/x/crypto v0.25.0 // indirect
-	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3
+	gocloud.dev v0.39.0
+	gocloud.dev/pubsub/natspubsub v0.39.0
+	gocloud.dev/pubsub/rabbitpubsub v0.39.0
+	golang.org/x/crypto v0.27.0 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
 	golang.org/x/image v0.18.0
-	golang.org/x/net v0.27.0
-	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/sys v0.22.0
-	golang.org/x/text v0.16.0 // indirect
-	golang.org/x/tools v0.23.0
-	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
-	google.golang.org/api v0.187.0
-	google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect
-	google.golang.org/grpc v1.64.1
+	golang.org/x/net v0.29.0
+	golang.org/x/oauth2 v0.23.0 // indirect
+	golang.org/x/sys v0.25.0
+	golang.org/x/text v0.18.0 // indirect
+	golang.org/x/tools v0.25.0
+	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
+	google.golang.org/api v0.199.0
+	google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
+	google.golang.org/grpc v1.67.0
 	google.golang.org/protobuf v1.34.2
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
-	modernc.org/libc v1.52.1 // indirect
 	modernc.org/mathutil v1.6.0
 	modernc.org/memory v1.8.0 // indirect
-	modernc.org/sqlite v1.30.2
+	modernc.org/sqlite v1.33.0
 	modernc.org/strutil v1.2.0
 	modernc.org/token v1.1.0 // indirect
 )
 
 require (
-	github.com/Jille/raft-grpc-transport v1.5.0
+	github.com/Jille/raft-grpc-transport v1.6.1
 	github.com/arangodb/go-driver v1.6.2
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.30.3
-	github.com/aws/aws-sdk-go-v2/config v1.27.16
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.18
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2
+	github.com/aws/aws-sdk-go-v2 v1.31.0
+	github.com/aws/aws-sdk-go-v2/config v1.27.33
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.32
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2
 	github.com/cognusion/imaging v1.0.1
 	github.com/fluent/fluent-logger-golang v1.9.0
-	github.com/getsentry/sentry-go v0.28.1
+	github.com/getsentry/sentry-go v0.29.0
 	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
-	github.com/hanwen/go-fuse/v2 v2.5.1
-	github.com/hashicorp/raft v1.7.0
+	github.com/hanwen/go-fuse/v2 v2.6.1
+	github.com/hashicorp/raft v1.7.1
 	github.com/hashicorp/raft-boltdb/v2 v2.3.0
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/parquet-go/parquet-go v0.23.0
 	github.com/rabbitmq/amqp091-go v1.10.0
-	github.com/rclone/rclone v1.66.0
+	github.com/rclone/rclone v1.68.1
 	github.com/rdleal/intervalst v1.4.0
-	github.com/redis/go-redis/v9 v9.5.1
-	github.com/schollz/progressbar/v3 v3.14.4
+	github.com/redis/go-redis/v9 v9.6.1
+	github.com/schollz/progressbar/v3 v3.16.0
 	github.com/shirou/gopsutil/v3 v3.24.5
 	github.com/tikv/client-go/v2 v2.0.7
-	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.4.2
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.68.1
-	go.etcd.io/etcd/client/pkg/v3 v3.5.14
+	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.77.1
+	go.etcd.io/etcd/client/pkg/v3 v3.5.16
 	go.uber.org/atomic v1.11.0
-	golang.org/x/sync v0.7.0
+	golang.org/x/sync v0.8.0
 	google.golang.org/grpc/security/advancedtls v1.0.0
 )
 
 require (
-	cloud.google.com/go/auth v0.6.1 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
-	cloud.google.com/go/compute/metadata v0.3.0 // indirect
-	cloud.google.com/go/iam v1.1.8 // indirect
+	cloud.google.com/go/auth v0.9.5 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
+	cloud.google.com/go/compute/metadata v0.5.2 // indirect
+	cloud.google.com/go/iam v1.2.0 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.2.2 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/Files-com/files-sdk-go/v3 v3.2.34 // indirect
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
 	github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect
+	github.com/ProtonMail/go-crypto v1.0.0 // indirect
 	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
 	github.com/ProtonMail/go-srp v0.0.7 // indirect
 	github.com/ProtonMail/gopenpgp/v2 v2.7.4 // indirect
 	github.com/PuerkitoBio/goquery v1.8.1 // indirect
-	github.com/Unknwon/goconfig v1.0.0 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.12 // indirect
-	github.com/aws/smithy-go v1.20.3 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.31.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 // indirect
+	github.com/aws/smithy-go v1.21.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.2 // indirect
+	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
 	github.com/buengese/sgzip v0.1.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
+	github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891 // indirect
 	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
@@ -217,26 +220,29 @@ require (
 	github.com/emersion/go-message v0.18.0 // indirect
 	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
 	github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
-	github.com/fatih/color v1.14.1 // indirect
+	github.com/fatih/color v1.16.0 // indirect
 	github.com/fclairamb/go-log v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/flynn/noise v1.0.1 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.4 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
-	github.com/go-chi/chi/v5 v5.0.11 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-chi/chi/v5 v5.1.0 // indirect
+	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-resty/resty/v2 v2.11.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
-	github.com/google/s2a-go v0.1.7 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/google/s2a-go v0.1.8 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
-	github.com/hashicorp/go-hclog v1.6.2 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
+	github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/henrybear327/Proton-API-Bridge v1.0.0 // indirect
@@ -248,29 +254,30 @@ require (
 	github.com/jlaffaye/ftp v0.2.0 // indirect
 	github.com/jonboulle/clockwork v0.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9 // indirect
 	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
-	github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/lpar/date v1.0.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/montanaflynn/stats v0.7.0 // indirect
-	github.com/nats-io/nats.go v1.34.0 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/nats-io/nats.go v1.37.0 // indirect
 	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
-	github.com/ncw/swift/v2 v2.0.2 // indirect
+	github.com/ncw/swift/v2 v2.0.3 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/onsi/gomega v1.27.10 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.55.1 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.69.2 // indirect
+	github.com/panjf2000/ants/v2 v2.9.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
@@ -288,8 +295,10 @@ require (
 	github.com/relvacode/iso8601 v1.3.0 // indirect
 	github.com/rfjakob/eme v1.1.2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/samber/lo v1.47.0 // indirect
 	github.com/segmentio/encoding v0.4.0 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
@@ -306,40 +315,43 @@ require (
 	github.com/tklauser/go-sysconf v0.3.13 // indirect
 	github.com/tklauser/numcpus v0.7.0 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
+	github.com/unknwon/goconfig v1.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20240528144234-5d5a685e41f7 // indirect
 	github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	github.com/zeebo/errs v1.3.0 // indirect
-	go.etcd.io/bbolt v1.3.8 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.13 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
-	go.opentelemetry.io/otel v1.24.0 // indirect
-	go.opentelemetry.io/otel/metric v1.24.0 // indirect
-	go.opentelemetry.io/otel/trace v1.24.0 // indirect
+	go.etcd.io/bbolt v1.3.10 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.16 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
+	go.opentelemetry.io/otel v1.29.0 // indirect
+	go.opentelemetry.io/otel/metric v1.29.0 // indirect
+	go.opentelemetry.io/otel/trace v1.29.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/mod v0.19.0 // indirect
-	golang.org/x/term v0.22.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect
+	golang.org/x/mod v0.21.0 // indirect
+	golang.org/x/term v0.24.0 // indirect
+	golang.org/x/time v0.6.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
-	storj.io/common v0.0.0-20240111121419-ecae1362576c // indirect
-	storj.io/drpc v0.0.33 // indirect
+	moul.io/http2curl/v2 v2.3.0 // indirect
+	storj.io/common v0.0.0-20240812101423-26b53789c348 // indirect
+	storj.io/drpc v0.0.35-0.20240709171858-0075ac871661 // indirect
+	storj.io/eventkit v0.0.0-20240415002644-1d9596fee086 // indirect
 	storj.io/infectious v0.0.2 // indirect
-	storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect
-	storj.io/uplink v1.12.2 // indirect
+	storj.io/picobuf v0.0.3 // indirect
+	storj.io/uplink v1.13.1 // indirect
 )
 
 // replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.71"
+appVersion: "3.75"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.0
+version: 4.0.375

k8s/charts/seaweedfs/templates/ca-cert.yaml

@@ -14,6 +14,6 @@ spec:
   commonName: "{{ template "seaweedfs.name" . }}-root-ca"
   isCA: true
   issuerRef:
-    name: {{ template "seaweedfs.name" . }}-clusterissuer
-    kind: ClusterIssuer
+    name: {{ template "seaweedfs.name" . }}-issuer
+    kind: Issuer
 {{- end }}

k8s/charts/seaweedfs/templates/cert-issuer.yaml

@@ -1,8 +1,8 @@
 {{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
-kind: ClusterIssuer
+kind: Issuer
 metadata:
-  name: {{ template "seaweedfs.name" . }}-clusterissuer
+  name: {{ template "seaweedfs.name" . }}-issuer
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}

k8s/charts/seaweedfs/templates/cosi-bucket-class.yaml

@@ -0,0 +1,16 @@
+{{- if and .Values.cosi.enabled .Values.cosi.bucketClassName }}
+---
+kind: BucketClass
+apiVersion: objectstorage.k8s.io/v1alpha1
+metadata:
+  name: {{ .Values.cosi.bucketClassName }}
+driverName: {{ .Values.cosi.driverName }}
+deletionPolicy: Delete
+---
+kind: BucketAccessClass
+apiVersion: objectstorage.k8s.io/v1alpha1
+metadata:
+  name: {{ .Values.cosi.bucketClassName }}
+driverName: {{ .Values.cosi.driverName }}
+authenticationType: KEY
+{{- end }}

k8s/charts/seaweedfs/templates/cosi-cluster-role.yaml

@@ -0,0 +1,69 @@
+{{- if .Values.cosi.enabled }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+rules:
+- apiGroups: ["objectstorage.k8s.io"]
+  resources:
+    - "buckets"
+    - "bucketaccesses"
+    - "bucketclaims"
+    - "bucketaccessclasses"
+    - "buckets/status"
+    - "bucketaccesses/status"
+    - "bucketclaims/status"
+    - "bucketaccessclasses/status"
+  verbs:
+    - "get"
+    - "list"
+    - "watch"
+    - "update"
+    - "create"
+    - "delete"
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs:
+    - "get"
+    - "watch"
+    - "list"
+    - "delete"
+    - "update"
+    - "create"
+- apiGroups: [""]
+  resources:
+    - "secrets"
+    - "events"
+  verbs:
+    - "get"
+    - "list"
+    - "watch"
+    - "update"
+    - "create"
+    - "delete"
+    - "patch"
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

k8s/charts/seaweedfs/templates/cosi-deployment.yaml

@@ -0,0 +1,204 @@
+{{- if .Values.cosi.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "seaweedfs.name" . }}-objectstorage-provisioner
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+spec:
+  replicas: {{ .Values.cosi.replicas }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+      helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: objectstorage-provisioner
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+        helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: objectstorage-provisioner
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.cosi.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.cosi.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      restartPolicy: {{ default .Values.global.restartPolicy .Values.cosi.restartPolicy }}
+      {{- if .Values.cosi.tolerations }}
+      tolerations:
+        {{ tpl .Values.cosi.tolerations . | nindent 8 | trim }}
+      {{- end }}
+      {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
+      terminationGracePeriodSeconds: 10
+      {{- if .Values.cosi.priorityClassName }}
+      priorityClassName: {{ .Values.cosi.priorityClassName | quote }}
+      {{- end }}
+      enableServiceLinks: false
+      serviceAccountName: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner
+      {{- if .Values.cosi.initContainers }}
+      initContainers:
+        {{ tpl .Values.cosi.initContainers . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.cosi.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.cosi.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: seaweedfs-cosi-driver
+          image: "{{ .Values.cosi.image }}"
+          imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
+          env:
+            - name: DRIVERNAME
+              value: "{{ .Values.cosi.driverName }}"
+            - name: ENDPOINT
+              {{- if .Values.cosi.endpoint }}
+              value: "{{ .Values.cosi.endpoint }}"
+              {{- else if .Values.s3.ingress.enabled }}
+              value: "{{ printf "https://%s" .Values.s3.ingress.host }}"
+              {{- else if .Values.s3.enabled }}
+              value: "{{ printf "https://%s-s3.%s.svc" (include "seaweedfs.name" .) .Release.Namespace }}"
+              {{- else }}
+              value: "{{ printf "https://%s-filer.%s.svc" (include "seaweedfs.name" .) .Release.Namespace }}"
+              {{- end }}
+            {{- with .Values.cosi.region }}
+            - name: REGION
+              value: "{{ . }}"
+            {{- end }}
+            - name: SEAWEEDFS_FILER
+              value: "{{ template "seaweedfs.name" . }}-filer:{{ .Values.filer.grpcPort }}"
+            {{- if .Values.global.enableSecurity }}
+            - name: WEED_GRPC_CLIENT_KEY
+              value: /usr/local/share/ca-certificates/client/tls.key
+            - name: WEED_GRPC_CLIENT_CERT
+              value: /usr/local/share/ca-certificates/client/tls.crt
+            - name: WEED_GRPC_CA
+              value: /usr/local/share/ca-certificates/client/ca.crt
+            {{- end }}
+            {{- if .Values.cosi.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.cosi.extraEnvironmentVars }}
+            - name: {{ $key }}
+            {{- if kindIs "string" $value }}
+              value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
+            {{- end }}
+            {{- end }}
+            {{- if .Values.global.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
+            - name: {{ $key }}
+            {{- if kindIs "string" $value }}
+              value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
+            {{- end }}
+            {{- end }}
+          volumeMounts:
+            - mountPath: /var/lib/cosi
+              name: socket
+            {{- if .Values.cosi.enableAuth }}
+            - mountPath: /etc/sw
+              name: config-users
+              readOnly: true
+            {{- end }}
+            {{- if .Values.global.enableSecurity }}
+            - name: security-config
+              readOnly: true
+              mountPath: /etc/seaweedfs/security.toml
+              subPath: security.toml
+            - name: ca-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/ca/
+            - name: master-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/master/
+            - name: volume-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/volume/
+            - name: filer-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/filer/
+            - name: client-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/client/
+            {{- end }}
+            {{ tpl .Values.cosi.extraVolumeMounts . | nindent 12 | trim }}
+        - name: seaweedfs-cosi-sidecar
+          image: "{{ .Values.cosi.sidecar.image }}"
+          imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
+          args:
+            - {{ printf "--v=%s" (default "5" .Values.cosi.sidecar.logLevel) }}
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          volumeMounts:
+            - mountPath: /var/lib/cosi
+              name: socket
+          {{- with .Values.cosi.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- if .Values.cosi.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.cosi.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
+      {{- if .Values.cosi.sidecars }}
+      {{- include "common.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: socket
+          emptyDir: {}
+        {{- if .Values.cosi.enableAuth }}
+        - name: config-users
+          secret:
+            defaultMode: 420
+            {{- if .Values.cosi.existingConfigSecret }}
+            secretName: {{ .Values.cosi.existingConfigSecret }}
+            {{- else }}
+            secretName: seaweedfs-client-cert
+            {{- end }}
+        {{- end }}
+        {{- if .Values.global.enableSecurity }}
+        - name: security-config
+          configMap:
+            name: {{ template "seaweedfs.name" . }}-security-config
+        - name: ca-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-ca-cert
+        - name: master-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-master-cert
+        - name: volume-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-volume-cert
+        - name: filer-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-filer-cert
+        - name: client-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-client-cert
+        {{- end }}
+        {{ tpl .Values.cosi.extraVolumes . | indent 8 | trim }}
+      {{- if .Values.cosi.nodeSelector }}
+      nodeSelector:
+        {{ tpl .Values.cosi.nodeSelector . | indent 8 | trim }}
+      {{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/cosi-service-account.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.cosi.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+automountServiceAccountToken: {{ .Values.global.automountServiceAccountToken }}
+{{- end }}

k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@@ -249,6 +249,14 @@ spec:
               name: metrics
             - containerPort: {{ .Values.filer.grpcPort }}
               #name: swfs-filer-grpc
+            {{- if .Values.filer.s3.enabled }}
+            - containerPort: {{ .Values.filer.s3.port }}
+              name: swfs-s3
+            {{- if .Values.filer.s3.httpsPort }}
+            - containerPort: {{ .Values.filer.s3.httpsPort }}
+              name: swfs-s3-tls
+            {{- end }}
+            {{- end }}
           {{- if .Values.filer.readinessProbe.enabled }}
           readinessProbe:
             httpGet:

k8s/charts/seaweedfs/templates/s3-deployment.yaml

@@ -14,7 +14,6 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-      helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
       app.kubernetes.io/instance: {{ .Release.Name }}
       app.kubernetes.io/component: s3
   template:
@@ -176,9 +175,13 @@ spec:
           ports:
             - containerPort: {{ .Values.s3.port }}
               name: swfs-s3
+            {{- if .Values.s3.httpsPort }}
+            - containerPort: {{ .Values.s3.httpsPort }}
+              name: swfs-s3-tls
+            {{- end }}
             {{- if .Values.s3.metricsPort }}
             - containerPort: {{ .Values.s3.metricsPort }}
-              name: "metrics"
+              name: metrics
             {{- end }}
           {{- if .Values.s3.readinessProbe.enabled }}
           readinessProbe:

k8s/charts/seaweedfs/templates/volume-resize-hook.yaml

@@ -0,0 +1,114 @@
+{{- if and .Values.volume.enabled .Values.volume.resizeHook.enabled }}
+{{- $seaweedfsName := include "seaweedfs.name" $ }}
+{{- $replicas := int .Values.volume.replicas -}}
+{{- $statefulsetName := printf "%s-volume" $seaweedfsName -}}
+{{- $statefulset := (lookup "apps/v1" "StatefulSet" .Release.Namespace $statefulsetName) -}}
+
+{{/* Check for changes in volumeClaimTemplates */}}
+{{- $templateChangesRequired := false -}}
+{{- if $statefulset -}}
+  {{- range $dir := .Values.volume.dataDirs -}}
+    {{- if eq .type "persistentVolumeClaim" -}}
+      {{- $desiredSize := .size -}}
+      {{- range $statefulset.spec.volumeClaimTemplates -}}
+        {{- if and (eq .metadata.name $dir.name) (ne .spec.resources.requests.storage $desiredSize) -}}
+          {{- $templateChangesRequired = true -}}
+        {{- end -}}
+      {{- end -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+
+{{/* Check for the need for patching existing PVCs */}}
+{{- $pvcChangesRequired := false -}}
+{{- range $dir := .Values.volume.dataDirs -}}
+  {{- if eq .type "persistentVolumeClaim" -}}
+    {{- $desiredSize := .size -}}
+    {{- range $i, $e := until $replicas }}
+      {{- $pvcName := printf "%s-%s-volume-%d" $dir.name $seaweedfsName $e -}}
+      {{- $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) -}}
+      {{- if and $currentPVC (ne ($currentPVC.spec.resources.requests.storage | toString) $desiredSize) -}}
+        {{- $pvcChangesRequired = true -}}
+      {{- end -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+
+{{- if or $templateChangesRequired $pvcChangesRequired }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ $seaweedfsName }}-volume-resize-hook"
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-weight: "0"
+    helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
+spec:
+  template:
+    spec:
+      serviceAccountName: {{ $seaweedfsName }}-volume-resize-hook
+      restartPolicy: Never
+      backoffLimit: 1
+      containers:
+        - name: resize
+          image: {{ .Values.volume.resizeHook.image }}
+          command: ["sh", "-xec"]
+          args:
+            - |
+              {{- if $pvcChangesRequired -}}
+              {{- range $dir := .Values.volume.dataDirs -}}
+              {{- if eq .type "persistentVolumeClaim" -}}
+              {{- $desiredSize := .size -}}
+              {{- range $i, $e := until $replicas }}
+              kubectl patch pvc {{ printf "%s-%s-volume-%d" $dir.name $seaweedfsName $e }} -p '{"spec":{"resources":{"requests":{"storage":"{{ $desiredSize }}"}}}}'
+              {{- end }}
+              {{- end }}
+              {{- end }}
+              {{- end -}}
+
+              {{- if $templateChangesRequired }}
+              kubectl delete statefulset {{ $statefulsetName }} --cascade=orphan
+              {{- end }}
+{{- end }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ $seaweedfsName }}-volume-resize-hook
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-weight: "-5"
+    helm.sh/hook-delete-policy: before-hook-creation
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $seaweedfsName }}-volume-resize-hook
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-weight: "-5"
+    helm.sh/hook-delete-policy: before-hook-creation
+rules:
+  - apiGroups: ["apps"]
+    resources: ["statefulsets"]
+    verbs: ["delete", "get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["patch", "get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $seaweedfsName }}-volume-resize-hook
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-weight: "-5"
+    helm.sh/hook-delete-policy: before-hook-creation
+subjects:
+  - kind: ServiceAccount
+    name: {{ $seaweedfsName }}-volume-resize-hook
+roleRef:
+  kind: Role
+  name: {{ $seaweedfsName }}-volume-resize-hook
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

k8s/charts/seaweedfs/values.yaml

@@ -100,6 +100,15 @@ master:
     storageClass: ""
     hostPathPrefix: /ssd
 
+  # You may use ANY storage-class, example with local-path-provisioner
+  # Annotations are optional.
+  # logs:
+  #   type: "persistentVolumeClaim"
+  #   size: "24Ti"
+  #   storageClass: "local-path-provisioner"
+  #   annotations:
+  #     "key": "value"
+
   # You can also use emptyDir storage:
   #  logs:
   #    type: "emptyDir"
@@ -305,6 +314,11 @@ volume:
    #   size: "800Gi"
    #   maxVolumes: 0
 
+  # This will automatically create a job for patching Kubernetes resources if the dataDirs type is 'persistentVolumeClaim' and the size has changed.
+  resizeHook:
+    enabled: true
+    image: bitnami/kubectl
+
   # idx can be defined by:
   #
   # idx:
@@ -445,7 +459,7 @@ volume:
   livenessProbe:
     enabled: true
     httpGet:
-      path: /status
+      path: /healthz
       scheme: HTTP
     initialDelaySeconds: 20
     periodSeconds: 90
@@ -458,7 +472,7 @@ volume:
   readinessProbe:
     enabled: true
     httpGet:
-      path: /status
+      path: /healthz
       scheme: HTTP
     initialDelaySeconds: 15
     periodSeconds: 15
@@ -530,6 +544,15 @@ filer:
     storageClass: ""
     hostPathPrefix: /storage
 
+  # You may use ANY storage-class, example with local-path-provisioner
+  # Annotations are optional.
+  # logs:
+  #   type: "persistentVolumeClaim"
+  #   size: "24Ti"
+  #   storageClass: "local-path-provisioner"
+  #   annotations:
+  #     "key": "value"
+
   # You can also use emptyDir storage:
   #  logs:
   #    type: "emptyDir"
@@ -867,10 +890,36 @@ s3:
     annotations: {}
     tls: []
 
+# Deploy Kubernetes COSI Driver for SeaweedFS
+# Requires COSI CRDs and controller to be installed in the cluster
+# For more information, visit: https://container-object-storage-interface.github.io/docs/deployment-guide
+cosi:
+  enabled: false
+  image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.1.2"
+  driverName: "seaweedfs.objectstorage.k8s.io"
+  bucketClassName: "seaweedfs"
+  endpoint: ""
+  region: ""
+
+  sidecar:
+    image: gcr.io/k8s-staging-sig-storage/objectstorage-sidecar/objectstorage-sidecar:v20230130-v0.1.0-24-gc0cf995
+
+  # enable user & permission to s3 (need to inject to all services)
+  enableAuth: false
+  # set to the name of an existing kubernetes Secret with the s3 json config file
+  # should have a secret key called seaweedfs_s3_config with an inline json configure
+  existingConfigSecret: null
+
+  podSecurityContext: {}
+  containerSecurityContext: {}
+
+  extraVolumes: ""
+  extraVolumeMounts: ""
+
 certificates:
   commonName: "SeaweedFS CA"
   ipAddresses: []
-  keyAlgorithm: rsa
+  keyAlgorithm: RSA
   keySize: 2048
   duration: 2160h  # 90d
   renewBefore: 360h  # 15d

other/java/client/pom.xml

@@ -3,15 +3,32 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>com.github.chrislusf</groupId>
+    <groupId>com.seaweedfs</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.59</version>
+    <version>3.71</version>
 
-    <parent>
-        <groupId>org.sonatype.oss</groupId>
-        <artifactId>oss-parent</artifactId>
-        <version>9</version>
-    </parent>
+    <name>SeaweedFS Java Client</name>
+    <description>A java client for SeaweedFS.</description>
+    <url>https://github.com/seaweedfs/seaweedfs</url>
+    <licenses>
+        <license>
+            <name>The Apache License, Version 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+        </license>
+    </licenses>
+    <developers>
+        <developer>
+            <name>Chris Lu</name>
+            <email>chris.lu@gmail.com</email>
+            <organization>SeaweedFS</organization>
+            <organizationUrl>https://seaweedfs.com</organizationUrl>
+        </developer>
+    </developers>
+    <scm>
+        <connection>scm:git:git://github.com/seaweedfs/seaweedfs.git</connection>
+        <developerConnection>scm:git:ssh://github.com:seaweedfs/seaweedfs.git</developerConnection>
+        <url>https://github.com/seaweedfs/seaweedfs/tree/master</url>
+    </scm>
 
     <properties>
         <protobuf.version>3.16.3</protobuf.version>
@@ -75,12 +92,6 @@
         </dependency>
     </dependencies>
 
-    <distributionManagement>
-        <snapshotRepository>
-            <id>ossrh</id>
-            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
-        </snapshotRepository>
-    </distributionManagement>
     <build>
         <extensions>
             <extension>
@@ -134,14 +145,13 @@
                 </executions>
             </plugin>
             <plugin>
-                <groupId>org.sonatype.plugins</groupId>
-                <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.8</version>
+                <groupId>org.sonatype.central</groupId>
+                <artifactId>central-publishing-maven-plugin</artifactId>
+                <version>0.5.0</version>
                 <extensions>true</extensions>
                 <configuration>
-                    <serverId>ossrh</serverId>
-                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
-                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                    <publishingServerId>central</publishingServerId>
+                    <autoPublish>true</autoPublish>
                 </configuration>
             </plugin>
             <plugin>

other/java/client/pom.xml.deploy

@@ -3,21 +3,38 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>com.github.chrislusf</groupId>
+    <groupId>com.seaweedfs</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.59</version>
+    <version>3.71</version>
 
-    <parent>
-        <groupId>org.sonatype.oss</groupId>
-        <artifactId>oss-parent</artifactId>
-        <version>9</version>
-    </parent>
+    <name>SeaweedFS Java Client</name>
+    <description>A java client for SeaweedFS.</description>
+    <url>https://github.com/seaweedfs/seaweedfs</url>
+    <licenses>
+        <license>
+            <name>The Apache License, Version 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+        </license>
+    </licenses>
+    <developers>
+        <developer>
+            <name>Chris Lu</name>
+            <email>chris.lu@gmail.com</email>
+            <organization>SeaweedFS</organization>
+            <organizationUrl>https://seaweedfs.com</organizationUrl>
+        </developer>
+    </developers>
+    <scm>
+        <connection>scm:git:git://github.com/seaweedfs/seaweedfs.git</connection>
+        <developerConnection>scm:git:ssh://github.com:seaweedfs/seaweedfs.git</developerConnection>
+        <url>https://github.com/seaweedfs/seaweedfs/tree/master</url>
+    </scm>
 
     <properties>
-        <protobuf.version>3.9.1</protobuf.version>
+        <protobuf.version>3.16.3</protobuf.version>
         <!-- follow https://github.com/grpc/grpc-java -->
-        <grpc.version>1.23.0</grpc.version>
-        <guava.version>28.0-jre</guava.version>
+        <grpc.version>1.53.0</grpc.version>
+        <guava.version>32.0.0-jre</guava.version>
     </properties>
 
     <dependencies>
@@ -65,17 +82,16 @@
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>4.12</version>
+            <version>4.13.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+            <version>1.3.2</version>
+        </dependency>
     </dependencies>
 
-    <distributionManagement>
-        <snapshotRepository>
-            <id>ossrh</id>
-            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
-        </snapshotRepository>
-    </distributionManagement>
     <build>
         <extensions>
             <extension>
@@ -129,14 +145,13 @@
                 </executions>
             </plugin>
             <plugin>
-                <groupId>org.sonatype.plugins</groupId>
-                <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.8</version>
+                <groupId>org.sonatype.central</groupId>
+                <artifactId>central-publishing-maven-plugin</artifactId>
+                <version>0.5.0</version>
                 <extensions>true</extensions>
                 <configuration>
-                    <serverId>ossrh</serverId>
-                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
-                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                    <publishingServerId>central</publishingServerId>
+                    <autoPublish>true</autoPublish>
                 </configuration>
             </plugin>
             <plugin>

other/java/client/pom_debug.xml

@@ -3,9 +3,9 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>com.github.chrislusf</groupId>
+    <groupId>com.seaweedfs</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.59</version>
+    <version>3.71</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -103,7 +103,7 @@ public class SeaweedRead {
             chunkCache.setChunk(chunkView.fileId, chunkData);
         }
 
-        int len = (int) chunkView.size;
+        int len = (int) chunkView.size - (int) (startOffset - chunkView.logicOffset);
         LOG.debug("readChunkView fid:{} chunkData.length:{} chunkView.offset:{} chunkView[{};{}) startOffset:{}",
                 chunkView.fileId, chunkData.length, chunkView.offset, chunkView.logicOffset, chunkView.logicOffset + chunkView.size, startOffset);
         buf.put(chunkData, (int) (startOffset - chunkView.logicOffset + chunkView.offset), len);

other/java/client/src/main/proto/filer.proto

@@ -221,6 +221,7 @@ message DeleteEntryRequest {
     bool ignore_recursive_error = 6;
     bool is_from_other_cluster = 7;
     repeated int32 signatures = 8;
+    int64 if_not_modified_after = 9;
 }
 
 message DeleteEntryResponse {
@@ -441,6 +442,7 @@ message FilerConf {
         string data_node = 11;
         uint32 max_file_name_length = 12;
         bool disable_chunk_deletion = 13;
+        bool worm = 14;
     }
     repeated PathConf locations = 2;
 }

other/java/examples/pom.xml

@@ -9,15 +9,15 @@
     <version>1.0-SNAPSHOT</version>
     <dependencies>
         <dependency>
-            <groupId>com.github.chrislusf</groupId>
+            <groupId>com.seaweedfs</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>3.59</version>
+            <version>3.71</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
-            <groupId>com.github.chrislusf</groupId>
+            <groupId>com.seaweedfs</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>3.59</version>
+            <version>3.71</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

other/java/hdfs-over-ftp/pom.xml

@@ -49,7 +49,7 @@
             <version>1.1.1</version>
         </dependency>
         <dependency>
-            <groupId>com.github.chrislusf</groupId>
+            <groupId>com.seaweedfs</groupId>
             <artifactId>seaweedfs-hadoop3-client</artifactId>
             <version>1.6.2</version>
         </dependency>
@@ -117,4 +117,4 @@
         </plugins>
     </build>
 
-</project>
+</project>

other/java/hdfs2/dependency-reduced-pom.xml

@@ -1,15 +1,31 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <parent>
-    <artifactId>oss-parent</artifactId>
-    <groupId>org.sonatype.oss</groupId>
-    <version>9</version>
-    <relativePath>../pom.xml/pom.xml</relativePath>
-  </parent>
   <modelVersion>4.0.0</modelVersion>
-  <groupId>com.github.chrislusf</groupId>
+  <groupId>com.seaweedfs</groupId>
   <artifactId>seaweedfs-hadoop2-client</artifactId>
+  <name>SeaweedFS HDFS2 Client</name>
   <version>${seaweedfs.client.version}</version>
+  <description>A java client for SeaweedFS.</description>
+  <url>https://github.com/seaweedfs/seaweedfs</url>
+  <developers>
+    <developer>
+      <name>Chris Lu</name>
+      <email>chris.lu@gmail.com</email>
+      <organization>SeaweedFS</organization>
+      <organizationUrl>https://seaweedfs.com</organizationUrl>
+    </developer>
+  </developers>
+  <licenses>
+    <license>
+      <name>The Apache License, Version 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+    </license>
+  </licenses>
+  <scm>
+    <connection>scm:git:git://github.com/seaweedfs/seaweedfs.git</connection>
+    <developerConnection>scm:git:ssh://github.com:seaweedfs/seaweedfs.git</developerConnection>
+    <url>https://github.com/seaweedfs/seaweedfs/tree/master</url>
+  </scm>
   <build>
     <plugins>
       <plugin>
@@ -85,14 +101,13 @@
         </executions>
       </plugin>
       <plugin>
-        <groupId>org.sonatype.plugins</groupId>
-        <artifactId>nexus-staging-maven-plugin</artifactId>
-        <version>1.6.8</version>
+        <groupId>org.sonatype.central</groupId>
+        <artifactId>central-publishing-maven-plugin</artifactId>
+        <version>0.5.0</version>
         <extensions>true</extensions>
         <configuration>
-          <serverId>ossrh</serverId>
-          <nexusUrl>https://oss.sonatype.org/</nexusUrl>
-          <autoReleaseAfterClose>true</autoReleaseAfterClose>
+          <publishingServerId>central</publishingServerId>
+          <autoPublish>true</autoPublish>
         </configuration>
       </plugin>
       <plugin>
@@ -311,14 +326,8 @@
       </exclusions>
     </dependency>
   </dependencies>
-  <distributionManagement>
-    <snapshotRepository>
-      <id>ossrh</id>
-      <url>https://oss.sonatype.org/content/repositories/snapshots</url>
-    </snapshotRepository>
-  </distributionManagement>
   <properties>
-    <seaweedfs.client.version>3.59</seaweedfs.client.version>
+    <seaweedfs.client.version>3.71</seaweedfs.client.version>
     <hadoop.version>3.2.4</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,26 +5,37 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>3.59</seaweedfs.client.version>
+        <seaweedfs.client.version>3.71</seaweedfs.client.version>
         <hadoop.version>3.2.4</hadoop.version>
     </properties>
 
-    <groupId>com.github.chrislusf</groupId>
+    <groupId>com.seaweedfs</groupId>
     <artifactId>seaweedfs-hadoop2-client</artifactId>
     <version>${seaweedfs.client.version}</version>
 
-    <parent>
-        <groupId>org.sonatype.oss</groupId>
-        <artifactId>oss-parent</artifactId>
-        <version>9</version>
-    </parent>
+    <name>SeaweedFS HDFS2 Client</name>
+    <description>A java client for SeaweedFS.</description>
+    <url>https://github.com/seaweedfs/seaweedfs</url>
+    <licenses>
+        <license>
+            <name>The Apache License, Version 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+        </license>
+    </licenses>
+    <developers>
+        <developer>
+            <name>Chris Lu</name>
+            <email>chris.lu@gmail.com</email>
+            <organization>SeaweedFS</organization>
+            <organizationUrl>https://seaweedfs.com</organizationUrl>
+        </developer>
+    </developers>
+    <scm>
+        <connection>scm:git:git://github.com/seaweedfs/seaweedfs.git</connection>
+        <developerConnection>scm:git:ssh://github.com:seaweedfs/seaweedfs.git</developerConnection>
+        <url>https://github.com/seaweedfs/seaweedfs/tree/master</url>
+    </scm>
 
-    <distributionManagement>
-        <snapshotRepository>
-            <id>ossrh</id>
-            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
-        </snapshotRepository>
-    </distributionManagement>
     <build>
         <plugins>
             <plugin>
@@ -104,14 +115,13 @@
                 </executions>
             </plugin>
             <plugin>
-                <groupId>org.sonatype.plugins</groupId>
-                <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.8</version>
+                <groupId>org.sonatype.central</groupId>
+                <artifactId>central-publishing-maven-plugin</artifactId>
+                <version>0.5.0</version>
                 <extensions>true</extensions>
                 <configuration>
-                    <serverId>ossrh</serverId>
-                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
-                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                    <publishingServerId>central</publishingServerId>
+                    <autoPublish>true</autoPublish>
                 </configuration>
             </plugin>
             <plugin>
@@ -151,7 +161,7 @@
             <scope>provided</scope>
         </dependency>
         <dependency>
-            <groupId>com.github.chrislusf</groupId>
+            <groupId>com.seaweedfs</groupId>
             <artifactId>seaweedfs-client</artifactId>
             <version>${seaweedfs.client.version}</version>
         </dependency>

other/java/hdfs3/dependency-reduced-pom.xml

@@ -1,15 +1,31 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <parent>
-    <artifactId>oss-parent</artifactId>
-    <groupId>org.sonatype.oss</groupId>
-    <version>9</version>
-    <relativePath>../pom.xml/pom.xml</relativePath>
-  </parent>
   <modelVersion>4.0.0</modelVersion>
-  <groupId>com.github.chrislusf</groupId>
+  <groupId>com.seaweedfs</groupId>
   <artifactId>seaweedfs-hadoop3-client</artifactId>
+  <name>SeaweedFS HDFS3 Client</name>
   <version>${seaweedfs.client.version}</version>
+  <description>A java client for SeaweedFS.</description>
+  <url>https://github.com/seaweedfs/seaweedfs</url>
+  <developers>
+    <developer>
+      <name>Chris Lu</name>
+      <email>chris.lu@gmail.com</email>
+      <organization>SeaweedFS</organization>
+      <organizationUrl>https://seaweedfs.com</organizationUrl>
+    </developer>
+  </developers>
+  <licenses>
+    <license>
+      <name>The Apache License, Version 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+    </license>
+  </licenses>
+  <scm>
+    <connection>scm:git:git://github.com/seaweedfs/seaweedfs.git</connection>
+    <developerConnection>scm:git:ssh://github.com:seaweedfs/seaweedfs.git</developerConnection>
+    <url>https://github.com/seaweedfs/seaweedfs/tree/master</url>
+  </scm>
   <build>
     <plugins>
       <plugin>
@@ -85,14 +101,13 @@
         </executions>
       </plugin>
       <plugin>
-        <groupId>org.sonatype.plugins</groupId>
-        <artifactId>nexus-staging-maven-plugin</artifactId>
-        <version>1.6.8</version>
+        <groupId>org.sonatype.central</groupId>
+        <artifactId>central-publishing-maven-plugin</artifactId>
+        <version>0.5.0</version>
         <extensions>true</extensions>
         <configuration>
-          <serverId>ossrh</serverId>
-          <nexusUrl>https://oss.sonatype.org/</nexusUrl>
-          <autoReleaseAfterClose>true</autoReleaseAfterClose>
+          <publishingServerId>central</publishingServerId>
+          <autoPublish>true</autoPublish>
         </configuration>
       </plugin>
       <plugin>
@@ -311,14 +326,8 @@
       </exclusions>
     </dependency>
   </dependencies>
-  <distributionManagement>
-    <snapshotRepository>
-      <id>ossrh</id>
-      <url>https://oss.sonatype.org/content/repositories/snapshots</url>
-    </snapshotRepository>
-  </distributionManagement>
   <properties>
-    <seaweedfs.client.version>3.59</seaweedfs.client.version>
+    <seaweedfs.client.version>3.71</seaweedfs.client.version>
     <hadoop.version>3.2.4</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,26 +5,37 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>3.59</seaweedfs.client.version>
+        <seaweedfs.client.version>3.71</seaweedfs.client.version>
         <hadoop.version>3.2.4</hadoop.version>
     </properties>
 
-    <groupId>com.github.chrislusf</groupId>
+    <groupId>com.seaweedfs</groupId>
     <artifactId>seaweedfs-hadoop3-client</artifactId>
     <version>${seaweedfs.client.version}</version>
 
-    <parent>
-        <groupId>org.sonatype.oss</groupId>
-        <artifactId>oss-parent</artifactId>
-        <version>9</version>
-    </parent>
+    <name>SeaweedFS HDFS3 Client</name>
+    <description>A java client for SeaweedFS.</description>
+    <url>https://github.com/seaweedfs/seaweedfs</url>
+    <licenses>
+        <license>
+            <name>The Apache License, Version 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+        </license>
+    </licenses>
+    <developers>
+        <developer>
+            <name>Chris Lu</name>
+            <email>chris.lu@gmail.com</email>
+            <organization>SeaweedFS</organization>
+            <organizationUrl>https://seaweedfs.com</organizationUrl>
+        </developer>
+    </developers>
+    <scm>
+        <connection>scm:git:git://github.com/seaweedfs/seaweedfs.git</connection>
+        <developerConnection>scm:git:ssh://github.com:seaweedfs/seaweedfs.git</developerConnection>
+        <url>https://github.com/seaweedfs/seaweedfs/tree/master</url>
+    </scm>
 
-    <distributionManagement>
-        <snapshotRepository>
-            <id>ossrh</id>
-            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
-        </snapshotRepository>
-    </distributionManagement>
     <build>
         <plugins>
             <plugin>
@@ -104,14 +115,13 @@
                 </executions>
             </plugin>
             <plugin>
-                <groupId>org.sonatype.plugins</groupId>
-                <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.8</version>
+                <groupId>org.sonatype.central</groupId>
+                <artifactId>central-publishing-maven-plugin</artifactId>
+                <version>0.5.0</version>
                 <extensions>true</extensions>
                 <configuration>
-                    <serverId>ossrh</serverId>
-                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
-                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                    <publishingServerId>central</publishingServerId>
+                    <autoPublish>true</autoPublish>
                 </configuration>
             </plugin>
             <plugin>
@@ -151,7 +161,7 @@
             <scope>provided</scope>
         </dependency>
         <dependency>
-            <groupId>com.github.chrislusf</groupId>
+            <groupId>com.seaweedfs</groupId>
             <artifactId>seaweedfs-client</artifactId>
             <version>${seaweedfs.client.version}</version>
         </dependency>

weed/Makefile

@@ -26,8 +26,8 @@ debug_server:
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1
 
 debug_volume:
-	go build -gcflags="all=-N -l"
-	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- volume -dir=~/tmp/100 -port 8564 -max=30 -preStopSeconds=2
+	go build -tags=5BytesOffset -gcflags="all=-N -l"
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- volume -dir=/Users/chrislu/tmp/x/volume_data -port 8564 -max=1 -preStopSeconds=2
 
 debug_webdav:
 	go build -gcflags="all=-N -l"

weed/command/backup.go

@@ -138,7 +138,9 @@ func runBackup(cmd *Command, args []string) bool {
 
 	if datSize > stats.TailOffset {
 		// remove the old data
-		v.Destroy(false)
+		if err := v.Destroy(false); err != nil {
+			fmt.Printf("Error destroying volume: %v\n", err)
+		}
 		// recreate an empty volume
 		v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0)
 		if err != nil {

weed/command/filer.go

@@ -130,6 +130,7 @@ func init() {
 	filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
 	filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
 	filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
+	filerWebDavOptions.maxMB = cmdFiler.Flag.Int("webdav.maxMB", 4, "split files larger than the limit")
 	filerWebDavOptions.filerRootPath = cmdFiler.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
 
 	// start iam on filer
@@ -239,6 +240,9 @@ func runFiler(cmd *Command, args []string) bool {
 // GetCertificateWithUpdate Auto refreshing TSL certificate
 func (fo *FilerOptions) GetCertificateWithUpdate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
 	certs, err := fo.certProvider.KeyMaterial(context.Background())
+	if certs == nil {
+		return nil, err
+	}
 	return &certs.Certs[0], err
 }
 

weed/command/filer_copy.go

@@ -483,7 +483,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 		for _, chunk := range chunks {
 			fileIds = append(fileIds, chunk.FileId)
 		}
-		operation.DeleteFiles(func(_ context.Context) pb.ServerAddress {
+		operation.DeleteFileIds(func(_ context.Context) pb.ServerAddress {
 			return pb.ServerAddress(copy.masters[0])
 		}, false, worker.options.grpcDialOption, fileIds)
 		return uploadError

weed/command/filer_remote_gateway.go

@@ -71,7 +71,7 @@ var cmdFilerRemoteGateway = &Command{
 	filer.remote.gateway listens on filer local buckets update events. 
 	If any bucket is created, deleted, or updated, it will mirror the changes to remote object store.
 
-		weed filer.remote.sync -createBucketAt=cloud1
+		weed filer.remote.gateway -createBucketAt=cloud1
 
 `,
 }

weed/command/master.go

@@ -36,14 +36,15 @@ var (
 )
 
 type MasterOptions struct {
-	port              *int
-	portGrpc          *int
-	ip                *string
-	ipBind            *string
-	metaFolder        *string
-	peers             *string
-	volumeSizeLimitMB *uint
-	volumePreallocate *bool
+	port                       *int
+	portGrpc                   *int
+	ip                         *string
+	ipBind                     *string
+	metaFolder                 *string
+	peers                      *string
+	volumeSizeLimitMB          *uint
+	volumePreallocate          *bool
+	maxParallelVacuumPerServer *int
 	// pulseSeconds       *int
 	defaultReplication *string
 	garbageThreshold   *float64
@@ -70,6 +71,7 @@ func init() {
 	m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
 	m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
+	m.maxParallelVacuumPerServer = cmdMaster.Flag.Int("maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel per volume server")
 	// m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
 	m.defaultReplication = cmdMaster.Flag.String("defaultReplication", "", "Default replication type if not specified.")
 	m.garbageThreshold = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
@@ -132,8 +134,8 @@ func runMaster(cmd *Command, args []string) bool {
 		*m.metricsHttpIp = *m.ip
 	}
 	go stats_collect.StartMetricsServer(*m.metricsHttpIp, *m.metricsHttpPort)
+	go stats_collect.LoopPushingMetric("masterServer", util.JoinHostPort(*m.ip, *m.port), *m.metricsAddress, *m.metricsIntervalSec)
 	startMaster(m, masterWhiteList)
-
 	return true
 }
 
@@ -311,10 +313,11 @@ func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
 func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOption {
 	masterAddress := pb.NewServerAddress(*m.ip, *m.port, *m.portGrpc)
 	return &weed_server.MasterOption{
-		Master:            masterAddress,
-		MetaFolder:        *m.metaFolder,
-		VolumeSizeLimitMB: uint32(*m.volumeSizeLimitMB),
-		VolumePreallocate: *m.volumePreallocate,
+		Master:                     masterAddress,
+		MetaFolder:                 *m.metaFolder,
+		VolumeSizeLimitMB:          uint32(*m.volumeSizeLimitMB),
+		VolumePreallocate:          *m.volumePreallocate,
+		MaxParallelVacuumPerServer: *m.maxParallelVacuumPerServer,
 		// PulseSeconds:            *m.pulseSeconds,
 		DefaultReplicaPlacement: *m.defaultReplication,
 		GarbageThreshold:        *m.garbageThreshold,

weed/command/mount.go

@@ -56,8 +56,8 @@ func init() {
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
 	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers")
 	mountOptions.cacheDirForRead = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
-	mountOptions.cacheSizeMBForRead = cmdMount.Flag.Int64("cacheCapacityMB", 0, "file chunk read cache capacity in MB")
-	mountOptions.cacheDirForWrite = cmdMount.Flag.String("cacheDirWrite", os.TempDir(), "buffer writes mostly for large files")
+	mountOptions.cacheSizeMBForRead = cmdMount.Flag.Int64("cacheCapacityMB", 128, "file chunk read cache capacity in MB")
+	mountOptions.cacheDirForWrite = cmdMount.Flag.String("cacheDirWrite", "", "buffer writes mostly for large files")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
 	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
 	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")

weed/command/mount_std.go

@@ -6,6 +6,15 @@ package command
 import (
 	"context"
 	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"os/user"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/hanwen/go-fuse/v2/fuse"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mount"
@@ -17,14 +26,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"google.golang.org/grpc/reflection"
-	"net"
-	"net/http"
-	"os"
-	"os/user"
-	"runtime"
-	"strconv"
-	"strings"
-	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
@@ -216,6 +217,11 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		mountRoot = mountRoot[0 : len(mountRoot)-1]
 	}
 
+	cacheDirForWrite := *option.cacheDirForWrite
+	if cacheDirForWrite == "" {
+		cacheDirForWrite = *option.cacheDirForRead
+	}
+
 	seaweedFileSystem := mount.NewSeaweedFileSystem(&mount.Option{
 		MountDirectory:     dir,
 		FilerAddresses:     filerAddresses,
@@ -229,7 +235,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		ConcurrentWriters:  *option.concurrentWriters,
 		CacheDirForRead:    *option.cacheDirForRead,
 		CacheSizeMBForRead: *option.cacheSizeMBForRead,
-		CacheDirForWrite:   *option.cacheDirForWrite,
+		CacheDirForWrite:   cacheDirForWrite,
 		DataCenter:         *option.dataCenter,
 		Quota:              int64(*option.collectionQuota) * 1024 * 1024,
 		MountUid:           uid,
@@ -265,7 +271,11 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	reflection.Register(grpcS)
 	go grpcS.Serve(montSocketListener)
 
-	seaweedFileSystem.StartBackgroundTasks()
+	err = seaweedFileSystem.StartBackgroundTasks()
+	if err != nil {
+		fmt.Printf("failed to start background tasks: %v\n", err)
+		return false
+	}
 
 	glog.V(0).Infof("mounted %s%s to %v", *option.filer, mountRoot, dir)
 	glog.V(0).Infof("This is SeaweedFS version %s %s %s", util.Version(), runtime.GOOS, runtime.GOARCH)

weed/command/s3.go

@@ -180,8 +180,11 @@ func runS3(cmd *Command, args []string) bool {
 }
 
 // GetCertificateWithUpdate Auto refreshing TSL certificate
-func (S3opt *S3Options) GetCertificateWithUpdate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
-	certs, err := S3opt.certProvider.KeyMaterial(context.Background())
+func (s3opt *S3Options) GetCertificateWithUpdate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+	certs, err := s3opt.certProvider.KeyMaterial(context.Background())
+	if certs == nil {
+		return nil, err
+	}
 	return &certs.Certs[0], err
 }
 
@@ -328,6 +331,10 @@ func (s3opt *S3Options) startS3Server() bool {
 			ClientAuth:     clientAuth,
 			ClientCAs:      caCertPool,
 		}
+		err = security.FixTlsConfig(util.GetViper(), httpS.TLSConfig)
+		if err != nil {
+			glog.Fatalf("error with tls config: %v", err)
+		}
 		if *s3opt.portHttps == 0 {
 			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
 			if s3ApiLocalListener != nil {

weed/command/scaffold/filer.toml

@@ -285,7 +285,7 @@ password = ""
 ssl = false
 ssl_ca_file = ""
 ssl_cert_file = ""
-ssl_key_file = "
+ssl_key_file = ""
 insecure_skip_verify = false
 option_pool_size = 0
 database = "seaweedfs"

weed/command/server.go

@@ -92,6 +92,7 @@ func init() {
 	masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
 	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
+	masterOptions.maxParallelVacuumPerServer = cmdServer.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server")
 	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
 	masterOptions.garbageThreshold = cmdServer.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterOptions.metricsAddress = cmdServer.Flag.String("master.metrics.address", "", "Prometheus gateway address")

weed/command/update.go

@@ -20,8 +20,8 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-	"golang.org/x/net/context/ctxhttp"
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	"golang.org/x/net/context/ctxhttp"
 )
 
 //copied from https://github.com/restic/restic/tree/master/internal/selfupdate
@@ -309,7 +309,12 @@ func extractToFile(buf []byte, filename, target string) error {
 		trd := tar.NewReader(gr)
 		hdr, terr := trd.Next()
 		if terr != nil {
-			glog.Errorf("uncompress file(%s) failed:%s", hdr.Name, terr)
+			if hdr != nil {
+				glog.Errorf("uncompress file(%s) failed:%s", hdr.Name, terr)
+			} else {
+				glog.Errorf("uncompress file is nil, failed:%s", terr)
+			}
+
 			return terr
 		}
 		rd = trd

weed/filer/cassandra/cassandra_store.go

@@ -2,6 +2,7 @@ package cassandra
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"github.com/gocql/gocql"
 	"time"
@@ -129,13 +130,10 @@ func (store *CassandraStore) FindEntry(ctx context.Context, fullpath util.FullPa
 	if err := store.session.Query(
 		"SELECT meta FROM filemeta WHERE directory=? AND name=?",
 		dir, name).Scan(&data); err != nil {
-		if err != gocql.ErrNotFound {
+		if errors.Is(err, gocql.ErrNotFound) {
 			return nil, filer_pb.ErrNotFound
 		}
-	}
-
-	if len(data) == 0 {
-		return nil, filer_pb.ErrNotFound
+		return nil, err
 	}
 
 	entry = &filer.Entry{

weed/filer/elastic/v7/elastic_store.go

@@ -159,8 +159,16 @@ func (store *ElasticStore) FindEntry(ctx context.Context, fullpath weed_util.Ful
 func (store *ElasticStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
 	index := getIndex(fullpath, false)
 	id := weed_util.Md5String([]byte(fullpath))
-	if strings.Count(string(fullpath), "/") == 1 {
-		return store.deleteIndex(ctx, index)
+	strFullpath := string(fullpath)
+
+	// A top-level subdirectory refers to an Elasticsearch index.
+	// If we delete an entry at the top level, we should attempt to delete the corresponding Elasticsearch index.
+	if strings.Count(strFullpath, "/") == 1 {
+		entry, err2 := store.FindEntry(ctx, fullpath)
+		if err2 == nil && entry.IsDirectory() {
+			bucketIndex := indexPrefix + strFullpath[1:]
+			store.deleteIndex(ctx, bucketIndex)
+		}
 	}
 	return store.deleteEntry(ctx, index, id)
 }

weed/filer/filechunk_group.go

@@ -1,6 +1,7 @@
 package filer
 
 import (
+	"io"
 	"sync"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -10,7 +11,6 @@ import (
 
 type ChunkGroup struct {
 	lookupFn     wdclient.LookupFileIdFunctionType
-	chunkCache   chunk_cache.ChunkCache
 	sections     map[SectionIndex]*FileChunkSection
 	sectionsLock sync.RWMutex
 	readerCache  *ReaderCache
@@ -19,7 +19,6 @@ type ChunkGroup struct {
 func NewChunkGroup(lookupFn wdclient.LookupFileIdFunctionType, chunkCache chunk_cache.ChunkCache, chunks []*filer_pb.FileChunk) (*ChunkGroup, error) {
 	group := &ChunkGroup{
 		lookupFn:    lookupFn,
-		chunkCache:  chunkCache,
 		sections:    make(map[SectionIndex]*FileChunkSection),
 		readerCache: NewReaderCache(32, chunkCache, lookupFn),
 	}
@@ -46,6 +45,9 @@ func (group *ChunkGroup) AddChunk(chunk *filer_pb.FileChunk) error {
 }
 
 func (group *ChunkGroup) ReadDataAt(fileSize int64, buff []byte, offset int64) (n int, tsNs int64, err error) {
+	if offset >= fileSize {
+		return 0, 0, io.EOF
+	}
 
 	group.sectionsLock.RLock()
 	defer group.sectionsLock.RUnlock()

weed/filer/filer_conf.go

@@ -4,10 +4,11 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
+
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 	"google.golang.org/grpc"
-	"io"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -111,7 +112,7 @@ func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) (err error) {
 	return nil
 }
 
-func (fc *FilerConf) GetLocationConf(locationPrefix string)(locConf *filer_pb.FilerConf_PathConf, found bool) {
+func (fc *FilerConf) GetLocationConf(locationPrefix string) (locConf *filer_pb.FilerConf_PathConf, found bool) {
 	return fc.rules.Get([]byte(locationPrefix))
 }
 
@@ -188,6 +189,7 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
 	a.Rack = util.Nvl(b.Rack, a.Rack)
 	a.DataNode = util.Nvl(b.DataNode, a.DataNode)
 	a.DisableChunkDeletion = b.DisableChunkDeletion || a.DisableChunkDeletion
+	a.Worm = b.Worm || a.Worm
 }
 
 func (fc *FilerConf) ToProto() *filer_pb.FilerConf {

weed/filer/filer_delete_entry.go

@@ -17,7 +17,7 @@ const (
 type OnChunksFunc func([]*filer_pb.FileChunk) error
 type OnHardLinkIdsFunc func([]HardLinkId) error
 
-func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32) (err error) {
+func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32, ifNotModifiedAfter int64) (err error) {
 	if p == "/" {
 		return nil
 	}
@@ -26,6 +26,9 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 	if findErr != nil {
 		return findErr
 	}
+	if ifNotModifiedAfter > 0 && entry.Attr.Mtime.Unix() > ifNotModifiedAfter {
+		return nil
+	}
 	isDeleteCollection := f.isBucket(entry)
 	if entry.IsDirectory() {
 		// delete the folder children, not including the folder itself

weed/filer/filer_deletion.go

@@ -1,11 +1,12 @@
 package filer
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/storage"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"strings"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -22,6 +23,7 @@ func LookupByMasterClientFn(masterClient *wdclient.MasterClient) func(vids []str
 				locations = append(locations, operation.Location{
 					Url:       loc.Url,
 					PublicUrl: loc.PublicUrl,
+					GrpcPort:  loc.GrpcPort,
 				})
 			}
 			m[vid] = &operation.LookupResult{
@@ -53,7 +55,7 @@ func (f *Filer) loopProcessingDeletion() {
 					fileIds = fileIds[:0]
 				}
 				deletionCount = len(toDeleteFileIds)
-				_, err := operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, toDeleteFileIds, lookupFunc)
+				_, err := operation.DeleteFileIdsWithLookupVolumeId(f.GrpcDialOption, toDeleteFileIds, lookupFunc)
 				if err != nil {
 					if !strings.Contains(err.Error(), storage.ErrorDeleted.Error()) {
 						glog.V(0).Infof("deleting fileIds len=%d error: %v", deletionCount, err)

weed/filer/filerstore_wrapper.go

@@ -164,6 +164,9 @@ func (fsw *FilerStoreWrapper) FindEntry(ctx context.Context, fp util.FullPath) (
 	entry, err = actualStore.FindEntry(ctx, fp)
 	// glog.V(4).Infof("FindEntry %s: %v", fp, err)
 	if err != nil {
+		if fsw.CanDropWholeBucket() && strings.Contains(err.Error(), "Table") && strings.Contains(err.Error(), "doesn't exist") {
+			err = filer_pb.ErrNotFound
+		}
 		return nil, err
 	}
 

weed/filer/leveldb/leveldb_store.go

@@ -44,7 +44,7 @@ func (store *LevelDBStore) Initialize(configuration weed_util.Configuration, pre
 }
 
 func (store *LevelDBStore) initialize(dir string) (err error) {
-	glog.Infof("filer store dir: %s", dir)
+	glog.V(0).Infof("filer store dir: %s", dir)
 	os.MkdirAll(dir, 0755)
 	if err := weed_util.TestFolderWritable(dir); err != nil {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)

weed/filer/reader_at.go

@@ -199,7 +199,8 @@ func (c *ChunkReadAt) readChunkSliceAt(buffer []byte, chunkView *ChunkView, next
 		return fetchChunkRange(buffer, c.readerCache.lookupFileIdFn, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset))
 	}
 
-	n, err = c.readerCache.ReadChunkAt(buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), chunkView.ViewOffset == 0)
+	shouldCache := (uint64(chunkView.ViewOffset) + chunkView.ChunkSize) <= c.readerCache.chunkCache.GetMaxFilePartSizeInCache()
+	n, err = c.readerCache.ReadChunkAt(buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), shouldCache)
 	if c.lastChunkFid != chunkView.FileId {
 		if chunkView.OffsetInChunk == 0 { // start of a new chunk
 			if c.lastChunkFid != "" {

weed/filer/reader_at_test.go

@@ -31,6 +31,14 @@ func (m *mockChunkCache) ReadChunkAt(data []byte, fileId string, offset uint64)
 func (m *mockChunkCache) SetChunk(fileId string, data []byte) {
 }
 
+func (m *mockChunkCache) GetMaxFilePartSizeInCache() (uint64) {
+	return 0
+}
+
+func (m *mockChunkCache) IsInCache(fileId string, lockNeeded bool) (answer bool) {
+	return false
+}
+
 func TestReaderAt(t *testing.T) {
 
 	visibles := NewIntervalList[*VisibleInterval]()

weed/filer/reader_cache.go

@@ -6,10 +6,11 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 	"github.com/seaweedfs/seaweedfs/weed/util/mem"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
-	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
 type ReaderCache struct {
@@ -61,6 +62,10 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView]) {
 		if _, found := rc.downloaders[chunkView.FileId]; found {
 			continue
 		}
+		if rc.chunkCache.IsInCache(chunkView.FileId, true) {
+			glog.V(4).Infof("%s is in cache", chunkView.FileId)
+			continue
+		}
 
 		if len(rc.downloaders) >= rc.limit {
 			// abort when slots are filled
@@ -69,7 +74,8 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView]) {
 
 		// glog.V(4).Infof("prefetch %s offset %d", chunkView.FileId, chunkView.ViewOffset)
 		// cache this chunk if not yet
-		cacher := newSingleChunkCacher(rc, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int(chunkView.ChunkSize), false)
+		shouldCache := (uint64(chunkView.ViewOffset) + chunkView.ChunkSize) <= rc.chunkCache.GetMaxFilePartSizeInCache()
+		cacher := newSingleChunkCacher(rc, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int(chunkView.ChunkSize), shouldCache)
 		go cacher.startCaching()
 		<-cacher.cacheStartedCh
 		rc.downloaders[chunkView.FileId] = cacher

weed/filer_client/filer_client_accessor.go

@@ -0,0 +1,65 @@
+package filer_client
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"google.golang.org/grpc"
+	jsonpb "google.golang.org/protobuf/encoding/protojson"
+)
+
+type FilerClientAccessor struct {
+	GetFiler          func() pb.ServerAddress
+	GetGrpcDialOption func() grpc.DialOption
+}
+
+func (fca *FilerClientAccessor) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return pb.WithFilerClient(streamingMode, 0, fca.GetFiler(), fca.GetGrpcDialOption(), fn)
+}
+
+func (fca *FilerClientAccessor) SaveTopicConfToFiler(t *mq_pb.Topic, conf *mq_pb.ConfigureTopicResponse) error {
+
+	glog.V(0).Infof("save conf for topic %v to filer", t)
+
+	// save the topic configuration on filer
+	topicDir := fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
+	if err := fca.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		var buf bytes.Buffer
+		filer.ProtoToText(&buf, conf)
+		return filer.SaveInsideFiler(client, topicDir, "topic.conf", buf.Bytes())
+	}); err != nil {
+		return fmt.Errorf("save topic to %s: %v", topicDir, err)
+	}
+	return nil
+}
+
+func (fca *FilerClientAccessor) ReadTopicConfFromFiler(t topic.Topic) (conf *mq_pb.ConfigureTopicResponse, err error) {
+
+	glog.V(0).Infof("load conf for topic %v from filer", t)
+
+	topicDir := fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
+	if err = fca.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		data, err := filer.ReadInsideFiler(client, topicDir, "topic.conf")
+		if err == filer_pb.ErrNotFound {
+			return err
+		}
+		if err != nil {
+			return fmt.Errorf("read topic.conf of %v: %v", t, err)
+		}
+		// parse into filer conf object
+		conf = &mq_pb.ConfigureTopicResponse{}
+		if err = jsonpb.Unmarshal(data, conf); err != nil {
+			return fmt.Errorf("unmarshal topic %v conf: %v", t, err)
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return conf, nil
+}

weed/mount/filehandle.go

@@ -66,8 +66,8 @@ func (fh *FileHandle) FullPath() util.FullPath {
 	return fp
 }
 
-func (fh *FileHandle) GetEntry() *filer_pb.Entry {
-	return fh.entry.GetEntry()
+func (fh *FileHandle) GetEntry() *LockedEntry {
+	return fh.entry
 }
 
 func (fh *FileHandle) SetEntry(entry *filer_pb.Entry) {
@@ -90,13 +90,6 @@ func (fh *FileHandle) UpdateEntry(fn func(entry *filer_pb.Entry)) *filer_pb.Entr
 }
 
 func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) {
-	fh.entryLock.Lock()
-	defer fh.entryLock.Unlock()
-
-	if fh.entry == nil {
-		return
-	}
-
 	fh.entry.AppendChunks(chunks)
 }
 
@@ -105,9 +98,6 @@ func (fh *FileHandle) ReleaseHandle() {
 	fhActiveLock := fh.wfs.fhLockTable.AcquireLock("ReleaseHandle", fh.fh, util.ExclusiveLock)
 	defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
 
-	fh.entryLock.Lock()
-	defer fh.entryLock.Unlock()
-
 	fh.dirtyPages.Destroy()
 	if IsDebugFileReadWrite {
 		fh.mirrorFile.Close()

weed/mount/filehandle_map.go

@@ -1,6 +1,7 @@
 package mount
 
 import (
+	"github.com/seaweedfs/seaweedfs/weed/util"
 	"sync"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -8,7 +9,6 @@ import (
 
 type FileHandleToInode struct {
 	sync.RWMutex
-	nextFh   FileHandleId
 	inode2fh map[uint64]*FileHandle
 	fh2inode map[FileHandleId]uint64
 }
@@ -17,7 +17,6 @@ func NewFileHandleToInode() *FileHandleToInode {
 	return &FileHandleToInode{
 		inode2fh: make(map[uint64]*FileHandle),
 		fh2inode: make(map[FileHandleId]uint64),
-		nextFh:   0,
 	}
 }
 
@@ -43,14 +42,13 @@ func (i *FileHandleToInode) AcquireFileHandle(wfs *WFS, inode uint64, entry *fil
 	defer i.Unlock()
 	fh, found := i.inode2fh[inode]
 	if !found {
-		fh = newFileHandle(wfs, i.nextFh, inode, entry)
-		i.nextFh++
+		fh = newFileHandle(wfs, FileHandleId(util.RandomUint64()), inode, entry)
 		i.inode2fh[inode] = fh
 		i.fh2inode[fh.fh] = inode
 	} else {
 		fh.counter++
 	}
-	if fh.GetEntry() != entry {
+	if fh.GetEntry().GetEntry() != entry {
 		fh.SetEntry(entry)
 	}
 	return fh

weed/mount/filehandle_read.go

@@ -29,28 +29,27 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, e
 	fileFullPath := fh.FullPath()
 
 	entry := fh.GetEntry()
-	if entry == nil {
-		return 0, 0, io.EOF
-	}
 
 	if entry.IsInRemoteOnly() {
 		glog.V(4).Infof("download remote entry %s", fileFullPath)
-		newEntry, err := fh.downloadRemoteEntry(entry)
+		err := fh.downloadRemoteEntry(entry)
 		if err != nil {
 			glog.V(1).Infof("download remote entry %s: %v", fileFullPath, err)
 			return 0, 0, err
 		}
-		entry = newEntry
 	}
 
 	fileSize := int64(entry.Attributes.FileSize)
 	if fileSize == 0 {
-		fileSize = int64(filer.FileSize(entry))
+		fileSize = int64(filer.FileSize(entry.GetEntry()))
 	}
 
 	if fileSize == 0 {
 		glog.V(1).Infof("empty fh %v", fileFullPath)
 		return 0, 0, io.EOF
+	} else if offset >= fileSize {
+		glog.V(1).Infof("invalid read, fileSize %d, offset %d for %s", fileSize, offset, fileFullPath)
+		return 0, 0, io.EOF
 	}
 
 	if offset < int64(len(entry.Content)) {
@@ -70,7 +69,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, e
 	return int64(totalRead), ts, err
 }
 
-func (fh *FileHandle) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) {
+func (fh *FileHandle) downloadRemoteEntry(entry *LockedEntry) error {
 
 	fileFullPath := fh.FullPath()
 	dir, _ := fileFullPath.DirAndName()
@@ -88,12 +87,12 @@ func (fh *FileHandle) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entr
 			return fmt.Errorf("CacheRemoteObjectToLocalCluster file %s: %v", fileFullPath, err)
 		}
 
-		entry = resp.Entry
+		entry.SetEntry(resp.Entry)
 
 		fh.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, resp.Entry))
 
 		return nil
 	})
 
-	return entry, err
+	return err
 }

weed/mount/filer_conf.go

@@ -0,0 +1,126 @@
+package mount
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func (wfs *WFS) subscribeFilerConfEvents() (func(), error) {
+	now := time.Now()
+	confDir := filer.DirectoryEtcSeaweedFS
+	confName := filer.FilerConfName
+	confFullName := filepath.Join(filer.DirectoryEtcSeaweedFS, filer.FilerConfName)
+
+	// read current conf
+	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		content, err := filer.ReadInsideFiler(client, confDir, confName)
+		if err != nil {
+			return err
+		}
+
+		fc := filer.NewFilerConf()
+		if len(content) > 0 {
+			if err := fc.LoadFromBytes(content); err != nil {
+				return fmt.Errorf("parse %s: %v", confFullName, err)
+			}
+		}
+
+		wfs.FilerConf = fc
+
+		return nil
+	})
+	if err != nil {
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			glog.V(0).Infof("fuse filer conf %s not found", confFullName)
+		} else {
+			return nil, err
+		}
+	}
+
+	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		message := resp.EventNotification
+		if message.NewEntry == nil {
+			return nil
+		}
+
+		dir := resp.Directory
+		name := resp.EventNotification.NewEntry.Name
+
+		if dir != confDir || name != confName {
+			return nil
+		}
+
+		content := message.NewEntry.Content
+		fc := filer.NewFilerConf()
+		if len(content) > 0 {
+			if err = fc.LoadFromBytes(content); err != nil {
+				return fmt.Errorf("parse %s: %v", confFullName, err)
+			}
+		}
+
+		wfs.FilerConf = fc
+
+		return nil
+	}
+
+	metadataFollowOption := &pb.MetadataFollowOption{
+		ClientName:             "fuse",
+		ClientId:               wfs.signature,
+		ClientEpoch:            1,
+		SelfSignature:          0,
+		PathPrefix:             confFullName,
+		AdditionalPathPrefixes: nil,
+		StartTsNs:              now.UnixNano(),
+		StopTsNs:               0,
+		EventErrorType:         pb.FatalOnError,
+	}
+
+	return func() {
+		// sync new conf changes
+		util.RetryUntil("followFilerConfChanges", func() error {
+			metadataFollowOption.ClientEpoch++
+			i := atomic.LoadInt32(&wfs.option.filerIndex)
+			n := len(wfs.option.FilerAddresses)
+			err = pb.FollowMetadata(wfs.option.FilerAddresses[i], wfs.option.GrpcDialOption, metadataFollowOption, processEventFn)
+			if err == nil {
+				atomic.StoreInt32(&wfs.option.filerIndex, i)
+				return nil
+			}
+
+			i++
+			if i >= int32(n) {
+				i = 0
+			}
+
+			return err
+		}, func(err error) bool {
+			glog.V(0).Infof("fuse follow filer conf changes: %v", err)
+			return true
+		})
+	}, nil
+}
+
+func (wfs *WFS) wormEnabledForEntry(path util.FullPath, entry *filer_pb.Entry) bool {
+	if entry == nil || entry.Attributes == nil {
+		return false
+	}
+	if wfs.FilerConf == nil {
+		return false
+	}
+
+	rule := wfs.FilerConf.MatchStorageRule(string(path))
+	if !rule.Worm {
+		return false
+	}
+
+	return entry.Attributes.FileSize > 0 || entry.Attributes.Crtime != entry.Attributes.Mtime
+}

weed/mount/locked_entry.go

@@ -16,6 +16,8 @@ func (le *LockedEntry) GetEntry() *filer_pb.Entry {
 	return le.Entry
 }
 
+// SetEntry sets the entry of the LockedEntry
+// entry should never be nil
 func (le *LockedEntry) SetEntry(entry *filer_pb.Entry) {
 	le.Lock()
 	defer le.Unlock()

weed/mount/meta_cache/meta_cache.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"sync"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/filer/leveldb"
@@ -118,6 +119,9 @@ func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *fi
 	if err != nil {
 		return nil, err
 	}
+	if entry.TtlSec > 0 && entry.Crtime.Add(time.Duration(entry.TtlSec)).Before(time.Now()) {
+		return nil, filer_pb.ErrNotFound
+	}
 	mc.mapIdFromFilerToLocal(entry)
 	return
 }
@@ -143,6 +147,9 @@ func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.Full
 	}
 
 	_, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *filer.Entry) bool {
+		if entry.TtlSec > 0 && entry.Crtime.Add(time.Duration(entry.TtlSec)).Before(time.Now()) {
+			return true
+		}
 		mc.mapIdFromFilerToLocal(entry)
 		return eachEntryFunc(entry)
 	})

weed/mount/weedfs.go

@@ -2,6 +2,7 @@ package mount
 
 import (
 	"context"
+	"errors"
 	"math/rand"
 	"os"
 	"path"
@@ -74,11 +75,12 @@ type WFS struct {
 	signature         int32
 	concurrentWriters *util.LimitedConcurrentExecutor
 	inodeToPath       *InodeToPath
-	fhmap             *FileHandleToInode
-	dhmap             *DirectoryHandleToInode
+	fhMap             *FileHandleToInode
+	dhMap             *DirectoryHandleToInode
 	fuseServer        *fuse.Server
 	IsOverQuota       bool
 	fhLockTable       *util.LockTable[FileHandleId]
+	FilerConf         *filer.FilerConf
 }
 
 func NewSeaweedFileSystem(option *Option) *WFS {
@@ -87,8 +89,8 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 		option:        option,
 		signature:     util.RandomInt32(),
 		inodeToPath:   NewInodeToPath(util.FullPath(option.FilerMountRootPath)),
-		fhmap:         NewFileHandleToInode(),
-		dhmap:         NewDirectoryHandleToInode(),
+		fhMap:         NewFileHandleToInode(),
+		dhMap:         NewDirectoryHandleToInode(),
 		fhLockTable:   util.NewLockTable[FileHandleId](),
 	}
 
@@ -106,24 +108,21 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 			return wfs.inodeToPath.IsChildrenCached(path)
 		}, func(filePath util.FullPath, entry *filer_pb.Entry) {
 			// Find inode if it is not a deleted path
-			if inode, inode_found := wfs.inodeToPath.GetInode(filePath); inode_found {
+			if inode, inodeFound := wfs.inodeToPath.GetInode(filePath); inodeFound {
 				// Find open file handle
-				if fh, fh_found := wfs.fhmap.FindFileHandle(inode); fh_found {
+				if fh, fhFound := wfs.fhMap.FindFileHandle(inode); fhFound {
 					fhActiveLock := fh.wfs.fhLockTable.AcquireLock("invalidateFunc", fh.fh, util.ExclusiveLock)
 					defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
 
-					fh.entryLock.Lock()
-					defer fh.entryLock.Unlock()
-
 					// Recreate dirty pages
 					fh.dirtyPages.Destroy()
 					fh.dirtyPages = newPageWriter(fh, wfs.option.ChunkSizeLimit)
 
 					// Update handle entry
-					newentry, status := wfs.maybeLoadEntry(filePath)
+					newEntry, status := wfs.maybeLoadEntry(filePath)
 					if status == fuse.OK {
-						if fh.GetEntry() != newentry {
-							fh.SetEntry(newentry)
+						if fh.GetEntry().GetEntry() != newEntry {
+							fh.SetEntry(newEntry)
 						}
 					}
 				}
@@ -141,10 +140,19 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 	return wfs
 }
 
-func (wfs *WFS) StartBackgroundTasks() {
+func (wfs *WFS) StartBackgroundTasks() error {
+	fn, err := wfs.subscribeFilerConfEvents()
+	if err != nil {
+		return err
+	}
+
+	go fn()
+
 	startTime := time.Now()
 	go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano())
 	go wfs.loopCheckQuota()
+
+	return nil
 }
 
 func (wfs *WFS) String() string {
@@ -161,7 +169,7 @@ func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle
 		return
 	}
 	var found bool
-	if fh, found = wfs.fhmap.FindFileHandle(inode); found {
+	if fh, found = wfs.fhMap.FindFileHandle(inode); found {
 		entry = fh.UpdateEntry(func(entry *filer_pb.Entry) {
 			if entry != nil && fh.entry.Attributes == nil {
 				entry.Attributes = &filer_pb.FuseAttributes{}
@@ -196,7 +204,7 @@ func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, fuse.St
 	// read from async meta cache
 	meta_cache.EnsureVisited(wfs.metaCache, wfs, util.FullPath(dir))
 	cachedEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath)
-	if cacheErr == filer_pb.ErrNotFound {
+	if errors.Is(cacheErr, filer_pb.ErrNotFound) {
 		return nil, fuse.ENOENT
 	}
 	return cachedEntry.ToProtoEntry(), fuse.OK

weed/mount/weedfs_attr.go

@@ -23,7 +23,7 @@ func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse
 		wfs.setAttrByPbEntry(&out.Attr, inode, entry, true)
 		return status
 	} else {
-		if fh, found := wfs.fhmap.FindFileHandle(inode); found {
+		if fh, found := wfs.fhMap.FindFileHandle(inode); found {
 			out.AttrValid = 1
 			wfs.setAttrByPbEntry(&out.Attr, inode, fh.entry.GetEntry(), true)
 			out.Nlink = 0

weed/mount/weedfs_dir_lookup.go

@@ -57,9 +57,9 @@ func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name strin
 
 	inode := wfs.inodeToPath.Lookup(fullFilePath, localEntry.Crtime.Unix(), localEntry.IsDirectory(), len(localEntry.HardLinkId) > 0, localEntry.Inode, true)
 
-	if fh, found := wfs.fhmap.FindFileHandle(inode); found {
+	if fh, found := wfs.fhMap.FindFileHandle(inode); found {
 		fh.entryLock.RLock()
-		if entry := fh.GetEntry(); entry != nil {
+		if entry := fh.GetEntry().GetEntry(); entry != nil {
 			glog.V(4).Infof("lookup opened file %s size %d", dirPath.Child(localEntry.Name()), filer.FileSize(entry))
 			localEntry = filer.FromPbEntry(string(dirPath), entry)
 		}

weed/mount/weedfs_dir_read.go

@@ -6,6 +6,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 	"math"
 	"sync"
 )
@@ -43,35 +44,32 @@ func NewDirectoryHandleToInode() *DirectoryHandleToInode {
 }
 
 func (wfs *WFS) AcquireDirectoryHandle() (DirectoryHandleId, *DirectoryHandle) {
-	wfs.fhmap.Lock()
-	fh := wfs.fhmap.nextFh
-	wfs.fhmap.nextFh++
-	wfs.fhmap.Unlock()
+	fh := FileHandleId(util.RandomUint64())
 
-	wfs.dhmap.Lock()
-	defer wfs.dhmap.Unlock()
+	wfs.dhMap.Lock()
+	defer wfs.dhMap.Unlock()
 	dh := new(DirectoryHandle)
 	dh.reset()
-	wfs.dhmap.dir2inode[DirectoryHandleId(fh)] = dh
+	wfs.dhMap.dir2inode[DirectoryHandleId(fh)] = dh
 	return DirectoryHandleId(fh), dh
 }
 
 func (wfs *WFS) GetDirectoryHandle(dhid DirectoryHandleId) *DirectoryHandle {
-	wfs.dhmap.Lock()
-	defer wfs.dhmap.Unlock()
-	if dh, found := wfs.dhmap.dir2inode[dhid]; found {
+	wfs.dhMap.Lock()
+	defer wfs.dhMap.Unlock()
+	if dh, found := wfs.dhMap.dir2inode[dhid]; found {
 		return dh
 	}
 	dh := new(DirectoryHandle)
 	dh.reset()
-	wfs.dhmap.dir2inode[dhid] = dh
+	wfs.dhMap.dir2inode[dhid] = dh
 	return dh
 }
 
 func (wfs *WFS) ReleaseDirectoryHandle(dhid DirectoryHandleId) {
-	wfs.dhmap.Lock()
-	defer wfs.dhmap.Unlock()
-	delete(wfs.dhmap.dir2inode, dhid)
+	wfs.dhMap.Lock()
+	defer wfs.dhMap.Unlock()
+	delete(wfs.dhMap.dir2inode, dhid)
 }
 
 // Directory handling
@@ -171,9 +169,9 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPl
 				isEarlyTerminated = true
 				return false
 			}
-			if fh, found := wfs.fhmap.FindFileHandle(inode); found {
+			if fh, found := wfs.fhMap.FindFileHandle(inode); found {
 				glog.V(4).Infof("readdir opened file %s", dirPath.Child(dirEntry.Name))
-				entry = filer.FromPbEntry(string(dirPath), fh.GetEntry())
+				entry = filer.FromPbEntry(string(dirPath), fh.GetEntry().GetEntry())
 			}
 			wfs.outputFilerEntry(entryOut, inode, entry)
 		}

weed/mount/weedfs_file_io.go

@@ -62,9 +62,10 @@ import (
 */
 func (wfs *WFS) Open(cancel <-chan struct{}, in *fuse.OpenIn, out *fuse.OpenOut) (status fuse.Status) {
 	var fileHandle *FileHandle
-	fileHandle, status = wfs.AcquireHandle(in.NodeId, in.Uid, in.Gid)
+	fileHandle, status = wfs.AcquireHandle(in.NodeId, in.Flags, in.Uid, in.Gid)
 	if status == fuse.OK {
 		out.Fh = uint64(fileHandle.fh)
+		out.OpenFlags = in.Flags
 		// TODO https://github.com/libfuse/libfuse/blob/master/include/fuse_common.h#L64
 	}
 	return status

weed/mount/weedfs_file_lseek.go

@@ -38,10 +38,8 @@ func (wfs *WFS) Lseek(cancel <-chan struct{}, in *fuse.LseekIn, out *fuse.LseekO
 	// lock the file until the proper offset was calculated
 	fhActiveLock := fh.wfs.fhLockTable.AcquireLock("Lseek", fh.fh, util.SharedLock)
 	defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
-	fh.entryLock.RLock()
-	defer fh.entryLock.RUnlock()
 
-	fileSize := int64(filer.FileSize(fh.GetEntry()))
+	fileSize := int64(filer.FileSize(fh.GetEntry().GetEntry()))
 	offset := max(int64(in.Offset), 0)
 
 	glog.V(4).Infof(

weed/mount/weedfs_file_mkrm.go

@@ -130,6 +130,10 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
 		return code
 	}
 
+	if wfs.wormEnabledForEntry(entryFullPath, entry) {
+		return fuse.EPERM
+	}
+
 	// first, ensure the filer store can correctly delete
 	glog.V(3).Infof("remove file: %v", entryFullPath)
 	isDeleteData := entry != nil && entry.HardLinkCounter <= 1

weed/mount/weedfs_file_sync.go

@@ -3,13 +3,14 @@ package mount
 import (
 	"context"
 	"fmt"
+	"syscall"
+	"time"
+
 	"github.com/hanwen/go-fuse/v2/fuse"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-	"syscall"
-	"time"
 )
 
 /**
@@ -116,13 +117,8 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
 	defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
 
 	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		fh.entryLock.Lock()
-		defer fh.entryLock.Unlock()
 
 		entry := fh.GetEntry()
-		if entry == nil {
-			return nil
-		}
 		entry.Name = name // this flush may be just after a rename operation
 
 		if entry.Attributes != nil {
@@ -133,15 +129,12 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
 			if entry.Attributes.Gid == 0 {
 				entry.Attributes.Gid = gid
 			}
-			if entry.Attributes.Crtime == 0 {
-				entry.Attributes.Crtime = time.Now().Unix()
-			}
 			entry.Attributes.Mtime = time.Now().Unix()
 		}
 
 		request := &filer_pb.CreateEntryRequest{
 			Directory:                string(dir),
-			Entry:                    entry,
+			Entry:                    entry.GetEntry(),
 			Signatures:               []int32{wfs.signature},
 			SkipCheckParentDirectory: true,
 		}

weed/mount/weedfs_filehandle.go

@@ -3,22 +3,27 @@ package mount
 import (
 	"github.com/hanwen/go-fuse/v2/fuse"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
-func (wfs *WFS) AcquireHandle(inode uint64, uid, gid uint32) (fileHandle *FileHandle, status fuse.Status) {
+func (wfs *WFS) AcquireHandle(inode uint64, flags, uid, gid uint32) (fileHandle *FileHandle, status fuse.Status) {
 	var entry *filer_pb.Entry
-	_, _, entry, status = wfs.maybeReadEntry(inode)
+	var path util.FullPath
+	path, _, entry, status = wfs.maybeReadEntry(inode)
 	if status == fuse.OK {
+		if wfs.wormEnabledForEntry(path, entry) && flags&fuse.O_ANYWRITE != 0 {
+			return nil, fuse.EPERM
+		}
 		// need to AcquireFileHandle again to ensure correct handle counter
-		fileHandle = wfs.fhmap.AcquireFileHandle(wfs, inode, entry)
+		fileHandle = wfs.fhMap.AcquireFileHandle(wfs, inode, entry)
 	}
 	return
 }
 
 func (wfs *WFS) ReleaseHandle(handleId FileHandleId) {
-	wfs.fhmap.ReleaseByHandle(handleId)
+	wfs.fhMap.ReleaseByHandle(handleId)
 }
 
 func (wfs *WFS) GetHandle(handleId FileHandleId) *FileHandle {
-	return wfs.fhmap.GetFileHandle(handleId)
+	return wfs.fhMap.GetFileHandle(handleId)
 }

weed/mount/weedfs_forget.go

@@ -65,5 +65,5 @@ func (wfs *WFS) Forget(nodeid, nlookup uint64) {
 	wfs.inodeToPath.Forget(nodeid, nlookup, func(dir util.FullPath) {
 		wfs.metaCache.DeleteFolderChildren(context.Background(), dir)
 	})
-	wfs.fhmap.ReleaseByInode(nodeid)
+	wfs.fhMap.ReleaseByInode(nodeid)
 }

weed/mount/weedfs_link.go

@@ -25,7 +25,6 @@ When creating a link:
 
 /** Create a hard link to a file */
 func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *fuse.EntryOut) (code fuse.Status) {
-
 	if wfs.IsOverQuota {
 		return fuse.Status(syscall.ENOSPC)
 	}
@@ -49,6 +48,11 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
 		return status
 	}
 
+	// hardlink is not allowed in WORM mode
+	if wfs.wormEnabledForEntry(oldEntryPath, oldEntry) {
+		return fuse.EPERM
+	}
+
 	// update old file to hardlink mode
 	if len(oldEntry.HardLinkId) == 0 {
 		oldEntry.HardLinkId = filer.NewHardLinkId()

weed/mount/weedfs_rename.go

@@ -235,7 +235,7 @@ func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamR
 
 		sourceInode, targetInode := wfs.inodeToPath.MovePath(oldPath, newPath)
 		if sourceInode != 0 {
-			fh, foundFh := wfs.fhmap.FindFileHandle(sourceInode)
+			fh, foundFh := wfs.fhMap.FindFileHandle(sourceInode)
 			if foundFh {
 				if entry := fh.GetEntry(); entry != nil {
 					entry.Name = newName

weed/mq/broker/broker_connect.go

@@ -55,9 +55,9 @@ func (b *MessageQueueBroker) BrokerConnectToBalancer(brokerBalancer string, stop
 			})
 			if err != nil {
 				if err == io.EOF {
-					return err
+					// return err
 				}
-				return fmt.Errorf("send stats message: %v", err)
+				return fmt.Errorf("send stats message to balancer %s: %v", brokerBalancer, err)
 			}
 			// glog.V(3).Infof("sent stats: %+v", stats)
 

weed/mq/broker/broker_grpc_assign.go

@@ -36,7 +36,7 @@ func (b *MessageQueueBroker) AssignTopicPartitions(c context.Context, request *m
 	// if is leader, notify the followers to drain existing topic partition subscriptions
 	if request.IsLeader {
 		for _, brokerPartition := range request.BrokerPartitionAssignments {
-			for _, follower := range brokerPartition.FollowerBrokers {
+			if follower := brokerPartition.FollowerBroker; follower != "" {
 				err := pb.WithBrokerGrpcClient(false, follower, b.grpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
 					_, err := client.AssignTopicPartitions(context.Background(), request)
 					return err
@@ -79,11 +79,11 @@ func (b *MessageQueueBroker) assignTopicPartitionsToBrokers(ctx context.Context,
 						return fmt.Errorf("create topic %s %v on %s: %v", t, bpa.LeaderBroker, bpa.Partition, doCreateErr)
 					}
 				}
-				brokerStats, found := b.Balancer.Brokers.Get(bpa.LeaderBroker)
+				brokerStats, found := b.PubBalancer.Brokers.Get(bpa.LeaderBroker)
 				if !found {
 					brokerStats = pub_balancer.NewBrokerStats()
-					if !b.Balancer.Brokers.SetIfAbsent(bpa.LeaderBroker, brokerStats) {
-						brokerStats, _ = b.Balancer.Brokers.Get(bpa.LeaderBroker)
+					if !b.PubBalancer.Brokers.SetIfAbsent(bpa.LeaderBroker, brokerStats) {
+						brokerStats, _ = b.PubBalancer.Brokers.Get(bpa.LeaderBroker)
 					}
 				}
 				brokerStats.RegisterAssignment(t, bpa.Partition, isAdd)

weed/mq/broker/broker_grpc_balance.go

@@ -20,8 +20,8 @@ func (b *MessageQueueBroker) BalanceTopics(ctx context.Context, request *mq_pb.B
 
 	ret := &mq_pb.BalanceTopicsResponse{}
 
-	actions := b.Balancer.BalancePublishers()
-	err = b.Balancer.ExecuteBalanceAction(actions, b.grpcDialOption)
+	actions := b.PubBalancer.BalancePublishers()
+	err = b.PubBalancer.ExecuteBalanceAction(actions, b.grpcDialOption)
 
 	return ret, err
 }

weed/mq/broker/broker_grpc_configure.go

@@ -37,7 +37,7 @@ func (b *MessageQueueBroker) ConfigureTopic(ctx context.Context, request *mq_pb.
 
 	t := topic.FromPbTopic(request.Topic)
 	var readErr, assignErr error
-	resp, readErr = b.readTopicConfFromFiler(t)
+	resp, readErr = b.fca.ReadTopicConfFromFiler(t)
 	if readErr != nil {
 		glog.V(0).Infof("read topic %s conf: %v", request.Topic, readErr)
 	}
@@ -60,18 +60,18 @@ func (b *MessageQueueBroker) ConfigureTopic(ctx context.Context, request *mq_pb.
 		}
 	}
 	resp = &mq_pb.ConfigureTopicResponse{}
-	if b.Balancer.Brokers.IsEmpty() {
+	if b.PubBalancer.Brokers.IsEmpty() {
 		return nil, status.Errorf(codes.Unavailable, pub_balancer.ErrNoBroker.Error())
 	}
-	resp.BrokerPartitionAssignments = pub_balancer.AllocateTopicPartitions(b.Balancer.Brokers, request.PartitionCount)
+	resp.BrokerPartitionAssignments = pub_balancer.AllocateTopicPartitions(b.PubBalancer.Brokers, request.PartitionCount)
 	resp.RecordType = request.RecordType
 
 	// save the topic configuration on filer
-	if err := b.saveTopicConfToFiler(request.Topic, resp); err != nil {
+	if err := b.fca.SaveTopicConfToFiler(request.Topic, resp); err != nil {
 		return nil, fmt.Errorf("configure topic: %v", err)
 	}
 
-	b.Balancer.OnPartitionChange(request.Topic, resp.BrokerPartitionAssignments)
+	b.PubBalancer.OnPartitionChange(request.Topic, resp.BrokerPartitionAssignments)
 
 	glog.V(0).Infof("ConfigureTopic: topic %s partition assignments: %v", request.Topic, resp.BrokerPartitionAssignments)
 

weed/mq/broker/broker_grpc_lookup.go

@@ -26,7 +26,7 @@ func (b *MessageQueueBroker) LookupTopicBrokers(ctx context.Context, request *mq
 	ret := &mq_pb.LookupTopicBrokersResponse{}
 	conf := &mq_pb.ConfigureTopicResponse{}
 	ret.Topic = request.Topic
-	if conf, err = b.readTopicConfFromFiler(t); err != nil {
+	if conf, err = b.fca.ReadTopicConfFromFiler(t); err != nil {
 		glog.V(0).Infof("lookup topic %s conf: %v", request.Topic, err)
 	} else {
 		err = b.ensureTopicActiveAssignments(t, conf)
@@ -50,7 +50,7 @@ func (b *MessageQueueBroker) ListTopics(ctx context.Context, request *mq_pb.List
 
 	ret := &mq_pb.ListTopicsResponse{}
 	knownTopics := make(map[string]struct{})
-	for brokerStatsItem := range b.Balancer.Brokers.IterBuffered() {
+	for brokerStatsItem := range b.PubBalancer.Brokers.IterBuffered() {
 		_, brokerStats := brokerStatsItem.Key, brokerStatsItem.Val
 		for topicPartitionStatsItem := range brokerStats.TopicPartitionStats.IterBuffered() {
 			topicPartitionStat := topicPartitionStatsItem.Val

weed/mq/broker/broker_grpc_pub.go

@@ -36,9 +36,6 @@ import (
 // Each subscription may not get data. It can act as a backup.
 
 func (b *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_PublishMessageServer) error {
-	// 1. write to the volume server
-	// 2. find the topic metadata owning filer
-	// 3. write to the filer
 
 	req, err := stream.Recv()
 	if err != nil {

weed/mq/broker/broker_grpc_pub_balancer.go

@@ -1,6 +1,7 @@
 package broker
 
 import (
+	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"google.golang.org/grpc/codes"
@@ -14,32 +15,32 @@ func (b *MessageQueueBroker) PublisherToPubBalancer(stream mq_pb.SeaweedMessagin
 	}
 	req, err := stream.Recv()
 	if err != nil {
-		return err
+		return fmt.Errorf("receive init message: %v", err)
 	}
 
 	// process init message
 	initMessage := req.GetInit()
 	var brokerStats *pub_balancer.BrokerStats
 	if initMessage != nil {
-		brokerStats = b.Balancer.AddBroker(initMessage.Broker)
+		brokerStats = b.PubBalancer.AddBroker(initMessage.Broker)
 	} else {
 		return status.Errorf(codes.InvalidArgument, "balancer init message is empty")
 	}
 	defer func() {
-		b.Balancer.RemoveBroker(initMessage.Broker, brokerStats)
+		b.PubBalancer.RemoveBroker(initMessage.Broker, brokerStats)
 	}()
 
 	// process stats message
 	for {
 		req, err := stream.Recv()
 		if err != nil {
-			return err
+			return fmt.Errorf("receive stats message from %s: %v", initMessage.Broker, err)
 		}
 		if !b.isLockOwner() {
 			return status.Errorf(codes.Unavailable, "not current broker balancer")
 		}
 		if receivedStats := req.GetStats(); receivedStats != nil {
-			b.Balancer.OnBrokerStatsUpdated(initMessage.Broker, brokerStats, receivedStats)
+			b.PubBalancer.OnBrokerStatsUpdated(initMessage.Broker, brokerStats, receivedStats)
 			// glog.V(4).Infof("received from %v: %+v", initMessage.Broker, receivedStats)
 		}
 	}

weed/mq/broker/broker_grpc_sub.go

@@ -4,24 +4,30 @@ import (
 	"context"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/sub_coordinator"
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+	"io"
 	"time"
 )
 
-func (b *MessageQueueBroker) SubscribeMessage(req *mq_pb.SubscribeMessageRequest, stream mq_pb.SeaweedMessaging_SubscribeMessageServer) (err error) {
+func (b *MessageQueueBroker) SubscribeMessage(stream mq_pb.SeaweedMessaging_SubscribeMessageServer) error {
 
-	ctx := stream.Context()
-	clientName := fmt.Sprintf("%s/%s-%s", req.GetInit().ConsumerGroup, req.GetInit().ConsumerId, req.GetInit().ClientId)
-
-	initMessage := req.GetInit()
-	if initMessage == nil {
+	req, err := stream.Recv()
+	if err != nil {
+		return err
+	}
+	if req.GetInit() == nil {
 		glog.Errorf("missing init message")
 		return fmt.Errorf("missing init message")
 	}
 
+	ctx := stream.Context()
+	clientName := fmt.Sprintf("%s/%s-%s", req.GetInit().ConsumerGroup, req.GetInit().ConsumerId, req.GetInit().ClientId)
+
 	t := topic.FromPbTopic(req.GetInit().Topic)
 	partition := topic.FromPbPartition(req.GetInit().GetPartitionOffset().GetPartition())
 
@@ -47,11 +53,98 @@ func (b *MessageQueueBroker) SubscribeMessage(req *mq_pb.SubscribeMessageRequest
 		}
 	}()
 
-	var startPosition log_buffer.MessagePosition
-	if req.GetInit() != nil && req.GetInit().GetPartitionOffset() != nil {
-		startPosition = getRequestPosition(req.GetInit().GetPartitionOffset())
+	startPosition := b.getRequestPosition(req.GetInit())
+	imt := sub_coordinator.NewInflightMessageTracker(int(req.GetInit().Concurrency))
+
+	// connect to the follower
+	var subscribeFollowMeStream mq_pb.SeaweedMessaging_SubscribeFollowMeClient
+	glog.V(0).Infof("follower broker: %v", req.GetInit().FollowerBroker)
+	if req.GetInit().FollowerBroker != "" {
+		follower := req.GetInit().FollowerBroker
+		if followerGrpcConnection, err := pb.GrpcDial(ctx, follower, true, b.grpcDialOption); err != nil {
+			return fmt.Errorf("fail to dial %s: %v", follower, err)
+		} else {
+			defer func() {
+				println("closing SubscribeFollowMe connection", follower)
+				subscribeFollowMeStream.CloseSend()
+				// followerGrpcConnection.Close()
+			}()
+			followerClient := mq_pb.NewSeaweedMessagingClient(followerGrpcConnection)
+			if subscribeFollowMeStream, err = followerClient.SubscribeFollowMe(ctx); err != nil {
+				return fmt.Errorf("fail to subscribe to %s: %v", follower, err)
+			} else {
+				if err := subscribeFollowMeStream.Send(&mq_pb.SubscribeFollowMeRequest{
+					Message: &mq_pb.SubscribeFollowMeRequest_Init{
+						Init: &mq_pb.SubscribeFollowMeRequest_InitMessage{
+							Topic:         req.GetInit().Topic,
+							Partition:     req.GetInit().GetPartitionOffset().Partition,
+							ConsumerGroup: req.GetInit().ConsumerGroup,
+						},
+					},
+				}); err != nil {
+					return fmt.Errorf("fail to send init to %s: %v", follower, err)
+				}
+			}
+		}
+		glog.V(0).Infof("follower %s connected", follower)
 	}
 
+	go func() {
+		var lastOffset int64
+		for {
+			ack, err := stream.Recv()
+			if err != nil {
+				if err == io.EOF {
+					// the client has called CloseSend(). This is to ack the close.
+					stream.Send(&mq_pb.SubscribeMessageResponse{Message: &mq_pb.SubscribeMessageResponse_Ctrl{
+						Ctrl: &mq_pb.SubscribeMessageResponse_SubscribeCtrlMessage{
+							IsEndOfStream: true,
+						},
+					}})
+					break
+				}
+				glog.V(0).Infof("topic %v partition %v subscriber %s lastOffset %d error: %v", t, partition, clientName, lastOffset, err)
+				break
+			}
+			if ack.GetAck().Key == nil {
+				// skip ack for control messages
+				continue
+			}
+			imt.AcknowledgeMessage(ack.GetAck().Key, ack.GetAck().Sequence)
+			currentLastOffset := imt.GetOldestAckedTimestamp()
+			// fmt.Printf("%+v recv (%s,%d), oldest %d\n", partition, string(ack.GetAck().Key), ack.GetAck().Sequence, currentLastOffset)
+			if subscribeFollowMeStream != nil && currentLastOffset > lastOffset {
+				if err := subscribeFollowMeStream.Send(&mq_pb.SubscribeFollowMeRequest{
+					Message: &mq_pb.SubscribeFollowMeRequest_Ack{
+						Ack: &mq_pb.SubscribeFollowMeRequest_AckMessage{
+							TsNs: currentLastOffset,
+						},
+					},
+				}); err != nil {
+					glog.Errorf("Error sending ack to follower: %v", err)
+					break
+				}
+				lastOffset = currentLastOffset
+				// fmt.Printf("%+v forwarding ack %d\n", partition, lastOffset)
+			}
+		}
+		if lastOffset > 0 {
+			glog.V(0).Infof("saveConsumerGroupOffset %v %v %v %v", t, partition, req.GetInit().ConsumerGroup, lastOffset)
+			if err := b.saveConsumerGroupOffset(t, partition, req.GetInit().ConsumerGroup, lastOffset); err != nil {
+				glog.Errorf("saveConsumerGroupOffset partition %v lastOffset %d: %v", partition, lastOffset, err)
+			}
+		}
+		if subscribeFollowMeStream != nil {
+			if err := subscribeFollowMeStream.Send(&mq_pb.SubscribeFollowMeRequest{
+				Message: &mq_pb.SubscribeFollowMeRequest_Close{
+					Close: &mq_pb.SubscribeFollowMeRequest_CloseMessage{},
+				},
+			}); err != nil {
+				glog.Errorf("Error sending close to follower: %v", err)
+			}
+		}
+	}()
+
 	return localTopicPartition.Subscribe(clientName, startPosition, func() bool {
 		if !isConnected {
 			return false
@@ -81,6 +174,13 @@ func (b *MessageQueueBroker) SubscribeMessage(req *mq_pb.SubscribeMessageRequest
 		// reset the sleep interval count
 		sleepIntervalCount = 0
 
+		for imt.IsInflight(logEntry.Key) {
+			time.Sleep(137 * time.Millisecond)
+		}
+		if logEntry.Key != nil {
+			imt.EnflightMessage(logEntry.Key, logEntry.TsNs)
+		}
+
 		if err := stream.Send(&mq_pb.SubscribeMessageResponse{Message: &mq_pb.SubscribeMessageResponse_Data{
 			Data: &mq_pb.DataMessage{
 				Key:   logEntry.Key,
@@ -97,10 +197,21 @@ func (b *MessageQueueBroker) SubscribeMessage(req *mq_pb.SubscribeMessageRequest
 	})
 }
 
-func getRequestPosition(offset *mq_pb.PartitionOffset) (startPosition log_buffer.MessagePosition) {
+func (b *MessageQueueBroker) getRequestPosition(initMessage *mq_pb.SubscribeMessageRequest_InitMessage) (startPosition log_buffer.MessagePosition) {
+	if initMessage == nil {
+		return
+	}
+	offset := initMessage.GetPartitionOffset()
 	if offset.StartTsNs != 0 {
 		startPosition = log_buffer.NewMessagePosition(offset.StartTsNs, -2)
+		return
 	}
+	if storedOffset, err := b.readConsumerGroupOffset(initMessage); err == nil {
+		glog.V(0).Infof("resume from saved offset %v %v %v: %v", initMessage.Topic, initMessage.PartitionOffset.Partition, initMessage.ConsumerGroup, storedOffset)
+		startPosition = log_buffer.NewMessagePosition(storedOffset, -2)
+		return
+	}
+
 	if offset.StartType == mq_pb.PartitionOffsetStartType_EARLIEST {
 		startPosition = log_buffer.NewMessagePosition(1, -3)
 	} else if offset.StartType == mq_pb.PartitionOffsetStartType_LATEST {

weed/mq/broker/broker_grpc_sub_coordinator.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/sub_coordinator"
-	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -21,41 +20,42 @@ func (b *MessageQueueBroker) SubscriberToSubCoordinator(stream mq_pb.SeaweedMess
 	}
 
 	var cgi *sub_coordinator.ConsumerGroupInstance
+	var cg *sub_coordinator.ConsumerGroup
 	// process init message
 	initMessage := req.GetInit()
 	if initMessage != nil {
-		cgi = b.Coordinator.AddSubscriber(initMessage.ConsumerGroup, initMessage.ConsumerGroupInstanceId, initMessage.Topic)
+		cg, cgi, err = b.SubCoordinator.AddSubscriber(initMessage)
+		if err != nil {
+			return status.Errorf(codes.InvalidArgument, err.Error())
+		}
 		glog.V(0).Infof("subscriber %s/%s/%s connected", initMessage.ConsumerGroup, initMessage.ConsumerGroupInstanceId, initMessage.Topic)
 	} else {
 		return status.Errorf(codes.InvalidArgument, "subscriber init message is empty")
 	}
 	defer func() {
-		b.Coordinator.RemoveSubscriber(initMessage.ConsumerGroup, initMessage.ConsumerGroupInstanceId, initMessage.Topic)
+		b.SubCoordinator.RemoveSubscriber(initMessage)
 		glog.V(0).Infof("subscriber %s/%s/%s disconnected: %v", initMessage.ConsumerGroup, initMessage.ConsumerGroupInstanceId, initMessage.Topic, err)
 	}()
 
 	ctx := stream.Context()
 
 	go func() {
-		// try to load the partition assignment from filer
-		if conf, err := b.readTopicConfFromFiler(topic.FromPbTopic(initMessage.Topic)); err == nil {
-			// send partition assignment to subscriber
-			cgi.ResponseChan <- &mq_pb.SubscriberToSubCoordinatorResponse{
-				Message: &mq_pb.SubscriberToSubCoordinatorResponse_Assignment_{
-					Assignment: &mq_pb.SubscriberToSubCoordinatorResponse_Assignment{
-						PartitionAssignments: conf.BrokerPartitionAssignments,
-					},
-				},
-			}
-		}
-
 		// process ack messages
 		for {
-			_, err := stream.Recv()
+			req, err := stream.Recv()
 			if err != nil {
 				glog.V(0).Infof("subscriber %s/%s/%s receive: %v", initMessage.ConsumerGroup, initMessage.ConsumerGroupInstanceId, initMessage.Topic, err)
 			}
 
+			if ackUnAssignment := req.GetAckUnAssignment(); ackUnAssignment != nil {
+				glog.V(0).Infof("subscriber %s/%s/%s ack close of %v", initMessage.ConsumerGroup, initMessage.ConsumerGroupInstanceId, initMessage.Topic, ackUnAssignment)
+				cg.AckUnAssignment(cgi, ackUnAssignment)
+			}
+			if ackAssignment := req.GetAckAssignment(); ackAssignment != nil {
+				glog.V(0).Infof("subscriber %s/%s/%s ack assignment %v", initMessage.ConsumerGroup, initMessage.ConsumerGroupInstanceId, initMessage.Topic, ackAssignment)
+				cg.AckAssignment(cgi, ackAssignment)
+			}
+
 			select {
 			case <-ctx.Done():
 				err := ctx.Err()

weed/mq/broker/broker_grpc_sub_follow.go

@@ -0,0 +1,101 @@
+package broker
+
+import (
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"io"
+	"time"
+)
+
+func (b *MessageQueueBroker) SubscribeFollowMe(stream mq_pb.SeaweedMessaging_SubscribeFollowMeServer) (err error) {
+	var req *mq_pb.SubscribeFollowMeRequest
+	req, err = stream.Recv()
+	if err != nil {
+		return err
+	}
+	initMessage := req.GetInit()
+	if initMessage == nil {
+		return fmt.Errorf("missing init message")
+	}
+
+	// create an in-memory offset
+	var lastOffset int64
+
+	// follow each published messages
+	for {
+		// receive a message
+		req, err = stream.Recv()
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+				break
+			}
+			glog.V(0).Infof("topic %v partition %v subscribe stream error: %v", initMessage.Topic, initMessage.Partition, err)
+			break
+		}
+
+		// Process the received message
+		if ackMessage := req.GetAck(); ackMessage != nil {
+			lastOffset = ackMessage.TsNs
+			// println("sub follower got offset", lastOffset)
+		} else if closeMessage := req.GetClose(); closeMessage != nil {
+			glog.V(0).Infof("topic %v partition %v subscribe stream closed: %v", initMessage.Topic, initMessage.Partition, closeMessage)
+			return nil
+		} else {
+			glog.Errorf("unknown message: %v", req)
+		}
+	}
+
+	t, p := topic.FromPbTopic(initMessage.Topic), topic.FromPbPartition(initMessage.Partition)
+
+	if lastOffset > 0 {
+		err = b.saveConsumerGroupOffset(t, p, initMessage.ConsumerGroup, lastOffset)
+	}
+
+	glog.V(0).Infof("shut down follower for %v offset %d", initMessage, lastOffset)
+
+	return err
+}
+
+func (b *MessageQueueBroker) readConsumerGroupOffset(initMessage *mq_pb.SubscribeMessageRequest_InitMessage) (offset int64, err error) {
+	t, p := topic.FromPbTopic(initMessage.Topic), topic.FromPbPartition(initMessage.PartitionOffset.Partition)
+
+	topicDir := fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
+	partitionGeneration := time.Unix(0, p.UnixTimeNs).UTC().Format(topic.TIME_FORMAT)
+	partitionDir := fmt.Sprintf("%s/%s/%04d-%04d", topicDir, partitionGeneration, p.RangeStart, p.RangeStop)
+	offsetFileName := fmt.Sprintf("%s.offset", initMessage.ConsumerGroup)
+
+	err = b.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		data, err := filer.ReadInsideFiler(client, partitionDir, offsetFileName)
+		if err != nil {
+			return err
+		}
+		if len(data) != 8 {
+			return fmt.Errorf("no offset found")
+		}
+		offset = int64(util.BytesToUint64(data))
+		return nil
+	})
+	return offset, err
+}
+
+func (b *MessageQueueBroker) saveConsumerGroupOffset(t topic.Topic, p topic.Partition, consumerGroup string, offset int64) error {
+
+	topicDir := fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
+	partitionGeneration := time.Unix(0, p.UnixTimeNs).UTC().Format(topic.TIME_FORMAT)
+	partitionDir := fmt.Sprintf("%s/%s/%04d-%04d", topicDir, partitionGeneration, p.RangeStart, p.RangeStop)
+	offsetFileName := fmt.Sprintf("%s.offset", consumerGroup)
+
+	offsetBytes := make([]byte, 8)
+	util.Uint64toBytes(offsetBytes, uint64(offset))
+
+	return b.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		glog.V(0).Infof("saving topic %s partition %v consumer group %s offset %d", t, p, consumerGroup, offset)
+		return filer.SaveInsideFiler(client, partitionDir, offsetFileName, offsetBytes)
+	})
+}

weed/mq/broker/broker_server.go

@@ -2,6 +2,7 @@ package broker
 
 import (
 	"context"
+	"github.com/seaweedfs/seaweedfs/weed/filer_client"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
 	"github.com/seaweedfs/seaweedfs/weed/mq/sub_coordinator"
@@ -44,16 +45,17 @@ type MessageQueueBroker struct {
 	filers            map[pb.ServerAddress]struct{}
 	currentFiler      pb.ServerAddress
 	localTopicManager *topic.LocalTopicManager
-	Balancer          *pub_balancer.Balancer
+	PubBalancer       *pub_balancer.PubBalancer
 	lockAsBalancer    *cluster.LiveLock
-	Coordinator       *sub_coordinator.Coordinator
+	SubCoordinator    *sub_coordinator.SubCoordinator
 	accessLock        sync.Mutex
+	fca               *filer_client.FilerClientAccessor
 }
 
 func NewMessageBroker(option *MessageQueueBrokerOption, grpcDialOption grpc.DialOption) (mqBroker *MessageQueueBroker, err error) {
 
-	pub_broker_balancer := pub_balancer.NewBalancer()
-	coordinator := sub_coordinator.NewCoordinator(pub_broker_balancer)
+	pubBalancer := pub_balancer.NewPubBalancer()
+	subCoordinator := sub_coordinator.NewSubCoordinator()
 
 	mqBroker = &MessageQueueBroker{
 		option:            option,
@@ -61,13 +63,18 @@ func NewMessageBroker(option *MessageQueueBrokerOption, grpcDialOption grpc.Dial
 		MasterClient:      wdclient.NewMasterClient(grpcDialOption, option.FilerGroup, cluster.BrokerType, option.BrokerAddress(), option.DataCenter, option.Rack, *pb.NewServiceDiscoveryFromMap(option.Masters)),
 		filers:            make(map[pb.ServerAddress]struct{}),
 		localTopicManager: topic.NewLocalTopicManager(),
-		Balancer:          pub_broker_balancer,
-		Coordinator:       coordinator,
+		PubBalancer:       pubBalancer,
+		SubCoordinator:    subCoordinator,
 	}
+	fca := &filer_client.FilerClientAccessor{
+		GetFiler:          mqBroker.GetFiler,
+		GetGrpcDialOption: mqBroker.GetGrpcDialOption,
+	}
+	mqBroker.fca = fca
+	subCoordinator.FilerClientAccessor = fca
+
 	mqBroker.MasterClient.SetOnPeerUpdateFn(mqBroker.OnBrokerUpdate)
-	pub_broker_balancer.OnPartitionChange = mqBroker.Coordinator.OnPartitionChange
-	pub_broker_balancer.OnAddBroker = mqBroker.Coordinator.OnSubAddBroker
-	pub_broker_balancer.OnRemoveBroker = mqBroker.Coordinator.OnSubRemoveBroker
+	pubBalancer.OnPartitionChange = mqBroker.SubCoordinator.OnPartitionChange
 
 	go mqBroker.MasterClient.KeepConnectedToMaster(context.Background())
 
@@ -119,6 +126,10 @@ func (b *MessageQueueBroker) OnBrokerUpdate(update *master_pb.ClusterNodeUpdate,
 
 }
 
+func (b *MessageQueueBroker) GetGrpcDialOption() grpc.DialOption {
+	return b.grpcDialOption
+}
+
 func (b *MessageQueueBroker) GetFiler() pb.ServerAddress {
 	return b.currentFiler
 }

weed/mq/broker/broker_topic_conf_read_write.go

@@ -1,64 +1,16 @@
 package broker
 
 import (
-	"bytes"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
-	jsonpb "google.golang.org/protobuf/encoding/protojson"
 )
 
-func (b *MessageQueueBroker) saveTopicConfToFiler(t *mq_pb.Topic, conf *mq_pb.ConfigureTopicResponse) error {
-
-	glog.V(0).Infof("save conf for topic %v to filer", t)
-
-	// save the topic configuration on filer
-	topicDir := fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
-	if err := b.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		var buf bytes.Buffer
-		filer.ProtoToText(&buf, conf)
-		return filer.SaveInsideFiler(client, topicDir, "topic.conf", buf.Bytes())
-	}); err != nil {
-		return fmt.Errorf("save topic to %s: %v", topicDir, err)
-	}
-	return nil
-}
-
-// readTopicConfFromFiler reads the topic configuration from filer
-// this should only be run in broker leader, to ensure correct active broker list.
-func (b *MessageQueueBroker) readTopicConfFromFiler(t topic.Topic) (conf *mq_pb.ConfigureTopicResponse, err error) {
-
-	glog.V(0).Infof("load conf for topic %v from filer", t)
-
-	topicDir := fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
-	if err = b.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		data, err := filer.ReadInsideFiler(client, topicDir, "topic.conf")
-		if err == filer_pb.ErrNotFound {
-			return err
-		}
-		if err != nil {
-			return fmt.Errorf("read topic.conf of %v: %v", t, err)
-		}
-		// parse into filer conf object
-		conf = &mq_pb.ConfigureTopicResponse{}
-		if err = jsonpb.Unmarshal(data, conf); err != nil {
-			return fmt.Errorf("unmarshal topic %v conf: %v", t, err)
-		}
-		return nil
-	}); err != nil {
-		return nil, err
-	}
-
-	return conf, nil
-}
-
 func (b *MessageQueueBroker) GetOrGenerateLocalPartition(t topic.Topic, partition topic.Partition) (localTopicPartition *topic.LocalPartition, getOrGenError error) {
 	// get or generate a local partition
-	conf, readConfErr := b.readTopicConfFromFiler(t)
+	conf, readConfErr := b.fca.ReadTopicConfFromFiler(t)
 	if readConfErr != nil {
 		glog.Errorf("topic %v not found: %v", t, readConfErr)
 		return nil, fmt.Errorf("topic %v not found: %v", t, readConfErr)
@@ -100,10 +52,10 @@ func (b *MessageQueueBroker) genLocalPartitionFromFiler(t topic.Topic, partition
 
 func (b *MessageQueueBroker) ensureTopicActiveAssignments(t topic.Topic, conf *mq_pb.ConfigureTopicResponse) (err error) {
 	// also fix assignee broker if invalid
-	hasChanges := pub_balancer.EnsureAssignmentsToActiveBrokers(b.Balancer.Brokers, 1, conf.BrokerPartitionAssignments)
+	hasChanges := pub_balancer.EnsureAssignmentsToActiveBrokers(b.PubBalancer.Brokers, 1, conf.BrokerPartitionAssignments)
 	if hasChanges {
 		glog.V(0).Infof("topic %v partition updated assignments: %v", t, conf.BrokerPartitionAssignments)
-		if err = b.saveTopicConfToFiler(t.ToPbTopic(), conf); err != nil {
+		if err = b.fca.SaveTopicConfToFiler(t.ToPbTopic(), conf); err != nil {
 			return err
 		}
 	}

weed/mq/broker/broker_topic_partition_read_write.go

@@ -50,8 +50,7 @@ func (b *MessageQueueBroker) genLogFlushFunc(t topic.Topic, partition *mq_pb.Par
 			localPartition.NotifyLogFlushed(logBuffer.LastFlushTsNs)
 		}
 
-		println("flushing at", logBuffer.LastFlushTsNs, "to", targetFile, "size", len(buf))
-
+		glog.V(0).Infof("flushing at %d to %s size %d", logBuffer.LastFlushTsNs, targetFile, len(buf))
 	}
 }
 
@@ -105,7 +104,7 @@ func (b *MessageQueueBroker) genLogOnDiskReadFunc(t topic.Topic, partition *mq_p
 
 	eachFileFn := func(entry *filer_pb.Entry, eachLogEntryFn log_buffer.EachLogEntryFuncType, starTsNs, stopTsNs int64) (processedTsNs int64, err error) {
 		if len(entry.Content) > 0 {
-			glog.Warningf("this should not happen. unexpected content in %s/%s", partitionDir, entry.Name)
+			// skip .offset files
 			return
 		}
 		var urlStrings []string

weed/mq/client/cmd/weed_sub_kv/subscriber_kv.go

@@ -15,9 +15,11 @@ import (
 )
 
 var (
-	namespace   = flag.String("ns", "test", "namespace")
-	t           = flag.String("topic", "test", "topic")
-	seedBrokers = flag.String("brokers", "localhost:17777", "seed brokers")
+	namespace               = flag.String("ns", "test", "namespace")
+	t                       = flag.String("topic", "test", "topic")
+	seedBrokers             = flag.String("brokers", "localhost:17777", "seed brokers")
+	maxPartitionCount       = flag.Int("maxPartitionCount", 3, "max partition count")
+	perPartitionConcurrency = flag.Int("perPartitionConcurrency", 1, "per partition concurrency")
 
 	clientId = flag.Uint("client_id", uint(util.RandomInt32()), "client id")
 )
@@ -27,10 +29,11 @@ func main() {
 	util_http.InitGlobalHttpClient()
 
 	subscriberConfig := &sub_client.SubscriberConfiguration{
-		ClientId:                fmt.Sprintf("client-%d", *clientId),
 		ConsumerGroup:           "test",
 		ConsumerGroupInstanceId: fmt.Sprintf("client-%d", *clientId),
 		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
+		MaxPartitionCount:       int32(*maxPartitionCount),
+		PerPartitionConcurrency: int32(*perPartitionConcurrency),
 	}
 
 	contentConfig := &sub_client.ContentConfiguration{
@@ -39,18 +42,14 @@ func main() {
 		StartTime: time.Unix(1, 1),
 	}
 
-	processorConfig := sub_client.ProcessorConfiguration{
-		ConcurrentPartitionLimit: 3,
-	}
-
 	brokers := strings.Split(*seedBrokers, ",")
-	subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig)
+	subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig)
 
 	counter := 0
-	subscriber.SetEachMessageFunc(func(key, value []byte) (bool, error) {
+	subscriber.SetEachMessageFunc(func(key, value []byte) error {
 		counter++
 		println(string(key), "=>", string(value), counter)
-		return true, nil
+		return nil
 	})
 
 	subscriber.SetCompletionFunc(func() {

weed/mq/client/cmd/weed_sub_record/subscriber_record.go

@@ -17,9 +17,11 @@ import (
 )
 
 var (
-	namespace   = flag.String("ns", "test", "namespace")
-	t           = flag.String("topic", "test", "topic")
-	seedBrokers = flag.String("brokers", "localhost:17777", "seed brokers")
+	namespace               = flag.String("ns", "test", "namespace")
+	t                       = flag.String("topic", "test", "topic")
+	seedBrokers             = flag.String("brokers", "localhost:17777", "seed brokers")
+	maxPartitionCount       = flag.Int("maxPartitionCount", 3, "max partition count")
+	perPartitionConcurrency = flag.Int("perPartitionConcurrency", 1, "per partition concurrency")
 
 	clientId = flag.Uint("client_id", uint(util.RandomInt32()), "client id")
 )
@@ -53,10 +55,11 @@ func main() {
 	util_http.InitGlobalHttpClient()
 
 	subscriberConfig := &sub_client.SubscriberConfiguration{
-		ClientId:                fmt.Sprintf("client-%d", *clientId),
 		ConsumerGroup:           "test",
 		ConsumerGroupInstanceId: fmt.Sprintf("client-%d", *clientId),
 		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
+		MaxPartitionCount:       int32(*maxPartitionCount),
+		PerPartitionConcurrency: int32(*perPartitionConcurrency),
 	}
 
 	contentConfig := &sub_client.ContentConfiguration{
@@ -65,20 +68,17 @@ func main() {
 		StartTime: time.Unix(1, 1),
 	}
 
-	processorConfig := sub_client.ProcessorConfiguration{
-		ConcurrentPartitionLimit: 3,
-	}
-
 	brokers := strings.Split(*seedBrokers, ",")
-	subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig)
+	subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig)
 
 	counter := 0
-	subscriber.SetEachMessageFunc(func(key, value []byte) (bool, error) {
+	subscriber.SetEachMessageFunc(func(key, value []byte) error {
 		counter++
 		record := &schema_pb.RecordValue{}
 		proto.Unmarshal(value, record)
 		fmt.Printf("record: %v\n", record)
-		return true, nil
+		time.Sleep(1300 * time.Millisecond)
+		return nil
 	})
 
 	subscriber.SetCompletionFunc(func() {

weed/mq/client/pub_client/publish.go

@@ -51,6 +51,7 @@ func (p *TopicPublisher) FinishPublish() error {
 				TsNs: time.Now().UnixNano(),
 				Ctrl: &mq_pb.ControlMessage{
 					IsClose: true,
+					PublisherName: p.config.PublisherName,
 				},
 			})
 		}

weed/mq/client/pub_client/scheduler.go

@@ -142,11 +142,11 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
 	if err = publishClient.Send(&mq_pb.PublishMessageRequest{
 		Message: &mq_pb.PublishMessageRequest_Init{
 			Init: &mq_pb.PublishMessageRequest_InitMessage{
-				Topic:           p.config.Topic.ToPbTopic(),
-				Partition:       job.Partition,
-				AckInterval:     128,
-				FollowerBrokers: job.FollowerBrokers,
-				PublisherName:   p.config.PublisherName,
+				Topic:          p.config.Topic.ToPbTopic(),
+				Partition:      job.Partition,
+				AckInterval:    128,
+				FollowerBroker: job.FollowerBroker,
+				PublisherName:  p.config.PublisherName,
 			},
 		},
 	}); err != nil {

weed/mq/client/sub_client/connect_to_sub_coordinator.go

@@ -2,12 +2,9 @@ package sub_client
 
 import (
 	"context"
-	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
-	"io"
-	"sync"
 	"time"
 )
 
@@ -51,6 +48,7 @@ func (sub *TopicSubscriber) doKeepConnectedToSubCoordinator() {
 							ConsumerGroup:           sub.SubscriberConfig.ConsumerGroup,
 							ConsumerGroupInstanceId: sub.SubscriberConfig.ConsumerGroupInstanceId,
 							Topic:                   sub.ContentConfig.Topic.ToPbTopic(),
+							MaxPartitionCount:       sub.SubscriberConfig.MaxPartitionCount,
 						},
 					},
 				}); err != nil {
@@ -58,6 +56,16 @@ func (sub *TopicSubscriber) doKeepConnectedToSubCoordinator() {
 					return err
 				}
 
+				go func() {
+					for reply := range sub.brokerPartitionAssignmentAckChan {
+						glog.V(0).Infof("subscriber instance %s ack %+v", sub.SubscriberConfig.ConsumerGroupInstanceId, reply)
+						if err := stream.Send(reply); err != nil {
+							glog.V(0).Infof("subscriber %s reply: %v", sub.ContentConfig.Topic, err)
+							return
+						}
+					}
+				}()
+
 				// keep receiving messages from the sub coordinator
 				for {
 					resp, err := stream.Recv()
@@ -65,11 +73,8 @@ func (sub *TopicSubscriber) doKeepConnectedToSubCoordinator() {
 						glog.V(0).Infof("subscriber %s receive: %v", sub.ContentConfig.Topic, err)
 						return err
 					}
-					assignment := resp.GetAssignment()
-					if assignment != nil {
-						glog.V(0).Infof("subscriber %s receive assignment: %v", sub.ContentConfig.Topic, assignment)
-					}
-					sub.onEachAssignment(assignment)
+					sub.brokerPartitionAssignmentChan <- resp
+					glog.V(0).Infof("Received assignment: %+v", resp)
 				}
 
 				return nil
@@ -82,100 +87,3 @@ func (sub *TopicSubscriber) doKeepConnectedToSubCoordinator() {
 		time.Sleep(waitTime)
 	}
 }
-
-func (sub *TopicSubscriber) onEachAssignment(assignment *mq_pb.SubscriberToSubCoordinatorResponse_Assignment) {
-	if assignment == nil {
-		return
-	}
-	// process each partition, with a concurrency limit
-	var wg sync.WaitGroup
-	semaphore := make(chan struct{}, sub.ProcessorConfig.ConcurrentPartitionLimit)
-
-	for _, assigned := range assignment.PartitionAssignments {
-		wg.Add(1)
-		semaphore <- struct{}{}
-		go func(assigned *mq_pb.BrokerPartitionAssignment) {
-			defer wg.Done()
-			defer func() { <-semaphore }()
-			glog.V(0).Infof("subscriber %s/%s assigned partition %+v at %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, assigned.Partition, assigned.LeaderBroker)
-			err := sub.onEachPartition(assigned)
-			if err != nil {
-				glog.V(0).Infof("subscriber %s/%s partition %+v at %v: %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, assigned.Partition, assigned.LeaderBroker, err)
-			}
-		}(assigned)
-	}
-
-	wg.Wait()
-}
-
-func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssignment) error {
-	// connect to the partition broker
-	return pb.WithBrokerGrpcClient(true, assigned.LeaderBroker, sub.SubscriberConfig.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
-		subscribeClient, err := client.SubscribeMessage(context.Background(), &mq_pb.SubscribeMessageRequest{
-			Message: &mq_pb.SubscribeMessageRequest_Init{
-				Init: &mq_pb.SubscribeMessageRequest_InitMessage{
-					ConsumerGroup: sub.SubscriberConfig.ConsumerGroup,
-					ConsumerId:    sub.SubscriberConfig.ConsumerGroupInstanceId,
-					Topic:         sub.ContentConfig.Topic.ToPbTopic(),
-					PartitionOffset: &mq_pb.PartitionOffset{
-						Partition: assigned.Partition,
-						StartTsNs: sub.alreadyProcessedTsNs,
-						StartType: mq_pb.PartitionOffsetStartType_EARLIEST_IN_MEMORY,
-					},
-					Filter:          sub.ContentConfig.Filter,
-					FollowerBrokers: assigned.FollowerBrokers,
-				},
-			},
-		})
-
-		if err != nil {
-			return fmt.Errorf("create subscribe client: %v", err)
-		}
-
-		glog.V(0).Infof("subscriber %s/%s connected to partition %+v at %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, assigned.Partition, assigned.LeaderBroker)
-
-		if sub.OnCompletionFunc != nil {
-			defer sub.OnCompletionFunc()
-		}
-		defer func() {
-			subscribeClient.SendMsg(&mq_pb.SubscribeMessageRequest{
-				Message: &mq_pb.SubscribeMessageRequest_Ack{
-					Ack: &mq_pb.SubscribeMessageRequest_AckMessage{
-						Sequence: 0,
-					},
-				},
-			})
-			subscribeClient.CloseSend()
-		}()
-
-		for {
-			// glog.V(0).Infof("subscriber %s/%s/%s waiting for message", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup)
-			resp, err := subscribeClient.Recv()
-			if err != nil {
-				return fmt.Errorf("subscribe recv: %v", err)
-			}
-			if resp.Message == nil {
-				glog.V(0).Infof("subscriber %s/%s received nil message", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup)
-				continue
-			}
-			switch m := resp.Message.(type) {
-			case *mq_pb.SubscribeMessageResponse_Data:
-				shouldContinue, processErr := sub.OnEachMessageFunc(m.Data.Key, m.Data.Value)
-				if processErr != nil {
-					return fmt.Errorf("process error: %v", processErr)
-				}
-				sub.alreadyProcessedTsNs = m.Data.TsNs
-				if !shouldContinue {
-					return nil
-				}
-			case *mq_pb.SubscribeMessageResponse_Ctrl:
-				// glog.V(0).Infof("subscriber %s/%s/%s received control %+v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, m.Ctrl)
-				if m.Ctrl.IsEndOfStream || m.Ctrl.IsEndOfTopic {
-					return io.EOF
-				}
-			}
-		}
-
-		return nil
-	})
-}

weed/mq/client/sub_client/on_each_partition.go

@@ -0,0 +1,125 @@
+package sub_client
+
+import (
+	"context"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"io"
+)
+
+func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssignment, stopCh chan struct{}) error {
+	// connect to the partition broker
+	return pb.WithBrokerGrpcClient(true, assigned.LeaderBroker, sub.SubscriberConfig.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
+
+		subscribeClient, err := client.SubscribeMessage(context.Background())
+		if err != nil {
+			return fmt.Errorf("create subscribe client: %v", err)
+		}
+
+		perPartitionConcurrency := sub.SubscriberConfig.PerPartitionConcurrency
+		if perPartitionConcurrency <= 0 {
+			perPartitionConcurrency = 1
+		}
+
+		if err = subscribeClient.Send(&mq_pb.SubscribeMessageRequest{
+			Message: &mq_pb.SubscribeMessageRequest_Init{
+				Init: &mq_pb.SubscribeMessageRequest_InitMessage{
+					ConsumerGroup: sub.SubscriberConfig.ConsumerGroup,
+					ConsumerId:    sub.SubscriberConfig.ConsumerGroupInstanceId,
+					Topic:         sub.ContentConfig.Topic.ToPbTopic(),
+					PartitionOffset: &mq_pb.PartitionOffset{
+						Partition: assigned.Partition,
+						StartType: mq_pb.PartitionOffsetStartType_EARLIEST_IN_MEMORY,
+					},
+					Filter:         sub.ContentConfig.Filter,
+					FollowerBroker: assigned.FollowerBroker,
+					Concurrency:    perPartitionConcurrency,
+				},
+			},
+		}); err != nil {
+			glog.V(0).Infof("subscriber %s connected to partition %+v at %v: %v", sub.ContentConfig.Topic, assigned.Partition, assigned.LeaderBroker, err)
+		}
+
+		glog.V(0).Infof("subscriber %s connected to partition %+v at %v", sub.ContentConfig.Topic, assigned.Partition, assigned.LeaderBroker)
+
+		if sub.OnCompletionFunc != nil {
+			defer sub.OnCompletionFunc()
+		}
+
+		type KeyedOffset struct {
+			Key    []byte
+			Offset int64
+		}
+
+		partitionOffsetChan := make(chan KeyedOffset, 1024)
+		defer func() {
+			close(partitionOffsetChan)
+		}()
+		executors := util.NewLimitedConcurrentExecutor(int(perPartitionConcurrency))
+
+		go func() {
+			for {
+				select {
+				case <-stopCh:
+					subscribeClient.CloseSend()
+					return
+				case ack, ok := <-partitionOffsetChan:
+					if !ok {
+						subscribeClient.CloseSend()
+						return
+					}
+					subscribeClient.SendMsg(&mq_pb.SubscribeMessageRequest{
+						Message: &mq_pb.SubscribeMessageRequest_Ack{
+							Ack: &mq_pb.SubscribeMessageRequest_AckMessage{
+								Key:      ack.Key,
+								Sequence: ack.Offset,
+							},
+						},
+					})
+				}
+			}
+		}()
+
+		var lastErr error
+
+		for lastErr == nil {
+			// glog.V(0).Infof("subscriber %s/%s/%s waiting for message", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup)
+			resp, err := subscribeClient.Recv()
+			if err != nil {
+				return fmt.Errorf("subscribe recv: %v", err)
+			}
+			if resp.Message == nil {
+				glog.V(0).Infof("subscriber %s/%s received nil message", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup)
+				continue
+			}
+			switch m := resp.Message.(type) {
+			case *mq_pb.SubscribeMessageResponse_Data:
+				if m.Data.Ctrl != nil {
+					glog.V(2).Infof("subscriber %s received control from producer:%s isClose:%v", sub.SubscriberConfig.ConsumerGroup, m.Data.Ctrl.PublisherName, m.Data.Ctrl.IsClose)
+					continue
+				}
+				executors.Execute(func() {
+					processErr := sub.OnEachMessageFunc(m.Data.Key, m.Data.Value)
+					if processErr == nil {
+						partitionOffsetChan <- KeyedOffset{
+							Key:    m.Data.Key,
+							Offset: m.Data.TsNs,
+						}
+					} else {
+						lastErr = processErr
+					}
+				})
+			case *mq_pb.SubscribeMessageResponse_Ctrl:
+				// glog.V(0).Infof("subscriber %s/%s/%s received control %+v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, m.Ctrl)
+				if m.Ctrl.IsEndOfStream || m.Ctrl.IsEndOfTopic {
+					return io.EOF
+				}
+			}
+		}
+
+		return lastErr
+	})
+}

weed/mq/client/sub_client/subscribe.go

@@ -1,11 +1,121 @@
 package sub_client
 
+import (
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"sync"
+	"time"
+)
+
+type ProcessorState struct {
+	stopCh chan struct{}
+}
+
 // Subscribe subscribes to a topic's specified partitions.
 // If a partition is moved to another broker, the subscriber will automatically reconnect to the new broker.
 
 func (sub *TopicSubscriber) Subscribe() error {
+
+	go sub.startProcessors()
+
 	// loop forever
 	sub.doKeepConnectedToSubCoordinator()
 
 	return nil
 }
+
+func (sub *TopicSubscriber) startProcessors() {
+	// listen to the messages from the sub coordinator
+	// start one processor per partition
+	var wg sync.WaitGroup
+	semaphore := make(chan struct{}, sub.SubscriberConfig.MaxPartitionCount)
+
+	for message := range sub.brokerPartitionAssignmentChan {
+		if assigned := message.GetAssignment(); assigned != nil {
+			wg.Add(1)
+			semaphore <- struct{}{}
+
+			topicPartition := topic.FromPbPartition(assigned.PartitionAssignment.Partition)
+
+			// wait until no covering partition is still in progress
+			sub.waitUntilNoOverlappingPartitionInFlight(topicPartition)
+
+			// start a processors
+			stopChan := make(chan struct{})
+			sub.activeProcessorsLock.Lock()
+			sub.activeProcessors[topicPartition] = &ProcessorState{
+				stopCh: stopChan,
+			}
+			sub.activeProcessorsLock.Unlock()
+
+			go func(assigned *mq_pb.BrokerPartitionAssignment, topicPartition topic.Partition) {
+				defer func() {
+					sub.activeProcessorsLock.Lock()
+					delete(sub.activeProcessors, topicPartition)
+					sub.activeProcessorsLock.Unlock()
+
+					<-semaphore
+					wg.Done()
+				}()
+				glog.V(0).Infof("subscriber %s/%s assigned partition %+v at %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, assigned.Partition, assigned.LeaderBroker)
+				sub.brokerPartitionAssignmentAckChan <- &mq_pb.SubscriberToSubCoordinatorRequest{
+					Message: &mq_pb.SubscriberToSubCoordinatorRequest_AckAssignment{
+						AckAssignment: &mq_pb.SubscriberToSubCoordinatorRequest_AckAssignmentMessage{
+							Partition: assigned.Partition,
+						},
+					},
+				}
+				err := sub.onEachPartition(assigned, stopChan)
+				if err != nil {
+					glog.V(0).Infof("subscriber %s/%s partition %+v at %v: %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, assigned.Partition, assigned.LeaderBroker, err)
+				} else {
+					glog.V(0).Infof("subscriber %s/%s partition %+v at %v completed", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, assigned.Partition, assigned.LeaderBroker)
+				}
+				sub.brokerPartitionAssignmentAckChan <- &mq_pb.SubscriberToSubCoordinatorRequest{
+					Message: &mq_pb.SubscriberToSubCoordinatorRequest_AckUnAssignment{
+						AckUnAssignment: &mq_pb.SubscriberToSubCoordinatorRequest_AckUnAssignmentMessage{
+							Partition: assigned.Partition,
+						},
+					},
+				}
+			}(assigned.PartitionAssignment, topicPartition)
+		}
+		if unAssignment := message.GetUnAssignment(); unAssignment != nil {
+			topicPartition := topic.FromPbPartition(unAssignment.Partition)
+			sub.activeProcessorsLock.Lock()
+			if processor, found := sub.activeProcessors[topicPartition]; found {
+				close(processor.stopCh)
+				delete(sub.activeProcessors, topicPartition)
+			}
+			sub.activeProcessorsLock.Unlock()
+		}
+	}
+
+	wg.Wait()
+
+}
+
+func (sub *TopicSubscriber) waitUntilNoOverlappingPartitionInFlight(topicPartition topic.Partition) {
+	foundOverlapping := true
+	for foundOverlapping {
+		sub.activeProcessorsLock.Lock()
+		foundOverlapping = false
+		var overlappedPartition topic.Partition
+		for partition, _ := range sub.activeProcessors {
+			if partition.Overlaps(topicPartition) {
+				if partition.Equals(topicPartition) {
+					continue
+				}
+				foundOverlapping = true
+				overlappedPartition = partition
+				break
+			}
+		}
+		sub.activeProcessorsLock.Unlock()
+		if foundOverlapping {
+			glog.V(0).Infof("subscriber %s new partition %v waiting for partition %+v to complete", sub.ContentConfig.Topic, topicPartition, overlappedPartition)
+			time.Sleep(1 * time.Second)
+		}
+	}
+}

weed/mq/client/sub_client/subscriber.go

@@ -4,6 +4,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"google.golang.org/grpc"
+	"sync"
 	"time"
 )
 
@@ -11,10 +12,9 @@ type SubscriberConfiguration struct {
 	ClientId                string
 	ConsumerGroup           string
 	ConsumerGroupInstanceId string
-	GroupMinimumPeers       int32
-	GroupMaximumPeers       int32
-	BootstrapServers        []string
 	GrpcDialOption          grpc.DialOption
+	MaxPartitionCount       int32 // how many partitions to process concurrently
+	PerPartitionConcurrency int32 // how many messages to process concurrently per partition
 }
 
 type ContentConfiguration struct {
@@ -23,33 +23,31 @@ type ContentConfiguration struct {
 	StartTime time.Time
 }
 
-type ProcessorConfiguration struct {
-	ConcurrentPartitionLimit int // how many partitions to process concurrently
-}
-
-type OnEachMessageFunc func(key, value []byte) (shouldContinue bool, err error)
+type OnEachMessageFunc func(key, value []byte) (err error)
 type OnCompletionFunc func()
 
 type TopicSubscriber struct {
-	SubscriberConfig           *SubscriberConfiguration
-	ContentConfig              *ContentConfiguration
-	ProcessorConfig            *ProcessorConfiguration
-	brokerPartitionAssignments []*mq_pb.BrokerPartitionAssignment
-	OnEachMessageFunc          OnEachMessageFunc
-	OnCompletionFunc           OnCompletionFunc
-	bootstrapBrokers           []string
-	waitForMoreMessage         bool
-	alreadyProcessedTsNs       int64
+	SubscriberConfig                 *SubscriberConfiguration
+	ContentConfig                    *ContentConfiguration
+	brokerPartitionAssignmentChan    chan *mq_pb.SubscriberToSubCoordinatorResponse
+	brokerPartitionAssignmentAckChan chan *mq_pb.SubscriberToSubCoordinatorRequest
+	OnEachMessageFunc                OnEachMessageFunc
+	OnCompletionFunc                 OnCompletionFunc
+	bootstrapBrokers                 []string
+	waitForMoreMessage               bool
+	activeProcessors                 map[topic.Partition]*ProcessorState
+	activeProcessorsLock             sync.Mutex
 }
 
-func NewTopicSubscriber(bootstrapBrokers []string, subscriber *SubscriberConfiguration, content *ContentConfiguration, processor ProcessorConfiguration) *TopicSubscriber {
+func NewTopicSubscriber(bootstrapBrokers []string, subscriber *SubscriberConfiguration, content *ContentConfiguration) *TopicSubscriber {
 	return &TopicSubscriber{
-		SubscriberConfig:     subscriber,
-		ContentConfig:        content,
-		ProcessorConfig:      &processor,
-		bootstrapBrokers:     bootstrapBrokers,
-		waitForMoreMessage:   true,
-		alreadyProcessedTsNs: content.StartTime.UnixNano(),
+		SubscriberConfig:                 subscriber,
+		ContentConfig:                    content,
+		brokerPartitionAssignmentChan:    make(chan *mq_pb.SubscriberToSubCoordinatorResponse, 1024),
+		brokerPartitionAssignmentAckChan: make(chan *mq_pb.SubscriberToSubCoordinatorRequest, 1024),
+		bootstrapBrokers:                 bootstrapBrokers,
+		waitForMoreMessage:               true,
+		activeProcessors:                 make(map[topic.Partition]*ProcessorState),
 	}
 }
 

weed/mq/pub_balancer/allocate.go

@@ -48,20 +48,11 @@ func pickBrokers(brokers cmap.ConcurrentMap[string, *BrokerStats], count int32)
 	return pickedBrokers
 }
 
-// reservoir sampling select N brokers from the active brokers, with exclusion of the excluded brokers
-func pickBrokersExcluded(brokers []string, count int, excludedLeadBroker string, excludedBrokers []string) []string {
-	// convert the excluded brokers to a map
-	excludedBrokerMap := make(map[string]bool)
-	for _, broker := range excludedBrokers {
-		excludedBrokerMap[broker] = true
-	}
-	if excludedLeadBroker != "" {
-		excludedBrokerMap[excludedLeadBroker] = true
-	}
-
+// reservoir sampling select N brokers from the active brokers, with exclusion of the excluded broker
+func pickBrokersExcluded(brokers []string, count int, excludedLeadBroker string, excludedBroker string) []string {
 	pickedBrokers := make([]string, 0, count)
 	for i, broker := range brokers {
-		if _, found := excludedBrokerMap[broker]; found {
+		if broker == excludedBroker {
 			continue
 		}
 		if len(pickedBrokers) < count {
@@ -102,21 +93,15 @@ func EnsureAssignmentsToActiveBrokers(activeBrokers cmap.ConcurrentMap[string, *
 			assignment.LeaderBroker = ""
 			count++
 		}
-		for i := 0; i < followerCount; i++ {
-			if i >= len(assignment.FollowerBrokers) {
-				count++
-				continue
-			}
-			if assignment.FollowerBrokers[i] == "" {
-				count++
-			} else if _, found := activeBrokers.Get(assignment.FollowerBrokers[i]); !found {
-				assignment.FollowerBrokers[i] = ""
-				count++
-			}
+		if assignment.FollowerBroker == "" {
+			count++
+		} else if _, found := activeBrokers.Get(assignment.FollowerBroker); !found {
+			assignment.FollowerBroker = ""
+			count++
 		}
 
 		if count > 0 {
-			pickedBrokers := pickBrokersExcluded(candidates, count, assignment.LeaderBroker, assignment.FollowerBrokers)
+			pickedBrokers := pickBrokersExcluded(candidates, count, assignment.LeaderBroker, assignment.FollowerBroker)
 			i := 0
 			if assignment.LeaderBroker == "" {
 				if i < len(pickedBrokers) {
@@ -125,34 +110,13 @@ func EnsureAssignmentsToActiveBrokers(activeBrokers cmap.ConcurrentMap[string, *
 					hasChanges = true
 				}
 			}
-
-			hasEmptyFollowers := false
-			j := 0
-			for ; j < len(assignment.FollowerBrokers); j++ {
-				if assignment.FollowerBrokers[j] == "" {
+			if assignment.FollowerBroker == "" {
+				if i < len(pickedBrokers) {
+					assignment.FollowerBroker = pickedBrokers[i]
+					i++
 					hasChanges = true
-					if i < len(pickedBrokers) {
-						assignment.FollowerBrokers[j] = pickedBrokers[i]
-						i++
-					} else {
-						hasEmptyFollowers = true
-					}
 				}
 			}
-			if hasEmptyFollowers {
-				var followerBrokers []string
-				for _, follower := range assignment.FollowerBrokers {
-					if follower != "" {
-						followerBrokers = append(followerBrokers, follower)
-					}
-				}
-				assignment.FollowerBrokers = followerBrokers
-			}
-
-			if i < len(pickedBrokers) {
-				assignment.FollowerBrokers = append(assignment.FollowerBrokers, pickedBrokers[i:]...)
-				hasChanges = true
-			}
 		}
 
 	}

weed/mq/pub_balancer/allocate_test.go

@@ -97,9 +97,7 @@ func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
 					{
 						LeaderBroker: "",
 						Partition:    &mq_pb.Partition{},
-						FollowerBrokers: []string{
-							"localhost:2",
-						},
+						FollowerBroker: "localhost:2",
 					},
 				},
 			},
@@ -114,9 +112,7 @@ func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
 					{
 						LeaderBroker: "localhost:1",
 						Partition:    &mq_pb.Partition{},
-						FollowerBrokers: []string{
-							"",
-						},
+						FollowerBroker: "",
 					},
 				},
 			},
@@ -131,9 +127,7 @@ func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
 					{
 						LeaderBroker: "localhost:1",
 						Partition:    &mq_pb.Partition{},
-						FollowerBrokers: []string{
-							"localhost:200",
-						},
+						FollowerBroker: "localhost:200",
 					},
 				},
 			},
@@ -148,43 +142,7 @@ func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
 					{
 						LeaderBroker: "localhost:100",
 						Partition:    &mq_pb.Partition{},
-						FollowerBrokers: []string{
-							"localhost:200",
-						},
-					},
-				},
-			},
-			hasChanges: true,
-		},
-		{
-			name: "test missing two followers",
-			args: args{
-				activeBrokers: activeBrokers,
-				followerCount: 3,
-				assignments: []*mq_pb.BrokerPartitionAssignment{
-					{
-						LeaderBroker: "localhost:1",
-						Partition:    &mq_pb.Partition{},
-						FollowerBrokers: []string{
-							"localhost:2",
-						},
-					},
-				},
-			},
-			hasChanges: true,
-		},
-		{
-			name: "test missing some followers",
-			args: args{
-				activeBrokers: activeBrokers,
-				followerCount: 10,
-				assignments: []*mq_pb.BrokerPartitionAssignment{
-					{
-						LeaderBroker: "localhost:1",
-						Partition:    &mq_pb.Partition{},
-						FollowerBrokers: []string{
-							"localhost:2",
-						},
+						FollowerBroker: "localhost:200",
 					},
 				},
 			},
@@ -199,9 +157,7 @@ func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
 					{
 						LeaderBroker: "localhost:1",
 						Partition:    &mq_pb.Partition{},
-						FollowerBrokers: []string{
-							"localhost:2",
-						},
+						FollowerBroker: "localhost:2",
 					},
 				},
 			},
@@ -230,9 +186,7 @@ func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
 					{
 						LeaderBroker: "localhost:1",
 						Partition:    &mq_pb.Partition{},
-						FollowerBrokers: []string{
-							"localhost:2",
-						},
+						FollowerBroker: "localhost:2",
 					},
 				},
 			},

weed/mq/pub_balancer/balance.go

@@ -54,12 +54,12 @@ type BalanceActionCreate struct {
 
 // BalancePublishers check the stats of all brokers,
 // and balance the publishers to the brokers.
-func (balancer *Balancer) BalancePublishers() []BalanceAction {
+func (balancer *PubBalancer) BalancePublishers() []BalanceAction {
 	action := BalanceTopicPartitionOnBrokers(balancer.Brokers)
 	return []BalanceAction{action}
 }
 
-func (balancer *Balancer) ExecuteBalanceAction(actions []BalanceAction, grpcDialOption grpc.DialOption) (err error) {
+func (balancer *PubBalancer) ExecuteBalanceAction(actions []BalanceAction, grpcDialOption grpc.DialOption) (err error) {
 	for _, action := range actions {
 		switch action.(type) {
 		case *BalanceActionMove:

weed/mq/pub_balancer/balance_action.go

@@ -8,10 +8,10 @@ import (
 	"google.golang.org/grpc"
 )
 
-// Balancer <= PublisherToPubBalancer() <= Broker <=> Publish()
-// ExecuteBalanceActionMove from Balancer => AssignTopicPartitions() => Broker => Publish()
+// PubBalancer <= PublisherToPubBalancer() <= Broker <=> Publish()
+// ExecuteBalanceActionMove from PubBalancer => AssignTopicPartitions() => Broker => Publish()
 
-func (balancer *Balancer) ExecuteBalanceActionMove(move *BalanceActionMove, grpcDialOption grpc.DialOption) error {
+func (balancer *PubBalancer) ExecuteBalanceActionMove(move *BalanceActionMove, grpcDialOption grpc.DialOption) error {
 	if _, found := balancer.Brokers.Get(move.SourceBroker); !found {
 		return fmt.Errorf("source broker %s not found", move.SourceBroker)
 	}