WIP

Move `shell.ErrorWaitGroup` into a dedicated common file, to cleanly reuse across `weed shell` commands.

.github/workflows/binaries_release4.yml

@@ -36,7 +36,7 @@ jobs:
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}
           overwrite: true
-          build_flags: -tags elastic,gocdk,rclone,sqlite,tikv,ydb
+          build_flags: -tags elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
           ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
@@ -52,7 +52,7 @@ jobs:
           goarch: ${{ matrix.goarch }}
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
-          build_flags: -tags 5BytesOffset,elastic,gocdk,rclone,sqlite,tikv,ydb
+          build_flags: -tags 5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
           ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed

.github/workflows/container_dev.yml

@@ -20,7 +20,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -33,30 +33,30 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -21,7 +21,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -34,30 +34,30 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -20,7 +20,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -34,20 +34,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -21,7 +21,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -35,20 +35,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -21,7 +21,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -35,20 +35,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release4.yml

@@ -20,7 +20,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -34,25 +34,25 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build
-          build-args: TAGS=elastic,gocdk,rclone,sqlite,tikv,ydb
+          build-args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
           platforms: linux/amd64
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release5.yml

@@ -20,7 +20,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v3
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -34,25 +34,25 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v1
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v1
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v2
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build
-          build-args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tikv,ydb
+          build-args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
           platforms: linux/amd64
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/depsreview.yml

@@ -11,4 +11,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@0659a74c94536054bfa5aeb92241f70d680cc78e
+        uses: actions/dependency-review-action@38ecb5b593bf0eb19e335c03f97670f792489a8b

.github/workflows/go.yml

@@ -34,7 +34,7 @@ jobs:
         cd weed; go get -v -t -d ./...
 
     - name: Build
-      run: cd weed; go build -tags "elastic gocdk sqlite ydb tikv rclone" -v .
+      run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
 
     - name: Test
-      run: cd weed; go test -tags "elastic gocdk sqlite ydb tikv rclone" -v ./...
+      run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...

.github/workflows/test-s3-over-https-using-awscli.yml

@@ -0,0 +1,79 @@
+name: "test s3 over https using aws-cli"
+
+on:
+  push:
+    branches: [master, test-https-s3-awscli]
+  pull_request:
+    branches: [master, test-https-s3-awscli]
+
+env:
+  AWS_ACCESS_KEY_ID: some_access_key1
+  AWS_SECRET_ACCESS_KEY: some_secret_key1
+  AWS_ENDPOINT_URL: https://localhost:8443
+
+defaults:
+  run:
+    working-directory: weed
+
+jobs:
+  awscli-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ^1.24
+
+      - name: Build SeaweedFS
+        run: |
+          go build
+
+      - name: Start SeaweedFS
+        run: |
+          set -e
+          mkdir -p /tmp/data
+          ./weed server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json &
+          until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
+
+      - name: Setup Caddy
+        run: |
+          curl -fsSL "https://caddyserver.com/api/download?os=linux&arch=amd64" -o caddy
+          chmod +x caddy
+          ./caddy version
+          echo "{
+            auto_https disable_redirects
+            local_certs
+          }
+          localhost:8443 {
+            tls internal
+            reverse_proxy localhost:8333
+          }" > Caddyfile
+
+      - name: Start Caddy
+        run: |
+          ./caddy start
+          until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
+
+      - name: Create Bucket
+        run: |
+          aws --no-verify-ssl s3api create-bucket --bucket bucket
+
+      - name: Test PutObject
+        run: |
+          set -e
+          dd if=/dev/urandom of=generated bs=1M count=2
+          aws --no-verify-ssl s3api put-object --bucket bucket --key test-putobject --body generated
+          aws --no-verify-ssl s3api get-object --bucket bucket --key test-putobject downloaded
+          diff -q generated downloaded
+          rm -f generated downloaded
+
+      - name: Test Multi-part Upload
+        run: |
+          set -e
+          dd if=/dev/urandom of=generated bs=1M count=32
+          aws --no-verify-ssl s3 cp --no-progress generated s3://bucket/test-multipart
+          aws --no-verify-ssl s3 cp --no-progress s3://bucket/test-multipart downloaded
+          diff -q generated downloaded
+          rm -f generated downloaded

.gitignore

@@ -87,7 +87,6 @@ other/java/hdfs/dependency-reduced-pom.xml
 
 # binary file
 weed/weed
-weed/mq/client/cmd/weed_pub_kv/weed_pub
 docker/weed
 
 # test generated files
@@ -95,7 +94,4 @@ weed/*/*.jpg
 docker/weed_sub
 docker/weed_pub
 weed/mq/schema/example.parquet
-docker/weed_pub_kv
-docker/weed_pub_record
-docker/weed_sub_kv
-docker/weed_sub_record
+docker/agent_sub_record

Makefile

@@ -14,7 +14,7 @@ warp_install:
 	go install github.com/minio/warp@v0.7.6
 
 full_install:
-	cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone"
+	cd weed; go install -tags "elastic gocdk sqlite ydb tarantool tikv rclone"
 
 server: install
 	weed -v 0 server -s3 -filer -filer.maxMB=64 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324
@@ -34,4 +34,4 @@ benchmark_with_pprof: debug = 1
 benchmark_with_pprof: benchmark
 
 test:
-	cd weed; go test -tags "elastic gocdk sqlite ydb tikv rclone" -v ./...
+	cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...

docker/Dockerfile.go_build

@@ -1,4 +1,4 @@
-FROM golang:1.23-alpine as builder
+FROM golang:1.24-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/seaweedfs/
 RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs

docker/Dockerfile.rocksdb_dev_env

@@ -1,9 +1,9 @@
-FROM golang:1.23 as builder
+FROM golang:1.24 as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v8.10.0
+ENV ROCKSDB_VERSION v9.10.0
 
 # build RocksDB
 RUN cd /tmp && \

docker/Dockerfile.rocksdb_large

@@ -1,9 +1,9 @@
-FROM golang:1.23 as builder
+FROM golang:1.24 as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v8.10.0
+ENV ROCKSDB_VERSION v9.10.0
 
 # build RocksDB
 RUN cd /tmp && \

docker/Dockerfile.tarantool.dev_env

@@ -0,0 +1,17 @@
+FROM tarantool/tarantool:3.3.1 AS builder
+
+# install dependencies
+RUN apt update && \
+  apt install -y git unzip cmake tt=2.7.0
+
+# init tt dir structure, create dir for app, create symlink
+RUN tt init && \
+  mkdir app && \
+  ln -sfn ${PWD}/app/ ${PWD}/instances.enabled/app
+
+# copy cluster configs
+COPY tarantool /opt/tarantool/app
+
+# build app
+RUN tt build app
+

docker/Makefile

@@ -9,10 +9,8 @@ binary:
 	export SWCOMMIT=$(shell git rev-parse --short HEAD)
 	export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
 	cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w -extldflags -static $(SWLDFLAGS)" && mv weed ../docker/
-	cd ../weed/mq/client/cmd/weed_pub_kv && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_pub_kv ../../../../../docker/
-	cd ../weed/mq/client/cmd/weed_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_pub_record ../../../../../docker/
-	cd ../weed/mq/client/cmd/weed_sub_kv && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_sub_kv ../../../../../docker/
-	cd ../weed/mq/client/cmd/weed_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_sub_record ../../../../../docker/
+	cd ../other/mq_client_example/agent_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_pub_record ../../../docker/
+	cd ../other/mq_client_example/agent_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_sub_record ../../../docker/
 
 binary_race: options = -race
 binary_race: cgo = 1
@@ -24,7 +22,7 @@ build: binary
 build_e2e: binary_race
 	docker build --no-cache -t chrislusf/seaweedfs:e2e -f Dockerfile.e2e .
 
-go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset
+go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset,tarantool
 	docker build --build-arg TAGS=$(tags) --no-cache -t chrislusf/seaweedfs:go_build -f Dockerfile.go_build .
 
 go_build_large_disk:
@@ -39,6 +37,9 @@ build_rocksdb_local: build_rocksdb_dev_env
 build_rocksdb:
 	docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large .
 
+build_tarantool_dev_env:
+	docker build --no-cache -t chrislusf/tarantool_dev_env -f Dockerfile.tarantool.dev_env .
+
 s3tests_build:
 	docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
 
@@ -97,6 +98,9 @@ s3tests: build s3tests_build
 brokers: build
 	docker compose -f compose/local-brokers-compose.yml -p seaweedfs up
 
+agent: build
+	docker compose -f compose/local-mq-test.yml -p seaweedfs up
+
 filer_etcd: build
 	docker stack deploy -c compose/swarm-etcd.yml fs
 
@@ -105,9 +109,12 @@ test_etcd: build
 
 test_ydb: tags = ydb
 test_ydb: build
-	export
 	docker compose -f compose/test-ydb-filer.yml -p seaweedfs up
 
+test_tarantool: tags = tarantool
+test_tarantool: build_tarantool_dev_env build
+	docker compose -f compose/test-tarantool-filer.yml -p seaweedfs up
+
 clean:
 	rm ./weed
 

docker/compose/fluent.conf

@@ -0,0 +1,8 @@
+<source>
+  @type forward
+  port 24224
+</source>
+
+<match **>
+  @type stdout  # Output logs to container's stdout (visible via `docker logs`)
+</match>

docker/compose/local-auditlog-compose.yml

@@ -19,7 +19,9 @@ services:
     depends_on:
       - fluent
   fluent:
-    image: fluent/fluentd:v1.14
+    image: fluent/fluentd:v1.17
+    volumes:
+      - ./fluent.conf:/fluentd/etc/fluent.conf
     ports:
       - 24224:24224
   #s3tests:

docker/compose/local-mq-test.yml

@@ -1,5 +1,3 @@
-version: '3.9'
-
 services:
   server:
     image: chrislusf/seaweedfs:local
@@ -19,9 +17,16 @@ services:
     depends_on:
       server:
         condition: service_healthy
+  mq_agent:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 16777:16777
+    command: "mq.agent -broker=mq_broker:17777 -port=16777"
+    depends_on:
+      - mq_broker
   mq_client:
     image: chrislusf/seaweedfs:local
     # run a custom command instead of entrypoint
     command: "ls -al"
     depends_on:
-      - mq_broker
+      - mq_agent

docker/compose/test-tarantool-filer.yml

@@ -0,0 +1,30 @@
+version: '3.9'
+
+services:
+  tarantool:
+    image: chrislusf/tarantool_dev_env
+    entrypoint: "tt start app -i"
+    environment:
+      APP_USER_PASSWORD: "app"
+      CLIENT_USER_PASSWORD: "client"
+      REPLICATOR_USER_PASSWORD: "replicator"
+      STORAGE_USER_PASSWORD: "storage"
+    network_mode: "host"
+    ports:
+      - "3303:3303"
+
+  s3:
+    image: chrislusf/seaweedfs:local
+    command: "server -ip=127.0.0.1 -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false"
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
+    environment:
+      WEED_LEVELDB2_ENABLED: "false"
+      WEED_TARANTOOL_ENABLED: "true"
+      WEED_TARANTOOL_ADDRESS: "127.0.0.1:3303"
+      WEED_TARANTOOL_USER: "client"
+      WEED_TARANTOOL_PASSWORD: "client"
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+    network_mode: "host"
+    depends_on:
+      - tarantool

docker/compose/userstore.json

@@ -0,0 +1,37 @@
+[
+  {
+    "Username": "admin",
+    "Password": "myadminpassword",
+    "PublicKeys": [
+    ],
+    "HomeDir": "/",
+    "Permissions": {
+      "/": ["*"]
+    },
+    "Uid": 0,
+    "Gid": 0
+  },
+  {
+    "Username": "user1",
+    "Password": "myuser1password",
+    "PublicKeys": [""],
+    "HomeDir": "/user1",
+    "Permissions": {
+      "/user1": ["*"],
+      "/public": ["read", "list","write"]
+    },
+    "Uid": 1111,
+    "Gid": 1111
+  },
+  {
+    "Username": "readonly",
+    "Password": "myreadonlypassword",
+    "PublicKeys": [],
+    "HomeDir": "/public",
+    "Permissions": {
+      "/public": ["read", "list"]
+    },
+    "Uid": 1112,
+    "Gid": 1112
+  }
+]

docker/tarantool/app-scm-1.rockspec

@@ -0,0 +1,14 @@
+package = 'app'
+version = 'scm-1'
+source  = {
+    url = '/dev/null',
+}
+dependencies = {
+    'crud == 1.5.2-1',
+    'expirationd == 1.6.0-1',
+    'metrics-export-role == 0.3.0-1',
+    'vshard == 0.1.32-1'
+}
+build = {
+    type = 'none';
+}

docker/tarantool/config.yaml

@@ -0,0 +1,145 @@
+config:
+  context:
+    app_user_password:
+      from: env
+      env: APP_USER_PASSWORD
+    client_user_password:
+      from: env
+      env: CLIENT_USER_PASSWORD
+    replicator_user_password:
+      from: env
+      env: REPLICATOR_USER_PASSWORD
+    storage_user_password:
+      from: env
+      env: STORAGE_USER_PASSWORD
+
+credentials:
+  roles:
+    crud-role:
+      privileges:
+        - permissions: [ "execute" ]
+          lua_call: [ "crud.delete", "crud.get", "crud.upsert" ]
+  users:
+    app:
+      password: '{{ context.app_user_password }}'
+      roles: [ public, crud-role ]
+    client:
+      password: '{{ context.client_user_password }}'
+      roles: [ super ]
+    replicator:
+      password: '{{ context.replicator_user_password }}'
+      roles: [ replication ]
+    storage:
+      password: '{{ context.storage_user_password }}'
+      roles: [ sharding ]
+
+iproto:
+  advertise:
+    peer:
+      login: replicator
+    sharding:
+      login: storage
+
+sharding:
+  bucket_count: 10000
+
+metrics:
+  include: [ all ]
+  exclude: [ vinyl ]
+  labels:
+    alias: '{{ instance_name }}'
+
+
+groups:
+  storages:
+    roles:
+      - roles.crud-storage
+      - roles.expirationd
+      - roles.metrics-export
+    roles_cfg:
+      roles.expirationd:
+        cfg:
+          metrics: true
+        filer_metadata_task:
+          space: filer_metadata
+          is_expired: filer_metadata.is_expired
+          options:
+            atomic_iteration: true
+            force: true
+            index: 'expire_at_idx'
+            iterator_type: GT
+            start_key:
+              - 0
+            tuples_per_iteration: 10000
+    app:
+      module: storage
+    sharding:
+      roles: [ storage ]
+    replication:
+      failover: election
+    database:
+      use_mvcc_engine: true
+    replicasets:
+      storage-001:
+        instances:
+          storage-001-a:
+            roles_cfg:
+              roles.metrics-export:
+                http:
+                  - listen: '0.0.0.0:8081'
+                    endpoints:
+                      - path: /metrics/prometheus/
+                        format: prometheus
+                      - path: /metrics/json
+                        format: json
+            iproto:
+              listen:
+                - uri: 127.0.0.1:3301
+              advertise:
+                client: 127.0.0.1:3301
+          storage-001-b:
+            roles_cfg:
+              roles.metrics-export:
+                http:
+                  - listen: '0.0.0.0:8082'
+                    endpoints:
+                      - path: /metrics/prometheus/
+                        format: prometheus
+                      - path: /metrics/json
+                        format: json
+            iproto:
+              listen:
+                - uri: 127.0.0.1:3302
+              advertise:
+                client: 127.0.0.1:3302
+  routers:
+    roles:
+      - roles.crud-router
+      - roles.metrics-export
+    roles_cfg:
+      roles.crud-router:
+        stats: true
+        stats_driver: metrics
+        stats_quantiles: true
+    app:
+      module: router
+    sharding:
+      roles: [ router ]
+    replicasets:
+      router-001:
+        instances:
+          router-001-a:
+            roles_cfg:
+              roles.metrics-export:
+                http:
+                  - listen: '0.0.0.0:8083'
+                    endpoints:
+                      - path: /metrics/prometheus/
+                        format: prometheus
+                      - path: /metrics/json
+                        format: json
+            iproto:
+              listen:
+                - uri: 127.0.0.1:3303
+              advertise:
+                client: 127.0.0.1:3303

docker/tarantool/instances.yaml

@@ -0,0 +1,7 @@
+---
+storage-001-a:
+
+storage-001-b:
+
+router-001-a:
+

docker/tarantool/router.lua

@@ -0,0 +1,77 @@
+local vshard = require('vshard')
+local log = require('log')
+
+-- Bootstrap the vshard router.
+while true do
+    local ok, err = vshard.router.bootstrap({
+        if_not_bootstrapped = true,
+    })
+    if ok then
+        break
+    end
+    log.info(('Router bootstrap error: %s'):format(err))
+end
+
+-- functions for filer_metadata space
+local filer_metadata = {
+    delete_by_directory_idx = function(directory)
+        -- find all storages
+        local storages = require('vshard').router.routeall()
+        -- on each storage
+        for _, storage in pairs(storages) do
+            -- call local function
+            local result, err = storage:callrw('filer_metadata.delete_by_directory_idx', { directory })
+            -- check for error
+            if err then
+                error("Failed to call function on storage: " .. tostring(err))
+            end
+        end
+        -- return
+        return true
+    end,
+    find_by_directory_idx_and_name = function(dirPath, startFileName, includeStartFile, limit)
+        -- init results
+        local results = {}
+        -- find all storages
+        local storages = require('vshard').router.routeall()
+        -- on each storage
+        for _, storage in pairs(storages) do
+            -- call local function
+            local result, err = storage:callro('filer_metadata.find_by_directory_idx_and_name', {
+                dirPath,
+                startFileName,
+                includeStartFile,
+                limit
+            })
+            -- check for error
+            if err then
+                error("Failed to call function on storage: " .. tostring(err))
+            end
+            -- add to results
+            for _, tuple in ipairs(result) do
+                table.insert(results, tuple)
+            end
+        end
+        -- sort
+        table.sort(results, function(a, b) return a[3] < b[3] end)
+        -- apply limit
+        if #results > limit then
+            local limitedResults = {}
+            for i = 1, limit do
+                table.insert(limitedResults, results[i])
+            end
+            results = limitedResults
+        end
+        -- return
+        return results
+    end,
+}
+
+rawset(_G, 'filer_metadata', filer_metadata)
+
+-- register functions for filer_metadata space, set grants
+for name, _ in pairs(filer_metadata) do
+    box.schema.func.create('filer_metadata.' .. name, { if_not_exists = true })
+    box.schema.user.grant('app', 'execute', 'function', 'filer_metadata.' .. name, { if_not_exists = true })
+    box.schema.user.grant('client', 'execute', 'function', 'filer_metadata.' .. name, { if_not_exists = true })
+end

docker/tarantool/storage.lua

@@ -0,0 +1,97 @@
+box.watch('box.status', function()
+    if box.info.ro then
+        return
+    end
+
+    -- ====================================
+    -- key_value space
+    -- ====================================
+    box.schema.create_space('key_value', {
+        format = {
+            { name = 'key', type = 'string' },
+            { name = 'bucket_id', type = 'unsigned' },
+            { name = 'value', type = 'string' }
+        },
+        if_not_exists = true
+    })
+
+    -- create key_value space indexes
+    box.space.key_value:create_index('id', {type = 'tree', parts = { 'key' }, unique = true, if_not_exists = true})
+    box.space.key_value:create_index('bucket_id', { type = 'tree', parts = { 'bucket_id' }, unique = false, if_not_exists = true })
+
+    -- ====================================
+    -- filer_metadata space
+    -- ====================================
+    box.schema.create_space('filer_metadata', {
+        format = {
+            { name = 'directory', type = 'string' },
+            { name = 'bucket_id', type = 'unsigned' },
+            { name = 'name', type = 'string' },
+            { name = 'expire_at', type = 'unsigned' },
+            { name = 'data', type = 'string' }
+        },
+        if_not_exists = true
+    })
+
+    -- create filer_metadata space indexes
+    box.space.filer_metadata:create_index('id', {type = 'tree', parts = { 'directory', 'name' }, unique = true, if_not_exists = true})
+    box.space.filer_metadata:create_index('bucket_id', { type = 'tree', parts = { 'bucket_id' }, unique = false, if_not_exists = true })
+    box.space.filer_metadata:create_index('directory_idx', { type = 'tree', parts = { 'directory' }, unique = false, if_not_exists = true })
+    box.space.filer_metadata:create_index('name_idx', { type = 'tree', parts = { 'name' }, unique = false, if_not_exists = true })
+    box.space.filer_metadata:create_index('expire_at_idx', { type = 'tree', parts = { 'expire_at' }, unique = false, if_not_exists = true})
+end)
+
+-- functions for filer_metadata space
+local filer_metadata = {
+    delete_by_directory_idx = function(directory)
+        local space = box.space.filer_metadata
+        local index = space.index.directory_idx
+        -- for each finded directories
+        for _, tuple in index:pairs({ directory }, { iterator = 'EQ' }) do
+            space:delete({ tuple[1], tuple[3] })
+        end
+        return true
+    end,
+    find_by_directory_idx_and_name = function(dirPath, startFileName, includeStartFile, limit)
+        local space = box.space.filer_metadata
+        local directory_idx = space.index.directory_idx
+        -- choose filter name function
+        local filter_filename_func
+        if includeStartFile then
+            filter_filename_func = function(value) return value >= startFileName end
+        else
+            filter_filename_func = function(value) return value > startFileName end
+        end
+        -- init results
+        local results = {}
+        -- for each finded directories
+        for _, tuple in directory_idx:pairs({ dirPath }, { iterator = 'EQ' }) do
+            -- filter by name
+            if filter_filename_func(tuple[3]) then
+                table.insert(results, tuple)
+            end
+        end
+        -- sort
+        table.sort(results, function(a, b) return a[3] < b[3] end)
+        -- apply limit
+        if #results > limit then
+            local limitedResults = {}
+            for i = 1, limit do
+                table.insert(limitedResults, results[i])
+            end
+            results = limitedResults
+        end
+        -- return
+        return results
+    end,
+    is_expired = function(args, tuple)
+        return (tuple[4] > 0) and (require('fiber').time() > tuple[4])
+    end
+}
+
+-- register functions for filer_metadata space, set grants
+rawset(_G, 'filer_metadata', filer_metadata)
+for name, _ in pairs(filer_metadata) do
+    box.schema.func.create('filer_metadata.' .. name, { setuid = true, if_not_exists = true })
+    box.schema.user.grant('storage', 'execute', 'function', 'filer_metadata.' .. name, { if_not_exists = true })
+end

go.mod

@@ -1,13 +1,13 @@
 module github.com/seaweedfs/seaweedfs
 
-go 1.23
+go 1.24
 
-toolchain go1.23.3
+toolchain go1.24.1
 
 require (
-	cloud.google.com/go v0.118.1 // indirect
-	cloud.google.com/go/pubsub v1.47.0
-	cloud.google.com/go/storage v1.50.0
+	cloud.google.com/go v0.120.0 // indirect
+	cloud.google.com/go/pubsub v1.49.0
+	cloud.google.com/go/storage v1.51.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Shopify/sarama v1.38.1
@@ -29,13 +29,12 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
-	github.com/fclairamb/ftpserverlib v0.25.0
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/go-redsync/redsync/v4 v4.13.0
-	github.com/go-sql-driver/mysql v1.8.1
+	github.com/go-sql-driver/mysql v1.9.2
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.7.0
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/golang/protobuf v1.5.4
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.3
@@ -47,23 +46,20 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jcmturner/gofork v1.7.6 // indirect
 	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
 	github.com/jinzhu/copier v0.4.0
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
-	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/reedsolomon v1.12.4
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.9
-	github.com/linxGnu/grocksdb v1.9.8
-	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/linxGnu/grocksdb v1.9.9
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.11 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/olivere/elastic/v7 v7.0.32
@@ -72,18 +68,18 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/posener/complete v1.2.3
 	github.com/pquerna/cachecontrol v0.2.0
-	github.com/prometheus/client_golang v1.20.5
+	github.com/prometheus/client_golang v1.21.1
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
-	github.com/prometheus/procfs v0.15.1
+	github.com/prometheus/common v0.62.0 // indirect
+	github.com/prometheus/procfs v0.16.1
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
 	github.com/seaweedfs/raft v1.1.3
 	github.com/sirupsen/logrus v1.9.3 // indirect
-	github.com/spf13/afero v1.11.0 // indirect
-	github.com/spf13/cast v1.6.0 // indirect
-	github.com/spf13/viper v1.19.0
+	github.com/spf13/afero v1.12.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/viper v1.20.1
 	github.com/stretchr/testify v1.10.0
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
@@ -99,89 +95,93 @@ require (
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
 	go.etcd.io/etcd/client/v3 v3.5.18
-	go.mongodb.org/mongo-driver v1.17.2
+	go.mongodb.org/mongo-driver v1.17.3
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.40.0
-	gocloud.dev/pubsub/natspubsub v0.40.0
-	gocloud.dev/pubsub/rabbitpubsub v0.40.0
-	golang.org/x/crypto v0.32.0 // indirect
-	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
-	golang.org/x/image v0.23.0
-	golang.org/x/net v0.34.0
-	golang.org/x/oauth2 v0.25.0 // indirect
-	golang.org/x/sys v0.29.0
-	golang.org/x/text v0.21.0 // indirect
-	golang.org/x/tools v0.29.0
-	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
-	google.golang.org/api v0.218.0
-	google.golang.org/genproto v0.0.0-20250122153221-138b5a5a4fd4 // indirect
-	google.golang.org/grpc v1.70.0
-	google.golang.org/protobuf v1.36.4
+	go.uber.org/zap v1.27.0
+	gocloud.dev v0.41.0
+	gocloud.dev/pubsub/natspubsub v0.41.0
+	gocloud.dev/pubsub/rabbitpubsub v0.41.0
+	golang.org/x/crypto v0.38.0
+	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394
+	golang.org/x/image v0.24.0
+	golang.org/x/net v0.40.0
+	golang.org/x/oauth2 v0.29.0 // indirect
+	golang.org/x/sys v0.33.0
+	golang.org/x/text v0.25.0 // indirect
+	golang.org/x/tools v0.33.0
+	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
+	google.golang.org/api v0.230.0
+	google.golang.org/genproto v0.0.0-20250324211829-b45e905df463 // indirect
+	google.golang.org/grpc v1.72.0
+	google.golang.org/protobuf v1.36.6
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
 	modernc.org/mathutil v1.7.1
-	modernc.org/memory v1.8.0 // indirect
-	modernc.org/sqlite v1.34.5
-	modernc.org/strutil v1.2.0
+	modernc.org/memory v1.9.1 // indirect
+	modernc.org/sqlite v1.37.0
+	modernc.org/strutil v1.2.1
 )
 
 require (
 	github.com/Jille/raft-grpc-transport v1.6.1
-	github.com/arangodb/go-driver v1.6.5
+	github.com/arangodb/go-driver v1.6.6
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.36.0
-	github.com/aws/aws-sdk-go-v2/config v1.29.1
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.54
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2
+	github.com/aws/aws-sdk-go-v2 v1.36.3
+	github.com/aws/aws-sdk-go-v2/config v1.29.14
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.67
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2
 	github.com/cognusion/imaging v1.0.1
 	github.com/fluent/fluent-logger-golang v1.9.0
 	github.com/getsentry/sentry-go v0.31.1
-	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
 	github.com/hanwen/go-fuse/v2 v2.7.2
-	github.com/hashicorp/raft v1.7.2
+	github.com/hashicorp/raft v1.7.3
 	github.com/hashicorp/raft-boltdb/v2 v2.3.1
+	github.com/minio/crc64nvme v1.0.1
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/parquet-go/parquet-go v0.24.0
+	github.com/pkg/sftp v1.13.7
 	github.com/rabbitmq/amqp091-go v1.10.0
-	github.com/rclone/rclone v1.68.2
+	github.com/rclone/rclone v1.69.1
 	github.com/rdleal/intervalst v1.4.1
-	github.com/redis/go-redis/v9 v9.7.0
+	github.com/redis/go-redis/v9 v9.8.0
 	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/shirou/gopsutil/v3 v3.24.5
+	github.com/tarantool/go-tarantool/v2 v2.3.0
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.98.0
-	go.etcd.io/etcd/client/pkg/v3 v3.5.18
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.108.3
+	go.etcd.io/etcd/client/pkg/v3 v3.5.21
 	go.uber.org/atomic v1.11.0
-	golang.org/x/sync v0.10.0
+	golang.org/x/sync v0.14.0
 	google.golang.org/grpc/security/advancedtls v1.0.0
 )
 
 require (
-	cel.dev/expr v0.19.0 // indirect
-	cloud.google.com/go/auth v0.14.0 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect
+	cel.dev/expr v0.22.1 // indirect
+	cloud.google.com/go/auth v0.16.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.6.0 // indirect
-	cloud.google.com/go/iam v1.3.1 // indirect
-	cloud.google.com/go/monitoring v1.23.0 // indirect
+	cloud.google.com/go/iam v1.4.2 // indirect
+	cloud.google.com/go/monitoring v1.24.1 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.2.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.4.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
-	github.com/Files-com/files-sdk-go/v3 v3.2.34 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
+	github.com/Files-com/files-sdk-go/v3 v3.2.97 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
-	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
 	github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
-	github.com/ProtonMail/go-crypto v1.0.0 // indirect
+	github.com/ProtonMail/go-crypto v1.1.3 // indirect
 	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
 	github.com/ProtonMail/go-srp v0.0.7 // indirect
 	github.com/ProtonMail/gopenpgp/v2 v2.7.4 // indirect
@@ -191,62 +191,66 @@ require (
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.31.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect
-	github.com/aws/smithy-go v1.22.2 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.69 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.34.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect
+	github.com/aws/smithy-go v1.22.3 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.2 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
 	github.com/buengese/sgzip v0.1.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
-	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
-	github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891 // indirect
-	github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
+	github.com/cloudinary/cloudinary-go/v2 v2.9.0 // indirect
+	github.com/cloudsoda/go-smb2 v0.0.0-20241223203758-52b943b88fd6 // indirect
+	github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect
 	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
+	github.com/creasty/defaults v1.7.0 // indirect
 	github.com/cronokirby/saferith v0.33.0 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
+	github.com/ebitengine/purego v0.8.1 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/emersion/go-message v0.18.0 // indirect
 	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
 	github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
-	github.com/envoyproxy/go-control-plane v0.13.1 // indirect
-	github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
+	github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/fatih/color v1.16.0 // indirect
-	github.com/fclairamb/go-log v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/flynn/noise v1.0.1 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.4 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.7 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
 	github.com/go-chi/chi/v5 v5.1.0 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-resty/resty/v2 v2.11.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
+	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
@@ -261,12 +265,12 @@ require (
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
-	github.com/jlaffaye/ftp v0.2.0 // indirect
-	github.com/jonboulle/clockwork v0.3.0 // indirect
+	github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 // indirect
+	github.com/jonboulle/clockwork v0.5.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
@@ -279,18 +283,20 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/nats-io/nats.go v1.37.0 // indirect
-	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/nats-io/nats.go v1.40.1 // indirect
+	github.com/nats-io/nkeys v0.4.10 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/ncw/swift/v2 v2.0.3 // indirect
 	github.com/nxadm/tail v1.4.11 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/onsi/ginkgo/v2 v2.19.0 // indirect
+	github.com/onsi/gomega v1.34.1 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.69.2 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.80.0 // indirect
 	github.com/panjf2000/ants/v2 v2.9.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
@@ -299,8 +305,7 @@ require (
 	github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/pkg/sftp v1.13.6 // indirect
-	github.com/pkg/xattr v0.4.9 // indirect
+	github.com/pkg/xattr v0.4.10 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
@@ -308,18 +313,20 @@ require (
 	github.com/rfjakob/eme v1.1.2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
-	github.com/sagikazarmark/locafero v0.4.0 // indirect
-	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sagikazarmark/locafero v0.7.0 // indirect
 	github.com/samber/lo v1.47.0 // indirect
+	github.com/shirou/gopsutil/v4 v4.24.12 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
 	github.com/sony/gobreaker v0.5.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 // indirect
+	github.com/t3rm1n4l/go-mega v0.0.0-20241213150454-ec0027fb0002 // indirect
+	github.com/tarantool/go-iproto v1.1.0 // indirect
 	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
 	github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
 	github.com/tinylib/msgp v1.1.8 // indirect
@@ -327,6 +334,8 @@ require (
 	github.com/tklauser/numcpus v0.7.0 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
 	github.com/unknwon/goconfig v1.0.0 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
 	github.com/ydb-platform/ydb-go-genproto v0.0.0-20241112172322-ea1f63298f77 // indirect
@@ -335,31 +344,28 @@ require (
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
-	github.com/zeebo/errs v1.3.0 // indirect
+	github.com/zeebo/errs v1.4.0 // indirect
 	go.etcd.io/bbolt v1.3.10 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.18 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/contrib/detectors/gcp v1.33.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
-	go.opentelemetry.io/otel v1.34.0 // indirect
-	go.opentelemetry.io/otel/metric v1.34.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.34.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect
-	go.opentelemetry.io/otel/trace v1.34.0 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
+	go.opentelemetry.io/otel v1.35.0 // indirect
+	go.opentelemetry.io/otel/metric v1.35.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.35.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
+	go.opentelemetry.io/otel/trace v1.35.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/mod v0.22.0 // indirect
-	golang.org/x/term v0.28.0 // indirect
-	golang.org/x/time v0.9.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250124145028-65684f501c47 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/time v0.11.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250414145226-207652e42e2e // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	modernc.org/libc v1.55.3 // indirect
+	modernc.org/libc v1.62.1 // indirect
 	moul.io/http2curl/v2 v2.3.0 // indirect
 	storj.io/common v0.0.0-20240812101423-26b53789c348 // indirect
 	storj.io/drpc v0.0.35-0.20240709171858-0075ac871661 // indirect
@@ -370,8 +376,3 @@ require (
 )
 
 // replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
-
-replace (
-	github.com/boltdb/bolt => go.etcd.io/bbolt v1.3.10
-	go.etcd.io/bbolt => github.com/etcd-io/bbolt v1.3.10
-)

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.84"
+appVersion: "3.87"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.384
+version: 4.0.388

k8s/charts/seaweedfs/templates/_helpers.tpl

@@ -73,6 +73,16 @@ Inject extra environment vars in the format key:value, if populated
 {{- end -}}
 {{- end -}}
 
+{{/* Return the proper sftp image */}}
+{{- define "sftp.image" -}}
+{{- if .Values.sftp.imageOverride -}}
+{{- $imageOverride := .Values.sftp.imageOverride -}}
+{{- printf "%s" $imageOverride -}}
+{{- else -}}
+{{- include "common.image" . }}
+{{- end -}}
+{{- end -}}
+
 {{/* Return the proper volume image */}}
 {{- define "volume.image" -}}
 {{- if .Values.volume.imageOverride -}}
@@ -88,7 +98,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- $registryName := default .Values.image.registry .Values.global.registry | toString -}}
 {{- $repositoryName := .Values.image.repository | toString -}}
 {{- $name := .Values.global.imageName | toString -}}
-{{- $tag := .Chart.AppVersion | toString -}}
+{{- $tag := default .Chart.AppVersion .Values.image.tag  | toString -}}
 {{- if $registryName -}}
 {{- printf "%s/%s%s:%s" $registryName $repositoryName $name $tag -}}
 {{- else -}}
@@ -134,14 +144,17 @@ Inject extra environment vars in the format key:value, if populated
 
 {{/* Return the proper imagePullSecrets */}}
 {{- define "seaweedfs.imagePullSecrets" -}}
-{{- if .Values.global.imagePullSecrets }}
-{{- if kindIs "string" .Values.global.imagePullSecrets }}
+{{- with .Values.global.imagePullSecrets }}
 imagePullSecrets:
-  - name: {{ .Values.global.imagePullSecrets }}
-{{- else }}
-imagePullSecrets:
-{{- range .Values.global.imagePullSecrets }}
+{{- if kindIs "string" . }}
   - name: {{ . }}
+{{- else }}
+{{- range . }}
+  {{- if kindIs "string" . }}
+  - name: {{ . }}
+  {{- else }}
+  - {{ toYaml . }}
+  {{- end}}
 {{- end }}
 {{- end }}
 {{- end }}
@@ -165,3 +178,23 @@ Usage:
     {{- $value }}
 {{- end }}
 {{- end -}}
+
+
+{{/*
+getOrGeneratePassword will check if a password exists in a secret and return it,
+or generate a new random password if it doesn't exist.
+*/}}
+{{- define "getOrGeneratePassword" -}}
+{{- $params := . -}}
+{{- $namespace := $params.namespace -}}
+{{- $secretName := $params.secretName -}}
+{{- $key := $params.key -}}
+{{- $length := default 16 $params.length -}}
+
+{{- $existingSecret := lookup "v1" "Secret" $namespace $secretName -}}
+{{- if and $existingSecret (index $existingSecret.data $key) -}}
+  {{- index $existingSecret.data $key | b64dec -}}
+{{- else -}}
+  {{- randAlphaNum $length -}}
+{{- end -}}
+{{- end -}}

k8s/charts/seaweedfs/templates/cosi-deployment.yaml

@@ -14,7 +14,6 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-      helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
       app.kubernetes.io/instance: {{ .Release.Name }}
       app.kubernetes.io/component: objectstorage-provisioner
   template:
@@ -140,6 +139,10 @@ spec:
               mountPath: /usr/local/share/ca-certificates/client/
             {{- end }}
             {{ tpl .Values.cosi.extraVolumeMounts . | nindent 12 | trim }}
+          {{- with .Values.cosi.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
         - name: seaweedfs-cosi-sidecar
           image: "{{ .Values.cosi.sidecar.image }}"
           imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
@@ -153,7 +156,7 @@ spec:
           volumeMounts:
             - mountPath: /var/lib/cosi
               name: socket
-          {{- with .Values.cosi.resources }}
+          {{- with .Values.cosi.sidecar.resources }}
           resources:
             {{- toYaml . | nindent 12 }}
           {{- end }}
@@ -173,7 +176,7 @@ spec:
             {{- if .Values.cosi.existingConfigSecret }}
             secretName: {{ .Values.cosi.existingConfigSecret }}
             {{- else }}
-            secretName: seaweedfs-client-cert
+            secretName: seaweedfs-s3-secret
             {{- end }}
         {{- end }}
         {{- if .Values.global.enableSecurity }}

k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@@ -173,7 +173,7 @@ spec:
               {{- end }}
               -dirListLimit={{ .Values.filer.dirListLimit }} \
               {{- if .Values.global.enableReplication }}
-              -defaultReplicaPlacement={{ .Values.global.replicationPlacment }} \
+              -defaultReplicaPlacement={{ .Values.global.replicationPlacement }} \
               {{- else }}
               -defaultReplicaPlacement={{ .Values.filer.defaultReplicaPlacement }} \
               {{- end }}
@@ -209,7 +209,7 @@ spec:
               -s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
               -s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
               {{- end }}
-              {{- if .Values.filer.s3.allowEmptyFolder }}
+              {{- if eq (typeOf .Values.filer.s3.allowEmptyFolder) "bool" }}
               -s3.allowEmptyFolder={{ .Values.filer.s3.allowEmptyFolder }} \
               {{- end }}
               {{- if .Values.filer.s3.enableAuth }}

k8s/charts/seaweedfs/templates/master-statefulset.yaml

@@ -139,7 +139,7 @@ spec:
               -mdir=/data \
               -ip.bind={{ .Values.master.ipBind }} \
               {{- if .Values.global.enableReplication }}
-              -defaultReplication={{ .Values.global.replicationPlacment }} \
+              -defaultReplication={{ .Values.global.replicationPlacement }} \
               {{- else }}
               -defaultReplication={{ .Values.master.defaultReplication }} \
               {{- end }}

k8s/charts/seaweedfs/templates/s3-deployment.yaml

@@ -134,7 +134,7 @@ spec:
               {{- if .Values.s3.domainName }}
               -domainName={{ .Values.s3.domainName }} \
               {{- end }}
-              {{- if .Values.s3.allowEmptyFolder }}
+              {{- if eq (typeOf .Values.s3.allowEmptyFolder) "bool" }}
               -allowEmptyFolder={{ .Values.s3.allowEmptyFolder }} \
               {{- end }}
               {{- if .Values.s3.enableAuth }}

k8s/charts/seaweedfs/templates/s3-secret.yaml

@@ -1,8 +1,8 @@
 {{- if or (and .Values.filer.s3.enabled .Values.filer.s3.enableAuth (not .Values.filer.s3.existingConfigSecret)) (and .Values.s3.enabled .Values.s3.enableAuth (not .Values.s3.existingConfigSecret)) }}
-{{- $access_key_admin := randAlphaNum 16 -}}
-{{- $secret_key_admin := randAlphaNum 32 -}}
-{{- $access_key_read := randAlphaNum 16 -}}
-{{- $secret_key_read := randAlphaNum 32 -}}
+{{- $access_key_admin := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-s3-secret" "key" "admin_access_key_id" "length" 20) -}}
+{{- $secret_key_admin := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-s3-secret" "key" "admin_secret_access_key" "length" 40) -}}
+{{- $access_key_read := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-s3-secret" "key" "read_access_key_id" "length" 20) -}}
+{{- $secret_key_read := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-s3-secret" "key" "read_secret_access_key" "length" 40) -}}
 apiVersion: v1
 kind: Secret
 type: Opaque
@@ -11,7 +11,7 @@ metadata:
   namespace: {{ .Release.Namespace }}
   annotations:
     "helm.sh/resource-policy": keep
-    "helm.sh/hook": "pre-install"
+    "helm.sh/hook": "pre-install,pre-upgrade"
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
@@ -32,4 +32,4 @@ stringData:
   s3_auditLogConfig.json: |
     {{ toJson .Values.s3.auditLogConfig | nindent 4 }}
   {{- end }}
-{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/sftp-deployment.yaml

@@ -0,0 +1,292 @@
+{{- if .Values.sftp.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "seaweedfs.name" . }}-sftp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Values.sftp.annotations }}
+  annotations:
+    {{- toYaml .Values.sftp.annotations | nindent 4 }}
+{{- end }}
+spec:
+  replicas: {{ .Values.sftp.replicas }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: sftp
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+        helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: sftp
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.sftp.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.sftp.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      restartPolicy: {{ default .Values.global.restartPolicy .Values.sftp.restartPolicy }}
+      {{- if .Values.sftp.tolerations }}
+      tolerations:
+        {{ tpl .Values.sftp.tolerations . | nindent 8 | trim }}
+      {{- end }}
+      {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
+      terminationGracePeriodSeconds: 10
+      {{- if .Values.sftp.priorityClassName }}
+      priorityClassName: {{ .Values.sftp.priorityClassName | quote }}
+      {{- end }}
+      enableServiceLinks: false
+      {{- if .Values.sftp.serviceAccountName }}
+      serviceAccountName: {{ .Values.sftp.serviceAccountName | quote }}
+      {{- end }}
+      {{- if .Values.sftp.initContainers }}
+      initContainers:
+        {{ tpl .Values.sftp.initContainers . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.sftp.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.sftp.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: seaweedfs
+          image: {{ template "sftp.image" . }}
+          imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: SEAWEEDFS_FULLNAME
+              value: "{{ template "seaweedfs.name" . }}"
+            {{- if .Values.sftp.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.sftp.extraEnvironmentVars }}
+            - name: {{ $key }}
+            {{- if kindIs "string" $value }}
+              value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
+            {{- end }}
+            {{- end }}
+            {{- if .Values.global.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
+            - name: {{ $key }}
+            {{- if kindIs "string" $value }}
+              value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
+            {{- end }}
+            {{- end }}
+          command:
+            - "/bin/sh"
+            - "-ec"
+            - |
+              exec /usr/bin/weed \
+              {{- if or (eq .Values.sftp.logs.type "hostPath") (eq .Values.sftp.logs.type "emptyDir") }}
+              -logdir=/logs \
+              {{- else }}
+              -logtostderr=true \
+              {{- end }}
+              {{- if .Values.sftp.loggingOverrideLevel }}
+              -v={{ .Values.sftp.loggingOverrideLevel }} \
+              {{- else }}
+              -v={{ .Values.global.loggingLevel }} \
+              {{- end }}
+              sftp \
+              -ip.bind={{ .Values.sftp.bindAddress }} \
+              -port={{ .Values.sftp.port }} \
+              {{- if .Values.sftp.metricsPort }}
+              -metricsPort={{ .Values.sftp.metricsPort }} \
+              {{- end }}
+              {{- if .Values.sftp.metricsIp }}
+              -metricsIp={{ .Values.sftp.metricsIp }} \
+              {{- end }}
+              {{- if .Values.sftp.sshPrivateKey }}
+              -sshPrivateKey={{ .Values.sftp.sshPrivateKey }} \
+              {{- end }}
+              {{- if .Values.sftp.hostKeysFolder }}
+              -hostKeysFolder={{ .Values.sftp.hostKeysFolder }} \
+              {{- end }}
+              {{- if .Values.sftp.authMethods }}
+              -authMethods={{ .Values.sftp.authMethods }} \
+              {{- end }}
+              {{- if .Values.sftp.maxAuthTries }}
+              -maxAuthTries={{ .Values.sftp.maxAuthTries }} \
+              {{- end }}
+              {{- if .Values.sftp.bannerMessage }}
+              -bannerMessage="{{ .Values.sftp.bannerMessage }}" \
+              {{- end }}
+              {{- if .Values.sftp.loginGraceTime }}
+              -loginGraceTime={{ .Values.sftp.loginGraceTime }} \
+              {{- end }}
+              {{- if .Values.sftp.clientAliveInterval }}
+              -clientAliveInterval={{ .Values.sftp.clientAliveInterval }} \
+              {{- end }}
+              {{- if .Values.sftp.clientAliveCountMax }}
+              -clientAliveCountMax={{ .Values.sftp.clientAliveCountMax }} \
+              {{- end }}
+              {{- if .Values.sftp.dataCenter }}
+              -dataCenter={{ .Values.sftp.dataCenter }} \
+              {{- end }}
+              {{- if .Values.sftp.localSocket }}
+              -localSocket={{ .Values.sftp.localSocket }} \
+              {{- end }}
+              {{- if .Values.global.enableSecurity }}
+              -cert.file=/usr/local/share/ca-certificates/client/tls.crt \
+              -key.file=/usr/local/share/ca-certificates/client/tls.key \
+              {{- end }}
+              -userStoreFile=/etc/sw/seaweedfs_sftp_config \
+              -filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
+          volumeMounts:
+            {{- if or (eq .Values.sftp.logs.type "hostPath") (eq .Values.sftp.logs.type "emptyDir") }}
+            - name: logs
+              mountPath: "/logs/"
+            {{- end }}
+            {{- if .Values.sftp.enableAuth }}
+            - mountPath: /etc/sw
+              name: config-users
+              readOnly: true
+            {{- end }}
+            - mountPath: /etc/sw/ssh
+              name: config-ssh
+              readOnly: true
+            {{- if .Values.global.enableSecurity }}
+            - name: security-config
+              readOnly: true
+              mountPath: /etc/seaweedfs/security.toml
+              subPath: security.toml
+            - name: ca-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/ca/
+            - name: master-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/master/
+            - name: volume-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/volume/
+            - name: filer-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/filer/
+            - name: client-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/client/
+            {{- end }}
+            {{ tpl .Values.sftp.extraVolumeMounts . | nindent 12 | trim }}
+          ports:
+            - containerPort: {{ .Values.sftp.port }}
+              name: swfs-sftp
+            {{- if .Values.sftp.metricsPort }}
+            - containerPort: {{ .Values.sftp.metricsPort }}
+              name: metrics
+            {{- end }}
+          {{- if .Values.sftp.readinessProbe.enabled }}
+          readinessProbe:
+            tcpSocket:
+              port: {{ .Values.sftp.port }}
+            initialDelaySeconds: {{ .Values.sftp.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.sftp.readinessProbe.periodSeconds }}
+            successThreshold: {{ .Values.sftp.readinessProbe.successThreshold }}
+            failureThreshold: {{ .Values.sftp.readinessProbe.failureThreshold }}
+            timeoutSeconds: {{ .Values.sftp.readinessProbe.timeoutSeconds }}
+          {{- end }}
+          {{- if .Values.sftp.livenessProbe.enabled }}
+          livenessProbe:
+            tcpSocket:
+              port: {{ .Values.sftp.port }}
+            initialDelaySeconds: {{ .Values.sftp.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.sftp.livenessProbe.periodSeconds }}
+            successThreshold: {{ .Values.sftp.livenessProbe.successThreshold }}
+            failureThreshold: {{ .Values.sftp.livenessProbe.failureThreshold }}
+            timeoutSeconds: {{ .Values.sftp.livenessProbe.timeoutSeconds }}
+          {{- end }}
+          {{- with .Values.sftp.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- if .Values.sftp.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.sftp.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
+      {{- if .Values.sftp.sidecars }}
+      {{- include "common.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
+      {{- end }}
+      volumes:
+        {{- if .Values.sftp.enableAuth }}
+        - name: config-users
+          secret:
+            defaultMode: 420
+            {{- if .Values.sftp.existingConfigSecret }}
+            secretName: {{ .Values.sftp.existingConfigSecret }}
+            {{- else }}
+            secretName: seaweedfs-sftp-secret
+            {{- end }}
+        {{- end }}
+        - name: config-ssh
+          secret:
+            defaultMode: 420
+            {{- if .Values.sftp.existingSshConfigSecret }}
+            secretName: {{ .Values.sftp.existingSshConfigSecret }}
+            {{- else }}
+            secretName: seaweedfs-sftp-ssh-secret
+            {{- end }}
+        {{- if eq .Values.sftp.logs.type "hostPath" }}
+        - name: logs
+          hostPath:
+            path: {{ .Values.sftp.logs.hostPathPrefix }}/logs/seaweedfs/sftp
+            type: DirectoryOrCreate
+        {{- end }}
+        {{- if eq .Values.sftp.logs.type "emptyDir" }}
+        - name: logs
+          emptyDir: {}
+        {{- end }}
+        {{- if .Values.global.enableSecurity }}
+        - name: security-config
+          configMap:
+            name: {{ template "seaweedfs.name" . }}-security-config
+        - name: ca-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-ca-cert
+        - name: master-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-master-cert
+        - name: volume-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-volume-cert
+        - name: filer-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-filer-cert
+        - name: client-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-client-cert
+        {{- end }}
+        {{ tpl .Values.sftp.extraVolumes . | indent 8 | trim }}
+      {{- if .Values.sftp.nodeSelector }}
+      nodeSelector:
+        {{ tpl .Values.sftp.nodeSelector . | indent 8 | trim }}
+      {{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/sftp-secret.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.sftp.enabled }}
+{{- $admin_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-sftp-secret" "key" "admin_password" 20) -}}
+{{- $read_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-sftp-secret" "key" "readonly_password" 20) -}}
+{{- $public_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" "seaweedfs-sftp-secret" "key" "public_user_password" 20) -}}
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: seaweedfs-sftp-secret
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/resource-policy": keep
+    "helm.sh/hook": "pre-install,pre-upgrade"
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: sftp
+stringData:
+  admin_password: {{ $admin_pwd }}
+  readonly_password: {{ $read_user_pwd }}
+  public_user_password: {{ $public_user_pwd }}
+  seaweedfs_sftp_config: '[{"Username":"admin","Password":"{{ $admin_pwd }}","PublicKeys":[],"HomeDir":"/","Permissions":{"/":["read","write","list"]},"Uid":0,"Gid":0},{"Username":"readonly_user","Password":"{{ $read_user_pwd }}","PublicKeys":[],"HomeDir":"/","Permissions":{"/":["read","list"]},"Uid":1112,"Gid":1112},{"Username":"public_user","Password":"{{ $public_user_pwd }}","PublicKeys":[],"HomeDir":"/public","Permissions":{"/public":["write","read","list"]},"Uid":1113,"Gid":1113}]'
+  seaweedfs_sftp_ssh_private_key: |
+    -----BEGIN OPENSSH PRIVATE KEY-----
+    b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+    QyNTUxOQAAACDH4McwcDphteXVullu6q7ephEN1N60z+w0qZw0UVW8OwAAAJDjxkmk48ZJ
+    pAAAAAtzc2gtZWQyNTUxOQAAACDH4McwcDphteXVullu6q7ephEN1N60z+w0qZw0UVW8Ow
+    AAAEAeVy/4+gf6rjj2jla/AHqJpC1LcS5hn04IUs4q+iVq/MfgxzBwOmG15dW6WW7qrt6m
+    EQ3U3rTP7DSpnDRRVbw7AAAADHNla291ckAwMDY2NwE=
+    -----END OPENSSH PRIVATE KEY-----
+{{- end }}

k8s/charts/seaweedfs/templates/sftp-service.yaml

@@ -0,0 +1,39 @@
+{{- if .Values.sftp.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "seaweedfs.name" . }}-sftp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    app.kubernetes.io/component: sftp
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- if .Values.sftp.annotations }}
+  annotations:
+    {{- toYaml .Values.sftp.annotations | nindent 4 }}
+{{- end }}
+spec:
+  type: {{ .Values.sftp.service.type | default "ClusterIP" }}
+  internalTrafficPolicy: {{ .Values.sftp.internalTrafficPolicy | default "Cluster" }}
+  ports:
+  - name: "swfs-sftp"
+    port: {{ .Values.sftp.port }}
+    targetPort: {{ .Values.sftp.port }}
+    protocol: TCP
+    {{- if and (eq (.Values.sftp.service.type | default "ClusterIP") "NodePort") .Values.sftp.service.nodePort }}
+    nodePort: {{ .Values.sftp.service.nodePort }}
+    {{- end }}
+{{- if .Values.sftp.metricsPort }}
+  - name: "metrics"
+    port: {{ .Values.sftp.metricsPort }}
+    targetPort: {{ .Values.sftp.metricsPort }}
+    protocol: TCP
+    {{- if and (eq (.Values.sftp.service.type | default "ClusterIP") "NodePort") .Values.sftp.service.metricsNodePort }}
+    nodePort: {{ .Values.sftp.service.metricsNodePort }}
+    {{- end }}
+{{- end }}
+  selector:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    app.kubernetes.io/component: sftp
+{{- end }}

k8s/charts/seaweedfs/templates/sftp-servicemonitor.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.sftp.enabled }}
+{{- if .Values.sftp.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "seaweedfs.name" . }}-sftp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: sftp
+    {{- with .Values.global.monitoring.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+{{- if .Values.sftp.annotations }}
+  annotations:
+    {{- toYaml .Values.sftp.annotations | nindent 4 }}
+{{- end }}
+spec:
+  endpoints:
+    - interval: 30s
+      port: metrics
+      scrapeTimeout: 5s
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+      app.kubernetes.io/component: sftp
+{{- end }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/values.yaml

@@ -27,13 +27,13 @@ global:
     gatewayHost: null
     gatewayPort: null
     additionalLabels: {}
-  # if enabled will use global.replicationPlacment and override master & filer defaultReplicaPlacement config
+  # if enabled will use global.replicationPlacement and override master & filer defaultReplicaPlacement config
   enableReplication: false
   #  replication type is XYZ:
   # X number of replica in other data centers
   # Y number of replica in other racks in the same data center
   # Z number of replica in other servers in the same rack
-  replicationPlacment: "001"
+  replicationPlacement: "001"
   extraEnvironmentVars:
     WEED_CLUSTER_DEFAULT: "sw"
     WEED_CLUSTER_SW_MASTER: "seaweedfs-master.seaweedfs:9333"
@@ -46,6 +46,7 @@ global:
 image:
   registry: ""
   repository: ""
+  tag: ""
 
 master:
   enabled: true
@@ -291,7 +292,7 @@ volume:
   # For each data disk you may use ANY storage-class, example with local-path-provisioner
   # Annotations are optional.
   #  dataDirs:
-  #   - name: data:
+  #   - name: data
   #     type: "persistentVolumeClaim"
   #     size: "24Ti"
   #     storageClass: "local-path-provisioner"
@@ -948,7 +949,82 @@ s3:
     # additional ingress annotations for the s3 endpoint
     annotations: {}
     tls: []
+sftp:
+  enabled: false
+  imageOverride: null
+  restartPolicy: null
+  replicas: 1
+  bindAddress: 0.0.0.0
+  port: 2022  # Default SFTP port
+  metricsPort: 9327
+  metricsIp: ""  # If empty, defaults to bindAddress
+  service:
+    type: ClusterIP  # Can be ClusterIP, NodePort, LoadBalancer
+    nodePort: null  # Optional: specific nodePort for SFTP
+    metricsNodePort: null  # Optional: specific nodePort for metrics
+  loggingOverrideLevel: null
 
+  # SSH server configuration
+  sshPrivateKey: "/etc/sw/seaweedfs_sftp_ssh_private_key"  # Path to the SSH private key file for host authentication
+  hostKeysFolder: "/etc/sw/ssh"  # path to folder containing SSH private key files for host authentication
+  authMethods: "password,publickey"  # Comma-separated list of allowed auth methods: password, publickey, keyboard-interactive
+  maxAuthTries: 6  # Maximum number of authentication attempts per connection
+  bannerMessage: "SeaweedFS SFTP Server"  # Message displayed before authentication
+  loginGraceTime: "2m"  # Timeout for authentication
+  clientAliveInterval: "5s"  # Interval for sending keep-alive messages
+  clientAliveCountMax: 3  # Maximum number of missed keep-alive messages before disconnecting
+  dataCenter: ""  # Prefer to read and write to volumes in this data center
+  localSocket: ""  # Default to /tmp/seaweedfs-sftp-<port>.sock
+
+  # User authentication
+  enableAuth: false
+  # Set to the name of an existing kubernetes Secret with the sftp json config file
+  # Should have a secret key called seaweedfs_sftp_config with an inline json config
+  existingConfigSecret: null
+  # Set to the name of an existing kubernetes Secret with the list of ssh private keys for sftp
+  existingSshConfigSecret: null
+
+  # Additional resources
+  sidecars: []
+  initContainers: ""
+  extraVolumes: ""
+  extraVolumeMounts: ""
+  podLabels: {}
+  podAnnotations: {}
+  annotations: {}
+  resources: {}
+  tolerations: ""
+  nodeSelector: |
+    kubernetes.io/arch: amd64
+  priorityClassName: ""
+  serviceAccountName: ""
+  podSecurityContext: {}
+  containerSecurityContext: {}
+
+  logs:
+    type: "hostPath"
+    hostPathPrefix: /storage
+
+  extraEnvironmentVars: {}
+
+  # Health checks
+  # Health checks for SFTP - using tcpSocket instead of httpGet
+  livenessProbe:
+    enabled: true
+    initialDelaySeconds: 20
+    periodSeconds: 60
+    successThreshold: 1
+    failureThreshold: 20
+    timeoutSeconds: 10
+
+  # Health checks for SFTP - using tcpSocket instead of httpGet
+  readinessProbe:
+    enabled: true
+    initialDelaySeconds: 15
+    periodSeconds: 15
+    successThreshold: 1
+    failureThreshold: 100
+    timeoutSeconds: 10
 # Deploy Kubernetes COSI Driver for SeaweedFS
 # Requires COSI CRDs and controller to be installed in the cluster
 # For more information, visit: https://container-object-storage-interface.github.io/docs/deployment-guide
@@ -962,6 +1038,11 @@ cosi:
 
   sidecar:
     image: gcr.io/k8s-staging-sig-storage/objectstorage-sidecar/objectstorage-sidecar:v20230130-v0.1.0-24-gc0cf995
+    # Resource requests, limits, etc. for the server cluster placement. This
+    # should map directly to the value of the resources field for a PodSpec,
+    # formatted as a multi-line string. By default no direct resource request
+    # is made.
+    resources: {}
 
   # enable user & permission to s3 (need to inject to all services)
   enableAuth: false
@@ -975,6 +1056,12 @@ cosi:
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  # Resource requests, limits, etc. for the server cluster placement. This
+  # should map directly to the value of the resources field for a PodSpec,
+  # formatted as a multi-line string. By default no direct resource request
+  # is made.
+  resources: {}
+
 certificates:
   commonName: "SeaweedFS CA"
   ipAddresses: []

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -14,11 +14,13 @@ public class FilerClient extends FilerGrpcClient {
     private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
 
     public FilerClient(String filerHost, int filerGrpcPort) {
-        super(filerHost, filerGrpcPort-10000, filerGrpcPort);
+        this(filerHost, filerGrpcPort-10000, filerGrpcPort, "");
     }
+    public FilerClient(String filerHost, int filerGrpcPort, String cn) { this(filerHost, filerGrpcPort-10000, filerGrpcPort, cn); }
+    public FilerClient(String filerHost, int filerPort, int filerGrpcPort) { this(filerHost, filerPort, filerGrpcPort, ""); }
 
-    public FilerClient(String filerHost, int filerPort, int filerGrpcPort) {
-        super(filerHost, filerPort, filerGrpcPort);
+    public FilerClient(String filerHost, int filerPort, int filerGrpcPort, String cn) {
+        super(filerHost, filerPort, filerGrpcPort, cn);
     }
 
     public static String toFileId(FilerProto.FileId fid) {

other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java

@@ -8,7 +8,6 @@ import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.net.ssl.SSLException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Random;
@@ -17,14 +16,12 @@ import java.util.concurrent.TimeUnit;
 public class FilerGrpcClient {
 
     private static final Logger logger = LoggerFactory.getLogger(FilerGrpcClient.class);
-    static SslContext sslContext;
+    private static final SslContext sslContext;
+    private static final String protocol;
 
     static {
-        try {
-            sslContext = FilerSslContext.loadSslContext();
-        } catch (SSLException e) {
-            logger.warn("failed to load ssl context", e);
-        }
+        sslContext = FilerSecurityContext.getGrpcSslContext();
+        protocol = FilerSecurityContext.isHttpSecurityEnabled() ? "https" : "http";
     }
 
     public final int VOLUME_SERVER_ACCESS_DIRECT = 0;
@@ -42,19 +39,27 @@ public class FilerGrpcClient {
     private int volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT;
     private String filerAddress;
 
-    public FilerGrpcClient(String host, int port, int grpcPort) {
-        this(host, port, grpcPort, sslContext);
+    public FilerGrpcClient(String host, int port, int grpcPort, String cn) {
+        this(host, port, grpcPort, cn, sslContext);
     }
 
-    public FilerGrpcClient(String host, int port, int grpcPort, SslContext sslContext) {
+    public FilerGrpcClient(String host, int port, int grpcPort, String cn, SslContext sslContext) {
 
         this(sslContext == null ?
-                ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext()
+                ManagedChannelBuilder.forAddress(host, grpcPort)
+                        .usePlaintext()
                         .maxInboundMessageSize(1024 * 1024 * 1024) :
-                NettyChannelBuilder.forAddress(host, grpcPort)
-                        .maxInboundMessageSize(1024 * 1024 * 1024)
-                        .negotiationType(NegotiationType.TLS)
-                        .sslContext(sslContext));
+                cn.isEmpty() ?
+                    NettyChannelBuilder.forAddress(host, grpcPort)
+                            .maxInboundMessageSize(1024 * 1024 * 1024)
+                            .negotiationType(NegotiationType.TLS)
+                            .sslContext(sslContext) :
+                    NettyChannelBuilder.forAddress(host, grpcPort)
+                            .maxInboundMessageSize(1024 * 1024 * 1024)
+                            .negotiationType(NegotiationType.TLS)
+                            .overrideAuthority(cn) //will not check hostname of the filer server
+                            .sslContext(sslContext)
+                );
 
         filerAddress = SeaweedUtil.joinHostPort(host, port);
 
@@ -130,12 +135,11 @@ public class FilerGrpcClient {
     public String getChunkUrl(String chunkId, String url, String publicUrl) {
         switch (this.volumeServerAccess) {
             case VOLUME_SERVER_ACCESS_PUBLIC_URL:
-                return String.format("http://%s/%s", publicUrl, chunkId);
+                return String.format("%s://%s/%s", protocol, publicUrl, chunkId);
             case VOLUME_SERVER_ACCESS_FILER_PROXY:
-                return String.format("http://%s/?proxyChunkId=%s", this.filerAddress, chunkId);
+                return String.format("%s://%s/?proxyChunkId=%s", protocol, this.filerAddress, chunkId);
             default:
-                return String.format("http://%s/%s", url, chunkId);
+                return String.format("%s://%s/%s", protocol, url, chunkId);
         }
     }
-
 }

other/java/client/src/main/java/seaweedfs/client/FilerSecurityContext.java

@@ -0,0 +1,164 @@
+package seaweedfs.client;
+
+import com.google.common.base.Strings;
+import com.moandjiezana.toml.Toml;
+import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
+import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.http.ssl.SSLContexts;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.*;
+import java.io.File;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+
+public abstract class FilerSecurityContext extends SslContext {
+//extends Netty SslContext to access its protected static utility methods in
+//buildHttpSslContext()
+
+    private static final Logger logger = LoggerFactory.getLogger(FilerSecurityContext.class);
+    private static boolean grpcSecurityEnabled;
+    private static boolean httpSecurityEnabled;
+    private static SslContext grpcSslContext;
+    private static SSLContext httpSslContext;
+
+    private static String grpcTrustCertCollectionFilePath;
+    private static String grpcClientCertChainFilePath;
+    private static String grpcClientPrivateKeyFilePath;
+
+    private static String httpTrustCertCollectionFilePath;
+    private static String httpClientCertChainFilePath;
+    private static String httpClientPrivateKeyFilePath;
+
+
+    static {
+        String securityFileName = "security.toml";
+        String home = System.getProperty("user.home");
+        File f1 = new File("./"+securityFileName);
+        File f2 = new File(home + "/.seaweedfs/"+securityFileName);
+        File f3 = new File("/etc/seaweedfs/"+securityFileName);
+
+        File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null;
+
+        if (securityFile==null){
+            logger.debug("Security file not found");
+            grpcSecurityEnabled = false;
+            httpSecurityEnabled = false;
+        } else {
+
+            Toml toml = new Toml().read(securityFile);
+            logger.debug("reading ssl setup from {}", securityFile);
+
+            grpcTrustCertCollectionFilePath = toml.getString("grpc.ca");
+            logger.debug("loading gRPC ca from {}", grpcTrustCertCollectionFilePath);
+            grpcClientCertChainFilePath = toml.getString("grpc.client.cert");
+            logger.debug("loading gRPC client ca from {}", grpcClientCertChainFilePath);
+            grpcClientPrivateKeyFilePath = toml.getString("grpc.client.key");
+            logger.debug("loading gRPC client key from {}", grpcClientPrivateKeyFilePath);
+
+            if (Strings.isNullOrEmpty(grpcClientCertChainFilePath) && Strings.isNullOrEmpty(grpcClientPrivateKeyFilePath)) {
+                logger.debug("gRPC private key file locations not set");
+                grpcSecurityEnabled = false;
+            } else {
+                try {
+                    grpcSslContext = buildGrpcSslContext();
+                    grpcSecurityEnabled = true;
+                } catch (Exception e) {
+                    logger.warn("Couldn't initialize gRPC security context, filer operations are likely to fail!", e);
+                    grpcSslContext = null;
+                    grpcSecurityEnabled = false;
+                }
+            }
+
+            if (toml.getBoolean("https.client.enabled")) {
+                httpTrustCertCollectionFilePath = toml.getString("https.client.ca");
+                logger.debug("loading HTTP ca from {}", httpTrustCertCollectionFilePath);
+                httpClientCertChainFilePath = toml.getString("https.client.cert");
+                logger.debug("loading HTTP client ca from {}", httpClientCertChainFilePath);
+                httpClientPrivateKeyFilePath = toml.getString("https.client.key");
+                logger.debug("loading HTTP client key from {}", httpClientPrivateKeyFilePath);
+
+                if (Strings.isNullOrEmpty(httpClientCertChainFilePath) && Strings.isNullOrEmpty(httpClientPrivateKeyFilePath)) {
+                    logger.debug("HTTP private key file locations not set");
+                    httpSecurityEnabled = false;
+                } else {
+                    try {
+                        httpSslContext = buildHttpSslContext();
+                        httpSecurityEnabled = true;
+                    } catch (Exception e) {
+                        logger.warn("Couldn't initialize HTTP security context, volume operations are likely to fail!", e);
+                        httpSslContext = null;
+                        httpSecurityEnabled = false;
+                    }
+                }
+            } else {
+                httpSecurityEnabled = false;
+            }
+        }
+        // possibly fix the format https://netty.io/wiki/sslcontextbuilder-and-private-key.html
+    }
+
+    public static boolean isGrpcSecurityEnabled() {
+        return grpcSecurityEnabled;
+    }
+
+    public static boolean isHttpSecurityEnabled() {
+        return httpSecurityEnabled;
+    }
+
+    public static SslContext getGrpcSslContext() {
+        return grpcSslContext;
+    }
+
+    public static SSLContext getHttpSslContext() {
+        return httpSslContext;
+    }
+
+    private static SslContext buildGrpcSslContext() throws SSLException {
+        SslContextBuilder builder = GrpcSslContexts.forClient();
+        if (grpcTrustCertCollectionFilePath != null) {
+            builder.trustManager(new File(grpcTrustCertCollectionFilePath));
+        }
+        if (grpcClientCertChainFilePath != null && grpcClientPrivateKeyFilePath != null) {
+            builder.keyManager(new File(grpcClientCertChainFilePath), new File(grpcClientPrivateKeyFilePath));
+        }
+
+        return builder.build();
+    }
+
+    private static SSLContext buildHttpSslContext() throws GeneralSecurityException, IOException {
+        SSLContextBuilder builder = SSLContexts.custom();
+
+        if (httpTrustCertCollectionFilePath != null) {
+            final X509Certificate[] trustCerts = toX509Certificates(new File(httpTrustCertCollectionFilePath));
+            final KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
+            ks.load(null, null);
+
+            int i = 0;
+            for (X509Certificate cert: trustCerts) {
+                String alias = Integer.toString(++i);
+                ks.setCertificateEntry(alias, cert);
+            }
+
+            builder.loadTrustMaterial(ks, null);
+        }
+
+        if (httpClientCertChainFilePath != null && httpClientPrivateKeyFilePath != null) {
+            final X509Certificate[] keyCerts = toX509Certificates(new File(httpClientCertChainFilePath));
+            final PrivateKey key = toPrivateKey(new File(httpClientPrivateKeyFilePath), null);
+            char[] emptyPassword = new char[0];
+            final KeyStore ks = buildKeyStore(keyCerts, key, emptyPassword, null);
+            logger.debug("Loaded {} key certificates", ks.size());
+            builder.loadKeyMaterial(ks, emptyPassword);
+        }
+
+        return builder.build();
+    }
+}

other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java

@@ -1,64 +0,0 @@
-package seaweedfs.client;
-
-import com.google.common.base.Strings;
-import com.moandjiezana.toml.Toml;
-import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
-import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
-import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
-import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.SSLException;
-import java.io.File;
-
-public class FilerSslContext {
-
-    private static final Logger logger = LoggerFactory.getLogger(FilerSslContext.class);
-
-    public static SslContext loadSslContext() throws SSLException {
-        String securityFileName = "security.toml";
-        String home = System.getProperty("user.home");
-        File f1 = new File("./"+securityFileName);
-        File f2 = new File(home + "/.seaweedfs/"+securityFileName);
-        File f3 = new File("/etc/seaweedfs/"+securityFileName);
-
-        File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null;
-
-        if (securityFile==null){
-            return null;
-        }
-
-        Toml toml = new Toml().read(securityFile);
-        logger.debug("reading ssl setup from {}", securityFile);
-
-        String trustCertCollectionFilePath = toml.getString("grpc.ca");
-        logger.debug("loading ca from {}", trustCertCollectionFilePath);
-        String clientCertChainFilePath = toml.getString("grpc.client.cert");
-        logger.debug("loading client ca from {}", clientCertChainFilePath);
-        String clientPrivateKeyFilePath = toml.getString("grpc.client.key");
-        logger.debug("loading client key from {}", clientPrivateKeyFilePath);
-
-        if (Strings.isNullOrEmpty(clientPrivateKeyFilePath) && Strings.isNullOrEmpty(clientPrivateKeyFilePath)){
-            return null;
-        }
-
-        // possibly fix the format https://netty.io/wiki/sslcontextbuilder-and-private-key.html
-
-        return buildSslContext(trustCertCollectionFilePath, clientCertChainFilePath, clientPrivateKeyFilePath);
-    }
-
-
-    private static SslContext buildSslContext(String trustCertCollectionFilePath,
-                                             String clientCertChainFilePath,
-                                             String clientPrivateKeyFilePath) throws SSLException {
-        SslContextBuilder builder = GrpcSslContexts.forClient();
-        if (trustCertCollectionFilePath != null) {
-            builder.trustManager(new File(trustCertCollectionFilePath));
-        }
-        if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
-            builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
-        }
-        return builder.trustManager(InsecureTrustManagerFactory.INSTANCE).build();
-    }
-}

other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java

@@ -1,27 +1,64 @@
 package seaweedfs.client;
 
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.impl.DefaultConnectionReuseStrategy;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLContext;
 
 public class SeaweedUtil {
 
-    static PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
+    private static final Logger logger = LoggerFactory.getLogger(SeaweedUtil.class);
+    static PoolingHttpClientConnectionManager cm;
     static CloseableHttpClient httpClient;
 
     static {
+        //Apache HTTP client has a terrible API that makes you configure everything twice
+        //NoopHostnameVerifier is required because SeaweedFS doesn't verify hostnames
+        //and the servers are likely to have TLS certificates that do not match their hosts
+        if (FilerSecurityContext.isHttpSecurityEnabled()) {
+            SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(
+                    FilerSecurityContext.getHttpSslContext(),
+                    NoopHostnameVerifier.INSTANCE);
+
+            Registry<ConnectionSocketFactory> socketFactoryRegistry =
+                    RegistryBuilder.<ConnectionSocketFactory>create()
+                            .register("https", sslSocketFactory)
+                            .register("http", new PlainConnectionSocketFactory())
+                            .build();
+            cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+        } else {
+            cm = new PoolingHttpClientConnectionManager();
+        }
+
         // Increase max total connection to 200
         cm.setMaxTotal(200);
         // Increase default max connection per route to 20
         cm.setDefaultMaxPerRoute(20);
 
-        httpClient = HttpClientBuilder.create()
+        HttpClientBuilder builder = HttpClientBuilder.create()
                 .setConnectionManager(cm)
                 .setConnectionReuseStrategy(DefaultConnectionReuseStrategy.INSTANCE)
-                .setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE)
-                .build();
+                .setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE);
+
+        if (FilerSecurityContext.isHttpSecurityEnabled()) {
+            builder.setSSLContext(FilerSecurityContext.getHttpSslContext());
+            builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
+        }
+
+        httpClient = builder.build();
     }
 
     public static CloseableHttpClient getClosableHttpClient() {

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -29,6 +29,7 @@ public class SeaweedFileSystem extends FileSystem {
     public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
     public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access";
     public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
+    public static final String FS_SEAWEED_FILER_CN = "fs.seaweed.filer.cn";
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
 
@@ -63,8 +64,9 @@ public class SeaweedFileSystem extends FileSystem {
         setConf(conf);
         this.uri = uri;
 
-        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf);
+        String cn = conf.get(FS_SEAWEED_FILER_CN, "");
 
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, cn, conf);
     }
 
     @Override

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -27,8 +27,8 @@ public class SeaweedFileSystemStore {
     private FilerClient filerClient;
     private Configuration conf;
 
-    public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) {
-        filerClient = new FilerClient(host, port, grpcPort);
+    public SeaweedFileSystemStore(String host, int port, int grpcPort, String cn, Configuration conf) {
+        filerClient = new FilerClient(host, port, grpcPort, cn);
         this.conf = conf;
         String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
         if (volumeServerAccessMode.equals("publicUrl")) {
@@ -36,7 +36,6 @@ public class SeaweedFileSystemStore {
         } else if (volumeServerAccessMode.equals("filerProxy")) {
             filerClient.setAccessVolumeServerByFilerProxy();
         }
-
     }
 
     public void close() {

other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -29,6 +29,7 @@ public class SeaweedFileSystem extends FileSystem {
     public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
     public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access";
     public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
+    public static final String FS_SEAWEED_FILER_CN = "fs.seaweed.filer.cn";
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
 
@@ -63,8 +64,9 @@ public class SeaweedFileSystem extends FileSystem {
         setConf(conf);
         this.uri = uri;
 
-        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf);
+        String cn = conf.get(FS_SEAWEED_FILER_CN, "");
 
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, cn, conf);
     }
 
     @Override

other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -27,8 +27,8 @@ public class SeaweedFileSystemStore {
     private FilerClient filerClient;
     private Configuration conf;
 
-    public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) {
-        filerClient = new FilerClient(host, port, grpcPort);
+    public SeaweedFileSystemStore(String host, int port, int grpcPort, String cn, Configuration conf) {
+        filerClient = new FilerClient(host, port, grpcPort, cn);
         this.conf = conf;
         String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
         if (volumeServerAccessMode.equals("publicUrl")) {
@@ -36,7 +36,6 @@ public class SeaweedFileSystemStore {
         } else if (volumeServerAccessMode.equals("filerProxy")) {
             filerClient.setAccessVolumeServerByFilerProxy();
         }
-
     }
 
     public void close() {

other/mq_client_example/agent_pub_record/agent_pub_record.go

@@ -3,9 +3,9 @@ package main
 import (
 	"flag"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/other/mq_client_example/example"
 	"github.com/seaweedfs/seaweedfs/weed/mq/client/agent_client"
 	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
-	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
 	"log"
 	"sync"
 	"sync/atomic"
@@ -20,13 +20,26 @@ var (
 
 	clientName = flag.String("client", "c1", "client name")
 
-	namespace   = flag.String("ns", "test", "namespace")
-	t           = flag.String("t", "test", "t")
-	seedBrokers = flag.String("brokers", "localhost:17777", "seed brokers")
+	namespace = flag.String("ns", "test", "namespace")
+	t         = flag.String("t", "test", "t")
+	agent     = flag.String("agent", "localhost:16777", "mq agent address")
 
 	counter int32
 )
 
+func genMyRecord(id int32) *example.MyRecord {
+	return &example.MyRecord{
+		Key:    []byte(fmt.Sprintf("key-%s-%d", *clientName, id)),
+		Field1: []byte(fmt.Sprintf("field1-%s-%d", *clientName, id)),
+		Field2: fmt.Sprintf("field2-%s-%d", *clientName, id),
+		Field3: id,
+		Field4: int64(id),
+		Field5: float32(id),
+		Field6: float64(id),
+		Field7: id%2 == 0,
+	}
+}
+
 func doPublish(publisher *agent_client.PublishSession, id int) {
 	startTime := time.Now()
 	for {
@@ -49,58 +62,12 @@ func doPublish(publisher *agent_client.PublishSession, id int) {
 	log.Printf("Publisher %s-%d finished in %s", *clientName, id, elapsed)
 }
 
-type MyRecord struct {
-	Key    []byte
-	Field1 []byte
-	Field2 string
-	Field3 int32
-	Field4 int64
-	Field5 float32
-	Field6 float64
-	Field7 bool
-}
-
-func genMyRecord(id int32) *MyRecord {
-	return &MyRecord{
-		Key:    []byte(fmt.Sprintf("key-%s-%d", *clientName, id)),
-		Field1: []byte(fmt.Sprintf("field1-%s-%d", *clientName, id)),
-		Field2: fmt.Sprintf("field2-%s-%d", *clientName, id),
-		Field3: id,
-		Field4: int64(id),
-		Field5: float32(id),
-		Field6: float64(id),
-		Field7: id%2 == 0,
-	}
-}
-
-func (r *MyRecord) ToRecordValue() *schema_pb.RecordValue {
-	return schema.RecordBegin().
-		SetBytes("key", r.Key).
-		SetBytes("field1", r.Field1).
-		SetString("field2", r.Field2).
-		SetInt32("field3", r.Field3).
-		SetInt64("field4", r.Field4).
-		SetFloat("field5", r.Field5).
-		SetDouble("field6", r.Field6).
-		SetBool("field7", r.Field7).
-		RecordEnd()
-}
-
 func main() {
 	flag.Parse()
 
-	recordType := schema.RecordTypeBegin().
-		WithField("key", schema.TypeBytes).
-		WithField("field1", schema.TypeBytes).
-		WithField("field2", schema.TypeString).
-		WithField("field3", schema.TypeInt32).
-		WithField("field4", schema.TypeInt64).
-		WithField("field5", schema.TypeFloat).
-		WithField("field6", schema.TypeDouble).
-		WithField("field7", schema.TypeBoolean).
-		RecordTypeEnd()
+	recordType := example.MyRecordType()
 
-	session, err := agent_client.NewPublishSession("localhost:16777", schema.NewSchema(*namespace, *t, recordType), *partitionCount, *clientName)
+	session, err := agent_client.NewPublishSession(*agent, schema.NewSchema(*namespace, *t, recordType), *partitionCount, *clientName)
 	if err != nil {
 		log.Printf("failed to create session: %v", err)
 		return

other/mq_client_example/agent_sub_record/agent_sub_record.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/other/mq_client_example/example"
+	"github.com/seaweedfs/seaweedfs/weed/mq/client/agent_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"log"
+	"time"
+)
+
+var (
+	namespace         = flag.String("ns", "test", "namespace")
+	t                 = flag.String("topic", "test", "topic")
+	agent             = flag.String("agent", "localhost:16777", "mq agent address")
+	maxPartitionCount = flag.Int("maxPartitionCount", 3, "max partition count")
+	slidingWindowSize = flag.Int("slidingWindowSize", 1, "per partition concurrency")
+	timeAgo           = flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
+
+	clientId = flag.Uint("client_id", uint(util.RandomInt32()), "client id")
+)
+
+func main() {
+	flag.Parse()
+
+	// determine the start of the messages
+	var startTsNs int64
+	startType := schema_pb.OffsetType_RESUME_OR_EARLIEST
+	if *timeAgo > 0 {
+		startTsNs = time.Now().Add(-*timeAgo).UnixNano()
+		startType = schema_pb.OffsetType_EXACT_TS_NS
+	}
+
+	session, err := agent_client.NewSubscribeSession(*agent, &agent_client.SubscribeOption{
+		ConsumerGroup:           "test",
+		ConsumerGroupInstanceId: fmt.Sprintf("client-%d", *clientId),
+		Topic:                   topic.NewTopic(*namespace, *t),
+		OffsetType:              startType,
+		OffsetTsNs:              startTsNs,
+		Filter:                  "",
+		MaxSubscribedPartitions: int32(*maxPartitionCount),
+		SlidingWindowSize:       int32(*slidingWindowSize),
+	})
+	if err != nil {
+		log.Printf("new subscribe session: %v", err)
+		return
+	}
+	defer session.CloseSession()
+
+	counter := 0
+	session.SubscribeMessageRecord(func(key []byte, recordValue *schema_pb.RecordValue) {
+		counter++
+		record := example.FromRecordValue(recordValue)
+		fmt.Printf("%d %s %v\n", counter, string(key), record.Field2)
+	}, func() {
+		log.Printf("done received %d messages", counter)
+	})
+
+}

other/mq_client_example/example/my_record.go

@@ -0,0 +1,56 @@
+package example
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+type MyRecord struct {
+	Key    []byte
+	Field1 []byte
+	Field2 string
+	Field3 int32
+	Field4 int64
+	Field5 float32
+	Field6 float64
+	Field7 bool
+}
+
+func MyRecordType() *schema_pb.RecordType {
+	return schema.RecordTypeBegin().
+		WithField("key", schema.TypeBytes).
+		WithField("field1", schema.TypeBytes).
+		WithField("field2", schema.TypeString).
+		WithField("field3", schema.TypeInt32).
+		WithField("field4", schema.TypeInt64).
+		WithField("field5", schema.TypeFloat).
+		WithField("field6", schema.TypeDouble).
+		WithField("field7", schema.TypeBoolean).
+		RecordTypeEnd()
+}
+
+func (r *MyRecord) ToRecordValue() *schema_pb.RecordValue {
+	return schema.RecordBegin().
+		SetBytes("key", r.Key).
+		SetBytes("field1", r.Field1).
+		SetString("field2", r.Field2).
+		SetInt32("field3", r.Field3).
+		SetInt64("field4", r.Field4).
+		SetFloat("field5", r.Field5).
+		SetDouble("field6", r.Field6).
+		SetBool("field7", r.Field7).
+		RecordEnd()
+}
+
+func FromRecordValue(recordValue *schema_pb.RecordValue) *MyRecord {
+	return &MyRecord{
+		Key:    recordValue.Fields["key"].GetBytesValue(),
+		Field1: recordValue.Fields["field1"].GetBytesValue(),
+		Field2: recordValue.Fields["field2"].GetStringValue(),
+		Field3: recordValue.Fields["field3"].GetInt32Value(),
+		Field4: recordValue.Fields["field4"].GetInt64Value(),
+		Field5: recordValue.Fields["field5"].GetFloatValue(),
+		Field6: recordValue.Fields["field6"].GetDoubleValue(),
+		Field7: recordValue.Fields["field7"].GetBoolValue(),
+	}
+}

replace_glog.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Find all Go files containing glog calls
+files=$(grep -l "glog\." --include="*.go" -r .)
+
+# Check if any files were found
+if [ -z "$files" ]; then
+    echo "No files found containing glog calls"
+    exit 0
+fi
+
+# Print the files that will be modified
+echo "The following files will be modified:"
+echo "$files"
+echo
+
+# Ask for confirmation
+read -p "Do you want to proceed with the replacement? (y/n) " -n 1 -r
+echo
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "Operation cancelled"
+    exit 1
+fi
+
+# Make the replacements
+for file in $files; do
+    echo "Processing $file"
+    # Replace all glog function calls with log
+    sed -i '' 's/glog\./log\./g' "$file"
+done
+
+echo "Replacement complete!" 

replace_glog_calls.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Find all Go files containing glog.V calls
+files=$(grep -l "glog.V" --include="*.go" -r .)
+
+# Check if any files were found
+if [ -z "$files" ]; then
+    echo "No files found containing glog.V calls"
+    exit 0
+fi
+
+# Print the files that will be modified
+echo "The following files will be modified:"
+echo "$files"
+echo
+
+# Ask for confirmation
+read -p "Do you want to proceed with the replacement? (y/n) " -n 1 -r
+echo
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "Operation cancelled"
+    exit 1
+fi
+
+# Make the replacements
+for file in $files; do
+    echo "Processing $file"
+    # Replace glog.V(n).Info with log.V(n).Info for n=0-4
+    for level in {0..4}; do
+        # Replace Info calls
+        sed -i '' "s/glog.V($level).Info/log.V($level).Info/g" "$file"
+        # Replace Infof calls
+        sed -i '' "s/glog.V($level).Infof/log.V($level).Infof/g" "$file"
+    done
+done
+
+echo "Replacement complete!" 

replace_imports.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Find all Go files containing the old import path
+files=$(grep -l "github.com/seaweedfs/seaweedfs/weed/glog" --include="*.go" -r .)
+
+# Check if any files were found
+if [ -z "$files" ]; then
+    echo "No files found containing the old import path"
+    exit 0
+fi
+
+# Print the files that will be modified
+echo "The following files will be modified:"
+echo "$files"
+echo
+
+# Ask for confirmation
+read -p "Do you want to proceed with the replacement? (y/n) " -n 1 -r
+echo
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "Operation cancelled"
+    exit 1
+fi
+
+# Make the replacements
+for file in $files; do
+    echo "Processing $file"
+    # Use sed to replace the import path
+    sed -i '' 's|github.com/seaweedfs/seaweedfs/weed/glog|github.com/seaweedfs/seaweedfs/weed/util/log|g' "$file"
+done
+
+echo "Replacement complete!" 

swap_log_levels.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Find all Go files containing log.V calls
+files=$(grep -l "log.V" --include="*.go" -r .)
+
+if [ -z "$files" ]; then
+    echo "No files found containing log.V calls"
+    exit 0
+fi
+
+# Create a temporary file for sed operations
+temp_file=$(mktemp)
+
+# Process each file
+for file in $files; do
+    echo "Processing $file"
+    
+    # First, replace log.V(-1) with a temporary placeholder
+    sed 's/log\.V(-1)/__TEMP_NEG_ONE__/g' "$file" > "$temp_file"
+    mv "$temp_file" "$file"
+    
+    # Replace log.V(4) with log.V(-1)
+    sed 's/log\.V(4)/log.V(-1)/g' "$file" > "$temp_file"
+    mv "$temp_file" "$file"
+    
+    # Replace the temporary placeholder with log.V(4)
+    sed 's/__TEMP_NEG_ONE__/log.V(4)/g' "$file" > "$temp_file"
+    mv "$temp_file" "$file"
+    
+    # Swap log.V(0) and log.V(3)
+    sed 's/log\.V(0)/__TEMP_ZERO__/g' "$file" > "$temp_file"
+    mv "$temp_file" "$file"
+    sed 's/log\.V(3)/log.V(0)/g' "$file" > "$temp_file"
+    mv "$temp_file" "$file"
+    sed 's/__TEMP_ZERO__/log.V(3)/g' "$file" > "$temp_file"
+    mv "$temp_file" "$file"
+    
+    # Swap log.V(1) and log.V(2)
+    sed 's/log\.V(1)/__TEMP_ONE__/g' "$file" > "$temp_file"
+    mv "$temp_file" "$file"
+    sed 's/log\.V(2)/log.V(1)/g' "$file" > "$temp_file"
+    mv "$temp_file" "$file"
+    sed 's/__TEMP_ONE__/log.V(2)/g' "$file" > "$temp_file"
+    mv "$temp_file" "$file"
+    
+    # Replace any other log.V(n) with log.V(-1)
+    sed -E 's/log\.V\([5-9][0-9]*\)/log.V(-1)/g' "$file" > "$temp_file"
+    mv "$temp_file" "$file"
+done
+
+# Clean up
+rm -f "$temp_file"
+
+echo "Log level swapping completed!" 

unmaintained/change_superblock/change_superblock.go

@@ -7,7 +7,7 @@ import (
 	"path"
 	"strconv"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
@@ -49,7 +49,7 @@ func main() {
 	}
 	datFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".dat"), os.O_RDWR, 0644)
 	if err != nil {
-		glog.Fatalf("Open Volume Data File [ERROR]: %v", err)
+		log.Fatalf("Open Volume Data File [ERROR]: %v", err)
 	}
 	datBackend := backend.NewDiskFile(datFile)
 	defer datBackend.Close()
@@ -57,7 +57,7 @@ func main() {
 	superBlock, err := super_block.ReadSuperBlock(datBackend)
 
 	if err != nil {
-		glog.Fatalf("cannot parse existing super block: %v", err)
+		log.Fatalf("cannot parse existing super block: %v", err)
 	}
 
 	fmt.Printf("Current Volume Replication: %s\n", superBlock.ReplicaPlacement)
@@ -69,7 +69,7 @@ func main() {
 		replica, err := super_block.NewReplicaPlacementFromString(*targetReplica)
 
 		if err != nil {
-			glog.Fatalf("cannot parse target replica %s: %v", *targetReplica, err)
+			log.Fatalf("cannot parse target replica %s: %v", *targetReplica, err)
 		}
 
 		fmt.Printf("Changing replication to: %s\n", replica)
@@ -82,7 +82,7 @@ func main() {
 		ttl, err := needle.ReadTTL(*targetTTL)
 
 		if err != nil {
-			glog.Fatalf("cannot parse target ttl %s: %v", *targetTTL, err)
+			log.Fatalf("cannot parse target ttl %s: %v", *targetTTL, err)
 		}
 
 		fmt.Printf("Changing ttl to: %s\n", ttl)
@@ -96,7 +96,7 @@ func main() {
 		header := superBlock.Bytes()
 
 		if n, e := datBackend.WriteAt(header, 0); n == 0 || e != nil {
-			glog.Fatalf("cannot write super block: %v", e)
+			log.Fatalf("cannot write super block: %v", e)
 		}
 
 		fmt.Println("Change Applied.")

unmaintained/diff_volume_servers/diff_volume_servers.go

@@ -10,7 +10,7 @@ import (
 	"math"
 	"os"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -49,7 +49,7 @@ func main() {
 	vid := uint32(*volumeId)
 	servers := pb.ServerAddresses(*serversStr).ToAddresses()
 	if len(servers) < 2 {
-		glog.Fatalf("You must specify more than 1 server\n")
+		log.Fatalf("You must specify more than 1 server\n")
 	}
 	var referenceServer pb.ServerAddress
 	var maxOffset int64
@@ -57,7 +57,7 @@ func main() {
 	for _, addr := range servers {
 		files, offset, err := getVolumeFiles(vid, addr)
 		if err != nil {
-			glog.Fatalf("Failed to copy idx from volume server %s\n", err)
+			log.Fatalf("Failed to copy idx from volume server %s\n", err)
 		}
 		allFiles[addr] = files
 		if offset > maxOffset {
@@ -103,7 +103,7 @@ func main() {
 				id, err = getNeedleFileId(vid, nid, addr)
 			}
 			if err != nil {
-				glog.Fatalf("Failed to get needle info %d from volume server %s\n", nid, err)
+				log.Fatalf("Failed to get needle info %d from volume server %s\n", nid, err)
 			}
 			fmt.Println(id, addr, diffMsg)
 		}

unmaintained/disk/disk_status.go

@@ -0,0 +1,57 @@
+//go:build !windows && !openbsd && !netbsd && !plan9 && !solaris
+// +build !windows,!openbsd,!netbsd,!plan9,!solaris
+
+package main
+
+import (
+	"log"
+	"syscall"
+)
+
+// go run unmaintained/disk/disk_status.go
+
+type DiskStatus struct {
+	Dir         string  `protobuf:"bytes,1,opt,name=dir,proto3" json:"dir,omitempty"`
+	All         uint64  `protobuf:"varint,2,opt,name=all,proto3" json:"all,omitempty"`
+	Used        uint64  `protobuf:"varint,3,opt,name=used,proto3" json:"used,omitempty"`
+	Free        uint64  `protobuf:"varint,4,opt,name=free,proto3" json:"free,omitempty"`
+	PercentFree float32 `protobuf:"fixed32,5,opt,name=percent_free,json=percentFree,proto3" json:"percent_free,omitempty"`
+	PercentUsed float32 `protobuf:"fixed32,6,opt,name=percent_used,json=percentUsed,proto3" json:"percent_used,omitempty"`
+	DiskType    string  `protobuf:"bytes,7,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"`
+
+	// new fields about avail blocks
+	Avail        uint64  `protobuf:"varint,4,opt,name=avail,proto3" json:"avail,omitempty"`
+	PercentAvail float32 `protobuf:"fixed32,5,opt,name=percent_avail,json=percentAvail,proto3" json:"percent_avail,omitempty"`
+}
+
+func main() {
+	dirs := []string{"/mnt/sdb", "/mnt/sdc", "/mnt/sdd", "/mnt/sde", "/mnt/sdf", "/mnt/sdg", "/mnt/sdh", "/mnt/sdi", "/mnt/sdj"}
+	// dirs := []string{"/mnt/sdb"}
+	for _, dir := range dirs {
+		disk := &DiskStatus{Dir: dir}
+		fillInDiskStatus(disk)
+
+		// bytes, _ := json.Marshal(disk)
+		// log.Printf("disk status %s", bytes)
+		log.Printf("disk: %s avail: %f free: %f", disk.Dir, disk.PercentAvail, disk.PercentFree)
+	}
+}
+
+func fillInDiskStatus(disk *DiskStatus) {
+	fs := syscall.Statfs_t{}
+	err := syscall.Statfs(disk.Dir, &fs)
+	if err != nil {
+		return
+	}
+
+	disk.All = fs.Blocks * uint64(fs.Bsize)
+	disk.Free = fs.Bfree * uint64(fs.Bsize)
+	disk.Used = disk.All - disk.Free
+	disk.PercentFree = float32((float64(disk.Free) / float64(disk.All)) * 100)
+	disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100)
+
+	// avail blocks
+	disk.Avail = fs.Bavail * uint64(fs.Bsize)
+	disk.PercentAvail = float32((float64(disk.Avail) / float64(disk.All)) * 100)
+	return
+}

unmaintained/fix_dat/fix_dat.go

@@ -8,7 +8,7 @@ import (
 	"path"
 	"strconv"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
@@ -45,26 +45,26 @@ func main() {
 	}
 	indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
 	if err != nil {
-		glog.Fatalf("Read Volume Index %v", err)
+		log.Fatalf("Read Volume Index %v", err)
 	}
 	defer indexFile.Close()
 	datFileName := path.Join(*fixVolumePath, fileName+".dat")
 	datFile, err := os.OpenFile(datFileName, os.O_RDONLY, 0644)
 	if err != nil {
-		glog.Fatalf("Read Volume Data %v", err)
+		log.Fatalf("Read Volume Data %v", err)
 	}
 	datBackend := backend.NewDiskFile(datFile)
 	defer datBackend.Close()
 
 	newDatFile, err := os.Create(path.Join(*fixVolumePath, fileName+".dat_fixed"))
 	if err != nil {
-		glog.Fatalf("Write New Volume Data %v", err)
+		log.Fatalf("Write New Volume Data %v", err)
 	}
 	defer newDatFile.Close()
 
 	superBlock, err := super_block.ReadSuperBlock(datBackend)
 	if err != nil {
-		glog.Fatalf("Read Volume Data superblock %v", err)
+		log.Fatalf("Read Volume Data superblock %v", err)
 	}
 	newDatFile.Write(superBlock.Bytes())
 

unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go

@@ -3,7 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -38,7 +38,7 @@ func main() {
 			return nil
 		}
 		name := event.EventNotification.NewEntry.Name
-		glog.V(0).Infof("=> %s ts:%+v", name, time.Unix(0, event.TsNs))
+		log.V(3).Infof("=> %s ts:%+v", name, time.Unix(0, event.TsNs))
 		id := name[4:]
 		if x, err := strconv.Atoi(id); err == nil {
 			if x != expected {
@@ -59,7 +59,7 @@ func startGenerateMetadata() {
 
 		for i := 0; i < *n; i++ {
 			name := fmt.Sprintf("file%d", i)
-			glog.V(0).Infof("write %s/%s", *dir, name)
+			log.V(3).Infof("write %s/%s", *dir, name)
 			if err := filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{
 				Directory: *dir,
 				Entry: &filer_pb.Entry{

unmaintained/remove_duplicate_fids/remove_duplicate_fids.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/storage"
 	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
@@ -50,7 +50,7 @@ func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset in
 		newFileName := filepath.Join(*volumePath, "dat_fixed")
 		newDatFile, err := os.Create(newFileName)
 		if err != nil {
-			glog.Fatalf("Write New Volume Data %v", err)
+			log.Fatalf("Write New Volume Data %v", err)
 		}
 		scanner.datBackend = backend.NewDiskFile(newDatFile)
 		scanner.datBackend.WriteAt(scanner.block.Bytes(), 0)
@@ -59,7 +59,7 @@ func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset in
 	checksum := Checksum(n)
 
 	if scanner.hashes[checksum] {
-		glog.V(0).Infof("duplicate checksum:%s fid:%d,%s%x @ offset:%d", checksum, *volumeId, n.Id, n.Cookie, offset)
+		log.V(3).Infof("duplicate checksum:%s fid:%d,%s%x @ offset:%d", checksum, *volumeId, n.Id, n.Cookie, offset)
 		return nil
 	}
 	scanner.hashes[checksum] = true
@@ -85,13 +85,13 @@ func main() {
 
 	if _, err := os.Stat(scanner.dir); err != nil {
 		if err := os.MkdirAll(scanner.dir, os.ModePerm); err != nil {
-			glog.Fatalf("could not create output dir : %s", err)
+			log.Fatalf("could not create output dir : %s", err)
 		}
 	}
 
 	err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)
 	if err != nil {
-		glog.Fatalf("Reading Volume File [ERROR] %s\n", err)
+		log.Fatalf("Reading Volume File [ERROR] %s\n", err)
 	}
 
 }

unmaintained/see_dat/see_dat.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/util"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/storage"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
@@ -34,7 +34,7 @@ func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
 
 func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
 	t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
-	glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v name %s",
+	log.V(3).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v name %s",
 		*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t, n.Name)
 	return nil
 }
@@ -48,6 +48,6 @@ func main() {
 	scanner := &VolumeFileScanner4SeeDat{}
 	err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)
 	if err != nil {
-		glog.Fatalf("Reading Volume File [ERROR] %s\n", err)
+		log.Fatalf("Reading Volume File [ERROR] %s\n", err)
 	}
 }

unmaintained/see_idx/see_idx.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/util"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
@@ -36,7 +36,7 @@ func main() {
 	}
 	indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
 	if err != nil {
-		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
+		log.Fatalf("Create Volume Index [ERROR] %s\n", err)
 	}
 	defer indexFile.Close()
 

weed/Makefile

@@ -41,6 +41,10 @@ debug_mq:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 mq.broker
 
+debug_mq_agent:
+	go build -gcflags="all=-N -l"
+	dlv --listen=:2346 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 mq.agent -broker=localhost:17777
+
 debug_filer_copy:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 filer.backup -filer=localhost:8888 -filerProxy -timeAgo=10h

weed/cluster/lock_client.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -73,7 +73,7 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
 		for {
 			if isLocked {
 				if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err != nil {
-					glog.V(0).Infof("Lost lock %s: %v", key, err)
+					log.V(3).Infof("Lost lock %s: %v", key, err)
 					isLocked = false
 				}
 			} else {
@@ -82,7 +82,7 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
 				}
 			}
 			if lockOwner != lock.LockOwner() && lock.LockOwner() != "" {
-				glog.V(0).Infof("Lock owner changed from %s to %s", lockOwner, lock.LockOwner())
+				log.V(3).Infof("Lock owner changed from %s to %s", lockOwner, lock.LockOwner())
 				onLockOwnerChange(lock.LockOwner())
 				lockOwner = lock.LockOwner()
 			}
@@ -102,7 +102,7 @@ func (lock *LiveLock) retryUntilLocked(lockDuration time.Duration) {
 		return lock.AttemptToLock(lockDuration)
 	}, func(err error) (shouldContinue bool) {
 		if err != nil {
-			glog.Warningf("create lock %s: %s", lock.key, err)
+			log.Warningf("create lock %s: %s", lock.key, err)
 		}
 		return lock.renewToken == ""
 	})

weed/cluster/lock_manager/distributed_lock_manager.go

@@ -2,7 +2,7 @@ package lock_manager
 
 import (
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"time"
 )
@@ -56,7 +56,7 @@ func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, move
 	}
 	if movedTo != dlm.Host {
 		servers := dlm.LockRing.GetSnapshot()
-		glog.V(0).Infof("lock %s not on current %s but on %s from %v", key, dlm.Host, movedTo, servers)
+		log.V(3).Infof("lock %s not on current %s but on %s from %v", key, dlm.Host, movedTo, servers)
 		return
 	}
 	owner, err = dlm.lockManager.GetLockOwner(key)

weed/cluster/lock_manager/lock_manager.go

@@ -3,7 +3,7 @@ package lock_manager
 import (
 	"fmt"
 	"github.com/google/uuid"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"sync"
 	"time"
 )
@@ -38,19 +38,19 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
 	lm.accessLock.Lock()
 	defer lm.accessLock.Unlock()
 
-	glog.V(4).Infof("lock %s %v %v %v", path, time.Unix(0, expiredAtNs), token, owner)
+	log.V(-1).Infof("lock %s %v %v %v", path, time.Unix(0, expiredAtNs), token, owner)
 
 	if oldValue, found := lm.locks[path]; found {
 		if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < time.Now().UnixNano() {
 			// lock is expired, set to a new lock
 			if token != "" {
-				glog.V(4).Infof("lock expired key %s non-empty token %v owner %v ts %s", path, token, owner, time.Unix(0, oldValue.ExpiredAtNs))
+				log.V(-1).Infof("lock expired key %s non-empty token %v owner %v ts %s", path, token, owner, time.Unix(0, oldValue.ExpiredAtNs))
 				err = LockErrorNonEmptyTokenOnExpiredLock
 				return
 			} else {
 				// new lock
 				renewToken = uuid.New().String()
-				glog.V(4).Infof("key %s new token %v owner %v", path, renewToken, owner)
+				log.V(-1).Infof("key %s new token %v owner %v", path, renewToken, owner)
 				lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
 				return
 			}
@@ -60,30 +60,30 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
 		if oldValue.Token == token {
 			// token matches, renew the lock
 			renewToken = uuid.New().String()
-			glog.V(4).Infof("key %s old token %v owner %v => %v owner %v", path, oldValue.Token, oldValue.Owner, renewToken, owner)
+			log.V(-1).Infof("key %s old token %v owner %v => %v owner %v", path, oldValue.Token, oldValue.Owner, renewToken, owner)
 			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
 			return
 		} else {
 			if token == "" {
 				// new lock
-				glog.V(4).Infof("key %s locked by %v", path, oldValue.Owner)
+				log.V(-1).Infof("key %s locked by %v", path, oldValue.Owner)
 				err = fmt.Errorf("lock already owned by %v", oldValue.Owner)
 				return
 			}
-			glog.V(4).Infof("key %s expected token %v owner %v received %v from %v", path, oldValue.Token, oldValue.Owner, token, owner)
+			log.V(-1).Infof("key %s expected token %v owner %v received %v from %v", path, oldValue.Token, oldValue.Owner, token, owner)
 			err = fmt.Errorf("lock: token mismatch")
 			return
 		}
 	} else {
-		glog.V(4).Infof("key %s no lock owner %v", path, owner)
+		log.V(-1).Infof("key %s no lock owner %v", path, owner)
 		if token == "" {
 			// new lock
-			glog.V(4).Infof("key %s new token %v owner %v", path, token, owner)
+			log.V(-1).Infof("key %s new token %v owner %v", path, token, owner)
 			renewToken = uuid.New().String()
 			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
 			return
 		} else {
-			glog.V(4).Infof("key %s non-empty token %v owner %v", path, token, owner)
+			log.V(-1).Infof("key %s non-empty token %v owner %v", path, token, owner)
 			err = LockErrorNonEmptyTokenOnNewLock
 			return
 		}
@@ -99,13 +99,13 @@ func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, err e
 		if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < now.UnixNano() {
 			// lock is expired, delete it
 			isUnlocked = true
-			glog.V(4).Infof("key %s expired at %v", path, time.Unix(0, oldValue.ExpiredAtNs))
+			log.V(-1).Infof("key %s expired at %v", path, time.Unix(0, oldValue.ExpiredAtNs))
 			delete(lm.locks, path)
 			return
 		}
 		if oldValue.Token == token {
 			isUnlocked = true
-			glog.V(4).Infof("key %s unlocked with %v", path, token)
+			log.V(-1).Infof("key %s unlocked with %v", path, token)
 			delete(lm.locks, path)
 			return
 		} else {
@@ -130,7 +130,7 @@ func (lm *LockManager) CleanUp() {
 				continue
 			}
 			if now > value.ExpiredAtNs {
-				glog.V(4).Infof("key %s expired at %v", key, time.Unix(0, value.ExpiredAtNs))
+				log.V(-1).Infof("key %s expired at %v", key, time.Unix(0, value.ExpiredAtNs))
 				delete(lm.locks, key)
 			}
 		}
@@ -148,12 +148,12 @@ func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Loc
 
 	for key, lock := range lm.locks {
 		if now > lock.ExpiredAtNs {
-			glog.V(4).Infof("key %s expired at %v", key, time.Unix(0, lock.ExpiredAtNs))
+			log.V(-1).Infof("key %s expired at %v", key, time.Unix(0, lock.ExpiredAtNs))
 			delete(lm.locks, key)
 			continue
 		}
 		if selectFn(key) {
-			glog.V(4).Infof("key %s selected and deleted", key)
+			log.V(-1).Infof("key %s selected and deleted", key)
 			delete(lm.locks, key)
 			lock.Key = key
 			locks = append(locks, lock)

weed/cluster/lock_manager/lock_ring.go

@@ -1,7 +1,7 @@
 package lock_manager
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"sort"
@@ -40,11 +40,11 @@ func (r *LockRing) SetTakeSnapshotCallback(onTakeSnapshot func(snapshot []pb.Ser
 // AddServer adds a server to the ring
 // if the previous snapshot passed the snapshot interval, create a new snapshot
 func (r *LockRing) AddServer(server pb.ServerAddress) {
-	glog.V(0).Infof("add server %v", server)
+	log.V(3).Infof("add server %v", server)
 	r.Lock()
 
 	if _, found := r.candidateServers[server]; found {
-		glog.V(0).Infof("add server: already exists %v", server)
+		log.V(3).Infof("add server: already exists %v", server)
 		r.Unlock()
 		return
 	}
@@ -56,7 +56,7 @@ func (r *LockRing) AddServer(server pb.ServerAddress) {
 }
 
 func (r *LockRing) RemoveServer(server pb.ServerAddress) {
-	glog.V(0).Infof("remove server %v", server)
+	log.V(3).Infof("remove server %v", server)
 
 	r.Lock()
 

weed/cluster/master_client.go

@@ -3,7 +3,7 @@ package cluster
 import (
 	"context"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"google.golang.org/grpc"
@@ -17,7 +17,7 @@ func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOp
 			FilerGroup: filerGroup,
 		})
 
-		glog.V(0).Infof("the cluster has %d %s\n", len(resp.ClusterNodes), clientType)
+		log.V(3).Infof("the cluster has %d %s\n", len(resp.ClusterNodes), clientType)
 		for _, node := range resp.ClusterNodes {
 			existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{
 				NodeType:    FilerType,
@@ -28,7 +28,7 @@ func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOp
 		}
 		return err
 	}); grpcErr != nil {
-		glog.V(0).Infof("connect to %s: %v", master, grpcErr)
+		log.V(3).Infof("connect to %s: %v", master, grpcErr)
 	}
 	return
 }

weed/command/benchmark.go

@@ -17,7 +17,7 @@ import (
 
 	"google.golang.org/grpc"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -123,7 +123,7 @@ func runBenchmark(cmd *Command, args []string) bool {
 	if *b.cpuprofile != "" {
 		f, err := os.Create(*b.cpuprofile)
 		if err != nil {
-			glog.Fatal(err)
+			log.Fatal(err)
 		}
 		pprof.StartCPUProfile(f)
 		defer pprof.StopCPUProfile()
@@ -316,7 +316,7 @@ func readFiles(fileIdLineChan chan string, s *stat) {
 func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan bool) {
 	file, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
 	if err != nil {
-		glog.Fatalf("File to create file %s: %s\n", fileName, err)
+		log.Fatalf("File to create file %s: %s\n", fileName, err)
 	}
 	defer file.Close()
 
@@ -335,7 +335,7 @@ func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan b
 func readFileIds(fileName string, fileIdLineChan chan string) {
 	file, err := os.Open(fileName) // For read access.
 	if err != nil {
-		glog.Fatalf("File to read file %s: %s\n", fileName, err)
+		log.Fatalf("File to read file %s: %s\n", fileName, err)
 	}
 	defer file.Close()
 

weed/command/command.go

@@ -32,6 +32,7 @@ var Commands = []*Command{
 	cmdMaster,
 	cmdMasterFollower,
 	cmdMount,
+	cmdMqAgent,
 	cmdMqBroker,
 	cmdS3,
 	cmdScaffold,
@@ -42,6 +43,7 @@ var Commands = []*Command{
 	cmdVersion,
 	cmdVolume,
 	cmdWebDav,
+	cmdSftp,
 }
 
 type Command struct {

weed/command/compact.go

@@ -1,7 +1,7 @@
 package command
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/storage"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -43,15 +43,15 @@ func runCompact(cmd *Command, args []string) bool {
 	vid := needle.VolumeId(*compactVolumeId)
 	v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, 0, 0)
 	if err != nil {
-		glog.Fatalf("Load Volume [ERROR] %s\n", err)
+		log.Fatalf("Load Volume [ERROR] %s\n", err)
 	}
 	if *compactMethod == 0 {
 		if err = v.Compact(preallocate, 0); err != nil {
-			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
+			log.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	} else {
 		if err = v.Compact2(preallocate, 0, nil); err != nil {
-			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
+			log.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	}
 

weed/command/export.go

@@ -13,7 +13,7 @@ import (
 	"text/template"
 	"time"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/storage"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
@@ -111,11 +111,11 @@ func (scanner *VolumeFileScanner4Export) VisitNeedle(n *needle.Needle, offset in
 	vid := scanner.vid
 
 	nv, ok := needleMap.Get(n.Id)
-	glog.V(3).Infof("key %d offset %d size %d disk_size %d compressed %v ok %v nv %+v",
+	log.V(0).Infof("key %d offset %d size %d disk_size %d compressed %v ok %v nv %+v",
 		n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed(), ok, nv)
 	if *showDeleted && n.Size > 0 || ok && nv.Size.IsValid() && nv.Offset.ToActualOffset() == offset {
 		if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
-			glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
+			log.V(0).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
 				n.LastModified, newerThanUnix)
 			return nil
 		}
@@ -139,9 +139,9 @@ func (scanner *VolumeFileScanner4Export) VisitNeedle(n *needle.Needle, offset in
 				printNeedle(vid, n, scanner.version, true, offset, n.DiskSize(scanner.version))
 			}
 		}
-		glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
+		log.V(1).Infof("This seems deleted %d size %d", n.Id, n.Size)
 	} else {
-		glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
+		log.V(1).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
 	}
 	return nil
 }
@@ -178,7 +178,7 @@ func runExport(cmd *Command, args []string) bool {
 			outputFile = os.Stdout
 		} else {
 			if outputFile, err = os.Create(*output); err != nil {
-				glog.Fatalf("cannot open output tar %s: %s", *output, err)
+				log.Fatalf("cannot open output tar %s: %s", *output, err)
 			}
 		}
 		defer outputFile.Close()
@@ -201,7 +201,7 @@ func runExport(cmd *Command, args []string) bool {
 	defer needleMap.Close()
 
 	if err := needleMap.LoadFromIdx(path.Join(util.ResolvePath(*export.dir), fileName+".idx")); err != nil {
-		glog.Fatalf("cannot load needle map from %s.idx: %s", fileName, err)
+		log.Fatalf("cannot load needle map from %s.idx: %s", fileName, err)
 	}
 
 	volumeFileScanner := &VolumeFileScanner4Export{
@@ -215,7 +215,7 @@ func runExport(cmd *Command, args []string) bool {
 
 	err = storage.ScanVolumeFile(util.ResolvePath(*export.dir), *export.collection, vid, storage.NeedleMapInMemory, volumeFileScanner)
 	if err != nil && err != io.EOF {
-		glog.Errorf("Export Volume File [ERROR] %s\n", err)
+		log.Errorf("Export Volume File [ERROR] %s\n", err)
 	}
 	return true
 }

weed/command/filer.go

@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/filer"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
@@ -35,6 +35,8 @@ var (
 	filerWebDavOptions WebDavOption
 	filerStartIam      *bool
 	filerIamOptions    IamOptions
+	filerStartSftp     *bool
+	filerSftpOptions   SftpOptions
 )
 
 type FilerOptions struct {
@@ -119,6 +121,10 @@ func init() {
 	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
 	filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 	filerS3Options.localSocket = cmdFiler.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
+	filerS3Options.tlsCACertificate = cmdFiler.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")
+	filerS3Options.tlsVerifyClientCert = cmdFiler.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate")
+	filerS3Options.bindIp = cmdFiler.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")
+	filerS3Options.idleTimeout = cmdFiler.Flag.Int("s3.idleTimeout", 10, "connection idle seconds")
 
 	// start webdav on filer
 	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
@@ -137,6 +143,19 @@ func init() {
 	filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
 	filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
 	filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
+
+	filerStartSftp = cmdFiler.Flag.Bool("sftp", false, "whether to start the SFTP server")
+	filerSftpOptions.port = cmdFiler.Flag.Int("sftp.port", 2022, "SFTP server listen port")
+	filerSftpOptions.sshPrivateKey = cmdFiler.Flag.String("sftp.sshPrivateKey", "", "path to the SSH private key file for host authentication")
+	filerSftpOptions.hostKeysFolder = cmdFiler.Flag.String("sftp.hostKeysFolder", "", "path to folder containing SSH private key files for host authentication")
+	filerSftpOptions.authMethods = cmdFiler.Flag.String("sftp.authMethods", "password,publickey", "comma-separated list of allowed auth methods: password, publickey, keyboard-interactive")
+	filerSftpOptions.maxAuthTries = cmdFiler.Flag.Int("sftp.maxAuthTries", 6, "maximum number of authentication attempts per connection")
+	filerSftpOptions.bannerMessage = cmdFiler.Flag.String("sftp.bannerMessage", "SeaweedFS SFTP Server - Unauthorized access is prohibited", "message displayed before authentication")
+	filerSftpOptions.loginGraceTime = cmdFiler.Flag.Duration("sftp.loginGraceTime", 2*time.Minute, "timeout for authentication")
+	filerSftpOptions.clientAliveInterval = cmdFiler.Flag.Duration("sftp.clientAliveInterval", 5*time.Second, "interval for sending keep-alive messages")
+	filerSftpOptions.clientAliveCountMax = cmdFiler.Flag.Int("sftp.clientAliveCountMax", 3, "maximum number of missed keep-alive messages before disconnecting")
+	filerSftpOptions.userStoreFile = cmdFiler.Flag.String("sftp.userStoreFile", "", "path to JSON file containing user credentials and permissions")
+	filerSftpOptions.localSocket = cmdFiler.Flag.String("sftp.localSocket", "", "default to /tmp/seaweedfs-sftp-<port>.sock")
 }
 
 func filerLongDesc() string {
@@ -195,7 +214,9 @@ func runFiler(cmd *Command, args []string) bool {
 	startDelay := time.Duration(2)
 	if *filerStartS3 {
 		filerS3Options.filer = &filerAddress
-		filerS3Options.bindIp = f.bindIp
+		if *filerS3Options.bindIp == "" {
+			filerS3Options.bindIp = f.bindIp
+		}
 		filerS3Options.localFilerSocket = f.localSocket
 		if *f.dataCenter != "" && *filerS3Options.dataCenter == "" {
 			filerS3Options.dataCenter = f.dataCenter
@@ -209,6 +230,7 @@ func runFiler(cmd *Command, args []string) bool {
 
 	if *filerStartWebDav {
 		filerWebDavOptions.filer = &filerAddress
+		filerWebDavOptions.ipBind = f.bindIp
 
 		if *filerWebDavOptions.disk == "" {
 			filerWebDavOptions.disk = f.diskType
@@ -228,6 +250,18 @@ func runFiler(cmd *Command, args []string) bool {
 			time.Sleep(delay * time.Second)
 			filerIamOptions.startIamServer()
 		}(startDelay)
+		startDelay++
+	}
+
+	if *filerStartSftp {
+		sftpOptions.filer = &filerAddress
+		if *f.dataCenter != "" && *filerSftpOptions.dataCenter == "" {
+			filerSftpOptions.dataCenter = f.dataCenter
+		}
+		go func(delay time.Duration) {
+			time.Sleep(delay * time.Second)
+			sftpOptions.startSftpServer()
+		}(startDelay)
 	}
 
 	f.masters = pb.ServerAddresses(*f.mastersString).ToServiceDiscovery()
@@ -290,44 +324,44 @@ func (fo *FilerOptions) startFiler() {
 		AllowedOrigins:        strings.Split(*fo.allowedOrigins, ","),
 	})
 	if nfs_err != nil {
-		glog.Fatalf("Filer startup error: %v", nfs_err)
+		log.Fatalf("Filer startup error: %v", nfs_err)
 	}
 
 	if *fo.publicPort != 0 {
 		publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
-		glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
+		log.V(3).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
 		publicListener, localPublicListener, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
 		if e != nil {
-			glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
+			log.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
 		}
 		go func() {
 			if e := http.Serve(publicListener, publicVolumeMux); e != nil {
-				glog.Fatalf("Volume server fail to serve public: %v", e)
+				log.Fatalf("Volume server fail to serve public: %v", e)
 			}
 		}()
 		if localPublicListener != nil {
 			go func() {
 				if e := http.Serve(localPublicListener, publicVolumeMux); e != nil {
-					glog.Errorf("Volume server fail to serve public: %v", e)
+					log.Errorf("Volume server fail to serve public: %v", e)
 				}
 			}()
 		}
 	}
 
-	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
+	log.V(3).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
 	filerListener, filerLocalListener, e := util.NewIpAndLocalListeners(
 		*fo.bindIp, *fo.port,
 		time.Duration(10)*time.Second,
 	)
 	if e != nil {
-		glog.Fatalf("Filer listener error: %v", e)
+		log.Fatalf("Filer listener error: %v", e)
 	}
 
 	// starting grpc server
 	grpcPort := *fo.portGrpc
 	grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0)
 	if err != nil {
-		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
+		log.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
 	}
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer"))
 	filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
@@ -344,13 +378,13 @@ func (fo *FilerOptions) startFiler() {
 			localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port)
 		}
 		if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
-			glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
+			log.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
 		}
 		go func() {
 			// start on local unix socket
 			filerSocketListener, err := net.Listen("unix", localSocket)
 			if err != nil {
-				glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
+				log.Fatalf("Failed to listen on %s: %v", localSocket, err)
 			}
 			httpS.Serve(filerSocketListener)
 		}()
@@ -368,14 +402,14 @@ func (fo *FilerOptions) startFiler() {
 			RefreshDuration: security.CredRefreshingInterval,
 		}
 		if fo.certProvider, err = pemfile.NewProvider(pemfileOptions); err != nil {
-			glog.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err)
+			log.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err)
 		}
 
 		caCertPool := x509.NewCertPool()
 		if caCertFile != "" {
 			caCertFile, err := os.ReadFile(caCertFile)
 			if err != nil {
-				glog.Fatalf("error reading CA certificate: %v", err)
+				log.Fatalf("error reading CA certificate: %v", err)
 			}
 			caCertPool.AppendCertsFromPEM(caCertFile)
 		}
@@ -394,23 +428,23 @@ func (fo *FilerOptions) startFiler() {
 		if filerLocalListener != nil {
 			go func() {
 				if err := httpS.ServeTLS(filerLocalListener, "", ""); err != nil {
-					glog.Errorf("Filer Fail to serve: %v", e)
+					log.Errorf("Filer Fail to serve: %v", e)
 				}
 			}()
 		}
 		if err := httpS.ServeTLS(filerListener, "", ""); err != nil {
-			glog.Fatalf("Filer Fail to serve: %v", e)
+			log.Fatalf("Filer Fail to serve: %v", e)
 		}
 	} else {
 		if filerLocalListener != nil {
 			go func() {
 				if err := httpS.Serve(filerLocalListener); err != nil {
-					glog.Errorf("Filer Fail to serve: %v", e)
+					log.Errorf("Filer Fail to serve: %v", e)
 				}
 			}()
 		}
 		if err := httpS.Serve(filerListener); err != nil {
-			glog.Fatalf("Filer Fail to serve: %v", e)
+			log.Fatalf("Filer Fail to serve: %v", e)
 		}
 	}
 }

weed/command/filer_backup.go

@@ -3,7 +3,7 @@ package command
 import (
 	"errors"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/replication/source"
@@ -78,7 +78,7 @@ func runFilerBackup(cmd *Command, args []string) bool {
 		clientEpoch++
 		err := doFilerBackup(grpcDialOption, &filerBackupOptions, clientId, clientEpoch)
 		if err != nil {
-			glog.Errorf("backup from %s: %v", *filerBackupOptions.filer, err)
+			log.Errorf("backup from %s: %v", *filerBackupOptions.filer, err)
 			time.Sleep(1747 * time.Millisecond)
 		}
 	}
@@ -118,14 +118,14 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 	if timeAgo.Milliseconds() == 0 {
 		lastOffsetTsNs, err := getOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId))
 		if err != nil {
-			glog.V(0).Infof("starting from %v", startFrom)
+			log.V(3).Infof("starting from %v", startFrom)
 		} else {
 			startFrom = time.Unix(0, lastOffsetTsNs)
-			glog.V(0).Infof("resuming from %v", startFrom)
+			log.V(3).Infof("resuming from %v", startFrom)
 		}
 	} else {
 		startFrom = time.Now().Add(-timeAgo)
-		glog.V(0).Infof("start time is set to %v", startFrom)
+		log.V(3).Infof("start time is set to %v", startFrom)
 	}
 
 	// create filer sink
@@ -146,7 +146,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 				return nil
 			}
 			if errors.Is(err, http.ErrNotFound) {
-				glog.V(0).Infof("got 404 error, ignore it: %s", err.Error())
+				log.V(3).Infof("got 404 error, ignore it: %s", err.Error())
 				return nil
 			}
 			return err
@@ -156,7 +156,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 	}
 
 	processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error {
-		glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, lastTsNs), float64(counter)/float64(3))
+		log.V(3).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, lastTsNs), float64(counter)/float64(3))
 		return setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), lastTsNs)
 	})
 
@@ -167,7 +167,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 				time.Sleep(time.Hour * 24)
 				key := util.Join(targetPath, now.Add(-1*time.Hour*24*time.Duration(*filerBackupOptions.retentionDays)).Format("2006-01-02"))
 				_ = dataSink.DeleteEntry(util.Join(targetPath, key), true, true, nil)
-				glog.V(0).Infof("incremental backup delete directory:%s", key)
+				log.V(3).Infof("incremental backup delete directory:%s", key)
 			}
 		}()
 	}

weed/command/filer_meta_backup.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
 	"reflect"
@@ -64,13 +64,13 @@ func runFilerMetaBackup(cmd *Command, args []string) bool {
 	v.SetConfigFile(*metaBackup.backupFilerConfig)
 
 	if err := v.ReadInConfig(); err != nil { // Handle errors reading the config file
-		glog.Fatalf("Failed to load %s file: %v\nPlease use this command to generate the a %s.toml file\n"+
+		log.Fatalf("Failed to load %s file: %v\nPlease use this command to generate the a %s.toml file\n"+
 			"    weed scaffold -config=%s -output=.\n\n\n",
 			*metaBackup.backupFilerConfig, err, "backup_filer", "filer")
 	}
 
 	if err := metaBackup.initStore(v); err != nil {
-		glog.V(0).Infof("init backup filer store: %v", err)
+		log.V(3).Infof("init backup filer store: %v", err)
 		return true
 	}
 
@@ -81,13 +81,13 @@ func runFilerMetaBackup(cmd *Command, args []string) bool {
 	}
 
 	if *metaBackup.restart || missingPreviousBackup {
-		glog.V(0).Infof("traversing metadata tree...")
+		log.V(3).Infof("traversing metadata tree...")
 		startTime := time.Now()
 		if err := metaBackup.traverseMetadata(); err != nil {
-			glog.Errorf("traverse meta data: %v", err)
+			log.Errorf("traverse meta data: %v", err)
 			return true
 		}
-		glog.V(0).Infof("metadata copied up to %v", startTime)
+		log.V(3).Infof("metadata copied up to %v", startTime)
 		if err := metaBackup.setOffset(startTime); err != nil {
 			startTime = time.Now()
 		}
@@ -96,7 +96,7 @@ func runFilerMetaBackup(cmd *Command, args []string) bool {
 	for {
 		err := metaBackup.streamMetadataBackup()
 		if err != nil {
-			glog.Errorf("filer meta backup from %s: %v", *metaBackup.filerAddress, err)
+			log.Errorf("filer meta backup from %s: %v", *metaBackup.filerAddress, err)
 			time.Sleep(1747 * time.Millisecond)
 		}
 	}
@@ -111,9 +111,9 @@ func (metaBackup *FilerMetaBackupOptions) initStore(v *viper.Viper) error {
 		if v.GetBool(store.GetName() + ".enabled") {
 			store = reflect.New(reflect.ValueOf(store).Elem().Type()).Interface().(filer.FilerStore)
 			if err := store.Initialize(v, store.GetName()+"."); err != nil {
-				glog.Fatalf("failed to initialize store for %s: %+v", store.GetName(), err)
+				log.Fatalf("failed to initialize store for %s: %+v", store.GetName(), err)
 			}
-			glog.V(0).Infof("configured filer store to %s", store.GetName())
+			log.V(3).Infof("configured filer store to %s", store.GetName())
 			hasDefaultStoreConfigured = true
 			metaBackup.store = filer.NewFilerStoreWrapper(store)
 			break
@@ -155,7 +155,7 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
 	if err != nil {
 		startTime = time.Now()
 	}
-	glog.V(0).Infof("streaming from %v", startTime)
+	log.V(3).Infof("streaming from %v", startTime)
 
 	store := metaBackup.store
 
@@ -192,7 +192,7 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
 
 	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
 		lastTime := time.Unix(0, lastTsNs)
-		glog.V(0).Infof("meta backup %s progressed to %v %0.2f/sec", *metaBackup.filerAddress, lastTime, float64(counter)/float64(3))
+		log.V(3).Infof("meta backup %s progressed to %v %0.2f/sec", *metaBackup.filerAddress, lastTime, float64(counter)/float64(3))
 		return metaBackup.setOffset(lastTime)
 	})
 

weed/command/filer_remote_gateway.go

@@ -3,7 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
@@ -115,7 +115,7 @@ func runFilerRemoteGateway(cmd *Command, args []string) bool {
 		return remoteGatewayOptions.followBucketUpdatesAndUploadToRemote(filerSource)
 	}, func(err error) bool {
 		if err != nil {
-			glog.Errorf("synchronize %s: %v", remoteGatewayOptions.bucketsDir, err)
+			log.Errorf("synchronize %s: %v", remoteGatewayOptions.bucketsDir, err)
 		}
 		return true
 	})

weed/command/filer_remote_gateway_buckets.go

@@ -3,7 +3,7 @@ package command
 import (
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
@@ -43,7 +43,7 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo
 			return nil
 		}
 		now := time.Now().UnixNano()
-		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
+		log.V(3).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
 		lastLogTsNs = now
 		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, offsetTsNs)
 	})
@@ -78,12 +78,12 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 		}
 		if option.mappings.PrimaryBucketStorageName != "" && *option.createBucketAt == "" {
 			*option.createBucketAt = option.mappings.PrimaryBucketStorageName
-			glog.V(0).Infof("%s is set as the primary remote storage", *option.createBucketAt)
+			log.V(3).Infof("%s is set as the primary remote storage", *option.createBucketAt)
 		}
 		if len(option.mappings.Mappings) == 1 && *option.createBucketAt == "" {
 			for k := range option.mappings.Mappings {
 				*option.createBucketAt = k
-				glog.V(0).Infof("%s is set as the only remote storage", *option.createBucketAt)
+				log.V(3).Infof("%s is set as the only remote storage", *option.createBucketAt)
 			}
 		}
 		if *option.createBucketAt == "" {
@@ -132,7 +132,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 			bucketName = remoteLocation.Bucket
 		}
 
-		glog.V(0).Infof("create bucket %s", bucketName)
+		log.V(3).Infof("create bucket %s", bucketName)
 		if err := client.CreateBucket(bucketName); err != nil {
 			return fmt.Errorf("create bucket %s in %s: %v", bucketName, remoteConf.Name, err)
 		}
@@ -150,7 +150,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 			return fmt.Errorf("findRemoteStorageClient %s: %v", entry.Name, err)
 		}
 
-		glog.V(0).Infof("delete remote bucket %s", remoteStorageMountLocation.Bucket)
+		log.V(3).Infof("delete remote bucket %s", remoteStorageMountLocation.Bucket)
 		if err := client.DeleteBucket(remoteStorageMountLocation.Bucket); err != nil {
 			return fmt.Errorf("delete remote bucket %s: %v", remoteStorageMountLocation.Bucket, err)
 		}
@@ -219,17 +219,17 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 			if err != nil {
 				return err
 			}
-			glog.V(2).Infof("create: %+v", resp)
+			log.V(1).Infof("create: %+v", resp)
 			if !shouldSendToRemote(message.NewEntry) {
-				glog.V(2).Infof("skipping creating: %+v", resp)
+				log.V(1).Infof("skipping creating: %+v", resp)
 				return nil
 			}
 			dest := toRemoteStorageLocation(bucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
 			if message.NewEntry.IsDirectory {
-				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
+				log.V(3).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
 				return client.WriteDirectory(dest, message.NewEntry)
 			}
-			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
+			log.V(3).Infof("create %s", remote_storage.FormatLocation(dest))
 			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
 			if writeErr != nil {
 				return writeErr
@@ -248,13 +248,13 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 			if err != nil {
 				return err
 			}
-			glog.V(2).Infof("delete: %+v", resp)
+			log.V(1).Infof("delete: %+v", resp)
 			dest := toRemoteStorageLocation(bucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
 			if message.OldEntry.IsDirectory {
-				glog.V(0).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
+				log.V(3).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
 				return client.RemoveDirectory(dest)
 			}
-			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
+			log.V(3).Infof("delete %s", remote_storage.FormatLocation(dest))
 			return client.DeleteFile(dest)
 		}
 		if message.OldEntry != nil && message.NewEntry != nil {
@@ -278,7 +278,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 			newBucket, newRemoteStorageMountLocation, newRemoteStorage, newOk := option.detectBucketInfo(message.NewParentPath)
 			if oldOk && newOk {
 				if !shouldSendToRemote(message.NewEntry) {
-					glog.V(2).Infof("skipping updating: %+v", resp)
+					log.V(1).Infof("skipping updating: %+v", resp)
 					return nil
 				}
 				client, err := remote_storage.GetRemoteStorage(oldRemoteStorage)
@@ -292,7 +292,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 						return nil
 					}
 					if message.OldEntry.RemoteEntry != nil && filer.IsSameData(message.OldEntry, message.NewEntry) {
-						glog.V(2).Infof("update meta: %+v", resp)
+						log.V(1).Infof("update meta: %+v", resp)
 						oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation)
 						return client.UpdateFileMetadata(oldDest, message.OldEntry, message.NewEntry)
 					} else {
@@ -316,14 +316,14 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				if message.OldEntry.IsDirectory {
 					return client.RemoveDirectory(oldDest)
 				}
-				glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
+				log.V(3).Infof("delete %s", remote_storage.FormatLocation(oldDest))
 				if err := client.DeleteFile(oldDest); err != nil {
 					return err
 				}
 			}
 			if newOk {
 				if !shouldSendToRemote(message.NewEntry) {
-					glog.V(2).Infof("skipping updating: %+v", resp)
+					log.V(1).Infof("skipping updating: %+v", resp)
 					return nil
 				}
 				client, err := remote_storage.GetRemoteStorage(newRemoteStorage)
@@ -375,13 +375,13 @@ func (option *RemoteGatewayOptions) detectBucketInfo(actualDir string) (bucket u
 	var isMounted bool
 	remoteStorageMountLocation, isMounted = option.mappings.Mappings[string(bucket)]
 	if !isMounted {
-		glog.Warningf("%s is not mounted", bucket)
+		log.Warningf("%s is not mounted", bucket)
 		return "", nil, nil, false
 	}
 	var hasClient bool
 	remoteConf, hasClient = option.remoteConfs[remoteStorageMountLocation.Name]
 	if !hasClient {
-		glog.Warningf("%s mounted to un-configured %+v", bucket, remoteStorageMountLocation)
+		log.Warningf("%s mounted to un-configured %+v", bucket, remoteStorageMountLocation)
 		return "", nil, nil, false
 	}
 	return bucket, remoteStorageMountLocation, remoteConf, true
@@ -422,7 +422,7 @@ func (option *RemoteGatewayOptions) collectRemoteStorageConf() (err error) {
 	}, "", false, math.MaxUint32)
 
 	if option.mappings.PrimaryBucketStorageName == "" && len(option.remoteConfs) == 1 {
-		glog.V(0).Infof("%s is set to the default remote storage", lastConfName)
+		log.V(3).Infof("%s is set to the default remote storage", lastConfName)
 		option.mappings.PrimaryBucketStorageName = lastConfName
 	}
 

weed/command/filer_remote_sync.go

@@ -2,7 +2,7 @@ package command
 
 import (
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/replication/source"
@@ -94,7 +94,7 @@ func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
 			return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir)
 		}, func(err error) bool {
 			if err != nil {
-				glog.Errorf("synchronize %s: %v", dir, err)
+				log.Errorf("synchronize %s: %v", dir, err)
 			}
 			return true
 		})

weed/command/filer_remote_sync_dir.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/filer"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
@@ -57,7 +57,7 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
 		}
 		// use processor.processedTsWatermark instead of the lastTsNs from the most recent job
 		now := time.Now().UnixNano()
-		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
+		log.V(3).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
 		lastLogTsNs = now
 		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, offsetTsNs)
 	})
@@ -103,10 +103,10 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 			}
 			if remoteLoc, found := mappings.Mappings[mountedDir]; found {
 				if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
-					glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
+					log.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
 				}
 			} else {
-				glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
+				log.V(3).Infof("unmounted %s exiting ...", mountedDir)
 				os.Exit(0)
 			}
 		}
@@ -142,17 +142,17 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 			if !filer.HasData(message.NewEntry) {
 				return nil
 			}
-			glog.V(2).Infof("create: %+v", resp)
+			log.V(1).Infof("create: %+v", resp)
 			if !shouldSendToRemote(message.NewEntry) {
-				glog.V(2).Infof("skipping creating: %+v", resp)
+				log.V(1).Infof("skipping creating: %+v", resp)
 				return nil
 			}
 			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
 			if message.NewEntry.IsDirectory {
-				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
+				log.V(3).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
 				return client.WriteDirectory(dest, message.NewEntry)
 			}
-			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
+			log.V(3).Infof("create %s", remote_storage.FormatLocation(dest))
 			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
 			if writeErr != nil {
 				return writeErr
@@ -160,13 +160,13 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 			return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
 		}
 		if filer_pb.IsDelete(resp) {
-			glog.V(2).Infof("delete: %+v", resp)
+			log.V(1).Infof("delete: %+v", resp)
 			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
 			if message.OldEntry.IsDirectory {
-				glog.V(0).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
+				log.V(3).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
 				return client.RemoveDirectory(dest)
 			}
-			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
+			log.V(3).Infof("delete %s", remote_storage.FormatLocation(dest))
 			return client.DeleteFile(dest)
 		}
 		if message.OldEntry != nil && message.NewEntry != nil {
@@ -176,7 +176,7 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 			oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
 			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
 			if !shouldSendToRemote(message.NewEntry) {
-				glog.V(2).Infof("skipping updating: %+v", resp)
+				log.V(1).Infof("skipping updating: %+v", resp)
 				return nil
 			}
 			if message.NewEntry.IsDirectory {
@@ -184,12 +184,12 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 			}
 			if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
 				if filer.IsSameData(message.OldEntry, message.NewEntry) {
-					glog.V(2).Infof("update meta: %+v", resp)
+					log.V(1).Infof("update meta: %+v", resp)
 					return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry)
 				}
 			}
-			glog.V(2).Infof("update: %+v", resp)
-			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
+			log.V(1).Infof("update: %+v", resp)
+			log.V(3).Infof("delete %s", remote_storage.FormatLocation(oldDest))
 			if err := client.DeleteFile(oldDest); err != nil {
 				if isMultipartUploadFile(resp.Directory, message.OldEntry.Name) {
 					return nil
@@ -211,7 +211,7 @@ func retriedWriteFile(client remote_storage.RemoteStorageClient, filerSource *so
 	var writeErr error
 	err = util.Retry("writeFile", func() error {
 		reader := filer.NewFileReader(filerSource, newEntry)
-		glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
+		log.V(3).Infof("create %s", remote_storage.FormatLocation(dest))
 		remoteEntry, writeErr = client.WriteFile(dest, newEntry, reader)
 		if writeErr != nil {
 			return writeErr
@@ -219,7 +219,7 @@ func retriedWriteFile(client remote_storage.RemoteStorageClient, filerSource *so
 		return nil
 	})
 	if err != nil {
-		glog.Errorf("write to %s: %v", dest, err)
+		log.Errorf("write to %s: %v", dest, err)
 	}
 	return
 }
@@ -232,7 +232,7 @@ func collectLastSyncOffset(filerClient filer_pb.FilerClient, grpcDialOption grpc
 	if timeAgo == 0 {
 		mountedDirEntry, err := filer_pb.GetEntry(filerClient, util.FullPath(mountedDir))
 		if err != nil {
-			glog.V(0).Infof("get mounted directory %s: %v", mountedDir, err)
+			log.V(3).Infof("get mounted directory %s: %v", mountedDir, err)
 			return time.Now()
 		}
 
@@ -240,7 +240,7 @@ func collectLastSyncOffset(filerClient filer_pb.FilerClient, grpcDialOption grpc
 		if mountedDirEntry != nil {
 			if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 {
 				lastOffsetTs = time.Unix(0, lastOffsetTsNs)
-				glog.V(0).Infof("resume from %v", lastOffsetTs)
+				log.V(3).Infof("resume from %v", lastOffsetTs)
 			} else {
 				lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0)
 			}

weed/command/filer_replication.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"strings"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/replication"
 	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
 	"github.com/seaweedfs/seaweedfs/weed/replication/sub"
@@ -42,10 +42,10 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 	for _, input := range sub.NotificationInputs {
 		if config.GetBool("notification." + input.GetName() + ".enabled") {
 			if err := input.Initialize(config, "notification."+input.GetName()+"."); err != nil {
-				glog.Fatalf("Failed to initialize notification input for %s: %+v",
+				log.Fatalf("Failed to initialize notification input for %s: %+v",
 					input.GetName(), err)
 			}
-			glog.V(0).Infof("Configure notification input to %s", input.GetName())
+			log.V(3).Infof("Configure notification input to %s", input.GetName())
 			notificationInput = input
 			break
 		}
@@ -63,7 +63,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 			fromDir := config.GetString("source.filer.directory")
 			toDir := config.GetString("sink.filer.directory")
 			if strings.HasPrefix(toDir, fromDir) {
-				glog.Fatalf("recursive replication! source directory %s includes the sink directory %s", fromDir, toDir)
+				log.Fatalf("recursive replication! source directory %s includes the sink directory %s", fromDir, toDir)
 			}
 		}
 	}
@@ -83,7 +83,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 	for {
 		key, m, onSuccessFn, onFailureFn, err := notificationInput.ReceiveMessage()
 		if err != nil {
-			glog.Errorf("receive %s: %+v", key, err)
+			log.Errorf("receive %s: %+v", key, err)
 			if onFailureFn != nil {
 				onFailureFn()
 			}
@@ -97,19 +97,19 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 			continue
 		}
 		if m.OldEntry != nil && m.NewEntry == nil {
-			glog.V(1).Infof("delete: %s", key)
+			log.V(2).Infof("delete: %s", key)
 		} else if m.OldEntry == nil && m.NewEntry != nil {
-			glog.V(1).Infof("add: %s", key)
+			log.V(2).Infof("add: %s", key)
 		} else {
-			glog.V(1).Infof("modify: %s", key)
+			log.V(2).Infof("modify: %s", key)
 		}
 		if err = replicator.Replicate(context.Background(), key, m); err != nil {
-			glog.Errorf("replicate %s: %+v", key, err)
+			log.Errorf("replicate %s: %+v", key, err)
 			if onFailureFn != nil {
 				onFailureFn()
 			}
 		} else {
-			glog.V(1).Infof("replicated %s", key)
+			log.V(2).Infof("replicated %s", key)
 			if onSuccessFn != nil {
 				onSuccessFn()
 			}
@@ -123,10 +123,10 @@ func findSink(config *util.ViperProxy) sink.ReplicationSink {
 	for _, sk := range sink.Sinks {
 		if config.GetBool("sink." + sk.GetName() + ".enabled") {
 			if err := sk.Initialize(config, "sink."+sk.GetName()+"."); err != nil {
-				glog.Fatalf("Failed to initialize sink for %s: %+v",
+				log.Fatalf("Failed to initialize sink for %s: %+v",
 					sk.GetName(), err)
 			}
-			glog.V(0).Infof("Configure sink to %s", sk.GetName())
+			log.V(3).Infof("Configure sink to %s", sk.GetName())
 			dataSink = sk
 			break
 		}
@@ -141,7 +141,7 @@ func validateOneEnabledInput(config *util.ViperProxy) {
 			if enabledInput == "" {
 				enabledInput = input.GetName()
 			} else {
-				glog.Fatalf("Notification input is enabled for both %s and %s", enabledInput, input.GetName())
+				log.Fatalf("Notification input is enabled for both %s and %s", enabledInput, input.GetName())
 			}
 		}
 	}

weed/command/filer_sync.go

@@ -10,7 +10,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/replication"
@@ -133,13 +133,13 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	// read a filer signature
 	aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOption, filerA)
 	if aFilerErr != nil {
-		glog.Errorf("get filer 'a' signature %d error from %s to %s: %v", aFilerSignature, *syncOptions.filerA, *syncOptions.filerB, aFilerErr)
+		log.Errorf("get filer 'a' signature %d error from %s to %s: %v", aFilerSignature, *syncOptions.filerA, *syncOptions.filerB, aFilerErr)
 		return true
 	}
 	// read b filer signature
 	bFilerSignature, bFilerErr := replication.ReadFilerSignature(grpcDialOption, filerB)
 	if bFilerErr != nil {
-		glog.Errorf("get filer 'b' signature %d error from %s to %s: %v", bFilerSignature, *syncOptions.filerA, *syncOptions.filerB, bFilerErr)
+		log.Errorf("get filer 'b' signature %d error from %s to %s: %v", bFilerSignature, *syncOptions.filerA, *syncOptions.filerB, bFilerErr)
 		return true
 	}
 
@@ -148,7 +148,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 		// set synchronization start timestamp to offset
 		initOffsetError := initOffsetFromTsMs(grpcDialOption, filerB, aFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath))
 		if initOffsetError != nil {
-			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError)
+			log.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError)
 			os.Exit(2)
 		}
 		for {
@@ -174,7 +174,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 				aFilerSignature,
 				bFilerSignature)
 			if err != nil {
-				glog.Errorf("sync from %s to %s: %v", *syncOptions.filerA, *syncOptions.filerB, err)
+				log.Errorf("sync from %s to %s: %v", *syncOptions.filerA, *syncOptions.filerB, err)
 				time.Sleep(1747 * time.Millisecond)
 			}
 		}
@@ -185,7 +185,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 		// set synchronization start timestamp to offset
 		initOffsetError := initOffsetFromTsMs(grpcDialOption, filerA, bFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath))
 		if initOffsetError != nil {
-			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError)
+			log.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError)
 			os.Exit(2)
 		}
 		go func() {
@@ -212,7 +212,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 					bFilerSignature,
 					aFilerSignature)
 				if err != nil {
-					glog.Errorf("sync from %s to %s: %v", *syncOptions.filerB, *syncOptions.filerA, err)
+					log.Errorf("sync from %s to %s: %v", *syncOptions.filerB, *syncOptions.filerA, err)
 					time.Sleep(2147 * time.Millisecond)
 				}
 			}
@@ -236,7 +236,7 @@ func initOffsetFromTsMs(grpcDialOption grpc.DialOption, targetFiler pb.ServerAdd
 	if setOffsetErr != nil {
 		return setOffsetErr
 	}
-	glog.Infof("setOffset from timestamp ms success! start offset: %d from %s to %s", fromTsNs, *syncOptions.filerA, *syncOptions.filerB)
+	log.Infof("setOffset from timestamp ms success! start offset: %d from %s to %s", fromTsNs, *syncOptions.filerA, *syncOptions.filerB)
 	return nil
 }
 
@@ -250,7 +250,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 		return err
 	}
 
-	glog.V(0).Infof("start sync %s(%d) => %s(%d) from %v(%d)", sourceFiler, sourceFilerSignature, targetFiler, targetFilerSignature, time.Unix(0, sourceFilerOffsetTsNs), sourceFilerOffsetTsNs)
+	log.V(3).Infof("start sync %s(%d) => %s(%d) from %v(%d)", sourceFiler, sourceFilerSignature, targetFiler, targetFilerSignature, time.Unix(0, sourceFilerOffsetTsNs), sourceFilerOffsetTsNs)
 
 	// create filer sink
 	filerSource := &source.FilerSource{}
@@ -273,7 +273,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 	}
 
 	if concurrency < 0 || concurrency > 1024 {
-		glog.Warningf("invalid concurrency value, using default: %d", DefaultConcurrencyLimit)
+		log.Warningf("invalid concurrency value, using default: %d", DefaultConcurrencyLimit)
 		concurrency = DefaultConcurrencyLimit
 	}
 	processor := NewMetadataProcessor(processEventFn, concurrency, sourceFilerOffsetTsNs)
@@ -290,7 +290,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 		}
 		// use processor.processedTsWatermark instead of the lastTsNs from the most recent job
 		now := time.Now().UnixNano()
-		glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
+		log.V(3).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
 		lastLogTsNs = now
 		// collect synchronous offset
 		statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(offsetTsNs))
@@ -397,7 +397,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 		}
 
 		if debug {
-			glog.V(0).Infof("received %v", resp)
+			log.V(3).Infof("received %v", resp)
 		}
 
 		if isMultipartUploadDir(resp.Directory + "/") {

weed/command/filer_sync_jobs.go

@@ -1,7 +1,7 @@
 package command
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -46,7 +46,7 @@ func (t *MetadataProcessor) AddSyncJob(resp *filer_pb.SubscribeMetadataResponse)
 		if err := util.Retry("metadata processor", func() error {
 			return t.fn(resp)
 		}); err != nil {
-			glog.Errorf("process %v: %v", resp, err)
+			log.Errorf("process %v: %v", resp, err)
 		}
 
 		t.activeJobsLock.Lock()

weed/command/fix.go

@@ -8,7 +8,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/storage"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
@@ -53,7 +53,7 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
 }
 
 func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
-	glog.V(2).Infof("key %v offset %d size %d disk_size %d compressed %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed())
+	log.V(1).Infof("key %v offset %d size %d disk_size %d compressed %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed())
 	if n.Size.IsValid() {
 		if pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size); pe != nil {
 			return fmt.Errorf("saved %d with error %v", n.Size, pe)
@@ -64,7 +64,7 @@ func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64
 				return fmt.Errorf("saved deleted %d with error %v", n.Size, pe)
 			}
 		} else {
-			glog.V(2).Infof("skipping deleted file ...")
+			log.V(1).Infof("skipping deleted file ...")
 			return scanner.nm.Delete(n.Id)
 		}
 	}
@@ -163,19 +163,19 @@ func doFixOneVolume(basepath string, baseFileName string, collection string, vol
 	if err := storage.ScanVolumeFile(basepath, collection, vid, storage.NeedleMapInMemory, scanner); err != nil {
 		err := fmt.Errorf("scan .dat File: %v", err)
 		if *fixIgnoreError {
-			glog.Error(err)
+			log.Error(err)
 		} else {
-			glog.Fatal(err)
+			log.Fatal(err)
 		}
 	}
 
 	if err := SaveToIdx(scanner, indexFileName); err != nil {
 		err := fmt.Errorf("save to .idx File: %v", err)
 		if *fixIgnoreError {
-			glog.Error(err)
+			log.Error(err)
 		} else {
 			os.Remove(indexFileName)
-			glog.Fatal(err)
+			log.Fatal(err)
 		}
 	}
 }

weed/command/iam.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/iamapi"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -56,14 +56,14 @@ func (iamopt *IamOptions) startIamServer() bool {
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
 			}
-			glog.V(0).Infof("IAM read filer configuration: %s", resp)
+			log.V(3).Infof("IAM read filer configuration: %s", resp)
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *iamopt.filer, filerAddress.ToGrpcAddress())
+			log.V(3).Infof("wait to connect to filer %s grpc address %s", *iamopt.filer, filerAddress.ToGrpcAddress())
 			time.Sleep(time.Second)
 		} else {
-			glog.V(0).Infof("connected to filer %s grpc address %s", *iamopt.filer, filerAddress.ToGrpcAddress())
+			log.V(3).Infof("connected to filer %s grpc address %s", *iamopt.filer, filerAddress.ToGrpcAddress())
 			break
 		}
 	}
@@ -76,9 +76,9 @@ func (iamopt *IamOptions) startIamServer() bool {
 		Port:           *iamopt.port,
 		GrpcDialOption: grpcDialOption,
 	})
-	glog.V(0).Info("NewIamApiServer created")
+	log.V(3).Info("NewIamApiServer created")
 	if iamApiServer_err != nil {
-		glog.Fatalf("IAM API Server startup error: %v", iamApiServer_err)
+		log.Fatalf("IAM API Server startup error: %v", iamApiServer_err)
 	}
 
 	httpS := &http.Server{Handler: router}
@@ -86,19 +86,19 @@ func (iamopt *IamOptions) startIamServer() bool {
 	listenAddress := fmt.Sprintf(":%d", *iamopt.port)
 	iamApiListener, iamApiLocalListener, err := util.NewIpAndLocalListeners(*iamopt.ip, *iamopt.port, time.Duration(10)*time.Second)
 	if err != nil {
-		glog.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err)
+		log.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err)
 	}
 
-	glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", util.Version(), *iamopt.port)
+	log.V(3).Infof("Start Seaweed IAM API Server %s at http port %d", util.Version(), *iamopt.port)
 	if iamApiLocalListener != nil {
 		go func() {
 			if err = httpS.Serve(iamApiLocalListener); err != nil {
-				glog.Errorf("IAM API Server Fail to serve: %v", err)
+				log.Errorf("IAM API Server Fail to serve: %v", err)
 			}
 		}()
 	}
 	if err = httpS.Serve(iamApiListener); err != nil {
-		glog.Fatalf("IAM API Server Fail to serve: %v", err)
+		log.Fatalf("IAM API Server Fail to serve: %v", err)
 	}
 
 	return true

weed/command/imports.go

@@ -31,6 +31,7 @@ import (
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/redis2"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/redis3"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/sqlite"
+	_ "github.com/seaweedfs/seaweedfs/weed/filer/tarantool"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/tikv"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/ydb"
 )

weed/command/master.go

@@ -22,7 +22,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
@@ -117,12 +117,12 @@ func runMaster(cmd *Command, args []string) bool {
 		os.MkdirAll(*m.metaFolder, 0755)
 	}
 	if err := util.TestFolderWritable(util.ResolvePath(*m.metaFolder)); err != nil {
-		glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *m.metaFolder, err)
+		log.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *m.metaFolder, err)
 	}
 
 	masterWhiteList := util.StringSplit(*m.whiteList, ",")
 	if *m.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
-		glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
+		log.Fatalf("volumeSizeLimitMB should be smaller than 30000")
 	}
 
 	switch {
@@ -160,10 +160,10 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers)
 	listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
-	glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
+	log.V(3).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
 	masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0)
 	if e != nil {
-		glog.Fatalf("Master startup error: %v", e)
+		log.Fatalf("Master startup error: %v", e)
 	}
 
 	// start raftServer
@@ -183,16 +183,16 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	var err error
 	if *masterOption.raftHashicorp {
 		if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
-			glog.Fatalf("NewHashicorpRaftServer: %s", err)
+			log.Fatalf("NewHashicorpRaftServer: %s", err)
 		}
 	} else {
 		raftServer, err = weed_server.NewRaftServer(raftServerOption)
 		if raftServer == nil {
-			glog.Fatalf("please verify %s is writable, see https://github.com/seaweedfs/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
+			log.Fatalf("please verify %s is writable, see https://github.com/seaweedfs/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
 		}
 	}
 	ms.SetRaftServer(raftServer)
-	r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods(http.MethodGet)
+	r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods(http.MethodGet, http.MethodHead)
 	r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods(http.MethodGet, http.MethodHead)
 	if *masterOption.raftHashicorp {
 		r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods(http.MethodGet)
@@ -201,7 +201,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	grpcPort := *masterOption.portGrpc
 	grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOption.ipBind, grpcPort, 0)
 	if err != nil {
-		glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
+		log.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
 	}
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
 	master_pb.RegisterSeaweedServer(grpcS, ms)
@@ -211,7 +211,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		protobuf.RegisterRaftServer(grpcS, raftServer)
 	}
 	reflection.Register(grpcS)
-	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOption.ipBind, grpcPort)
+	log.V(3).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOption.ipBind, grpcPort)
 	if grpcLocalL != nil {
 		go grpcS.Serve(grpcLocalL)
 	}
@@ -279,7 +279,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 }
 
 func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers string) (masterAddress pb.ServerAddress, cleanedPeers []pb.ServerAddress) {
-	glog.V(0).Infof("current: %s:%d peers:%s", masterIp, masterPort, peers)
+	log.V(3).Infof("current: %s:%d peers:%s", masterIp, masterPort, peers)
 	masterAddress = pb.NewServerAddress(masterIp, masterPort, masterGrpcPort)
 	cleanedPeers = pb.ServerAddresses(peers).ToAddresses()
 
@@ -295,7 +295,7 @@ func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers strin
 		cleanedPeers = append(cleanedPeers, masterAddress)
 	}
 	if len(cleanedPeers)%2 == 0 {
-		glog.Fatalf("Only odd number of masters are supported: %+v", cleanedPeers)
+		log.Fatalf("Only odd number of masters are supported: %+v", cleanedPeers)
 	}
 	return
 }

weed/command/master_follower.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/gorilla/mux"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
@@ -99,13 +99,13 @@ func startMasterFollower(masterOptions MasterOptions) {
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("failed to talk to filer %v: %v", masters, err)
-			glog.V(0).Infof("wait for %d seconds ...", i+1)
+			log.V(3).Infof("failed to talk to filer %v: %v", masters, err)
+			log.V(3).Infof("wait for %d seconds ...", i+1)
 			time.Sleep(time.Duration(i+1) * time.Second)
 		}
 	}
 	if err != nil {
-		glog.Errorf("failed to talk to filer %v: %v", masters, err)
+		log.Errorf("failed to talk to filer %v: %v", masters, err)
 		return
 	}
 
@@ -119,22 +119,22 @@ func startMasterFollower(masterOptions MasterOptions) {
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, option, masters)
 	listeningAddress := util.JoinHostPort(*masterOptions.ipBind, *masterOptions.port)
-	glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
+	log.V(3).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
 	masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOptions.ipBind, *masterOptions.port, 0)
 	if e != nil {
-		glog.Fatalf("Master startup error: %v", e)
+		log.Fatalf("Master startup error: %v", e)
 	}
 
 	// starting grpc server
 	grpcPort := *masterOptions.portGrpc
 	grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOptions.ipBind, grpcPort, 0)
 	if err != nil {
-		glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
+		log.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
 	}
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
 	master_pb.RegisterSeaweedServer(grpcS, ms)
 	reflection.Register(grpcS)
-	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOptions.ip, grpcPort)
+	log.V(3).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOptions.ip, grpcPort)
 	if grpcLocalL != nil {
 		go grpcS.Serve(grpcLocalL)
 	}

weed/command/mount_linux.go

@@ -3,7 +3,7 @@ package command
 import (
 	"bufio"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"io"
 	"os"
 	"strings"
@@ -144,7 +144,7 @@ func checkMountPointAvailable(dir string) bool {
 
 	if mounted, err := mounted(mountPoint); err != nil || mounted {
 		if err != nil {
-			glog.Errorf("check %s: %v", mountPoint, err)
+			log.Errorf("check %s: %v", mountPoint, err)
 		}
 		return false
 	}

weed/command/mount_std.go

@@ -16,7 +16,7 @@ import (
 	"time"
 
 	"github.com/hanwen/go-fuse/v2/fuse"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/mount"
 	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
 	"github.com/seaweedfs/seaweedfs/weed/mount/unmount"
@@ -61,7 +61,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	// basic checks
 	chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB
 	if chunkSizeLimitMB <= 0 {
-		fmt.Printf("Please specify a reasonable buffer size.")
+		fmt.Printf("Please specify a reasonable buffer size.\n")
 		return false
 	}
 
@@ -81,13 +81,13 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("failed to talk to filer %v: %v", filerAddresses, err)
-			glog.V(0).Infof("wait for %d seconds ...", i+1)
+			log.V(3).Infof("failed to talk to filer %v: %v", filerAddresses, err)
+			log.V(3).Infof("wait for %d seconds ...", i+1)
 			time.Sleep(time.Duration(i+1) * time.Second)
 		}
 	}
 	if err != nil {
-		glog.Errorf("failed to talk to filer %v: %v", filerAddresses, err)
+		log.Errorf("failed to talk to filer %v: %v", filerAddresses, err)
 		return true
 	}
 
@@ -111,11 +111,11 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		*option.localSocket = fmt.Sprintf("/tmp/seaweedfs-mount-%d.sock", mountDirHash)
 	}
 	if err := os.Remove(*option.localSocket); err != nil && !os.IsNotExist(err) {
-		glog.Fatalf("Failed to remove %s, error: %s", *option.localSocket, err.Error())
+		log.Fatalf("Failed to remove %s, error: %s", *option.localSocket, err.Error())
 	}
 	montSocketListener, err := net.Listen("unix", *option.localSocket)
 	if err != nil {
-		glog.Fatalf("Failed to listen on %s: %v", *option.localSocket, err)
+		log.Fatalf("Failed to listen on %s: %v", *option.localSocket, err)
 	}
 
 	// detect mount folder mode
@@ -158,7 +158,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 
 	// Ensure target mount point availability
 	if isValid := checkMountPointAvailable(dir); !isValid {
-		glog.Fatalf("Target mount point is not available: %s, please check!", dir)
+		log.Fatalf("Target mount point is not available: %s, please check!", dir)
 		return true
 	}
 
@@ -262,7 +262,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 
 	server, err := fuse.NewServer(seaweedFileSystem, dir, fuseMountOptions)
 	if err != nil {
-		glog.Fatalf("Mount fail: %v", err)
+		log.Fatalf("Mount fail: %v", err)
 	}
 	grace.OnInterrupt(func() {
 		unmount.Unmount(dir)
@@ -279,10 +279,12 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		return false
 	}
 
-	glog.V(0).Infof("mounted %s%s to %v", *option.filer, mountRoot, dir)
-	glog.V(0).Infof("This is SeaweedFS version %s %s %s", util.Version(), runtime.GOOS, runtime.GOARCH)
+	log.V(3).Infof("mounted %s%s to %v", *option.filer, mountRoot, dir)
+	log.V(3).Infof("This is SeaweedFS version %s %s %s", util.Version(), runtime.GOOS, runtime.GOARCH)
 
 	server.Serve()
 
+	seaweedFileSystem.ClearCacheDir()
+
 	return true
 }

weed/command/mq_agent.go

@@ -5,7 +5,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pb"
 	"google.golang.org/grpc/reflection"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -26,12 +26,12 @@ type MessageQueueAgentOptions struct {
 func init() {
 	cmdMqAgent.Run = runMqAgent // break init cycle
 	mqAgentOptions.brokersString = cmdMqAgent.Flag.String("broker", "localhost:17777", "comma-separated message queue brokers")
-	mqAgentOptions.ip = cmdMqAgent.Flag.String("ip", "localhost", "message queue agent host address")
+	mqAgentOptions.ip = cmdMqAgent.Flag.String("ip", "", "message queue agent host address")
 	mqAgentOptions.port = cmdMqAgent.Flag.Int("port", 16777, "message queue agent gRPC server port")
 }
 
 var cmdMqAgent = &Command{
-	UsageLine: "mq.agent [-port=6377] [-master=<ip:port>]",
+	UsageLine: "mq.agent [-port=16777] [-broker=<ip:port>]",
 	Short:     "<WIP> start a message queue agent",
 	Long: `start a message queue agent
 
@@ -62,8 +62,9 @@ func (mqAgentOpt *MessageQueueAgentOptions) startQueueAgent() bool {
 	// start grpc listener
 	grpcL, _, err := util.NewIpAndLocalListeners(*mqAgentOpt.ip, *mqAgentOpt.port, 0)
 	if err != nil {
-		glog.Fatalf("failed to listen on grpc port %d: %v", *mqAgentOpt.port, err)
+		log.Fatalf("failed to listen on grpc port %d: %v", *mqAgentOpt.port, err)
 	}
+	log.Infof("Start Seaweed Message Queue Agent on %s:%d", *mqAgentOpt.ip, *mqAgentOpt.port)
 	grpcS := pb.NewGrpcServer()
 	mq_agent_pb.RegisterSeaweedMessagingAgentServer(grpcS, agentServer)
 	reflection.Register(grpcS)

weed/command/mq_broker.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/mq/broker"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
@@ -79,13 +79,13 @@ func (mqBrokerOpt *MessageQueueBrokerOptions) startQueueServer() bool {
 		Port:               *mqBrokerOpt.port,
 	}, grpcDialOption)
 	if err != nil {
-		glog.Fatalf("failed to create new message broker for queue server: %v", err)
+		log.Fatalf("failed to create new message broker for queue server: %v", err)
 	}
 
 	// start grpc listener
 	grpcL, _, err := util.NewIpAndLocalListeners("", *mqBrokerOpt.port, 0)
 	if err != nil {
-		glog.Fatalf("failed to listen on grpc port %d: %v", *mqBrokerOpt.port, err)
+		log.Fatalf("failed to listen on grpc port %d: %v", *mqBrokerOpt.port, err)
 	}
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.msg_broker"))
 	mq_pb.RegisterSeaweedMessagingServer(grpcS, qs)

weed/command/s3.go

@@ -25,7 +25,7 @@ import (
 
 	"github.com/gorilla/mux"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/s3api"
 	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -57,6 +57,7 @@ type S3Options struct {
 	dataCenter                *string
 	localSocket               *string
 	certProvider              certprovider.Provider
+	idleTimeout               *int
 }
 
 func init() {
@@ -81,6 +82,7 @@ func init() {
 	s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 	s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path")
 	s3StandaloneOptions.localSocket = cmdS3.Flag.String("localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
+	s3StandaloneOptions.idleTimeout = cmdS3.Flag.Int("idleTimeout", 10, "connection idle seconds")
 }
 
 var cmdS3 = &Command{
@@ -210,14 +212,14 @@ func (s3opt *S3Options) startS3Server() bool {
 			filerBucketsPath = resp.DirBuckets
 			filerGroup = resp.FilerGroup
 			metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec)
-			glog.V(0).Infof("S3 read filer buckets dir: %s", filerBucketsPath)
+			log.V(3).Infof("S3 read filer buckets dir: %s", filerBucketsPath)
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *s3opt.filer, filerAddress.ToGrpcAddress())
+			log.V(3).Infof("wait to connect to filer %s grpc address %s", *s3opt.filer, filerAddress.ToGrpcAddress())
 			time.Sleep(time.Second)
 		} else {
-			glog.V(0).Infof("connected to filer %s grpc address %s", *s3opt.filer, filerAddress.ToGrpcAddress())
+			log.V(3).Infof("connected to filer %s grpc address %s", *s3opt.filer, filerAddress.ToGrpcAddress())
 			break
 		}
 	}
@@ -244,7 +246,7 @@ func (s3opt *S3Options) startS3Server() bool {
 		FilerGroup:                filerGroup,
 	})
 	if s3ApiServer_err != nil {
-		glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
+		log.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
 	}
 
 	httpS := &http.Server{Handler: router}
@@ -262,22 +264,23 @@ func (s3opt *S3Options) startS3Server() bool {
 			localSocket = fmt.Sprintf("/tmp/seaweedfs-s3-%d.sock", *s3opt.port)
 		}
 		if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
-			glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
+			log.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
 		}
 		go func() {
 			// start on local unix socket
 			s3SocketListener, err := net.Listen("unix", localSocket)
 			if err != nil {
-				glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
+				log.Fatalf("Failed to listen on %s: %v", localSocket, err)
 			}
 			httpS.Serve(s3SocketListener)
 		}()
 	}
 
 	listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port)
-	s3ApiListener, s3ApiLocalListener, err := util.NewIpAndLocalListeners(*s3opt.bindIp, *s3opt.port, time.Duration(10)*time.Second)
+	s3ApiListener, s3ApiLocalListener, err := util.NewIpAndLocalListeners(
+		*s3opt.bindIp, *s3opt.port, time.Duration(*s3opt.idleTimeout)*time.Second)
 	if err != nil {
-		glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
+		log.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
 	}
 
 	if len(*s3opt.auditLogConfig) > 0 {
@@ -291,7 +294,7 @@ func (s3opt *S3Options) startS3Server() bool {
 	grpcPort := *s3opt.portGrpc
 	grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*s3opt.bindIp, grpcPort, 0)
 	if err != nil {
-		glog.Fatalf("s3 failed to listen on grpc port %d: %v", grpcPort, err)
+		log.Fatalf("s3 failed to listen on grpc port %d: %v", grpcPort, err)
 	}
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.s3"))
 	s3_pb.RegisterSeaweedS3Server(grpcS, s3ApiServer)
@@ -308,21 +311,21 @@ func (s3opt *S3Options) startS3Server() bool {
 			RefreshDuration: security.CredRefreshingInterval,
 		}
 		if s3opt.certProvider, err = pemfile.NewProvider(pemfileOptions); err != nil {
-			glog.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err)
+			log.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err)
 		}
 
 		caCertPool := x509.NewCertPool()
-		if *s3Options.tlsCACertificate != "" {
+		if *s3opt.tlsCACertificate != "" {
 			// load CA certificate file and add it to list of client CAs
 			caCertFile, err := ioutil.ReadFile(*s3opt.tlsCACertificate)
 			if err != nil {
-				glog.Fatalf("error reading CA certificate: %v", err)
+				log.Fatalf("error reading CA certificate: %v", err)
 			}
 			caCertPool.AppendCertsFromPEM(caCertFile)
 		}
 
 		clientAuth := tls.NoClientCert
-		if *s3Options.tlsVerifyClientCert {
+		if *s3opt.tlsVerifyClientCert {
 			clientAuth = tls.RequireAndVerifyClientCert
 		}
 
@@ -333,49 +336,49 @@ func (s3opt *S3Options) startS3Server() bool {
 		}
 		err = security.FixTlsConfig(util.GetViper(), httpS.TLSConfig)
 		if err != nil {
-			glog.Fatalf("error with tls config: %v", err)
+			log.Fatalf("error with tls config: %v", err)
 		}
 		if *s3opt.portHttps == 0 {
-			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
+			log.V(3).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
 			if s3ApiLocalListener != nil {
 				go func() {
 					if err = httpS.ServeTLS(s3ApiLocalListener, "", ""); err != nil {
-						glog.Fatalf("S3 API Server Fail to serve: %v", err)
+						log.Fatalf("S3 API Server Fail to serve: %v", err)
 					}
 				}()
 			}
 			if err = httpS.ServeTLS(s3ApiListener, "", ""); err != nil {
-				glog.Fatalf("S3 API Server Fail to serve: %v", err)
+				log.Fatalf("S3 API Server Fail to serve: %v", err)
 			}
 		} else {
-			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.portHttps)
+			log.V(3).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.portHttps)
 			s3ApiListenerHttps, s3ApiLocalListenerHttps, _ := util.NewIpAndLocalListeners(
-				*s3opt.bindIp, *s3opt.portHttps, time.Duration(10)*time.Second)
+                *s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second)
 			if s3ApiLocalListenerHttps != nil {
 				go func() {
 					if err = httpS.ServeTLS(s3ApiLocalListenerHttps, "", ""); err != nil {
-						glog.Fatalf("S3 API Server Fail to serve: %v", err)
+						log.Fatalf("S3 API Server Fail to serve: %v", err)
 					}
 				}()
 			}
 			go func() {
 				if err = httpS.ServeTLS(s3ApiListenerHttps, "", ""); err != nil {
-					glog.Fatalf("S3 API Server Fail to serve: %v", err)
+					log.Fatalf("S3 API Server Fail to serve: %v", err)
 				}
 			}()
 		}
 	}
 	if *s3opt.tlsPrivateKey == "" || *s3opt.portHttps > 0 {
-		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.Version(), *s3opt.port)
+		log.V(3).Infof("Start Seaweed S3 API Server %s at http port %d", util.Version(), *s3opt.port)
 		if s3ApiLocalListener != nil {
 			go func() {
 				if err = httpS.Serve(s3ApiLocalListener); err != nil {
-					glog.Fatalf("S3 API Server Fail to serve: %v", err)
+					log.Fatalf("S3 API Server Fail to serve: %v", err)
 				}
 			}()
 		}
 		if err = httpS.Serve(s3ApiListener); err != nil {
-			glog.Fatalf("S3 API Server Fail to serve: %v", err)
+			log.Fatalf("S3 API Server Fail to serve: %v", err)
 		}
 	}
 

weed/command/scaffold/filer.toml

@@ -139,12 +139,13 @@ connection_max_lifetime_seconds = 0
 enableUpsert = true
 upsertQuery = """UPSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)"""
 
-[cassandra]
+[cassandra2]
 # CREATE TABLE filemeta (
+#    dirhash bigint,
 #    directory varchar,
 #    name varchar,
 #    meta blob,
-#    PRIMARY KEY (directory, name)
+#    PRIMARY KEY ((dirhash, directory), name)
 # ) WITH CLUSTERING ORDER BY (name ASC);
 enabled = false
 keyspace = "seaweedfs"
@@ -170,6 +171,10 @@ enabled = false
 address = "localhost:6379"
 password = ""
 database = 0
+enable_tls = false
+ca_cert_path = ""
+client_cert_path = ""
+client_key_path = ""
 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
 superLargeDirectories = []
 
@@ -180,6 +185,10 @@ masterName = "master"
 username = ""
 password = ""
 database = 0
+enable_tls = false
+ca_cert_path = ""
+client_cert_path = ""
+client_key_path = ""
 
 [redis_cluster2]
 enabled = false
@@ -192,6 +201,10 @@ addresses = [
     "localhost:30006",
 ]
 password = ""
+enable_tls = false
+ca_cert_path = ""
+client_cert_path = ""
+client_key_path = ""
 # allows reads from slave servers or the master, but all writes still go to the master
 readOnly = false
 # automatically use the closest Redis server for reads
@@ -199,11 +212,16 @@ routeByLatency = false
 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
 superLargeDirectories = []
 
+# The following lua redis stores uses lua to ensure atomicity
 [redis_lua]
 enabled = false
 address = "localhost:6379"
 password = ""
 database = 0
+enable_tls = false
+ca_cert_path = ""
+client_cert_path = ""
+client_key_path = ""
 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
 superLargeDirectories = []
 
@@ -214,6 +232,10 @@ masterName = "master"
 username = ""
 password = ""
 database = 0
+enable_tls = false
+ca_cert_path = ""
+client_cert_path = ""
+client_key_path = ""
 
 [redis_lua_cluster]
 enabled = false
@@ -226,6 +248,10 @@ addresses = [
     "localhost:30006",
 ]
 password = ""
+enable_tls = false
+ca_cert_path = ""
+client_cert_path = ""
+client_key_path = ""
 # allows reads from slave servers or the master, but all writes still go to the master
 readOnly = false
 # automatically use the closest Redis server for reads
@@ -233,36 +259,6 @@ routeByLatency = false
 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
 superLargeDirectories = []
 
-[redis3] # beta
-enabled = false
-address = "localhost:6379"
-password = ""
-database = 0
-
-[redis3_sentinel]
-enabled = false
-addresses = ["172.22.12.7:26379","172.22.12.8:26379","172.22.12.9:26379"]
-masterName = "master"
-username = ""
-password = ""
-database = 0
-
-[redis_cluster3] # beta
-enabled = false
-addresses = [
-    "localhost:30001",
-    "localhost:30002",
-    "localhost:30003",
-    "localhost:30004",
-    "localhost:30005",
-    "localhost:30006",
-]
-password = ""
-# allows reads from slave servers or the master, but all writes still go to the master
-readOnly = false
-# automatically use the closest Redis server for reads
-routeByLatency = false
-
 [etcd]
 enabled = false
 servers = "localhost:2379"
@@ -333,7 +329,7 @@ dialTimeOut = 10
 ##########################
 # To add path-specific filer store:
 #
-# 1. Add a name following the store type separated by a dot ".". E.g., cassandra.tmp
+# 1. Add a name following the store type separated by a dot ".". E.g., cassandra2.tmp
 # 2. Add a location configuration. E.g., location = "/tmp/"
 # 3. Copy and customize all other configurations.
 #     Make sure they are not the same if using the same store type!
@@ -365,3 +361,10 @@ cert_path=""
 key_path=""
 # The name list used to verify the cn name
 verify_cn=""
+
+[tarantool]
+address = "localhost:3301"
+user = "guest"
+password = ""
+timeout = "5s"
+maxReconnects = 1000

weed/command/server.go

@@ -9,7 +9,7 @@ import (
 
 	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
@@ -28,6 +28,7 @@ var (
 	masterOptions   MasterOptions
 	filerOptions    FilerOptions
 	s3Options       S3Options
+	sftpOptions     SftpOptions
 	iamOptions      IamOptions
 	webdavOptions   WebDavOption
 	mqBrokerOptions MessageQueueBrokerOptions
@@ -73,6 +74,7 @@ var (
 	isStartingVolumeServer = cmdServer.Flag.Bool("volume", true, "whether to start volume server")
 	isStartingFiler        = cmdServer.Flag.Bool("filer", false, "whether to start filer")
 	isStartingS3           = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
+	isStartingSftp         = cmdServer.Flag.Bool("sftp", false, "whether to start Sftp server")
 	isStartingIam          = cmdServer.Flag.Bool("iam", false, "whether to start IAM service")
 	isStartingWebDav       = cmdServer.Flag.Bool("webdav", false, "whether to start WebDAV gateway")
 	isStartingMqBroker     = cmdServer.Flag.Bool("mq.broker", false, "whether to start message queue broker")
@@ -156,7 +158,20 @@ func init() {
 	s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
 	s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 	s3Options.localSocket = cmdServer.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
+	s3Options.bindIp = cmdServer.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")
+	s3Options.idleTimeout = cmdServer.Flag.Int("s3.idleTimeout", 10, "connection idle seconds")
 
+	sftpOptions.port = cmdServer.Flag.Int("sftp.port", 2022, "SFTP server listen port")
+	sftpOptions.sshPrivateKey = cmdServer.Flag.String("sftp.sshPrivateKey", "", "path to the SSH private key file for host authentication")
+	sftpOptions.hostKeysFolder = cmdServer.Flag.String("sftp.hostKeysFolder", "", "path to folder containing SSH private key files for host authentication")
+	sftpOptions.authMethods = cmdServer.Flag.String("sftp.authMethods", "password,publickey", "comma-separated list of allowed auth methods: password, publickey, keyboard-interactive")
+	sftpOptions.maxAuthTries = cmdServer.Flag.Int("sftp.maxAuthTries", 6, "maximum number of authentication attempts per connection")
+	sftpOptions.bannerMessage = cmdServer.Flag.String("sftp.bannerMessage", "SeaweedFS SFTP Server - Unauthorized access is prohibited", "message displayed before authentication")
+	sftpOptions.loginGraceTime = cmdServer.Flag.Duration("sftp.loginGraceTime", 2*time.Minute, "timeout for authentication")
+	sftpOptions.clientAliveInterval = cmdServer.Flag.Duration("sftp.clientAliveInterval", 5*time.Second, "interval for sending keep-alive messages")
+	sftpOptions.clientAliveCountMax = cmdServer.Flag.Int("sftp.clientAliveCountMax", 3, "maximum number of missed keep-alive messages before disconnecting")
+	sftpOptions.userStoreFile = cmdServer.Flag.String("sftp.userStoreFile", "", "path to JSON file containing user credentials and permissions")
+	sftpOptions.localSocket = cmdServer.Flag.String("sftp.localSocket", "", "default to /tmp/seaweedfs-sftp-<port>.sock")
 	iamOptions.port = cmdServer.Flag.Int("iam.port", 8111, "iam server http listen port")
 
 	webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port")
@@ -188,6 +203,9 @@ func runServer(cmd *Command, args []string) bool {
 	if *isStartingS3 {
 		*isStartingFiler = true
 	}
+	if *isStartingSftp {
+		*isStartingFiler = true
+	}
 	if *isStartingIam {
 		*isStartingFiler = true
 	}
@@ -218,9 +236,15 @@ func runServer(cmd *Command, args []string) bool {
 	filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToServiceDiscovery()
 	filerOptions.ip = serverIp
 	filerOptions.bindIp = serverBindIp
-	s3Options.bindIp = serverBindIp
+	if *s3Options.bindIp == "" {
+		s3Options.bindIp = serverBindIp
+	}
+	if sftpOptions.bindIp == nil || *sftpOptions.bindIp == "" {
+		sftpOptions.bindIp = serverBindIp
+	}
 	iamOptions.ip = serverBindIp
 	iamOptions.masters = masterOptions.peers
+	webdavOptions.ipBind = serverBindIp
 	serverOptions.v.ip = serverIp
 	serverOptions.v.bindIp = serverBindIp
 	serverOptions.v.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
@@ -241,11 +265,13 @@ func runServer(cmd *Command, args []string) bool {
 	mqBrokerOptions.dataCenter = serverDataCenter
 	mqBrokerOptions.rack = serverRack
 	s3Options.dataCenter = serverDataCenter
+	sftpOptions.dataCenter = serverDataCenter
 	filerOptions.disableHttp = serverDisableHttp
 	masterOptions.disableHttp = serverDisableHttp
 
 	filerAddress := string(pb.NewServerAddress(*serverIp, *filerOptions.port, *filerOptions.portGrpc))
 	s3Options.filer = &filerAddress
+	sftpOptions.filer = &filerAddress
 	iamOptions.filer = &filerAddress
 	webdavOptions.filer = &filerAddress
 	mqBrokerOptions.filerGroup = filerOptions.filerGroup
@@ -255,14 +281,14 @@ func runServer(cmd *Command, args []string) bool {
 	folders := strings.Split(*volumeDataFolders, ",")
 
 	if *masterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
-		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
+		log.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
 	}
 
 	if *masterOptions.metaFolder == "" {
 		*masterOptions.metaFolder = folders[0]
 	}
 	if err := util.TestFolderWritable(util.ResolvePath(*masterOptions.metaFolder)); err != nil {
-		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterOptions.metaFolder, err)
+		log.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterOptions.metaFolder, err)
 	}
 	filerOptions.defaultLevelDbDirectory = masterOptions.metaFolder
 
@@ -286,6 +312,14 @@ func runServer(cmd *Command, args []string) bool {
 		}()
 	}
 
+	if *isStartingSftp {
+		go func() {
+			time.Sleep(2 * time.Second)
+			sftpOptions.localSocket = filerOptions.localSocket
+			sftpOptions.startSftpServer()
+		}()
+	}
+
 	if *isStartingIam {
 		go func() {
 			time.Sleep(2 * time.Second)

weed/command/sftp.go

@@ -0,0 +1,193 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"runtime"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	filer_pb "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/sftpd"
+	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var (
+	sftpOptionsStandalone SftpOptions
+)
+
+// SftpOptions holds configuration options for the SFTP server.
+type SftpOptions struct {
+	filer               *string
+	bindIp              *string
+	port                *int
+	sshPrivateKey       *string
+	hostKeysFolder      *string
+	authMethods         *string
+	maxAuthTries        *int
+	bannerMessage       *string
+	loginGraceTime      *time.Duration
+	clientAliveInterval *time.Duration
+	clientAliveCountMax *int
+	userStoreFile       *string
+	dataCenter          *string
+	metricsHttpPort     *int
+	metricsHttpIp       *string
+	localSocket         *string
+}
+
+// cmdSftp defines the SFTP command similar to the S3 command.
+var cmdSftp = &Command{
+	UsageLine: "sftp [-port=2022] [-filer=<ip:port>] [-sshPrivateKey=</path/to/private_key>]",
+	Short:     "start an SFTP server that is backed by a SeaweedFS filer",
+	Long: `Start an SFTP server that leverages the SeaweedFS filer service to handle file operations.
+
+Instead of reading from or writing to a local filesystem, all file operations
+are routed through the filer (filer_pb) gRPC API. This allows you to centralize
+your file management in SeaweedFS.
+	`,
+}
+
+func init() {
+	// Register the command to avoid cyclic dependencies.
+	cmdSftp.Run = runSftp
+
+	sftpOptionsStandalone.filer = cmdSftp.Flag.String("filer", "localhost:8888", "filer server address (ip:port)")
+	sftpOptionsStandalone.bindIp = cmdSftp.Flag.String("ip.bind", "0.0.0.0", "ip address to bind SFTP server")
+	sftpOptionsStandalone.port = cmdSftp.Flag.Int("port", 2022, "SFTP server listen port")
+	sftpOptionsStandalone.sshPrivateKey = cmdSftp.Flag.String("sshPrivateKey", "", "path to the SSH private key file for host authentication")
+	sftpOptionsStandalone.hostKeysFolder = cmdSftp.Flag.String("hostKeysFolder", "", "path to folder containing SSH private key files for host authentication")
+	sftpOptionsStandalone.authMethods = cmdSftp.Flag.String("authMethods", "password,publickey", "comma-separated list of allowed auth methods: password, publickey, keyboard-interactive")
+	sftpOptionsStandalone.maxAuthTries = cmdSftp.Flag.Int("maxAuthTries", 6, "maximum number of authentication attempts per connection")
+	sftpOptionsStandalone.bannerMessage = cmdSftp.Flag.String("bannerMessage", "SeaweedFS SFTP Server - Unauthorized access is prohibited", "message displayed before authentication")
+	sftpOptionsStandalone.loginGraceTime = cmdSftp.Flag.Duration("loginGraceTime", 2*time.Minute, "timeout for authentication")
+	sftpOptionsStandalone.clientAliveInterval = cmdSftp.Flag.Duration("clientAliveInterval", 5*time.Second, "interval for sending keep-alive messages")
+	sftpOptionsStandalone.clientAliveCountMax = cmdSftp.Flag.Int("clientAliveCountMax", 3, "maximum number of missed keep-alive messages before disconnecting")
+	sftpOptionsStandalone.userStoreFile = cmdSftp.Flag.String("userStoreFile", "", "path to JSON file containing user credentials and permissions")
+	sftpOptionsStandalone.dataCenter = cmdSftp.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
+	sftpOptionsStandalone.metricsHttpPort = cmdSftp.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
+	sftpOptionsStandalone.metricsHttpIp = cmdSftp.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
+	sftpOptionsStandalone.localSocket = cmdSftp.Flag.String("localSocket", "", "default to /tmp/seaweedfs-sftp-<port>.sock")
+}
+
+// runSftp is the command entry point.
+func runSftp(cmd *Command, args []string) bool {
+	// Load security configuration as done in other SeaweedFS services.
+	util.LoadSecurityConfiguration()
+
+	// Configure metrics
+	switch {
+	case *sftpOptionsStandalone.metricsHttpIp != "":
+		// nothing to do, use sftpOptionsStandalone.metricsHttpIp
+	case *sftpOptionsStandalone.bindIp != "":
+		*sftpOptionsStandalone.metricsHttpIp = *sftpOptionsStandalone.bindIp
+	}
+	go stats_collect.StartMetricsServer(*sftpOptionsStandalone.metricsHttpIp, *sftpOptionsStandalone.metricsHttpPort)
+
+	return sftpOptionsStandalone.startSftpServer()
+}
+
+func (sftpOpt *SftpOptions) startSftpServer() bool {
+	filerAddress := pb.ServerAddress(*sftpOpt.filer)
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	// metrics read from the filer
+	var metricsAddress string
+	var metricsIntervalSec int
+	var filerGroup string
+
+	// Connect to the filer service and try to retrieve basic configuration.
+	for {
+		err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			if err != nil {
+				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
+			}
+			metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec)
+			filerGroup = resp.FilerGroup
+			log.V(3).Infof("SFTP read filer configuration, using filer at: %s", filerAddress)
+			return nil
+		})
+		if err != nil {
+			log.V(3).Infof("Waiting to connect to filer %s grpc address %s...", *sftpOpt.filer, filerAddress.ToGrpcAddress())
+			time.Sleep(time.Second)
+		} else {
+			log.V(3).Infof("Connected to filer %s grpc address %s", *sftpOpt.filer, filerAddress.ToGrpcAddress())
+			break
+		}
+	}
+
+	go stats_collect.LoopPushingMetric("sftp", stats_collect.SourceName(uint32(*sftpOpt.port)), metricsAddress, metricsIntervalSec)
+
+	// Parse auth methods
+	var authMethods []string
+	if *sftpOpt.authMethods != "" {
+		authMethods = util.StringSplit(*sftpOpt.authMethods, ",")
+	}
+
+	// Create a new SFTP service instance with all options
+	service := sftpd.NewSFTPService(&sftpd.SFTPServiceOptions{
+		GrpcDialOption:      grpcDialOption,
+		DataCenter:          *sftpOpt.dataCenter,
+		FilerGroup:          filerGroup,
+		Filer:               filerAddress,
+		SshPrivateKey:       *sftpOpt.sshPrivateKey,
+		HostKeysFolder:      *sftpOpt.hostKeysFolder,
+		AuthMethods:         authMethods,
+		MaxAuthTries:        *sftpOpt.maxAuthTries,
+		BannerMessage:       *sftpOpt.bannerMessage,
+		LoginGraceTime:      *sftpOpt.loginGraceTime,
+		ClientAliveInterval: *sftpOpt.clientAliveInterval,
+		ClientAliveCountMax: *sftpOpt.clientAliveCountMax,
+		UserStoreFile:       *sftpOpt.userStoreFile,
+	})
+
+	// Set up Unix socket if on non-Windows platforms
+	if runtime.GOOS != "windows" {
+		localSocket := *sftpOpt.localSocket
+		if localSocket == "" {
+			localSocket = fmt.Sprintf("/tmp/seaweedfs-sftp-%d.sock", *sftpOpt.port)
+		}
+		if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
+			log.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
+		}
+		go func() {
+			// start on local unix socket
+			sftpSocketListener, err := net.Listen("unix", localSocket)
+			if err != nil {
+				log.Fatalf("Failed to listen on %s: %v", localSocket, err)
+			}
+			if err := service.Serve(sftpSocketListener); err != nil {
+				log.Fatalf("Failed to serve SFTP on socket %s: %v", localSocket, err)
+			}
+		}()
+	}
+
+	// Start the SFTP service on TCP
+	listenAddress := fmt.Sprintf("%s:%d", *sftpOpt.bindIp, *sftpOpt.port)
+	sftpListener, sftpLocalListener, err := util.NewIpAndLocalListeners(*sftpOpt.bindIp, *sftpOpt.port, time.Duration(10)*time.Second)
+	if err != nil {
+		log.Fatalf("SFTP server listener on %s error: %v", listenAddress, err)
+	}
+
+	log.V(3).Infof("Start Seaweed SFTP Server %s at %s", util.Version(), listenAddress)
+
+	if sftpLocalListener != nil {
+		go func() {
+			if err := service.Serve(sftpLocalListener); err != nil {
+				log.Fatalf("SFTP Server failed to serve on local listener: %v", err)
+			}
+		}()
+	}
+
+	if err := service.Serve(sftpListener); err != nil {
+		log.Fatalf("SFTP Server failed to serve: %v", err)
+	}
+
+	return true
+}

weed/command/update.go

@@ -18,7 +18,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/log"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 	"golang.org/x/net/context/ctxhttp"
@@ -86,7 +86,7 @@ func runUpdate(cmd *Command, args []string) bool {
 
 	if *updateOpt.dir != "" {
 		if err := util.TestFolderWritable(util.ResolvePath(*updateOpt.dir)); err != nil {
-			glog.Fatalf("Check Folder(-dir) Writable %s : %s", *updateOpt.dir, err)
+			log.Fatalf("Check Folder(-dir) Writable %s : %s", *updateOpt.dir, err)
 			return false
 		}
 	} else {
@@ -101,16 +101,16 @@ func runUpdate(cmd *Command, args []string) bool {
 
 	if runtime.GOOS == "windows" {
 		if target == path {
-			glog.Fatalf("On windows, name of the new weed shouldn't be same to the original name.")
+			log.Fatalf("On windows, name of the new weed shouldn't be same to the original name.")
 			return false
 		}
 	}
 
-	glog.V(0).Infof("new weed will be saved to %s", target)
+	log.V(3).Infof("new weed will be saved to %s", target)
 
 	_, err := downloadRelease(context.Background(), target, *updateOpt.Version)
 	if err != nil {
-		glog.Errorf("unable to download weed: %v", err)
+		log.Errorf("unable to download weed: %v", err)
 		return false
 	}
 	return true
@@ -125,14 +125,14 @@ func downloadRelease(ctx context.Context, target string, ver string) (version st
 
 	if rel.Version == currentVersion {
 		if ver == "0" {
-			glog.V(0).Infof("weed is up to date")
+			log.V(3).Infof("weed is up to date")
 		} else {
-			glog.V(0).Infof("no need to download the same version of weed ")
+			log.V(3).Infof("no need to download the same version of weed ")
 		}
 		return currentVersion, nil
 	}
 
-	glog.V(0).Infof("download version: %s", rel.Version)
+	log.V(3).Infof("download version: %s", rel.Version)
 
 	largeDiskSuffix := ""
 	if util.VolumeSizeLimitGB == 8000 {
@@ -165,7 +165,7 @@ func downloadRelease(ctx context.Context, target string, ver string) (version st
 	md5Ctx.Write(buf)
 	binaryMd5 := md5Ctx.Sum(nil)
 	if hex.EncodeToString(binaryMd5) != string(md5Val[0:32]) {
-		glog.Errorf("md5:'%s' '%s'", hex.EncodeToString(binaryMd5), string(md5Val[0:32]))
+		log.Errorf("md5:'%s' '%s'", hex.EncodeToString(binaryMd5), string(md5Val[0:32]))
 		err = fmt.Errorf("binary md5sum doesn't match")
 		return "", err
 	}
@@ -174,7 +174,7 @@ func downloadRelease(ctx context.Context, target string, ver string) (version st
 	if err != nil {
 		return "", err
 	} else {
-		glog.V(0).Infof("successfully updated weed to version %v\n", rel.Version)
+		log.V(3).Infof("successfully updated weed to version %v\n", rel.Version)
 	}
 
 	return rel.Version, nil
@@ -228,7 +228,7 @@ func GitHubLatestRelease(ctx context.Context, ver string, owner, repo string) (R
 	}
 	if ver == "0" {
 		release = releaseList[0]
-		glog.V(0).Infof("latest version is %v\n", release.TagName)
+		log.V(3).Infof("latest version is %v\n", release.TagName)
 	} else {
 		for _, r := range releaseList {
 			if r.TagName == ver {
@@ -287,7 +287,7 @@ func getGithubDataFile(ctx context.Context, assets []Asset, suffix string) (file
 		return "", nil, fmt.Errorf("unable to find file with suffix %v", suffix)
 	}
 
-	glog.V(0).Infof("download %v\n", filename)
+	log.V(3).Infof("download %v\n", filename)
 	data, err = getGithubData(ctx, url)
 	if err != nil {
 		return "", nil, err
@@ -310,9 +310,9 @@ func extractToFile(buf []byte, filename, target string) error {
 		hdr, terr := trd.Next()
 		if terr != nil {
 			if hdr != nil {
-				glog.Errorf("uncompress file(%s) failed:%s", hdr.Name, terr)
+				log.Errorf("uncompress file(%s) failed:%s", hdr.Name, terr)
 			} else {
-				glog.Errorf("uncompress file is nil, failed:%s", terr)
+				log.Errorf("uncompress file is nil, failed:%s", terr)
 			}
 
 			return terr
@@ -371,6 +371,6 @@ func extractToFile(buf []byte, filename, target string) error {
 		return err
 	}
 
-	glog.V(0).Infof("saved %d bytes in %v\n", n, target)
+	log.V(3).Infof("saved %d bytes in %v\n", n, target)
 	return os.Chmod(target, mode)
 }