2.74

fix mine detect

.github/workflows/binaries_dev.yml

@@ -20,22 +20,60 @@ jobs:
           assets: |
             weed-*
 
-  build_dev:
+  build_dev_linux:
     needs: cleanup
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        goos: [linux, windows, darwin, freebsd]
-        goarch: [amd64, arm, arm64]
-        exclude:
-          - goarch: arm
-            goos: darwin
-          - goarch: 386
-            goos: darwin
-          - goarch: arm
-            goos: windows
-          - goarch: arm64
-            goos: windows
+        goos: [linux]
+        goarch: [amd64]
+
+    steps:
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set BUILD_TIME env
+        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
+
+      - name: Go Release Binaries Large Disk
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          release_tag: dev
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed-large-disk
+          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
+
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          release_tag: dev
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed-normal-disk
+          asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
+
+  build_dev_darwin:
+    needs: build_dev_linux
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [darwin]
+        goarch: [amd64, arm64]
 
     steps:
 

.github/workflows/binaries_release0.yml

@@ -0,0 +1,54 @@
+# This is a basic workflow to help you get started with Actions
+
+name: "go: build versioned binaries for windows"
+
+on:
+  push:
+    tags:
+      - '*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+
+  build-release-binaries_windows:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [windows]
+        goarch: [amd64]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          # build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
+      - name: Go Release Large Disk Binaries
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"

.github/workflows/binaries_release1.yml

@@ -1,6 +1,6 @@
 # This is a basic workflow to help you get started with Actions
 
-name: "go: build versioned binaries"
+name: "go: build versioned binaries for linux"
 
 on:
   push:
@@ -13,21 +13,12 @@ on:
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
 
-  build-release-binaries:
+  build-release-binaries_linux:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        goos: [linux, windows, darwin, freebsd]
+        goos: [linux]
         goarch: [amd64, arm, arm64]
-        exclude:
-          - goarch: arm
-            goos: darwin
-          - goarch: 386
-            goos: darwin
-          - goarch: arm
-            goos: windows
-          - goarch: arm64
-            goos: windows
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:

.github/workflows/binaries_release2.yml

@@ -0,0 +1,54 @@
+# This is a basic workflow to help you get started with Actions
+
+name: "go: build versioned binaries for darwin"
+
+on:
+  push:
+    tags:
+      - '*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+
+  build-release-binaries_darwin:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [darwin]
+        goarch: [amd64, arm64]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          # build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
+      - name: Go Release Large Disk Binaries
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"

.github/workflows/binaries_release3.yml

@@ -0,0 +1,54 @@
+# This is a basic workflow to help you get started with Actions
+
+name: "go: build versioned binaries for freebsd"
+
+on:
+  push:
+    tags:
+      - '*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+
+  build-release-binaries_freebsd:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [freebsd]
+        goarch: [amd64, arm, arm64]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          # build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
+      - name: Go Release Large Disk Binaries
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"

.github/workflows/container_dev.yml

@@ -58,6 +58,6 @@ jobs:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
+          platforms: linux/amd64, linux/arm64
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release.yml

@@ -1,121 +0,0 @@
-name: "docker: build release containers"
-
-on:
-  push:
-    tags:
-      - '*'
-  workflow_dispatch: []
-
-jobs:
-  build-default-release-container:
-    runs-on: [ubuntu-latest]
-
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            chrislusf/seaweedfs
-            ghcr.io/chrislusf/seaweedfs
-          tags: |
-            type=ref,event=tag
-          flavor: |
-            latest=false
-          labels: |
-            org.opencontainers.image.title=seaweedfs
-            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
-            org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      -
-        name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
-          file: ./docker/Dockerfile.go_build
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-  build-large-release-container:
-    runs-on: [ubuntu-latest]
-
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            chrislusf/seaweedfs
-            ghcr.io/chrislusf/seaweedfs
-          tags: |
-            type=ref,event=tag,suffix=_large_disk
-          flavor: |
-            latest=false
-          labels: |
-            org.opencontainers.image.title=seaweedfs
-            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
-            org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      -
-        name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
-          file: ./docker/Dockerfile.go_build_large
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release1.yml

@@ -0,0 +1,54 @@
+name: "docker: build release containers for normal volume"
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch: []
+
+jobs:
+  build-default-release-container:
+    runs-on: [ubuntu-latest]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            chrislusf/seaweedfs
+          tags: |
+            type=ref,event=tag
+          flavor: |
+            latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.go_build
+          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release2.yml

@@ -0,0 +1,55 @@
+name: "docker: build release containers for large volume"
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch: []
+
+jobs:
+
+  build-large-release-container:
+    runs-on: [ubuntu-latest]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            chrislusf/seaweedfs
+          tags: |
+            type=ref,event=tag,suffix=_large_disk
+          flavor: |
+            latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.go_build_large
+          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release3.yml

@@ -0,0 +1,55 @@
+name: "docker: build release containers for rocksdb"
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch: []
+
+jobs:
+
+  build-large-release-container_rocksdb:
+    runs-on: [ubuntu-latest]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            chrislusf/seaweedfs
+          tags: |
+            type=ref,event=tag,suffix=_large_disk_rocksdb
+          flavor: |
+            latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.rocksdb_large
+          platforms: linux/amd64
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}

docker/Dockerfile.rocksdb_large

@@ -0,0 +1,59 @@
+FROM golang:1.17-buster as builder
+
+RUN apt-get update
+RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
+
+ENV ROCKSDB_VERSION v6.22.1
+
+# build RocksDB
+RUN cd /tmp && \
+    git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
+    cd rocksdb && \
+    make static_lib && \
+    make install-static
+
+ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
+ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
+
+# build SeaweedFS
+RUN mkdir -p /go/src/github.com/chrislusf/
+RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
+ARG BRANCH=${BRANCH:-master}
+RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
+RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
+  && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+  && go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
+
+
+FROM alpine AS final
+LABEL author="Chris Lu"
+COPY --from=builder /go/bin/weed /usr/bin/
+RUN mkdir -p /etc/seaweedfs
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+RUN apk add fuse snappy gflags
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+# webdav server http port
+EXPOSE 7333
+
+RUN mkdir -p /data/filerldb2
+
+VOLUME /data
+
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/Makefile

@@ -13,6 +13,12 @@ build: binary
 	docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
 	rm ./weed
 
+build_gorocksdb:
+	docker build --no-cache -t chrislusf/gorocksdb -f Dockerfile.go_rocksdb .
+
+build_rocksdb:
+	docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large .
+
 s3tests_build:
 	docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
 

docker/compose/local-cluster-compose.yml

@@ -6,25 +6,37 @@ services:
     ports:
       - 9333:9333
       - 19333:19333
-    command: "master -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m1"
+    command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
   master1:
     image: chrislusf/seaweedfs:local
     ports:
       - 9334:9334
       - 19334:19334
-    command: "master -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m2"
+    command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
   master2:
     image: chrislusf/seaweedfs:local
     ports:
       - 9335:9335
       - 19335:19335
-    command: "master -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m3"
+    command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
   volume1:
     image: chrislusf/seaweedfs:local
     ports:
       - 8080:8080
       - 18080:18080
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1 -disk=ssd1'
+    command: 'volume -dataCenter dc1 -rack v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -34,7 +46,7 @@ services:
     ports:
       - 8082:8082
       - 18082:18082
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1 -disk=ssd1'
+    command: 'volume -dataCenter dc2 -rack v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -44,7 +56,7 @@ services:
     ports:
       - 8083:8083
       - 18083:18083
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
+    command: 'volume -dataCenter dc3 -rack v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -54,7 +66,8 @@ services:
     ports:
       - 8888:8888
       - 18888:18888
-    command: 'filer -master="master0:9333,master1:9334,master2:9335"'
+      - 8111:8111
+    command: 'filer -defaultReplicaPlacement 100 -iam -master="master0:9333,master1:9334,master2:9335"'
     depends_on:
       - master0
       - master1
@@ -65,11 +78,11 @@ services:
     image: chrislusf/seaweedfs:local
     ports:
       - 8333:8333
-    command: 's3 -filer="filer:8888"'
+    command: '-v=9 s3 -filer="filer:8888"'
     depends_on:
       - master0
       - master1
       - master2
       - volume1
       - volume2
-      - filer
+      - filer

go.mod

@@ -8,7 +8,6 @@ require (
 	cloud.google.com/go/storage v1.16.1
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.14.0
-	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
 	github.com/OneOfOne/xxhash v1.2.2
 	github.com/Shopify/sarama v1.23.1
@@ -39,6 +38,7 @@ require (
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.4.4
+	github.com/go-redsync/redsync/v4 v4.4.1
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/go-zookeeper/zk v1.0.2 // indirect
@@ -54,11 +54,10 @@ require (
 	github.com/googleapis/gax-go v2.0.2+incompatible // indirect
 	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
 	github.com/gorilla/mux v1.7.4
-	github.com/gorilla/websocket v1.4.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
-	github.com/hashicorp/go-multierror v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.0 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
@@ -74,6 +73,7 @@ require (
 	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.0
+	github.com/linxGnu/grocksdb v1.6.38
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/mailru/easyjson v0.7.1 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
@@ -110,12 +110,11 @@ require (
 	github.com/spf13/viper v1.4.0
 	github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
 	github.com/stretchr/testify v1.7.0
+	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
 	github.com/tidwall/gjson v1.8.1
 	github.com/tidwall/match v1.0.3
 	github.com/tidwall/pretty v1.1.0 // indirect
-	github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210824090536-16d902a3c7e5
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
 	github.com/valyala/bytebufferpool v1.0.0
@@ -126,24 +125,24 @@ require (
 	github.com/xdg-go/scram v1.0.2 // indirect
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.etcd.io/etcd v3.3.25+incompatible
+	go.etcd.io/etcd/client/v3 v3.5.0
 	go.mongodb.org/mongo-driver v1.7.0
 	go.opencensus.io v0.23.0 // indirect
 	go.opentelemetry.io/otel v0.15.0 // indirect
 	gocloud.dev v0.20.0
 	gocloud.dev/pubsub/natspubsub v0.20.0
 	gocloud.dev/pubsub/rabbitpubsub v0.20.0
-	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect
+	golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
 	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf
+	golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/tools v0.1.5
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/api v0.56.0
+	google.golang.org/api v0.57.0
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 // indirect
+	google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 // indirect
 	google.golang.org/grpc v1.40.0
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
@@ -165,35 +164,23 @@ require (
 )
 
 require (
-	github.com/coreos/etcd v3.3.10+incompatible // indirect
-	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
+	cloud.google.com/go/kms v1.0.0 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
-	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.2 // indirect
 	github.com/mattn/go-runewidth v0.0.7 // indirect
-	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 // indirect
-	github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd // indirect
-	github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4 // indirect
-	github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 // indirect
-	github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307 // indirect
+	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d // indirect
-	github.com/twmb/murmur3 v1.1.3 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.0 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
-	go.etcd.io/etcd/client/v3 v3.5.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.17.0 // indirect
-	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )

k8s/helm_charts2/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.69"
-version: "2.69"
+appVersion: "2.74"
+version: "2.74"

k8s/helm_charts2/templates/filer-servicemonitor.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.filer.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -15,4 +16,5 @@ spec:
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: filer
+{{- end }}
 {{- end }}

k8s/helm_charts2/templates/ingress.yaml

@@ -2,6 +2,7 @@ apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
  name: ingress-{{ template "seaweedfs.name" . }}-filer
+ namespace: {{ .Release.Namespace }}
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.ingress.kubernetes.io/auth-type: "basic"
@@ -32,6 +33,7 @@ apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
   name: ingress-{{ template "seaweedfs.name" . }}-master
+  namespace: {{ .Release.Namespace }}
   annotations:
     kubernetes.io/ingress.class: "nginx"
     nginx.ingress.kubernetes.io/auth-type: "basic"

k8s/helm_charts2/templates/s3-servicemonitor.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.s3.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -15,4 +16,5 @@ spec:
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: s3
+{{- end }}
 {{- end }}

k8s/helm_charts2/templates/volume-servicemonitor.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.volume.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -15,4 +16,5 @@ spec:
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: volume
+{{- end }}
 {{- end }}

k8s/helm_charts2/values.yaml

@@ -161,7 +161,7 @@ volume:
 
   # Directories to store data files. dir[,dir]... (default "/tmp")
   dir: "/data"
-  # Directories to store index files. dir[,dir]... (default "/tmp")
+  # Directories to store index files. dir[,dir]... (default is the same as "dir")
   dir_idx: null
 
   # Maximum numbers of volumes, count[,count]...

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.8</version>
+    <version>1.6.9</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.8</version>
+    <version>1.6.9</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.8</version>
+    <version>1.6.9</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/src/main/java/seaweedfs/client/ChunkCache.java

@@ -15,7 +15,7 @@ public class ChunkCache {
         }
         this.cache = CacheBuilder.newBuilder()
                 .maximumSize(maxEntries)
-                .expireAfterAccess(1, TimeUnit.HOURS)
+                .expireAfterWrite(1, TimeUnit.HOURS)
                 .build();
     }
 

other/java/client/src/main/java/seaweedfs/client/ReadChunks.java

@@ -0,0 +1,109 @@
+package seaweedfs.client;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+public class ReadChunks {
+
+    public static List<SeaweedRead.VisibleInterval> readResolvedChunks(List<FilerProto.FileChunk> chunkList) throws IOException {
+        List<Point> points = new ArrayList<>(chunkList.size() * 2);
+        for (FilerProto.FileChunk chunk : chunkList) {
+            points.add(new Point(chunk.getOffset(), chunk, true));
+            points.add(new Point(chunk.getOffset() + chunk.getSize(), chunk, false));
+        }
+        Collections.sort(points, new Comparator<Point>() {
+            @Override
+            public int compare(Point a, Point b) {
+                int x = (int) (a.x - b.x);
+                if (a.x != b.x) {
+                    return (int) (a.x - b.x);
+                }
+                if (a.ts != b.ts) {
+                    return (int) (a.ts - b.ts);
+                }
+                if (!a.isStart) {
+                    return -1;
+                }
+                return 1;
+            }
+        });
+
+        long prevX = 0;
+        List<SeaweedRead.VisibleInterval> visibles = new ArrayList<>();
+        ArrayList<Point> queue = new ArrayList<>();
+        for (Point point : points) {
+            if (point.isStart) {
+                if (queue.size() > 0) {
+                    int lastIndex = queue.size() - 1;
+                    Point lastPoint = queue.get(lastIndex);
+                    if (point.x != prevX && lastPoint.ts < point.ts) {
+                        addToVisibles(visibles, prevX, lastPoint, point);
+                        prevX = point.x;
+                    }
+                }
+                // insert into queue
+                for (int i = queue.size(); i >= 0; i--) {
+                    if (i == 0 || queue.get(i - 1).ts <= point.ts) {
+                        if (i == queue.size()) {
+                            prevX = point.x;
+                        }
+                        queue.add(i, point);
+                        break;
+                    }
+                }
+            } else {
+                int lastIndex = queue.size() - 1;
+                int index = lastIndex;
+                Point startPoint = null;
+                for (; index >= 0; index--) {
+                    startPoint = queue.get(index);
+                    if (startPoint.ts == point.ts) {
+                        queue.remove(index);
+                        break;
+                    }
+                }
+                if (index == lastIndex && startPoint != null) {
+                    addToVisibles(visibles, prevX, startPoint, point);
+                    prevX = point.x;
+                }
+            }
+        }
+
+        return visibles;
+
+    }
+
+    private static void addToVisibles(List<SeaweedRead.VisibleInterval> visibles, long prevX, Point startPoint, Point point) {
+        if (prevX < point.x) {
+            FilerProto.FileChunk chunk = startPoint.chunk;
+            visibles.add(new SeaweedRead.VisibleInterval(
+                    prevX,
+                    point.x,
+                    chunk.getFileId(),
+                    chunk.getMtime(),
+                    prevX - chunk.getOffset(),
+                    chunk.getOffset() == prevX && chunk.getSize() == prevX - startPoint.x,
+                    chunk.getCipherKey().toByteArray(),
+                    chunk.getIsCompressed()
+            ));
+        }
+    }
+
+    static class Point {
+        long x;
+        long ts;
+        FilerProto.FileChunk chunk;
+        boolean isStart;
+
+        public Point(long x, FilerProto.FileChunk chunk, boolean isStart) {
+            this.x = x;
+            this.ts = chunk.getMtime();
+            this.chunk = chunk;
+            this.isStart = isStart;
+        }
+    }
+
+}

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -226,96 +226,8 @@ public class SeaweedRead {
 
         chunkList = FileChunkManifest.resolveChunkManifest(filerClient, chunkList);
 
-        FilerProto.FileChunk[] chunks = chunkList.toArray(new FilerProto.FileChunk[0]);
-        Arrays.sort(chunks, new Comparator<FilerProto.FileChunk>() {
-            @Override
-            public int compare(FilerProto.FileChunk a, FilerProto.FileChunk b) {
-                // if just a.getMtime() - b.getMtime(), it will overflow!
-                if (a.getMtime() < b.getMtime()) {
-                    return -1;
-                } else if (a.getMtime() > b.getMtime()) {
-                    return 1;
-                }
-                return 0;
-            }
-        });
+        return ReadChunks.readResolvedChunks(chunkList);
 
-        List<VisibleInterval> visibles = new ArrayList<>();
-        for (FilerProto.FileChunk chunk : chunks) {
-            List<VisibleInterval> newVisibles = new ArrayList<>();
-            visibles = mergeIntoVisibles(visibles, newVisibles, chunk);
-        }
-
-        return visibles;
-    }
-
-    private static List<VisibleInterval> mergeIntoVisibles(List<VisibleInterval> visibles,
-                                                           List<VisibleInterval> newVisibles,
-                                                           FilerProto.FileChunk chunk) {
-        VisibleInterval newV = new VisibleInterval(
-                chunk.getOffset(),
-                chunk.getOffset() + chunk.getSize(),
-                chunk.getFileId(),
-                chunk.getMtime(),
-                0,
-                true,
-                chunk.getCipherKey().toByteArray(),
-                chunk.getIsCompressed()
-        );
-
-        // easy cases to speed up
-        if (visibles.size() == 0) {
-            visibles.add(newV);
-            return visibles;
-        }
-        if (visibles.get(visibles.size() - 1).stop <= chunk.getOffset()) {
-            visibles.add(newV);
-            return visibles;
-        }
-
-        for (VisibleInterval v : visibles) {
-            if (v.start < chunk.getOffset() && chunk.getOffset() < v.stop) {
-                newVisibles.add(new VisibleInterval(
-                        v.start,
-                        chunk.getOffset(),
-                        v.fileId,
-                        v.modifiedTime,
-                        v.chunkOffset,
-                        false,
-                        v.cipherKey,
-                        v.isCompressed
-                ));
-            }
-            long chunkStop = chunk.getOffset() + chunk.getSize();
-            if (v.start < chunkStop && chunkStop < v.stop) {
-                newVisibles.add(new VisibleInterval(
-                        chunkStop,
-                        v.stop,
-                        v.fileId,
-                        v.modifiedTime,
-                        v.chunkOffset + (chunkStop - v.start),
-                        false,
-                        v.cipherKey,
-                        v.isCompressed
-                ));
-            }
-            if (chunkStop <= v.start || v.stop <= chunk.getOffset()) {
-                newVisibles.add(v);
-            }
-        }
-        newVisibles.add(newV);
-
-        // keep everything sorted
-        for (int i = newVisibles.size() - 1; i >= 0; i--) {
-            if (i > 0 && newV.start < newVisibles.get(i - 1).start) {
-                newVisibles.set(i, newVisibles.get(i - 1));
-            } else {
-                newVisibles.set(i, newV);
-                break;
-            }
-        }
-
-        return newVisibles;
     }
 
     public static String parseVolumeId(String fileId) {

other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java

@@ -15,7 +15,7 @@ public class VolumeIdCache {
         }
         this.cache = CacheBuilder.newBuilder()
                 .maximumSize(maxEntries)
-                .expireAfterAccess(5, TimeUnit.MINUTES)
+                .expireAfterWrite(5, TimeUnit.MINUTES)
                 .build();
     }
 

other/java/client/src/main/proto/filer.proto

@@ -30,6 +30,8 @@ service SeaweedFiler {
 
     rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
     }
+    rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) {
+    }
 
     rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
     }
@@ -225,6 +227,18 @@ message AtomicRenameEntryRequest {
 message AtomicRenameEntryResponse {
 }
 
+message StreamRenameEntryRequest {
+    string old_directory = 1;
+    string old_name = 2;
+    string new_directory = 3;
+    string new_name = 4;
+    repeated int32 signatures = 5;
+}
+message StreamRenameEntryResponse {
+    string directory = 1;
+    EventNotification event_notification = 2;
+    int64 ts_ns = 3;
+}
 message AssignVolumeRequest {
     int32 count = 1;
     string collection = 2;

other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java

@@ -6,6 +6,7 @@ import org.junit.Test;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Random;
 
 public class SeaweedReadTest {
 
@@ -13,17 +14,17 @@ public class SeaweedReadTest {
     public void testNonOverlappingVisibleIntervals() throws IOException {
         List<FilerProto.FileChunk> chunks = new ArrayList<>();
         chunks.add(FilerProto.FileChunk.newBuilder()
-            .setFileId("aaa")
-            .setOffset(0)
-            .setSize(100)
-            .setMtime(1000)
-            .build());
+                .setFileId("aaa")
+                .setOffset(0)
+                .setSize(100)
+                .setMtime(1000)
+                .build());
         chunks.add(FilerProto.FileChunk.newBuilder()
-            .setFileId("bbb")
-            .setOffset(100)
-            .setSize(133)
-            .setMtime(2000)
-            .build());
+                .setFileId("bbb")
+                .setOffset(100)
+                .setSize(133)
+                .setMtime(2000)
+                .build());
 
         List<SeaweedRead.VisibleInterval> visibleIntervals = SeaweedRead.nonOverlappingVisibleIntervals(null, chunks);
         for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
@@ -61,4 +62,106 @@ public class SeaweedReadTest {
 
     }
 
+
+    @Test
+    public void testReadResolvedChunks() throws IOException {
+        List<FilerProto.FileChunk> chunks = new ArrayList<>();
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("a")
+                .setOffset(0)
+                .setSize(100)
+                .setMtime(1)
+                .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("b")
+                .setOffset(50)
+                .setSize(100)
+                .setMtime(2)
+                .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("c")
+                .setOffset(200)
+                .setSize(50)
+                .setMtime(3)
+                .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("d")
+                .setOffset(250)
+                .setSize(50)
+                .setMtime(4)
+                .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("e")
+                .setOffset(175)
+                .setSize(100)
+                .setMtime(5)
+                .build());
+
+        List<SeaweedRead.VisibleInterval> visibleIntervals = ReadChunks.readResolvedChunks(chunks);
+        for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
+            System.out.println("visible:" + visibleInterval);
+        }
+
+        Assert.assertEquals(4, visibleIntervals.size());
+
+        SeaweedRead.VisibleInterval visibleInterval = visibleIntervals.get(0);
+        Assert.assertEquals(visibleInterval.start, 0);
+        Assert.assertEquals(visibleInterval.stop, 50);
+        Assert.assertEquals(visibleInterval.modifiedTime, 1);
+        Assert.assertEquals(visibleInterval.fileId, "a");
+
+        visibleInterval = visibleIntervals.get(1);
+        Assert.assertEquals(visibleInterval.start, 50);
+        Assert.assertEquals(visibleInterval.stop, 150);
+        Assert.assertEquals(visibleInterval.modifiedTime, 2);
+        Assert.assertEquals(visibleInterval.fileId, "b");
+
+        visibleInterval = visibleIntervals.get(2);
+        Assert.assertEquals(visibleInterval.start, 175);
+        Assert.assertEquals(visibleInterval.stop, 275);
+        Assert.assertEquals(visibleInterval.modifiedTime, 5);
+        Assert.assertEquals(visibleInterval.fileId, "e");
+
+        visibleInterval = visibleIntervals.get(3);
+        Assert.assertEquals(visibleInterval.start, 275);
+        Assert.assertEquals(visibleInterval.stop, 300);
+        Assert.assertEquals(visibleInterval.modifiedTime, 4);
+        Assert.assertEquals(visibleInterval.fileId, "d");
+
+    }
+
+
+    @Test
+    public void testRandomizedReadResolvedChunks() throws IOException {
+        Random random = new Random();
+        int limit = 1024*1024;
+        long[] array = new long[limit];
+        List<FilerProto.FileChunk> chunks = new ArrayList<>();
+        for (long ts=0;ts<1024;ts++){
+            int x = random.nextInt(limit);
+            int y = random.nextInt(limit);
+            int size = Math.min(Math.abs(x-y), 1024);
+            chunks.add(randomWrite(array, Math.min(x,y), size, ts));
+        }
+
+        List<SeaweedRead.VisibleInterval> visibleIntervals = ReadChunks.readResolvedChunks(chunks);
+        for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
+            System.out.println("visible:" + visibleInterval);
+            for (int i = (int) visibleInterval.start; i<visibleInterval.stop; i++) {
+                Assert.assertEquals(array[i], visibleInterval.modifiedTime);
+            }
+        }
+
+    }
+    private FilerProto.FileChunk randomWrite(long[] array, int start, int size, long ts) {
+        for (int i=start;i<start+size;i++) {
+            array[i] = ts;
+        }
+        return FilerProto.FileChunk.newBuilder()
+                .setFileId("")
+                .setOffset(start)
+                .setSize(size)
+                .setMtime(ts)
+                .build();
+    }
 }

other/java/examples/pom.xml

@@ -11,13 +11,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>1.6.8</version>
+            <version>1.6.9</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.6.8</version>
+            <version>1.6.9</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

other/java/hdfs2/dependency-reduced-pom.xml

@@ -301,7 +301,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.9</seaweedfs.client.version>
     <hadoop.version>2.9.2</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.9</seaweedfs.client.version>
         <hadoop.version>2.9.2</hadoop.version>
     </properties>
 

other/java/hdfs3/dependency-reduced-pom.xml

@@ -309,7 +309,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.9</seaweedfs.client.version>
     <hadoop.version>3.1.1</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.9</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 

test/s3/basic/basic_test.go

@@ -2,14 +2,15 @@ package basic
 
 import (
 	"fmt"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
-	"io/ioutil"
-	"os"
-	"strings"
-	"testing"
 )
 
 var (
@@ -108,8 +109,8 @@ func TestListBucket(t *testing.T) {
 func TestListObjectV2(t *testing.T) {
 
 	listObj, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
-		Bucket: aws.String(Bucket),
-		Prefix: aws.String("foo"),
+		Bucket:    aws.String(Bucket),
+		Prefix:    aws.String("foo"),
 		Delimiter: aws.String("/"),
 	})
 	if err != nil {
@@ -169,7 +170,7 @@ func TestObjectOp(t *testing.T) {
 		exitErrorf("Unable to get copy object, %v", err)
 	}
 
-	data, err := ioutil.ReadAll(getObj.Body)
+	data, err := io.ReadAll(getObj.Body)
 	if err != nil {
 		exitErrorf("Unable to read object data, %v", err)
 	}

test/s3/multipart/aws_upload.go

@@ -19,10 +19,10 @@ import (
 const (
 	maxPartSize        = int64(5 * 1024 * 1024)
 	maxRetries         = 3
-	awsAccessKeyID     = "Your access key"
-	awsSecretAccessKey = "Your secret key"
-	awsBucketRegion    = "S3 bucket region"
-	awsBucketName      = "newBucket"
+	awsAccessKeyID     = "any"
+	awsSecretAccessKey = "any"
+	awsBucketRegion    = "us‑west‑1"
+	awsBucketName      = "bucket1"
 )
 
 var (
@@ -37,7 +37,7 @@ func main() {
 	if err != nil {
 		fmt.Printf("bad credentials: %s", err)
 	}
-	cfg := aws.NewConfig().WithRegion(awsBucketRegion).WithCredentials(creds).WithDisableSSL(true).WithEndpoint("localhost:8333")
+	cfg := aws.NewConfig().WithRegion(awsBucketRegion).WithCredentials(creds).WithDisableSSL(true).WithEndpoint("localhost:8333").WithS3ForcePathStyle(true)
 	svc := s3.New(session.New(), cfg)
 
 	file, err := os.Open(*filename)

unmaintained/stream_read_volume/stream_read_volume.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"io"
+)
+
+var (
+	volumeServer       = flag.String("volumeServer", "localhost:8080", "a volume server")
+	volumeId         = flag.Int("volumeId", -1, "a volume id to stream read")
+	grpcDialOption   grpc.DialOption
+)
+
+func main() {
+	flag.Parse()
+
+	util.LoadConfiguration("security", false)
+	grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	vid := uint32(*volumeId)
+
+	eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error {
+		fmt.Printf("%d,%x%08x %d\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob))
+		return nil
+	}
+
+	err := operation.WithVolumeServerClient(pb.ServerAddress(*volumeServer), grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		copyFileClient, err := vs.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
+			VolumeIds:           []uint32{vid},
+		})
+		if err != nil {
+			return err
+		}
+		for {
+			resp, err := copyFileClient.Recv()
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			if err = eachNeedleFunc(resp); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		fmt.Printf("read %s: %v\n", *volumeServer, err)
+	}
+
+}
+

unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go

@@ -5,7 +5,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"mime/multipart"
@@ -45,7 +44,7 @@ func main() {
 			defer wg.Done()
 
 			client := &http.Client{Transport: &http.Transport{
-				MaxIdleConns:     1024,
+				MaxIdleConns:        1024,
 				MaxIdleConnsPerHost: 1024,
 			}}
 			r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(x)))
@@ -128,7 +127,7 @@ func uploadFileToFiler(client *http.Client, data []byte, filename, destination s
 		if err != nil {
 			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
 		}
-		io.Copy(ioutil.Discard, resp.Body)
+		io.Copy(io.Discard, resp.Body)
 		resp.Body.Close()
 	}
 

unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go

@@ -5,7 +5,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"mime/multipart"
@@ -36,7 +35,7 @@ func main() {
 
 	var fileNames []string
 
-	files, err := ioutil.ReadDir(*dir)
+	files, err := os.ReadDir(*dir)
 	if err != nil {
 		log.Fatalf("fail to read dir %v: %v", *dir, err)
 	}
@@ -142,7 +141,7 @@ func uploadFileToFiler(client *http.Client, filename, destination string) (size
 		if err != nil {
 			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
 		}
-		io.Copy(ioutil.Discard, resp.Body)
+		io.Copy(io.Discard, resp.Body)
 		resp.Body.Close()
 	}
 

weed/command/download.go

@@ -2,17 +2,17 @@ package command
 
 import (
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/pb"
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"google.golang.org/grpc"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
 	"strings"
 
+	"google.golang.org/grpc"
+
 	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -82,7 +82,7 @@ func downloadToFile(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpti
 	}
 	defer f.Close()
 	if isFileList {
-		content, err := ioutil.ReadAll(rc.Body)
+		content, err := io.ReadAll(rc.Body)
 		if err != nil {
 			return err
 		}
@@ -119,7 +119,7 @@ func fetchContent(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption
 		return "", nil, e
 	}
 	defer util.CloseResponse(rc)
-	content, e = ioutil.ReadAll(rc.Body)
+	content, e = io.ReadAll(rc.Body)
 	return
 }
 

weed/command/filer.go

@@ -62,7 +62,7 @@ func init() {
 	f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
 	f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to")
 	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
-	f.portGrpc = cmdFiler.Flag.Int("port.grpc", 18888, "filer server grpc listen port")
+	f.portGrpc = cmdFiler.Flag.Int("port.grpc", 0, "filer server grpc listen port")
 	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
 	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
@@ -87,7 +87,7 @@ func init() {
 	filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
 	filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
-	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", false, "allow empty folders")
+	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
 
 	// start webdav on filer
 	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
@@ -180,6 +180,9 @@ func (fo *FilerOptions) startFiler() {
 	if *fo.publicPort != 0 {
 		publicVolumeMux = http.NewServeMux()
 	}
+	if *fo.portGrpc == 0 {
+		*fo.portGrpc = 10000 + *fo.port
+	}
 
 	defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")
 

weed/command/filer_copy.go

@@ -3,9 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filer"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -16,14 +14,14 @@ import (
 
 	"google.golang.org/grpc"
 
-	"github.com/chrislusf/seaweedfs/weed/util/grace"
-
+	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/grace"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )
 
@@ -212,7 +210,7 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
 	}
 
 	if mode.IsDir() {
-		files, _ := ioutil.ReadDir(fileOrDir)
+		files, _ := os.ReadDir(fileOrDir)
 		for _, subFileOrDir := range files {
 			cleanedDestDirectory := filepath.Clean(destPath + fi.Name())
 			if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), cleanedDestDirectory+"/", fileCopyTaskChan); err != nil {
@@ -339,14 +337,14 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 	if task.fileMode&os.ModeDir == 0 && task.fileSize > 0 {
 
 		mimeType = detectMimeType(f)
-		data, err := ioutil.ReadAll(f)
+		data, err := io.ReadAll(f)
 		if err != nil {
 			return err
 		}
 
-		// assign a volume
-		err = util.Retry("assignVolume", func() error {
-			return pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err = util.Retry("upload", func() error {
+			// assign a volume
+			assignErr := pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 				request := &filer_pb.AssignVolumeRequest{
 					Count:       1,
@@ -369,35 +367,41 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 				}
 				return nil
 			})
+			if assignErr != nil {
+				return assignErr
+			}
+
+			// upload data
+			targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId
+			uploadOption := &operation.UploadOption{
+				UploadUrl:         targetUrl,
+				Filename:          fileName,
+				Cipher:            worker.options.cipher,
+				IsInputCompressed: false,
+				MimeType:          mimeType,
+				PairMap:           nil,
+				Jwt:               security.EncodedJwt(assignResult.Auth),
+			}
+			uploadResult, err := operation.UploadData(data, uploadOption)
+			if err != nil {
+				return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
+			}
+			if uploadResult.Error != "" {
+				return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
+			}
+			if *worker.options.verbose {
+				fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
+			}
+
+			fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName)
+			chunks = append(chunks, uploadResult.ToPbFileChunk(assignResult.FileId, 0))
+
+			return nil
 		})
 		if err != nil {
-			return fmt.Errorf("Failed to assign from %v: %v\n", worker.options.masters, err)
+			return fmt.Errorf("upload %v: %v\n", fileName, err)
 		}
 
-		targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId
-		uploadOption := &operation.UploadOption{
-			UploadUrl:         targetUrl,
-			Filename:          fileName,
-			Cipher:            worker.options.cipher,
-			IsInputCompressed: false,
-			MimeType:          mimeType,
-			PairMap:           nil,
-			Jwt:               security.EncodedJwt(assignResult.Auth),
-		}
-		uploadResult, err := operation.UploadData(data, uploadOption)
-		if err != nil {
-			return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
-		}
-		if uploadResult.Error != "" {
-			return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
-		}
-		if *worker.options.verbose {
-			fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
-		}
-
-		chunks = append(chunks, uploadResult.ToPbFileChunk(assignResult.FileId, 0))
-
-		fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName)
 	}
 
 	if err := pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

weed/command/filer_meta_tail.go

@@ -71,12 +71,12 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
 	}
 
 	shouldPrint := func(resp *filer_pb.SubscribeMetadataResponse) bool {
-		if filterFunc == nil {
-			return true
-		}
 		if resp.EventNotification.OldEntry == nil && resp.EventNotification.NewEntry == nil {
 			return false
 		}
+		if filterFunc == nil {
+			return true
+		}
 		if resp.EventNotification.OldEntry != nil && filterFunc(resp.Directory, resp.EventNotification.OldEntry.Name) {
 			return true
 		}

weed/command/imports.go

@@ -29,6 +29,6 @@ import (
 	_ "github.com/chrislusf/seaweedfs/weed/filer/postgres2"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/redis"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/redis2"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/redis3"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/sqlite"
-	_ "github.com/chrislusf/seaweedfs/weed/filer/tikv"
 )

weed/command/master.go

@@ -47,7 +47,7 @@ type MasterOptions struct {
 func init() {
 	cmdMaster.Run = runMaster // break init cycle
 	m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
-	m.portGrpc = cmdMaster.Flag.Int("port.grpc", 19333, "grpc listen port")
+	m.portGrpc = cmdMaster.Flag.Int("port.grpc", 0, "grpc listen port")
 	m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address, also used as identifier")
 	m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to")
 	m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
@@ -113,6 +113,10 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 
 	backend.LoadConfiguration(util.GetViper())
 
+	if *masterOption.portGrpc == 0 {
+		*masterOption.portGrpc = 10000 + *masterOption.port
+	}
+
 	myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.portGrpc, *masterOption.peers)
 
 	r := mux.NewRouter()

weed/command/master_follower.go

@@ -23,7 +23,7 @@ var (
 func init() {
 	cmdMasterFollower.Run = runMasterFollower // break init cycle
 	mf.port = cmdMasterFollower.Flag.Int("port", 9334, "http listen port")
-	mf.portGrpc = cmdMasterFollower.Flag.Int("port.grpc", 19334, "grpc listen port")
+	mf.portGrpc = cmdMasterFollower.Flag.Int("port.grpc", 0, "grpc listen port")
 	mf.ipBind = cmdMasterFollower.Flag.String("ip.bind", "", "ip address to bind to")
 	mf.peers = cmdMasterFollower.Flag.String("masters", "localhost:9333", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
 
@@ -70,6 +70,10 @@ func runMasterFollower(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("master", false)
 
+	if *mf.portGrpc == 0 {
+		*mf.portGrpc = 10000 + *mf.port
+	}
+
 	startMasterFollower(mf)
 
 	return true

weed/command/s3.go

@@ -42,7 +42,7 @@ func init() {
 	s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
 	s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
 	s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
-	s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", false, "allow empty folders")
+	s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders")
 }
 
 var cmdS3 = &Command{

weed/command/scaffold.go

@@ -2,9 +2,10 @@ package command
 
 import (
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/command/scaffold"
-	"io/ioutil"
+	"os"
 	"path/filepath"
+
+	"github.com/chrislusf/seaweedfs/weed/command/scaffold"
 )
 
 func init() {
@@ -55,7 +56,7 @@ func runScaffold(cmd *Command, args []string) bool {
 	}
 
 	if *outputPath != "" {
-		ioutil.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
+		os.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
 	} else {
 		fmt.Println(content)
 	}

weed/command/scaffold/filer.toml

@@ -185,6 +185,28 @@ routeByLatency = false
 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
 superLargeDirectories = []
 
+[redis3] # beta
+enabled = false
+address = "localhost:6379"
+password = ""
+database = 0
+
+[redis_cluster3] # beta
+enabled = false
+addresses = [
+    "localhost:30001",
+    "localhost:30002",
+    "localhost:30003",
+    "localhost:30004",
+    "localhost:30005",
+    "localhost:30006",
+]
+password = ""
+# allows reads from slave servers or the master, but all writes still go to the master
+readOnly = false
+# automatically use the closest Redis server for reads
+routeByLatency = false
+
 [etcd]
 enabled = false
 servers = "localhost:2379"
@@ -230,11 +252,3 @@ location = "/tmp/"
 address = "localhost:6379"
 password = ""
 database = 1
-
-[tikv]
-enabled = false
-# If you have many pd address, use ',' split then:
-#   pdaddrs = "pdhost1:2379, pdhost2:2379, pdhost3:2379"
-pdaddrs = "localhost:2379"
-# Concurrency for TiKV delete range
-deleterange_concurrency = 1

weed/command/scaffold/master.toml

@@ -23,10 +23,7 @@ default = "localhost:8888"    # used by maintenance scripts if the scripts needs
 
 
 [master.sequencer]
-type = "raft"     # Choose [raft|etcd|snowflake] type for storing the file id sequence
-# when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
-# example : http://127.0.0.1:2379,http://127.0.0.1:2389
-sequencer_etcd_urls = "http://127.0.0.1:2379"
+type = "raft"     # Choose [raft|snowflake] type for storing the file id sequence
 # when sequencer.type = snowflake, the snowflake id must be different from other masters
 sequencer_snowflake_id = 0     # any number between 1~1023
 

weed/command/server.go

@@ -86,7 +86,7 @@ func init() {
 	serverOptions.debugPort = cmdServer.Flag.Int("debug.port", 6060, "http port for debugging")
 
 	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
-	masterOptions.portGrpc = cmdServer.Flag.Int("master.port.grpc", 19333, "master server grpc listen port")
+	masterOptions.portGrpc = cmdServer.Flag.Int("master.port.grpc", 0, "master server grpc listen port")
 	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
 	masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
 	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
@@ -99,7 +99,7 @@ func init() {
 
 	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
-	filerOptions.portGrpc = cmdServer.Flag.Int("filer.port.grpc", 18888, "filer server grpc listen port")
+	filerOptions.portGrpc = cmdServer.Flag.Int("filer.port.grpc", 0, "filer server grpc listen port")
 	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
 	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
@@ -111,7 +111,7 @@ func init() {
 	filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
-	serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 18080, "volume server grpc listen port")
+	serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
 	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
 	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	serverOptions.v.diskType = cmdServer.Flag.String("volume.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
@@ -132,7 +132,7 @@ func init() {
 	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
 	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
-	s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", false, "allow empty folders")
+	s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
 
 	webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port")
 	webdavOptions.collection = cmdServer.Flag.String("webdav.collection", "", "collection to create the files")
@@ -168,16 +168,21 @@ func runServer(cmd *Command, args []string) bool {
 		*isStartingFiler = true
 	}
 
+	if *isStartingMasterServer {
+		_, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.portGrpc, *masterOptions.peers)
+		peers := strings.Join(pb.ToAddressStrings(peerList), ",")
+		masterOptions.peers = &peers
+	}
+
 	// ip address
 	masterOptions.ip = serverIp
 	masterOptions.ipBind = serverBindIp
-	_, masters := checkPeers(*masterOptions.ip, *masterOptions.port, *masterOptions.portGrpc, *masterOptions.peers)
-	filerOptions.masters = masters
+	filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
 	filerOptions.ip = serverIp
 	filerOptions.bindIp = serverBindIp
 	serverOptions.v.ip = serverIp
 	serverOptions.v.bindIp = serverBindIp
-	serverOptions.v.masters = masters
+	serverOptions.v.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
 	serverOptions.v.idleConnectionTimeout = serverTimeout
 	serverOptions.v.dataCenter = serverDataCenter
 	serverOptions.v.rack = serverRack

weed/command/volume.go

@@ -70,7 +70,7 @@ type VolumeServerOptions struct {
 func init() {
 	cmdVolume.Run = runVolume // break init cycle
 	v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
-	v.portGrpc = cmdVolume.Flag.Int("port.grpc", 18080, "grpc listen port")
+	v.portGrpc = cmdVolume.Flag.Int("port.grpc", 0, "grpc listen port")
 	v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
 	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
 	v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
@@ -197,6 +197,9 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	if *v.publicPort == 0 {
 		*v.publicPort = *v.port
 	}
+	if *v.portGrpc == 0 {
+		*v.portGrpc = 10000 + *v.port
+	}
 	if *v.publicUrl == "" {
 		*v.publicUrl = util.JoinHostPort(*v.ip, *v.publicPort)
 	}

weed/filer/filechunks.go

@@ -225,6 +225,12 @@ func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunction
 
 	chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks, startOffset, stopOffset)
 
+	visibles2 := readResolvedChunks(chunks)
+
+	if true {
+		return visibles2, err
+	}
+
 	sort.Slice(chunks, func(i, j int) bool {
 		if chunks[i].Mtime == chunks[j].Mtime {
 			filer_pb.EnsureFid(chunks[i])
@@ -246,9 +252,26 @@ func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunction
 
 	}
 
+	if len(visibles) != len(visibles2) {
+		fmt.Printf("different visibles size %d : %d\n", len(visibles), len(visibles2))
+	} else {
+		for i := 0; i < len(visibles); i++ {
+			checkDifference(visibles[i], visibles2[i])
+		}
+	}
+
 	return
 }
 
+func checkDifference(x, y VisibleInterval) {
+	if x.start != y.start ||
+		x.stop != y.stop ||
+		x.fileId != y.fileId ||
+		x.modifiedTime != y.modifiedTime {
+		fmt.Printf("different visible %+v : %+v\n", x, y)
+	}
+}
+
 // find non-overlapping visible intervals
 // visible interval map to one file chunk
 

weed/filer/filechunks_read.go

@@ -0,0 +1,119 @@
+package filer
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"sort"
+)
+
+func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) {
+
+	var points []*Point
+	for _, chunk := range chunks {
+		points = append(points, &Point{
+			x:       chunk.Offset,
+			ts:      chunk.Mtime,
+			chunk:   chunk,
+			isStart: true,
+		})
+		points = append(points, &Point{
+			x:       chunk.Offset + int64(chunk.Size),
+			ts:      chunk.Mtime,
+			chunk:   chunk,
+			isStart: false,
+		})
+	}
+	sort.Slice(points, func(i, j int) bool {
+		if points[i].x != points[j].x {
+			return points[i].x < points[j].x
+		}
+		if points[i].ts != points[j].ts {
+			return points[i].ts < points[j].ts
+		}
+		if !points[i].isStart {
+			return true
+		}
+		return false
+	})
+
+	var prevX int64
+	var queue []*Point
+	for _, point := range points {
+		if point.isStart {
+			if len(queue) > 0 {
+				lastIndex := len(queue) -1
+				lastPoint := queue[lastIndex]
+				if point.x != prevX && lastPoint.ts < point.ts {
+					visibles = addToVisibles(visibles, prevX, lastPoint, point)
+					prevX = point.x
+				}
+			}
+			// insert into queue
+			for i := len(queue); i >= 0; i-- {
+				if i == 0 || queue[i-1].ts <= point.ts {
+					if i == len(queue) {
+						prevX = point.x
+					}
+					queue = addToQueue(queue, i, point)
+					break
+				}
+			}
+		} else {
+			lastIndex := len(queue) - 1
+			index := lastIndex
+			var startPoint *Point
+			for ; index >= 0; index-- {
+				startPoint = queue[index]
+				if startPoint.ts == point.ts {
+					queue = removeFromQueue(queue, index)
+					break
+				}
+			}
+			if index == lastIndex && startPoint != nil {
+				visibles = addToVisibles(visibles, prevX, startPoint, point)
+				prevX = point.x
+			}
+		}
+	}
+
+	return
+}
+
+func removeFromQueue(queue []*Point, index int) []*Point {
+	for i := index; i < len(queue)-1; i++ {
+		queue[i] = queue[i+1]
+	}
+	queue = queue[:len(queue)-1]
+	return queue
+}
+
+func addToQueue(queue []*Point, index int, point *Point) []*Point {
+	queue = append(queue, point)
+	for i := len(queue) - 1; i > index; i-- {
+		queue[i], queue[i-1] = queue[i-1], queue[i]
+	}
+	return queue
+}
+
+func addToVisibles(visibles []VisibleInterval, prevX int64, startPoint *Point, point *Point) []VisibleInterval {
+	if prevX < point.x {
+		chunk := startPoint.chunk
+		visibles = append(visibles, VisibleInterval{
+			start:        prevX,
+			stop:         point.x,
+			fileId:       chunk.GetFileIdString(),
+			modifiedTime: chunk.Mtime,
+			chunkOffset:  prevX - chunk.Offset,
+			chunkSize:    chunk.Size,
+			cipherKey:    chunk.CipherKey,
+			isGzipped:    chunk.IsCompressed,
+		})
+	}
+	return visibles
+}
+
+type Point struct {
+	x       int64
+	ts      int64
+	chunk   *filer_pb.FileChunk
+	isStart bool
+}

weed/filer/filechunks_read_test.go

@@ -0,0 +1,210 @@
+package filer
+
+import (
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"math/rand"
+	"testing"
+)
+
+func TestReadResolvedChunks(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: "a",
+			Offset: 0,
+			Size:   100,
+			Mtime:  1,
+		},
+		{
+			FileId: "b",
+			Offset: 50,
+			Size:   100,
+			Mtime:  2,
+		},
+		{
+			FileId: "c",
+			Offset: 200,
+			Size:   50,
+			Mtime:  3,
+		},
+		{
+			FileId: "d",
+			Offset: 250,
+			Size:   50,
+			Mtime:  4,
+		},
+		{
+			FileId: "e",
+			Offset: 175,
+			Size:   100,
+			Mtime:  5,
+		},
+	}
+
+	visibles := readResolvedChunks(chunks)
+
+	for _, visible := range visibles {
+		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTime)
+	}
+
+}
+
+func TestRandomizedReadResolvedChunks(t *testing.T) {
+
+	var limit int64 = 1024*1024
+	array := make([]int64, limit)
+	var chunks []*filer_pb.FileChunk
+	for ts := int64(0); ts < 1024; ts++ {
+		x := rand.Int63n(limit)
+		y := rand.Int63n(limit)
+		size := x - y
+		if size < 0 {
+			size = -size
+		}
+		if size > 1024 {
+			size = 1024
+		}
+		start := x
+		if start > y {
+			start = y
+		}
+		chunks = append(chunks, randomWrite(array, start, size, ts))
+	}
+
+	visibles := readResolvedChunks(chunks)
+
+	for _, visible := range visibles {
+		for i := visible.start; i<visible.stop;i++{
+			if array[i] != visible.modifiedTime {
+				t.Errorf("position %d expected ts %d actual ts %d", i, array[i], visible.modifiedTime)
+			}
+		}
+	}
+
+	// fmt.Printf("visibles %d", len(visibles))
+
+}
+
+func randomWrite(array []int64, start int64, size int64, ts int64) *filer_pb.FileChunk {
+	for i := start; i < start+size; i++ {
+		array[i] = ts
+	}
+	// fmt.Printf("write [%d,%d) %d\n", start, start+size, ts)
+	return &filer_pb.FileChunk{
+		FileId: "",
+		Offset: start,
+		Size:   uint64(size),
+		Mtime:  ts,
+	}
+}
+
+func TestSequentialReadResolvedChunks(t *testing.T) {
+
+	var chunkSize int64 = 1024*1024*2
+	var chunks []*filer_pb.FileChunk
+	for ts := int64(0); ts < 13; ts++ {
+		chunks = append(chunks, &filer_pb.FileChunk{
+			FileId: "",
+			Offset: chunkSize*ts,
+			Size:   uint64(chunkSize),
+			Mtime:  1,
+		})
+	}
+
+	visibles := readResolvedChunks(chunks)
+
+	fmt.Printf("visibles %d", len(visibles))
+
+}
+
+func TestActualReadResolvedChunks(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: "5,e7b96fef48",
+			Offset: 0,
+			Size:   2097152,
+			Mtime:  1634447487595823000,
+		},
+		{
+			FileId: "5,e5562640b9",
+			Offset: 2097152,
+			Size:   2097152,
+			Mtime:  1634447487595826000,
+		},
+		{
+			FileId: "5,df033e0fe4",
+			Offset: 4194304,
+			Size:   2097152,
+			Mtime:  1634447487595827000,
+		},
+		{
+			FileId: "7,eb08148a9b",
+			Offset: 6291456,
+			Size:   2097152,
+			Mtime:  1634447487595827000,
+		},
+		{
+			FileId: "7,e0f92d1604",
+			Offset: 8388608,
+			Size:   2097152,
+			Mtime:  1634447487595828000,
+		},
+		{
+			FileId: "7,e33cb63262",
+			Offset: 10485760,
+			Size:   2097152,
+			Mtime:  1634447487595828000,
+		},
+		{
+			FileId: "5,ea98e40e93",
+			Offset: 12582912,
+			Size:   2097152,
+			Mtime:  1634447487595829000,
+		},
+		{
+			FileId: "5,e165661172",
+			Offset: 14680064,
+			Size:   2097152,
+			Mtime:  1634447487595829000,
+		},
+		{
+			FileId: "3,e692097486",
+			Offset: 16777216,
+			Size:   2097152,
+			Mtime:  1634447487595830000,
+		},
+		{
+			FileId: "3,e28e2e3cbd",
+			Offset: 18874368,
+			Size:   2097152,
+			Mtime:  1634447487595830000,
+		},
+		{
+			FileId: "3,e443974d4e",
+			Offset: 20971520,
+			Size:   2097152,
+			Mtime:  1634447487595830000,
+		},
+		{
+			FileId: "2,e815bed597",
+			Offset: 23068672,
+			Size:   2097152,
+			Mtime:  1634447487595831000,
+		},
+		{
+			FileId: "5,e94715199e",
+			Offset: 25165824,
+			Size:   1974736,
+			Mtime:  1634447487595832000,
+		},
+	}
+
+	visibles := readResolvedChunks(chunks)
+
+	for _, visible := range visibles {
+		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTime)
+	}
+
+}

weed/filer/filer.go

@@ -44,6 +44,7 @@ type Filer struct {
 	Signature           int32
 	FilerConf           *FilerConf
 	RemoteStorage       *FilerRemoteStorage
+	UniqueFileId        uint32
 }
 
 func NewFiler(masters []pb.ServerAddress, grpcDialOption grpc.DialOption,
@@ -54,6 +55,7 @@ func NewFiler(masters []pb.ServerAddress, grpcDialOption grpc.DialOption,
 		GrpcDialOption:      grpcDialOption,
 		FilerConf:           NewFilerConf(),
 		RemoteStorage:       NewFilerRemoteStorage(),
+		UniqueFileId:        uint32(util.RandomInt32()),
 	}
 	f.LocalMetaLogBuffer = log_buffer.NewLogBuffer("local", LogFlushInterval, f.logFlushFunc, notifyFn)
 	f.metaLogCollection = collection

weed/filer/filer_conf.go

@@ -3,6 +3,10 @@ package filer
 import (
 	"bytes"
 	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
+	"google.golang.org/grpc"
 	"io"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -26,6 +30,29 @@ type FilerConf struct {
 	rules ptrie.Trie
 }
 
+func ReadFilerConf(filerGrpcAddress pb.ServerAddress, grpcDialOption grpc.DialOption, masterClient *wdclient.MasterClient) (*FilerConf, error) {
+	var buf bytes.Buffer
+	if err := pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		if masterClient != nil {
+			return ReadEntry(masterClient, client, DirectoryEtcSeaweedFS, FilerConfName, &buf)
+		} else {
+			content, err := ReadInsideFiler(client, DirectoryEtcSeaweedFS, FilerConfName)
+			buf = *bytes.NewBuffer(content)
+			return err
+		}
+	}); err != nil && err != filer_pb.ErrNotFound {
+		return nil, fmt.Errorf("read %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err)
+	}
+
+	fc := NewFilerConf()
+	if buf.Len() > 0 {
+		if err := fc.LoadFromBytes(buf.Bytes()); err != nil {
+			return nil, fmt.Errorf("parse %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err)
+		}
+	}
+	return fc, nil
+}
+
 func NewFilerConf() (fc *FilerConf) {
 	fc = &FilerConf{
 		rules: ptrie.New(),
@@ -115,6 +142,18 @@ func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf
 	return pathConf
 }
 
+func (fc *FilerConf) GetCollectionTtls(collection string) (ttls map[string]string) {
+	ttls = make(map[string]string)
+	fc.rules.Walk(func(key []byte, value interface{}) bool {
+		t := value.(*filer_pb.FilerConf_PathConf)
+		if t.Collection == collection {
+			ttls[t.LocationPrefix] = t.GetTtl()
+		}
+		return true
+	})
+	return ttls
+}
+
 // merge if values in b is not empty, merge them into a
 func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
 	a.Collection = util.Nvl(b.Collection, a.Collection)

weed/filer/filer_notify.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math"
 	"strings"
 	"time"
 
@@ -92,8 +93,8 @@ func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {
 
 	startTime, stopTime = startTime.UTC(), stopTime.UTC()
 
-	targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.segment", SystemLogDir,
-		startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
+	targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.%08x", SystemLogDir,
+		startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(), f.UniqueFileId,
 		// startTime.Second(), startTime.Nanosecond(),
 	)
 
@@ -111,7 +112,7 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(
 
 	startTime = startTime.UTC()
 	startDate := fmt.Sprintf("%04d-%02d-%02d", startTime.Year(), startTime.Month(), startTime.Day())
-	startHourMinute := fmt.Sprintf("%02d-%02d.segment", startTime.Hour(), startTime.Minute())
+	startHourMinute := fmt.Sprintf("%02d-%02d", startTime.Hour(), startTime.Minute())
 
 	sizeBuf := make([]byte, 4)
 	startTsNs := startTime.UnixNano()
@@ -122,14 +123,15 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(
 	}
 	for _, dayEntry := range dayEntries {
 		// println("checking day", dayEntry.FullPath)
-		hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "", "", "")
+		hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, math.MaxInt32, "", "", "")
 		if listHourMinuteErr != nil {
 			return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr)
 		}
 		for _, hourMinuteEntry := range hourMinuteEntries {
 			// println("checking hh-mm", hourMinuteEntry.FullPath)
 			if dayEntry.Name() == startDate {
-				if strings.Compare(hourMinuteEntry.Name(), startHourMinute) < 0 {
+				hourMinute := util.FileNameBase(hourMinuteEntry.Name())
+				if strings.Compare(hourMinute, startHourMinute) < 0 {
 					continue
 				}
 			}

weed/filer/filer_notify_append.go

@@ -33,6 +33,8 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error {
 				Gid:    OS_GID,
 			},
 		}
+	} else if err != nil {
+		return fmt.Errorf("find %s: %v", fullpath, err)
 	} else {
 		offset = int64(TotalSize(entry.Chunks))
 	}

weed/filer/filer_search.go

@@ -23,15 +23,15 @@ func splitPattern(pattern string) (prefix string, restPattern string) {
 // For now, prefix and namePattern are mutually exclusive
 func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string) (entries []*Entry, hasMore bool, err error) {
 
+	if limit > math.MaxInt32-1 {
+		limit = math.MaxInt32 - 1
+	}
+
 	_, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, prefix, namePattern, namePatternExclude, func(entry *Entry) bool {
 		entries = append(entries, entry)
 		return true
 	})
 
-	if limit == math.MaxInt64 {
-		limit = math.MaxInt64 - 1
-	}
-
 	hasMore = int64(len(entries)) >= limit+1
 	if hasMore {
 		entries = entries[:limit]

weed/filer/filerstore_translate_path.go

@@ -3,6 +3,7 @@ package filer
 import (
 	"context"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"math"
 	"strings"
 )
 
@@ -120,6 +121,10 @@ func (t *FilerStorePathTranlator) ListDirectoryPrefixedEntries(ctx context.Conte
 
 	newFullPath := t.translatePath(dirPath)
 
+	if limit > math.MaxInt32-1 {
+		limit = math.MaxInt32 - 1
+	}
+
 	return t.actualStore.ListDirectoryPrefixedEntries(ctx, newFullPath, startFileName, includeStartFile, limit, prefix, func(entry *Entry) bool {
 		entry.FullPath = dirPath[:len(t.storeRoot)-1] + entry.FullPath
 		return eachEntryFunc(entry)

weed/filer/filerstore_wrapper.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/viant/ptrie"
+	"math"
 	"strings"
 	"time"
 
@@ -248,14 +249,18 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context,
 	defer func() {
 		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "prefixList").Observe(time.Since(start).Seconds())
 	}()
+	if limit > math.MaxInt32-1 {
+		limit = math.MaxInt32 - 1
+	}
 	glog.V(4).Infof("ListDirectoryPrefixedEntries %s from %s prefix %s limit %d", dirPath, startFileName, prefix, limit)
-	lastFileName, err = actualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, eachEntryFunc)
+	adjustedEntryFunc := func(entry *Entry) bool {
+		fsw.maybeReadHardLink(ctx, entry)
+		filer_pb.AfterEntryDeserialization(entry.Chunks)
+		return eachEntryFunc(entry)
+	}
+	lastFileName, err = actualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, adjustedEntryFunc)
 	if err == ErrUnsupportedListDirectoryPrefixed {
-		lastFileName, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, func(entry *Entry) bool {
-			fsw.maybeReadHardLink(ctx, entry)
-			filer_pb.AfterEntryDeserialization(entry.Chunks)
-			return eachEntryFunc(entry)
-		})
+		lastFileName, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, adjustedEntryFunc)
 	}
 	return lastFileName, err
 }

weed/filer/leveldb/leveldb_store_test.go

@@ -3,7 +3,6 @@ package leveldb
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"testing"
 	"time"
@@ -14,7 +13,7 @@ import (
 
 func TestCreateAndFind(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
 	store.initialize(dir)
@@ -68,7 +67,7 @@ func TestCreateAndFind(t *testing.T) {
 
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
 	store.initialize(dir)
@@ -91,7 +90,7 @@ func TestEmptyRoot(t *testing.T) {
 
 func BenchmarkInsertEntry(b *testing.B) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_bench")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_bench")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
 	store.initialize(dir)

weed/filer/leveldb2/leveldb2_store_test.go

@@ -2,7 +2,6 @@ package leveldb
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -12,7 +11,7 @@ import (
 
 func TestCreateAndFind(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDB2Store{}
 	store.initialize(dir, 2)
@@ -66,7 +65,7 @@ func TestCreateAndFind(t *testing.T) {
 
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDB2Store{}
 	store.initialize(dir, 2)

weed/filer/leveldb3/leveldb3_store_test.go

@@ -2,7 +2,6 @@ package leveldb
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -12,7 +11,7 @@ import (
 
 func TestCreateAndFind(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDB3Store{}
 	store.initialize(dir)
@@ -66,7 +65,7 @@ func TestCreateAndFind(t *testing.T) {
 
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDB3Store{}
 	store.initialize(dir)

weed/filer/meta_aggregator.go

@@ -110,7 +110,7 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self pb.ServerAddress, p
 		}
 		dir := event.Directory
 		// println("received meta change", dir, "size", len(data))
-		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, 0)
+		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
 		if maybeReplicateMetadataChange != nil {
 			maybeReplicateMetadataChange(event)
 		}

weed/filer/redis3/ItemList.go

@@ -0,0 +1,507 @@
+package redis3
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/util/skiplist"
+	"github.com/go-redis/redis/v8"
+)
+
+type ItemList struct {
+	skipList  *skiplist.SkipList
+	batchSize int
+	client    redis.UniversalClient
+	prefix    string
+}
+
+func newItemList(client redis.UniversalClient, prefix string, store skiplist.ListStore, batchSize int) *ItemList {
+	return &ItemList{
+		skipList:  skiplist.New(store),
+		batchSize: batchSize,
+		client:    client,
+		prefix:    prefix,
+	}
+}
+
+/*
+Be reluctant to create new nodes. Try to fit into either previous node or next node.
+Prefer to add to previous node.
+
+There are multiple cases after finding the name for greater or equal node
+ 1. found and node.Key == name
+    The node contains a batch with leading key the same as the name
+    nothing to do
+ 2. no such node found or node.Key > name
+
+    if no such node found
+      prevNode = list.LargestNode
+
+	// case 2.1
+    if previousNode contains name
+      nothing to do
+
+    // prefer to add to previous node
+    if prevNode != nil {
+		// case 2.2
+		if prevNode has capacity
+			prevNode.add name, and save
+			return
+		// case 2.3
+		split prevNode by name
+    }
+
+	// case 2.4
+	// merge into next node. Avoid too many nodes if adding data in reverse order.
+	if nextNode is not nil and nextNode has capacity
+	  delete nextNode.Key
+      nextNode.Key = name
+      nextNode.batch.add name
+      insert nodeNode.Key
+	  return
+
+	// case 2.5
+    if prevNode is nil
+      insert new node with key = name, value = batch{name}
+      return
+
+*/
+
+func (nl *ItemList) canAddMember(node *skiplist.SkipListElementReference, name string) (alreadyContains bool, nodeSize int, err error) {
+	ctx := context.Background()
+	pipe := nl.client.TxPipeline()
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	countOperation := pipe.ZLexCount(ctx, key, "-", "+")
+	scoreOperationt := pipe.ZScore(ctx, key, name)
+	if _, err = pipe.Exec(ctx); err != nil && err != redis.Nil {
+		return false, 0, err
+	}
+	if err == redis.Nil {
+		err = nil
+	}
+	alreadyContains = scoreOperationt.Err() == nil
+	nodeSize = int(countOperation.Val())
+	return
+}
+
+func (nl *ItemList) WriteName(name string) error {
+
+	lookupKey := []byte(name)
+	prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey)
+	if err != nil {
+		return err
+	}
+	// case 1: the name already exists as one leading key in the batch
+	if found && bytes.Compare(nextNode.Key, lookupKey) == 0 {
+		return nil
+	}
+
+	var prevNodeReference *skiplist.SkipListElementReference
+	if !found {
+		prevNodeReference = nl.skipList.GetLargestNodeReference()
+	}
+
+	if nextNode != nil && prevNode == nil {
+		prevNodeReference = nextNode.Prev
+	}
+
+	if prevNodeReference != nil {
+		alreadyContains, nodeSize, err := nl.canAddMember(prevNodeReference, name)
+		if err != nil {
+			return err
+		}
+		if alreadyContains {
+			// case 2.1
+			return nil
+		}
+
+		// case 2.2
+		if nodeSize < nl.batchSize {
+			return nl.NodeAddMember(prevNodeReference, name)
+		}
+
+		// case 2.3
+		x := nl.NodeInnerPosition(prevNodeReference, name)
+		y := nodeSize - x
+		addToX := x <= y
+		// add to a new node
+		if x == 0 || y == 0 {
+			if err := nl.ItemAdd(lookupKey, 0, name); err != nil {
+				return err
+			}
+			return nil
+		}
+		if addToX {
+			// collect names before name, add them to X
+			namesToX, err := nl.NodeRangeBeforeExclusive(prevNodeReference, name)
+			if err != nil {
+				return nil
+			}
+			// delete skiplist reference to old node
+			if _, err := nl.skipList.DeleteByKey(prevNodeReference.Key); err != nil {
+				return err
+			}
+			// add namesToY and name to a new X
+			namesToX = append(namesToX, name)
+			if err := nl.ItemAdd([]byte(namesToX[0]), 0, namesToX...); err != nil {
+				return nil
+			}
+			// remove names less than name from current Y
+			if err := nl.NodeDeleteBeforeExclusive(prevNodeReference, name); err != nil {
+				return nil
+			}
+
+			// point skip list to current Y
+			if err := nl.ItemAdd(lookupKey, prevNodeReference.ElementPointer); err != nil {
+				return nil
+			}
+			return nil
+		} else {
+			// collect names after name, add them to Y
+			namesToY, err := nl.NodeRangeAfterExclusive(prevNodeReference, name)
+			if err != nil {
+				return nil
+			}
+			// add namesToY and name to a new Y
+			namesToY = append(namesToY, name)
+			if err := nl.ItemAdd(lookupKey, 0, namesToY...); err != nil {
+				return nil
+			}
+			// remove names after name from current X
+			if err := nl.NodeDeleteAfterExclusive(prevNodeReference, name); err != nil {
+				return nil
+			}
+			return nil
+		}
+
+	}
+
+	// case 2.4
+	if nextNode != nil {
+		nodeSize := nl.NodeSize(nextNode.Reference())
+		if nodeSize < nl.batchSize {
+			if id, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil {
+				return err
+			} else {
+				if err := nl.ItemAdd(lookupKey, id, name); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+	}
+
+	// case 2.5
+	// now prevNode is nil
+	return nl.ItemAdd(lookupKey, 0, name)
+}
+
+/*
+// case 1: exists in nextNode
+if nextNode != nil && nextNode.Key == name {
+	remove from nextNode, update nextNode
+	// TODO: merge with prevNode if possible?
+	return
+}
+if nextNode is nil
+	prevNode = list.Largestnode
+if prevNode == nil and nextNode.Prev != nil
+	prevNode = load(nextNode.Prev)
+
+// case 2: does not exist
+// case 2.1
+if prevNode == nil {
+	return
+}
+// case 2.2
+if prevNameBatch does not contain name {
+	return
+}
+
+// case 3
+delete from prevNameBatch
+if prevNameBatch + nextNode < capacityList
+	// case 3.1
+	merge
+else
+	// case 3.2
+	update prevNode
+
+
+*/
+func (nl *ItemList) DeleteName(name string) error {
+	lookupKey := []byte(name)
+	prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey)
+	if err != nil {
+		return err
+	}
+
+	// case 1
+	if found && bytes.Compare(nextNode.Key, lookupKey) == 0 {
+		if _, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil {
+			return err
+		}
+		if err := nl.NodeDeleteMember(nextNode.Reference(), name); err != nil {
+			return err
+		}
+		minName := nl.NodeMin(nextNode.Reference())
+		if minName == "" {
+			return nl.NodeDelete(nextNode.Reference())
+		}
+		return nl.ItemAdd([]byte(minName), nextNode.Id)
+	}
+
+	if !found {
+		prevNode, err = nl.skipList.GetLargestNode()
+		if err != nil {
+			return err
+		}
+	}
+
+	if nextNode != nil && prevNode == nil {
+		prevNode, err = nl.skipList.LoadElement(nextNode.Prev)
+		if err != nil {
+			return err
+		}
+	}
+
+	// case 2
+	if prevNode == nil {
+		// case 2.1
+		return nil
+	}
+	if !nl.NodeContainsItem(prevNode.Reference(), name) {
+		return nil
+	}
+
+	// case 3
+	if err := nl.NodeDeleteMember(prevNode.Reference(), name); err != nil {
+		return err
+	}
+	prevSize := nl.NodeSize(prevNode.Reference())
+	if prevSize == 0 {
+		if _, err := nl.skipList.DeleteByKey(prevNode.Key); err != nil {
+			return err
+		}
+		return nil
+	}
+	nextSize := nl.NodeSize(nextNode.Reference())
+	if nextSize > 0 && prevSize+nextSize < nl.batchSize {
+		// case 3.1 merge nextNode and prevNode
+		if _, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil {
+			return err
+		}
+		nextNames, err := nl.NodeRangeBeforeExclusive(nextNode.Reference(), "")
+		if err != nil {
+			return err
+		}
+		if err := nl.NodeAddMember(prevNode.Reference(), nextNames...); err != nil {
+			return err
+		}
+		return nl.NodeDelete(nextNode.Reference())
+	} else {
+		// case 3.2 update prevNode
+		// no action to take
+		return nil
+	}
+
+	return nil
+}
+
+func (nl *ItemList) ListNames(startFrom string, visitNamesFn func(name string) bool) error {
+	lookupKey := []byte(startFrom)
+	prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey)
+	if err != nil {
+		return err
+	}
+	if found && bytes.Compare(nextNode.Key, lookupKey) == 0 {
+		prevNode = nil
+	}
+	if !found {
+		prevNode, err = nl.skipList.GetLargestNode()
+		if err != nil {
+			return err
+		}
+	}
+
+	if prevNode != nil {
+		if !nl.NodeScanIncluseiveAfter(prevNode.Reference(), startFrom, visitNamesFn) {
+			return nil
+		}
+	}
+
+	for nextNode != nil {
+		if !nl.NodeScanIncluseiveAfter(nextNode.Reference(), startFrom, visitNamesFn) {
+			return nil
+		}
+		nextNode, err = nl.skipList.LoadElement(nextNode.Next[0])
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (nl *ItemList) RemoteAllListElement() error {
+
+	t := nl.skipList
+
+	nodeRef := t.StartLevels[0]
+	for nodeRef != nil {
+		node, err := t.LoadElement(nodeRef)
+		if err != nil {
+			return err
+		}
+		if node == nil {
+			return nil
+		}
+		if err := t.DeleteElement(node); err != nil {
+			return err
+		}
+		if err := nl.NodeDelete(node.Reference()); err != nil {
+			return err
+		}
+		nodeRef = node.Next[0]
+	}
+	return nil
+
+}
+
+func (nl *ItemList) NodeContainsItem(node *skiplist.SkipListElementReference, item string) bool {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	_, err := nl.client.ZScore(context.Background(), key, item).Result()
+	if err == redis.Nil {
+		return false
+	}
+	if err == nil {
+		return true
+	}
+	return false
+}
+
+func (nl *ItemList) NodeSize(node *skiplist.SkipListElementReference) int {
+	if node == nil {
+		return 0
+	}
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	return int(nl.client.ZLexCount(context.Background(), key, "-", "+").Val())
+}
+
+func (nl *ItemList) NodeAddMember(node *skiplist.SkipListElementReference, names ...string) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	var members []*redis.Z
+	for _, name := range names {
+		members = append(members, &redis.Z{
+			Score:  0,
+			Member: name,
+		})
+	}
+	return nl.client.ZAddNX(context.Background(), key, members...).Err()
+}
+func (nl *ItemList) NodeDeleteMember(node *skiplist.SkipListElementReference, name string) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	return nl.client.ZRem(context.Background(), key, name).Err()
+}
+
+func (nl *ItemList) NodeDelete(node *skiplist.SkipListElementReference) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	return nl.client.Del(context.Background(), key).Err()
+}
+
+func (nl *ItemList) NodeInnerPosition(node *skiplist.SkipListElementReference, name string) int {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	return int(nl.client.ZLexCount(context.Background(), key, "-", "("+name).Val())
+}
+
+func (nl *ItemList) NodeMin(node *skiplist.SkipListElementReference) string {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	slice := nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{
+		Min:    "-",
+		Max:    "+",
+		Offset: 0,
+		Count:  1,
+	}).Val()
+	if len(slice) > 0 {
+		s := slice[0]
+		return s
+	}
+	return ""
+}
+
+func (nl *ItemList) NodeScanIncluseiveAfter(node *skiplist.SkipListElementReference, startFrom string, visitNamesFn func(name string) bool) bool {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if startFrom == "" {
+		startFrom = "-"
+	} else {
+		startFrom = "[" + startFrom
+	}
+	names := nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{
+		Min: startFrom,
+		Max: "+",
+	}).Val()
+	for _, n := range names {
+		if !visitNamesFn(n) {
+			return false
+		}
+	}
+	return true
+}
+
+func (nl *ItemList) NodeRangeBeforeExclusive(node *skiplist.SkipListElementReference, stopAt string) ([]string, error) {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if stopAt == "" {
+		stopAt = "+"
+	} else {
+		stopAt = "(" + stopAt
+	}
+	return nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{
+		Min: "-",
+		Max: stopAt,
+	}).Result()
+}
+func (nl *ItemList) NodeRangeAfterExclusive(node *skiplist.SkipListElementReference, startFrom string) ([]string, error) {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if startFrom == "" {
+		startFrom = "-"
+	} else {
+		startFrom = "(" + startFrom
+	}
+	return nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{
+		Min: startFrom,
+		Max: "+",
+	}).Result()
+}
+
+func (nl *ItemList) NodeDeleteBeforeExclusive(node *skiplist.SkipListElementReference, stopAt string) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if stopAt == "" {
+		stopAt = "+"
+	} else {
+		stopAt = "(" + stopAt
+	}
+	return nl.client.ZRemRangeByLex(context.Background(), key, "-", stopAt).Err()
+}
+func (nl *ItemList) NodeDeleteAfterExclusive(node *skiplist.SkipListElementReference, startFrom string) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if startFrom == "" {
+		startFrom = "-"
+	} else {
+		startFrom = "(" + startFrom
+	}
+	return nl.client.ZRemRangeByLex(context.Background(), key, startFrom, "+").Err()
+}
+
+func (nl *ItemList) ItemAdd(lookupKey []byte, idIfKnown int64, names ...string) error {
+	if id, err := nl.skipList.InsertByKey(lookupKey, idIfKnown, nil); err != nil {
+		return err
+	} else {
+		if len(names) > 0 {
+			return nl.NodeAddMember(&skiplist.SkipListElementReference{
+				ElementPointer: id,
+				Key:            lookupKey,
+			}, names...)
+		}
+	}
+	return nil
+}

weed/filer/redis3/item_list_serde.go

@@ -0,0 +1,75 @@
+package redis3
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util/skiplist"
+	"github.com/go-redis/redis/v8"
+	"github.com/golang/protobuf/proto"
+)
+
+func LoadItemList(data []byte, prefix string, client redis.UniversalClient, store skiplist.ListStore, batchSize int) *ItemList {
+
+	nl := &ItemList{
+		skipList:  skiplist.New(store),
+		batchSize: batchSize,
+		client:    client,
+		prefix:    prefix,
+	}
+
+	if len(data) == 0 {
+		return nl
+	}
+
+	message := &skiplist.SkipListProto{}
+	if err := proto.Unmarshal(data, message); err != nil {
+		glog.Errorf("loading skiplist: %v", err)
+	}
+	nl.skipList.MaxNewLevel = int(message.MaxNewLevel)
+	nl.skipList.MaxLevel = int(message.MaxLevel)
+	for i, ref := range message.StartLevels {
+		nl.skipList.StartLevels[i] = &skiplist.SkipListElementReference{
+			ElementPointer: ref.ElementPointer,
+			Key:            ref.Key,
+		}
+	}
+	for i, ref := range message.EndLevels {
+		nl.skipList.EndLevels[i] = &skiplist.SkipListElementReference{
+			ElementPointer: ref.ElementPointer,
+			Key:            ref.Key,
+		}
+	}
+	return nl
+}
+
+func (nl *ItemList) HasChanges() bool {
+	return nl.skipList.HasChanges
+}
+
+func (nl *ItemList) ToBytes() []byte {
+	message := &skiplist.SkipListProto{}
+	message.MaxNewLevel = int32(nl.skipList.MaxNewLevel)
+	message.MaxLevel = int32(nl.skipList.MaxLevel)
+	for _, ref := range nl.skipList.StartLevels {
+		if ref == nil {
+			break
+		}
+		message.StartLevels = append(message.StartLevels, &skiplist.SkipListElementReference{
+			ElementPointer: ref.ElementPointer,
+			Key:            ref.Key,
+		})
+	}
+	for _, ref := range nl.skipList.EndLevels {
+		if ref == nil {
+			break
+		}
+		message.EndLevels = append(message.EndLevels, &skiplist.SkipListElementReference{
+			ElementPointer: ref.ElementPointer,
+			Key:            ref.Key,
+		})
+	}
+	data, err := proto.Marshal(message)
+	if err != nil {
+		glog.Errorf("marshal skiplist: %v", err)
+	}
+	return data
+}

weed/filer/redis3/kv_directory_children.go

@@ -0,0 +1,138 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/go-redis/redis/v8"
+)
+
+const maxNameBatchSizeLimit = 1000000
+
+func insertChild(ctx context.Context, redisStore *UniversalRedis3Store, key string, name string) error {
+
+	// lock and unlock
+	mutex := redisStore.redsync.NewMutex(key + "lock")
+	if err := mutex.Lock(); err != nil {
+		return fmt.Errorf("lock %s: %v", key, err)
+	}
+	defer func() {
+		mutex.Unlock()
+	}()
+
+	client := redisStore.Client
+	data, err := client.Get(ctx, key).Result()
+	if err != nil {
+		if err != redis.Nil {
+			return fmt.Errorf("read %s: %v", key, err)
+		}
+	}
+	store := newSkipListElementStore(key, client)
+	nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit)
+
+	if err := nameList.WriteName(name); err != nil {
+		glog.Errorf("add %s %s: %v", key, name, err)
+		return err
+	}
+
+	if !nameList.HasChanges() {
+		return nil
+	}
+
+	if err := client.Set(ctx, key, nameList.ToBytes(), 0).Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func removeChild(ctx context.Context, redisStore *UniversalRedis3Store, key string, name string) error {
+
+	// lock and unlock
+	mutex := redisStore.redsync.NewMutex(key + "lock")
+	if err := mutex.Lock(); err != nil {
+		return fmt.Errorf("lock %s: %v", key, err)
+	}
+	defer mutex.Unlock()
+
+	client := redisStore.Client
+	data, err := client.Get(ctx, key).Result()
+	if err != nil {
+		if err != redis.Nil {
+			return fmt.Errorf("read %s: %v", key, err)
+		}
+	}
+	store := newSkipListElementStore(key, client)
+	nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit)
+
+	if err := nameList.DeleteName(name); err != nil {
+		return err
+	}
+	if !nameList.HasChanges() {
+		return nil
+	}
+
+	if err := client.Set(ctx, key, nameList.ToBytes(), 0).Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func removeChildren(ctx context.Context, redisStore *UniversalRedis3Store, key string, onDeleteFn func(name string) error) error {
+
+	// lock and unlock
+	mutex := redisStore.redsync.NewMutex(key + "lock")
+	if err := mutex.Lock(); err != nil {
+		return fmt.Errorf("lock %s: %v", key, err)
+	}
+	defer mutex.Unlock()
+
+	client := redisStore.Client
+	data, err := client.Get(ctx, key).Result()
+	if err != nil {
+		if err != redis.Nil {
+			return fmt.Errorf("read %s: %v", key, err)
+		}
+	}
+	store := newSkipListElementStore(key, client)
+	nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit)
+
+	if err = nameList.ListNames("", func(name string) bool {
+		if err := onDeleteFn(name); err != nil {
+			glog.Errorf("delete %s child %s: %v", key, name, err)
+			return false
+		}
+		return true
+	}); err != nil {
+		return err
+	}
+
+	if err = nameList.RemoteAllListElement(); err != nil {
+		return err
+	}
+
+	return nil
+
+}
+
+func listChildren(ctx context.Context, redisStore *UniversalRedis3Store, key string, startFileName string, eachFn func(name string) bool) error {
+	client := redisStore.Client
+	data, err := client.Get(ctx, key).Result()
+	if err != nil {
+		if err != redis.Nil {
+			return fmt.Errorf("read %s: %v", key, err)
+		}
+	}
+	store := newSkipListElementStore(key, client)
+	nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit)
+
+	if err = nameList.ListNames(startFileName, func(name string) bool {
+		return eachFn(name)
+	}); err != nil {
+		return err
+	}
+
+	return nil
+
+}

weed/filer/redis3/kv_directory_children_test.go

@@ -0,0 +1,210 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+	"github.com/go-redis/redis/v8"
+	"github.com/stvp/tempredis"
+	"strconv"
+	"testing"
+	"time"
+)
+
+var names = []string{
+	"cassandra.in.sh",
+	"cassandra",
+	"debug-cql.bat",
+	"nodetool",
+	"nodetool.bat",
+	"source-conf.ps1",
+	"sstableloader",
+	"sstableloader.bat",
+	"sstablescrub",
+	"sstablescrub.bat",
+	"sstableupgrade",
+	"sstableupgrade.bat",
+	"sstableutil",
+	"sstableutil.bat",
+	"sstableverify",
+	"sstableverify.bat",
+	"stop-server",
+	"stop-server.bat",
+	"stop-server.ps1",
+	"cassandra.in.bat",
+	"cqlsh.py",
+	"cqlsh",
+	"cassandra.ps1",
+	"cqlsh.bat",
+	"debug-cql",
+	"cassandra.bat",
+}
+
+func yTestNameList(t *testing.T) {
+	server, err := tempredis.Start(tempredis.Config{})
+	if err != nil {
+		panic(err)
+	}
+	defer server.Term()
+
+	client := redis.NewClient(&redis.Options{
+		Network: "unix",
+		Addr:    server.Socket(),
+	})
+
+	store := newSkipListElementStore("/yyy/bin", client)
+	var data []byte
+	for _, name := range names {
+		nameList := LoadItemList(data, "/yyy/bin", client, store, maxNameBatchSizeLimit)
+		nameList.WriteName(name)
+
+		nameList.ListNames("", func(name string) bool {
+			println(name)
+			return true
+		})
+
+		if nameList.HasChanges() {
+			data = nameList.ToBytes()
+		}
+		println()
+	}
+
+	nameList := LoadItemList(data, "/yyy/bin", client, store, maxNameBatchSizeLimit)
+	nameList.ListNames("", func(name string) bool {
+		println(name)
+		return true
+	})
+
+}
+
+func yBenchmarkNameList(b *testing.B) {
+
+	server, err := tempredis.Start(tempredis.Config{})
+	if err != nil {
+		panic(err)
+	}
+	defer server.Term()
+
+	client := redis.NewClient(&redis.Options{
+		Network: "unix",
+		Addr:    server.Socket(),
+	})
+
+	store := newSkipListElementStore("/yyy/bin", client)
+	var data []byte
+	for i := 0; i < b.N; i++ {
+		nameList := LoadItemList(data, "/yyy/bin", client, store, maxNameBatchSizeLimit)
+
+		nameList.WriteName(strconv.Itoa(i) + "namexxxxxxxxxxxxxxxxxxx")
+
+		if nameList.HasChanges() {
+			data = nameList.ToBytes()
+		}
+	}
+}
+
+func BenchmarkRedis(b *testing.B) {
+
+	server, err := tempredis.Start(tempredis.Config{})
+	if err != nil {
+		panic(err)
+	}
+	defer server.Term()
+
+	client := redis.NewClient(&redis.Options{
+		Network: "unix",
+		Addr:    server.Socket(),
+	})
+
+	for i := 0; i < b.N; i++ {
+		client.ZAddNX(context.Background(), "/yyy/bin", &redis.Z{Score: 0, Member: strconv.Itoa(i) + "namexxxxxxxxxxxxxxxxxxx"})
+	}
+}
+
+func xTestNameListAdd(t *testing.T) {
+
+	server, err := tempredis.Start(tempredis.Config{})
+	if err != nil {
+		panic(err)
+	}
+	defer server.Term()
+
+	client := redis.NewClient(&redis.Options{
+		Addr:     "localhost:6379",
+		Password: "",
+		DB:       0,
+	})
+
+	client.FlushAll(context.Background())
+
+	N := 364800
+
+	ts0 := time.Now()
+	store := newSkipListElementStore("/y", client)
+	var data []byte
+	nameList := LoadItemList(data, "/y", client, store, 100000)
+	for i := 0; i < N; i++ {
+		nameList.WriteName(fmt.Sprintf("%8d", i))
+	}
+
+	ts1 := time.Now()
+
+	for i := 0; i < N; i++ {
+		client.ZAddNX(context.Background(), "/x", &redis.Z{Score: 0, Member: fmt.Sprintf("name %8d", i)})
+	}
+	ts2 := time.Now()
+
+	fmt.Printf("%v %v", ts1.Sub(ts0), ts2.Sub(ts1))
+
+	/*
+		keys := client.Keys(context.Background(), "/*m").Val()
+		for _, k := range keys {
+			println("key", k)
+			for i, v := range client.ZRangeByLex(context.Background(), k, &redis.ZRangeBy{
+				Min:    "-",
+				Max:    "+",
+			}).Val() {
+				println(" ", i, v)
+			}
+		}
+	*/
+}
+
+func xBenchmarkNameList(b *testing.B) {
+
+	server, err := tempredis.Start(tempredis.Config{})
+	if err != nil {
+		panic(err)
+	}
+	defer server.Term()
+
+	client := redis.NewClient(&redis.Options{
+		Addr:     "localhost:6379",
+		Password: "",
+		DB:       0,
+	})
+
+	store := newSkipListElementStore("/yyy/bin", client)
+	var data []byte
+	for i := 0; i < b.N; i++ {
+		nameList := LoadItemList(data, "/yyy/bin", client, store, maxNameBatchSizeLimit)
+
+		nameList.WriteName(fmt.Sprintf("name %8d", i))
+
+		if nameList.HasChanges() {
+			data = nameList.ToBytes()
+		}
+	}
+}
+
+func xBenchmarkRedis(b *testing.B) {
+
+	client := redis.NewClient(&redis.Options{
+		Addr:     "localhost:6379",
+		Password: "",
+		DB:       0,
+	})
+
+	for i := 0; i < b.N; i++ {
+		client.ZAddNX(context.Background(), "/xxx/bin", &redis.Z{Score: 0, Member: fmt.Sprintf("name %8d", i)})
+	}
+}

weed/filer/redis3/redis_cluster_store.go

@@ -0,0 +1,45 @@
+package redis3
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/go-redis/redis/v8"
+	"github.com/go-redsync/redsync/v4"
+	"github.com/go-redsync/redsync/v4/redis/goredis/v8"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &RedisCluster3Store{})
+}
+
+type RedisCluster3Store struct {
+	UniversalRedis3Store
+}
+
+func (store *RedisCluster3Store) GetName() string {
+	return "redis_cluster3"
+}
+
+func (store *RedisCluster3Store) Initialize(configuration util.Configuration, prefix string) (err error) {
+
+	configuration.SetDefault(prefix+"useReadOnly", false)
+	configuration.SetDefault(prefix+"routeByLatency", false)
+
+	return store.initialize(
+		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetBool(prefix+"useReadOnly"),
+		configuration.GetBool(prefix+"routeByLatency"),
+	)
+}
+
+func (store *RedisCluster3Store) initialize(addresses []string, password string, readOnly, routeByLatency bool) (err error) {
+	store.Client = redis.NewClusterClient(&redis.ClusterOptions{
+		Addrs:          addresses,
+		Password:       password,
+		ReadOnly:       readOnly,
+		RouteByLatency: routeByLatency,
+	})
+	store.redsync = redsync.New(goredis.NewPool(store.Client))
+	return
+}

weed/filer/redis3/redis_store.go

@@ -0,0 +1,39 @@
+package redis3
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/go-redis/redis/v8"
+	"github.com/go-redsync/redsync/v4"
+	"github.com/go-redsync/redsync/v4/redis/goredis/v8"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &Redis3Store{})
+}
+
+type Redis3Store struct {
+	UniversalRedis3Store
+}
+
+func (store *Redis3Store) GetName() string {
+	return "redis3"
+}
+
+func (store *Redis3Store) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetString(prefix+"address"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetInt(prefix+"database"),
+	)
+}
+
+func (store *Redis3Store) initialize(hostPort string, password string, database int) (err error) {
+	store.Client = redis.NewClient(&redis.Options{
+		Addr:     hostPort,
+		Password: password,
+		DB:       database,
+	})
+	store.redsync = redsync.New(goredis.NewPool(store.Client))
+	return
+}

weed/filer/redis3/skiplist_element_store.go

@@ -0,0 +1,62 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util/skiplist"
+	"github.com/go-redis/redis/v8"
+	"github.com/golang/protobuf/proto"
+)
+
+type SkipListElementStore struct {
+	Prefix string
+	client redis.UniversalClient
+}
+
+var _ = skiplist.ListStore(&SkipListElementStore{})
+
+func newSkipListElementStore(prefix string, client redis.UniversalClient) *SkipListElementStore {
+	return &SkipListElementStore{
+		Prefix: prefix,
+		client: client,
+	}
+}
+
+func (m *SkipListElementStore) SaveElement(id int64, element *skiplist.SkipListElement) error {
+	key := fmt.Sprintf("%s%d", m.Prefix, id)
+	data, err := proto.Marshal(element)
+	if err != nil {
+		glog.Errorf("marshal %s: %v", key, err)
+	}
+	return m.client.Set(context.Background(), key, data, 0).Err()
+}
+
+func (m *SkipListElementStore) DeleteElement(id int64) error {
+	key := fmt.Sprintf("%s%d", m.Prefix, id)
+	return m.client.Del(context.Background(), key).Err()
+}
+
+func (m *SkipListElementStore) LoadElement(id int64) (*skiplist.SkipListElement, error) {
+	key := fmt.Sprintf("%s%d", m.Prefix, id)
+	data, err := m.client.Get(context.Background(), key).Result()
+	if err != nil {
+		if err == redis.Nil {
+			return nil, nil
+		}
+		return nil, err
+	}
+	t := &skiplist.SkipListElement{}
+	err = proto.Unmarshal([]byte(data), t)
+	if err == nil {
+		for i := 0; i < len(t.Next); i++ {
+			if t.Next[i].IsNil() {
+				t.Next[i] = nil
+			}
+		}
+		if t.Prev.IsNil() {
+			t.Prev = nil
+		}
+	}
+	return t, err
+}

weed/filer/redis3/universal_redis_store.go

@@ -0,0 +1,179 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+	"github.com/go-redsync/redsync/v4"
+	"time"
+
+	"github.com/go-redis/redis/v8"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+const (
+	DIR_LIST_MARKER = "\x00"
+)
+
+type UniversalRedis3Store struct {
+	Client  redis.UniversalClient
+	redsync *redsync.Redsync
+}
+
+func (store *UniversalRedis3Store) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *UniversalRedis3Store) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *UniversalRedis3Store) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *UniversalRedis3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+
+	if len(entry.Chunks) > 50 {
+		value = util.MaybeGzipData(value)
+	}
+
+	if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
+		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
+	}
+
+	dir, name := entry.FullPath.DirAndName()
+
+	if name != "" {
+		if err = insertChild(ctx, store, genDirectoryListKey(dir), name); err != nil {
+			return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err)
+		}
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis3Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *UniversalRedis3Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+
+	data, err := store.Client.Get(ctx, string(fullpath)).Result()
+	if err == redis.Nil {
+		return nil, filer_pb.ErrNotFound
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("get %s : %v", fullpath, err)
+	}
+
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data)))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *UniversalRedis3Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
+
+	_, err = store.Client.Del(ctx, genDirectoryListKey(string(fullpath))).Result()
+	if err != nil {
+		return fmt.Errorf("delete dir list %s : %v", fullpath, err)
+	}
+
+	_, err = store.Client.Del(ctx, string(fullpath)).Result()
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	dir, name := fullpath.DirAndName()
+
+	if name != "" {
+		if err = removeChild(ctx, store, genDirectoryListKey(dir), name); err != nil {
+			return fmt.Errorf("DeleteEntry %s in parent dir: %v", fullpath, err)
+		}
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis3Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
+
+	return removeChildren(ctx, store, genDirectoryListKey(string(fullpath)), func(name string) error {
+		path := util.NewFullPath(string(fullpath), name)
+		_, err = store.Client.Del(ctx, string(path)).Result()
+		if err != nil {
+			return fmt.Errorf("DeleteFolderChildren %s in parent dir: %v", fullpath, err)
+		}
+		// not efficient, but need to remove if it is a directory
+		store.Client.Del(ctx, genDirectoryListKey(string(path)))
+		return nil
+	})
+
+}
+
+func (store *UniversalRedis3Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
+}
+
+func (store *UniversalRedis3Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+
+	dirListKey := genDirectoryListKey(string(dirPath))
+	counter := int64(0)
+
+	err = listChildren(ctx, store, dirListKey, startFileName, func(fileName string) bool {
+		if startFileName != "" {
+			if !includeStartFile && startFileName == fileName {
+				return true
+			}
+		}
+
+		path := util.NewFullPath(string(dirPath), fileName)
+		entry, err := store.FindEntry(ctx, path)
+		lastFileName = fileName
+		if err != nil {
+			glog.V(0).Infof("list %s : %v", path, err)
+			if err == filer_pb.ErrNotFound {
+				return true
+			}
+		} else {
+			if entry.TtlSec > 0 {
+				if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+					store.Client.Del(ctx, string(path)).Result()
+					store.Client.ZRem(ctx, dirListKey, fileName).Result()
+					return true
+				}
+			}
+			counter++
+			if !eachEntryFunc(entry) {
+				return false
+			}
+			if counter >= limit {
+				return false
+			}
+		}
+		return true
+	})
+
+	return lastFileName, err
+}
+
+func genDirectoryListKey(dir string) (dirList string) {
+	return dir + DIR_LIST_MARKER
+}
+
+func (store *UniversalRedis3Store) Shutdown() {
+	store.Client.Close()
+}

weed/filer/redis3/universal_redis_store_kv.go

@@ -0,0 +1,42 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/go-redis/redis/v8"
+)
+
+func (store *UniversalRedis3Store) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+
+	_, err = store.Client.Set(ctx, string(key), value, 0).Result()
+
+	if err != nil {
+		return fmt.Errorf("kv put: %v", err)
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis3Store) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+
+	data, err := store.Client.Get(ctx, string(key)).Result()
+
+	if err == redis.Nil {
+		return nil, filer.ErrKvNotFound
+	}
+
+	return []byte(data), err
+}
+
+func (store *UniversalRedis3Store) KvDelete(ctx context.Context, key []byte) (err error) {
+
+	_, err = store.Client.Del(ctx, string(key)).Result()
+
+	if err != nil {
+		return fmt.Errorf("kv delete: %v", err)
+	}
+
+	return nil
+}

weed/filer/rocksdb/rocksdb_store.go

@@ -10,7 +10,7 @@ import (
 	"io"
 	"os"
 
-	"github.com/tecbot/gorocksdb"
+	gorocksdb "github.com/linxGnu/grocksdb"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"

weed/filer/rocksdb/rocksdb_store_test.go

@@ -1,3 +1,4 @@
+//go:build rocksdb
 // +build rocksdb
 
 package rocksdb
@@ -5,7 +6,6 @@ package rocksdb
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"testing"
 	"time"
@@ -16,7 +16,7 @@ import (
 
 func TestCreateAndFind(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &RocksDBStore{}
 	store.initialize(dir)
@@ -70,7 +70,7 @@ func TestCreateAndFind(t *testing.T) {
 
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &RocksDBStore{}
 	store.initialize(dir)
@@ -93,7 +93,7 @@ func TestEmptyRoot(t *testing.T) {
 
 func BenchmarkInsertEntry(b *testing.B) {
 	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_bench")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_bench")
 	defer os.RemoveAll(dir)
 	store := &RocksDBStore{}
 	store.initialize(dir)

weed/filer/rocksdb/rocksdb_ttl.go

@@ -5,7 +5,7 @@ package rocksdb
 import (
 	"time"
 
-	"github.com/tecbot/gorocksdb"
+	gorocksdb "github.com/linxGnu/grocksdb"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 )
@@ -38,3 +38,8 @@ func (t *TTLFilter) Filter(level int, key, val []byte) (remove bool, newVal []by
 func (t *TTLFilter) Name() string {
 	return "TTLFilter"
 }
+func (t *TTLFilter) SetIgnoreSnapshots(value bool) {
+}
+
+func (t *TTLFilter) Destroy() {
+}

weed/filer/tikv/tikv.go

@@ -1,5 +0,0 @@
-package tikv
-
-/*
- * This empty file is let go build can work without tikv tag
- */

weed/filer/tikv/tikv_store.go

@@ -1,389 +0,0 @@
-//go:build tikv
-// +build tikv
-
-package tikv
-
-import (
-	"bytes"
-	"context"
-	"crypto/sha1"
-	"fmt"
-	"io"
-	"strings"
-
-	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/tikv/client-go/v2/tikv"
-	"github.com/tikv/client-go/v2/txnkv"
-)
-
-var (
-	_ filer.FilerStore = ((*TikvStore)(nil))
-)
-
-func init() {
-	filer.Stores = append(filer.Stores, &TikvStore{})
-}
-
-type TikvStore struct {
-	client                 *tikv.KVStore
-	deleteRangeConcurrency int
-}
-
-// Basic APIs
-func (store *TikvStore) GetName() string {
-	return "tikv"
-}
-
-func (store *TikvStore) Initialize(config util.Configuration, prefix string) error {
-	pdAddrs := []string{}
-	pdAddrsStr := config.GetString(prefix + "pdaddrs")
-	for _, item := range strings.Split(pdAddrsStr, ",") {
-		pdAddrs = append(pdAddrs, strings.TrimSpace(item))
-	}
-	drc := config.GetInt(prefix + "deleterange_concurrency")
-	if drc <= 0 {
-		drc = 1
-	}
-	store.deleteRangeConcurrency = drc
-	return store.initialize(pdAddrs)
-}
-
-func (store *TikvStore) initialize(pdAddrs []string) error {
-	client, err := tikv.NewTxnClient(pdAddrs)
-	store.client = client
-	return err
-}
-
-func (store *TikvStore) Shutdown() {
-	err := store.client.Close()
-	if err != nil {
-		glog.V(0).Infof("Shutdown TiKV client got error: %v", err)
-	}
-}
-
-// ~ Basic APIs
-
-// Entry APIs
-func (store *TikvStore) InsertEntry(ctx context.Context, entry *filer.Entry) error {
-	dir, name := entry.DirAndName()
-	key := generateKey(dir, name)
-
-	value, err := entry.EncodeAttributesAndChunks()
-	if err != nil {
-		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
-	}
-	txn, err := store.getTxn(ctx)
-	if err != nil {
-		return err
-	}
-	err = txn.RunInTxn(func(txn *txnkv.KVTxn) error {
-		return txn.Set(key, value)
-	})
-	if err != nil {
-		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
-	}
-	return nil
-}
-
-func (store *TikvStore) UpdateEntry(ctx context.Context, entry *filer.Entry) error {
-	return store.InsertEntry(ctx, entry)
-}
-
-func (store *TikvStore) FindEntry(ctx context.Context, path util.FullPath) (*filer.Entry, error) {
-	dir, name := path.DirAndName()
-	key := generateKey(dir, name)
-
-	txn, err := store.getTxn(ctx)
-	if err != nil {
-		return nil, err
-	}
-	var value []byte = nil
-	err = txn.RunInTxn(func(txn *txnkv.KVTxn) error {
-		val, err := txn.Get(context.TODO(), key)
-		if err == nil {
-			value = val
-		}
-		return err
-	})
-
-	if isNotExists(err) || value == nil {
-		return nil, filer_pb.ErrNotFound
-	}
-
-	if err != nil {
-		return nil, fmt.Errorf("get %s : %v", path, err)
-	}
-
-	entry := &filer.Entry{
-		FullPath: path,
-	}
-	err = entry.DecodeAttributesAndChunks(value)
-	if err != nil {
-		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
-	}
-	return entry, nil
-}
-
-func (store *TikvStore) DeleteEntry(ctx context.Context, path util.FullPath) error {
-	dir, name := path.DirAndName()
-	key := generateKey(dir, name)
-
-	txn, err := store.getTxn(ctx)
-	if err != nil {
-		return err
-	}
-
-	err = txn.RunInTxn(func(txn *txnkv.KVTxn) error {
-		return txn.Delete(key)
-	})
-	if err != nil {
-		return fmt.Errorf("delete %s : %v", path, err)
-	}
-	return nil
-}
-
-// ~ Entry APIs
-
-// Directory APIs
-func (store *TikvStore) DeleteFolderChildren(ctx context.Context, path util.FullPath) error {
-	directoryPrefix := genDirectoryKeyPrefix(path, "")
-
-	txn, err := store.getTxn(ctx)
-	if err != nil {
-		return err
-	}
-	var (
-		startKey []byte = nil
-		endKey   []byte = nil
-	)
-	err = txn.RunInTxn(func(txn *txnkv.KVTxn) error {
-		iter, err := txn.Iter(directoryPrefix, nil)
-		if err != nil {
-			return err
-		}
-		defer iter.Close()
-		for iter.Valid() {
-			key := iter.Key()
-			endKey = key
-			if !bytes.HasPrefix(key, directoryPrefix) {
-				break
-			}
-			if startKey == nil {
-				startKey = key
-			}
-
-			err = iter.Next()
-			if err != nil {
-				return err
-			}
-		}
-		// Only one Key matched just delete it.
-		if startKey != nil && bytes.Equal(startKey, endKey) {
-			return txn.Delete(startKey)
-		}
-		return nil
-	})
-	if err != nil {
-		return fmt.Errorf("delete %s : %v", path, err)
-	}
-
-	if startKey != nil && endKey != nil && !bytes.Equal(startKey, endKey) {
-		// has startKey and endKey and they are not equals, so use delete range
-		_, err = store.client.DeleteRange(context.Background(), startKey, endKey, store.deleteRangeConcurrency)
-		if err != nil {
-			return fmt.Errorf("delete %s : %v", path, err)
-		}
-	}
-	return err
-}
-
-func (store *TikvStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (string, error) {
-	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
-}
-
-func (store *TikvStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (string, error) {
-	lastFileName := ""
-	directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix)
-	lastFileStart := directoryPrefix
-	if startFileName != "" {
-		lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
-	}
-
-	txn, err := store.getTxn(ctx)
-	if err != nil {
-		return lastFileName, err
-	}
-	err = txn.RunInTxn(func(txn *txnkv.KVTxn) error {
-		iter, err := txn.Iter(lastFileStart, nil)
-		if err != nil {
-			return err
-		}
-		defer iter.Close()
-		i := int64(0)
-		first := true
-		for iter.Valid() {
-			if first {
-				first = false
-				if !includeStartFile {
-					if iter.Valid() {
-						// Check first item is lastFileStart
-						if bytes.Equal(iter.Key(), lastFileStart) {
-							// Is lastFileStart and not include start file, just
-							// ignore it.
-							err = iter.Next()
-							if err != nil {
-								return err
-							}
-							continue
-						}
-					}
-				}
-			}
-			// Check for limitation
-			if limit > 0 {
-				i++
-				if i > limit {
-					break
-				}
-			}
-			// Validate key prefix
-			key := iter.Key()
-			if !bytes.HasPrefix(key, directoryPrefix) {
-				break
-			}
-			value := iter.Value()
-
-			// Start process
-			fileName := getNameFromKey(key)
-			if fileName != "" {
-				// Got file name, then generate the Entry
-				entry := &filer.Entry{
-					FullPath: util.NewFullPath(string(dirPath), fileName),
-				}
-				// Update lastFileName
-				lastFileName = fileName
-				// Check for decode value.
-				if decodeErr := entry.DecodeAttributesAndChunks(value); decodeErr != nil {
-					// Got error just return the error
-					glog.V(0).Infof("list %s : %v", entry.FullPath, err)
-					return err
-				}
-				// Run for each callback if return false just break the iteration
-				if !eachEntryFunc(entry) {
-					break
-				}
-			}
-			// End process
-
-			err = iter.Next()
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		return lastFileName, fmt.Errorf("prefix list %s : %v", dirPath, err)
-	}
-	return lastFileName, nil
-}
-
-// ~ Directory APIs
-
-// Transaction Related APIs
-func (store *TikvStore) BeginTransaction(ctx context.Context) (context.Context, error) {
-	tx, err := store.client.Begin()
-	if err != nil {
-		return ctx, err
-	}
-	return context.WithValue(ctx, "tx", tx), nil
-}
-
-func (store *TikvStore) CommitTransaction(ctx context.Context) error {
-	if tx, ok := ctx.Value("tx").(*txnkv.KVTxn); ok {
-		return tx.Commit(context.Background())
-	}
-	return nil
-}
-
-func (store *TikvStore) RollbackTransaction(ctx context.Context) error {
-	if tx, ok := ctx.Value("tx").(*txnkv.KVTxn); ok {
-		return tx.Rollback()
-	}
-	return nil
-}
-
-// ~ Transaction Related APIs
-
-// Transaction Wrapper
-type TxnWrapper struct {
-	*txnkv.KVTxn
-	inContext bool
-}
-
-func (w *TxnWrapper) RunInTxn(f func(txn *txnkv.KVTxn) error) error {
-	err := f(w.KVTxn)
-	if !w.inContext {
-		if err != nil {
-			w.KVTxn.Rollback()
-			return err
-		}
-		w.KVTxn.Commit(context.Background())
-		return nil
-	}
-	return err
-}
-
-func (store *TikvStore) getTxn(ctx context.Context) (*TxnWrapper, error) {
-	if tx, ok := ctx.Value("tx").(*txnkv.KVTxn); ok {
-		return &TxnWrapper{tx, true}, nil
-	}
-	txn, err := store.client.Begin()
-	if err != nil {
-		return nil, err
-	}
-	return &TxnWrapper{txn, false}, nil
-}
-
-// ~ Transaction Wrapper
-
-// Encoding Functions
-func hashToBytes(dir string) []byte {
-	h := sha1.New()
-	io.WriteString(h, dir)
-	b := h.Sum(nil)
-	return b
-}
-
-func generateKey(dirPath, fileName string) []byte {
-	key := hashToBytes(dirPath)
-	key = append(key, []byte(fileName)...)
-	return key
-}
-
-func getNameFromKey(key []byte) string {
-	return string(key[sha1.Size:])
-}
-
-func genDirectoryKeyPrefix(fullpath util.FullPath, startFileName string) (keyPrefix []byte) {
-	keyPrefix = hashToBytes(string(fullpath))
-	if len(startFileName) > 0 {
-		keyPrefix = append(keyPrefix, []byte(startFileName)...)
-	}
-	return keyPrefix
-}
-
-func isNotExists(err error) bool {
-	if err == nil {
-		return false
-	}
-	if err.Error() == "not exist" {
-		return true
-	}
-	return false
-}
-
-// ~ Encoding Functions

weed/filer/tikv/tikv_store_kv.go

@@ -1,50 +0,0 @@
-//go:build tikv
-// +build tikv
-
-package tikv
-
-import (
-	"context"
-
-	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/tikv/client-go/v2/txnkv"
-)
-
-func (store *TikvStore) KvPut(ctx context.Context, key []byte, value []byte) error {
-	tw, err := store.getTxn(ctx)
-	if err != nil {
-		return err
-	}
-	return tw.RunInTxn(func(txn *txnkv.KVTxn) error {
-		return txn.Set(key, value)
-	})
-}
-
-func (store *TikvStore) KvGet(ctx context.Context, key []byte) ([]byte, error) {
-	tw, err := store.getTxn(ctx)
-	if err != nil {
-		return nil, err
-	}
-	var data []byte = nil
-	err = tw.RunInTxn(func(txn *txnkv.KVTxn) error {
-		val, err := txn.Get(context.TODO(), key)
-		if err == nil {
-			data = val
-		}
-		return err
-	})
-	if isNotExists(err) {
-		return data, filer.ErrKvNotFound
-	}
-	return data, err
-}
-
-func (store *TikvStore) KvDelete(ctx context.Context, key []byte) error {
-	tw, err := store.getTxn(ctx)
-	if err != nil {
-		return err
-	}
-	return tw.RunInTxn(func(txn *txnkv.KVTxn) error {
-		return txn.Delete(key)
-	})
-}

weed/filesys/dir_rename.go

@@ -2,12 +2,10 @@ package filesys
 
 import (
 	"context"
-	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer"
-	"math"
-
 	"github.com/seaweedfs/fuse"
 	"github.com/seaweedfs/fuse/fs"
+	"io"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@@ -23,19 +21,12 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 
 	glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath)
 
-	// find local old entry
-	oldEntry, err := dir.wfs.metaCache.FindEntry(context.Background(), oldPath)
-	if err != nil {
-		glog.Errorf("dir Rename can not find source %s : %v", oldPath, err)
-		return fuse.ENOENT
-	}
-
 	// update remote filer
-	err = dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 
-		request := &filer_pb.AtomicRenameEntryRequest{
+		request := &filer_pb.StreamRenameEntryRequest{
 			OldDirectory: dir.FullPath(),
 			OldName:      req.OldName,
 			NewDirectory: newDir.FullPath(),
@@ -43,12 +34,28 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 			Signatures:   []int32{dir.wfs.signature},
 		}
 
-		_, err := client.AtomicRenameEntry(ctx, request)
+		stream, err := client.StreamRenameEntry(ctx, request)
 		if err != nil {
 			glog.Errorf("dir AtomicRenameEntry %s => %s : %v", oldPath, newPath, err)
 			return fuse.EXDEV
 		}
 
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				} else {
+					return recvErr
+				}
+			}
+
+			if err = dir.handleRenameResponse(ctx, resp); err != nil {
+				return err
+			}
+
+		}
+
 		return nil
 
 	})
@@ -57,23 +64,25 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 		return fuse.EIO
 	}
 
-	err = dir.moveEntry(context.Background(), util.FullPath(dir.FullPath()), oldEntry, util.FullPath(newDir.FullPath()), req.NewName)
-	if err != nil {
-		glog.V(0).Infof("dir local Rename %s => %s : %v", oldPath, newPath, err)
-		return fuse.EIO
-	}
-
 	return nil
 }
 
-func (dir *Dir) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string) error {
 
-	oldName := entry.Name()
+func (dir *Dir) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamRenameEntryResponse) error {
+	// comes from filer StreamRenameEntry, can only be create or delete entry
 
-	oldPath := oldParent.Child(oldName)
-	newPath := newParent.Child(newName)
-	if err := dir.moveSelfEntry(ctx, oldParent, entry, newParent, newName, func() error {
+	if resp.EventNotification.NewEntry != nil {
+		// with new entry, the old entry name also exists. This is the first step to create new entry
+		newEntry := filer.FromPbEntry(resp.EventNotification.NewParentPath, resp.EventNotification.NewEntry)
+		if err := dir.wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, "", newEntry); err != nil {
+			return err
+		}
 
+		oldParent, newParent := util.FullPath(resp.Directory), util.FullPath(resp.EventNotification.NewParentPath)
+		oldName, newName := resp.EventNotification.OldEntry.Name, resp.EventNotification.NewEntry.Name
+
+		oldPath := oldParent.Child(oldName)
+		newPath := newParent.Child(newName)
 		oldFsNode := NodeWithId(oldPath.AsInode())
 		newFsNode := NodeWithId(newPath.AsInode())
 		newDirNode, found := dir.wfs.Server.FindInternalNode(NodeWithId(newParent.AsInode()))
@@ -110,65 +119,13 @@ func (dir *Dir) moveEntry(ctx context.Context, oldParent util.FullPath, entry *f
 		}
 		dir.wfs.handlesLock.Unlock()
 
-		if entry.IsDirectory() {
-			if err := dir.moveFolderSubEntries(ctx, oldParent, oldName, newParent, newName); err != nil {
-				return err
-			}
-		}
-		return nil
-	}); err != nil {
-		return fmt.Errorf("fail to move %s => %s: %v", oldPath, newPath, err)
-	}
-
-	return nil
-}
-
-func (dir *Dir) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, oldName string, newParent util.FullPath, newName string) error {
-
-	currentDirPath := oldParent.Child(oldName)
-	newDirPath := newParent.Child(newName)
-
-	glog.V(1).Infof("moving folder %s => %s", currentDirPath, newDirPath)
-
-	var moveErr error
-	listErr := dir.wfs.metaCache.ListDirectoryEntries(ctx, currentDirPath, "", false, int64(math.MaxInt32), func(item *filer.Entry) bool {
-		moveErr = dir.moveEntry(ctx, currentDirPath, item, newDirPath, item.Name())
-		if moveErr != nil {
-			return false
-		}
-		return true
-	})
-	if listErr != nil {
-		return listErr
-	}
-	if moveErr != nil {
-		return moveErr
-	}
-
-	return nil
-}
-
-func (dir *Dir) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, moveFolderSubEntries func() error) error {
-
-	newPath := newParent.Child(newName)
-	oldPath := oldParent.Child(entry.Name())
-
-	entry.FullPath = newPath
-	if err := dir.wfs.metaCache.InsertEntry(ctx, entry); err != nil {
-		glog.V(0).Infof("dir Rename insert local %s => %s : %v", oldPath, newPath, err)
-		return fuse.EIO
-	}
-
-	if moveFolderSubEntries != nil {
-		if moveChildrenErr := moveFolderSubEntries(); moveChildrenErr != nil {
-			return moveChildrenErr
+	}else if resp.EventNotification.OldEntry != nil {
+		// without new entry, only old entry name exists. This is the second step to delete old entry
+		if err := dir.wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath(resp.Directory, resp.EventNotification.OldEntry.Name), nil); err != nil {
+			return err
 		}
 	}
 
-	if err := dir.wfs.metaCache.DeleteEntry(ctx, oldPath); err != nil {
-		glog.V(0).Infof("dir Rename delete local %s => %s : %v", oldPath, newPath, err)
-		return fuse.EIO
-	}
-
 	return nil
+
 }

weed/filesys/meta_cache/meta_cache.go

@@ -2,7 +2,6 @@ package meta_cache
 
 import (
 	"context"
-	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/filer/leveldb"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -122,7 +121,8 @@ func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.Full
 	//defer mc.RUnlock()
 
 	if !mc.visitedBoundary.HasVisited(dirPath) {
-		return fmt.Errorf("unsynchronized dir: %v", dirPath)
+		// if this request comes after renaming, it should be fine
+		glog.Warningf("unsynchronized dir: %v", dirPath)
 	}
 
 	_, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *filer.Entry) bool {

weed/filesys/meta_cache/meta_cache_subscribe.go

@@ -58,7 +58,7 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
 	}
 
 	util.RetryForever("followMetaUpdates", func() error {
-		return pb.WithFilerClientFollowMetadata(client, "mount", dir, lastTsNs, selfSignature, processEventFn, true)
+		return pb.WithFilerClientFollowMetadata(client, "mount", dir, &lastTsNs, selfSignature, processEventFn, true)
 	}, func(err error) bool {
 		glog.Errorf("follow metadata updates: %v", err)
 		return true

weed/images/orientation_test.go

@@ -1,7 +1,6 @@
 package images
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
 )
@@ -9,11 +8,11 @@ import (
 func TestXYZ(t *testing.T) {
 	fname := "sample1.jpg"
 
-	dat, _ := ioutil.ReadFile(fname)
+	dat, _ := os.ReadFile(fname)
 
 	fixed_data := FixJpgOrientation(dat)
 
-	ioutil.WriteFile("fixed1.jpg", fixed_data, 0644)
+	os.WriteFile("fixed1.jpg", fixed_data, 0644)
 
 	os.Remove("fixed1.jpg")
 

weed/images/resizing_test.go

@@ -2,7 +2,6 @@ package images
 
 import (
 	"bytes"
-	"io/ioutil"
 	"os"
 	"testing"
 )
@@ -10,13 +9,13 @@ import (
 func TestResizing(t *testing.T) {
 	fname := "sample2.webp"
 
-	dat, _ := ioutil.ReadFile(fname)
+	dat, _ := os.ReadFile(fname)
 
 	resized, _, _ := Resized(".webp", bytes.NewReader(dat), 100, 30, "")
 	buf := new(bytes.Buffer)
 	buf.ReadFrom(resized)
 
-	ioutil.WriteFile("resized1.png", buf.Bytes(), 0644)
+	os.WriteFile("resized1.png", buf.Bytes(), 0644)
 
 	os.Remove("resized1.png")
 

weed/messaging/broker/broker_grpc_server_subscribe.go

@@ -2,6 +2,7 @@ package broker
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
 	"io"
 	"strings"
@@ -141,7 +142,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs
 func (broker *MessageBroker) readPersistedLogBuffer(tp *TopicPartition, startTime time.Time, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (err error) {
 	startTime = startTime.UTC()
 	startDate := fmt.Sprintf("%04d-%02d-%02d", startTime.Year(), startTime.Month(), startTime.Day())
-	startHourMinute := fmt.Sprintf("%02d-%02d.segment", startTime.Hour(), startTime.Minute())
+	startHourMinute := fmt.Sprintf("%02d-%02d", startTime.Hour(), startTime.Minute())
 
 	sizeBuf := make([]byte, 4)
 	startTsNs := startTime.UnixNano()
@@ -153,7 +154,8 @@ func (broker *MessageBroker) readPersistedLogBuffer(tp *TopicPartition, startTim
 		dayDir := fmt.Sprintf("%s/%s", topicDir, dayEntry.Name)
 		return filer_pb.List(broker, dayDir, "", func(hourMinuteEntry *filer_pb.Entry, isLast bool) error {
 			if dayEntry.Name == startDate {
-				if strings.Compare(hourMinuteEntry.Name, startHourMinute) < 0 {
+				hourMinute := util.FileNameBase(hourMinuteEntry.Name)
+				if strings.Compare(hourMinute, startHourMinute) < 0 {
 					return nil
 				}
 			}

weed/operation/assign_file_id.go

@@ -66,6 +66,10 @@ func Assign(masterFn GetMasterFn, grpcDialOption grpc.DialOption, primaryRequest
 				return grpcErr
 			}
 
+			if resp.Error != "" {
+				return fmt.Errorf("assignRequest: %v", resp.Error)
+			}
+
 			ret.Count = resp.Count
 			ret.Fid = resp.Fid
 			ret.Url = resp.Location.Url
@@ -80,10 +84,6 @@ func Assign(masterFn GetMasterFn, grpcDialOption grpc.DialOption, primaryRequest
 				})
 			}
 
-			if resp.Error != "" {
-				return fmt.Errorf("assignRequest: %v", resp.Error)
-			}
-
 			return nil
 
 		})

weed/operation/chunked_file.go

@@ -4,9 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/pb"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"sort"
 	"sync"
@@ -14,6 +12,7 @@ import (
 	"google.golang.org/grpc"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -108,7 +107,7 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri
 		return written, err
 	}
 	defer func() {
-		io.Copy(ioutil.Discard, resp.Body)
+		io.Copy(io.Discard, resp.Body)
 		resp.Body.Close()
 	}()
 

weed/operation/upload_content.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"mime/multipart"
 	"net/http"
@@ -91,7 +90,7 @@ func doUpload(reader io.Reader, option *UploadOption) (uploadResult *UploadResul
 	if ok {
 		data = bytesReader.Bytes
 	} else {
-		data, err = ioutil.ReadAll(reader)
+		data, err = io.ReadAll(reader)
 		if err != nil {
 			err = fmt.Errorf("read input: %v", err)
 			return
@@ -278,7 +277,7 @@ func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize
 		return &ret, nil
 	}
 
-	resp_body, ra_err := ioutil.ReadAll(resp.Body)
+	resp_body, ra_err := io.ReadAll(resp.Body)
 	if ra_err != nil {
 		return nil, fmt.Errorf("read response body %v: %v", option.UploadUrl, ra_err)
 	}

weed/pb/filer.proto

@@ -30,6 +30,8 @@ service SeaweedFiler {
 
     rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
     }
+    rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) {
+    }
 
     rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
     }
@@ -225,6 +227,18 @@ message AtomicRenameEntryRequest {
 message AtomicRenameEntryResponse {
 }
 
+message StreamRenameEntryRequest {
+    string old_directory = 1;
+    string old_name = 2;
+    string new_directory = 3;
+    string new_name = 4;
+    repeated int32 signatures = 5;
+}
+message StreamRenameEntryResponse {
+    string directory = 1;
+    EventNotification event_notification = 2;
+    int64 ts_ns = 3;
+}
 message AssignVolumeRequest {
     int32 count = 1;
     string collection = 2;

weed/pb/filer_pb/filer_client.go

@@ -101,9 +101,13 @@ func SeaweedList(client SeaweedFilerClient, parentDirectoryPath, prefix string,
 func doSeaweedList(client SeaweedFilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunciton, startFrom string, inclusive bool, limit uint32) (err error) {
 	// Redundancy limit to make it correctly judge whether it is the last file.
 	redLimit := limit
-	if limit != math.MaxInt32 && limit != 0 {
+
+	if limit < math.MaxInt32 && limit != 0 {
 		redLimit = limit + 1
 	}
+	if redLimit > math.MaxInt32 {
+		redLimit = math.MaxInt32
+	}
 	request := &ListEntriesRequest{
 		Directory:          string(fullDirPath),
 		Prefix:             prefix,

weed/pb/filer_pb_tail.go

@@ -17,7 +17,7 @@ func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption,
 	processEventFn ProcessMetadataFunc, fatalOnError bool) error {
 
 	err := WithFilerClient(filerAddress, grpcDialOption, makeFunc(clientName,
-		pathPrefix, additionalPathPrefixes, lastTsNs, selfSignature, processEventFn, fatalOnError))
+		pathPrefix, additionalPathPrefixes, &lastTsNs, selfSignature, processEventFn, fatalOnError))
 	if err != nil {
 		return fmt.Errorf("subscribing filer meta change: %v", err)
 	}
@@ -25,7 +25,7 @@ func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption,
 }
 
 func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient,
-	clientName string, pathPrefix string, lastTsNs int64, selfSignature int32,
+	clientName string, pathPrefix string, lastTsNs *int64, selfSignature int32,
 	processEventFn ProcessMetadataFunc, fatalOnError bool) error {
 
 	err := filerClient.WithFilerClient(makeFunc(clientName,
@@ -37,7 +37,7 @@ func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient,
 	return nil
 }
 
-func makeFunc(clientName string, pathPrefix string, additionalPathPrefixes []string, lastTsNs int64, selfSignature int32,
+func makeFunc(clientName string, pathPrefix string, additionalPathPrefixes []string, lastTsNs *int64, selfSignature int32,
 	processEventFn ProcessMetadataFunc, fatalOnError bool) func(client filer_pb.SeaweedFilerClient) error {
 	return func(client filer_pb.SeaweedFilerClient) error {
 		ctx, cancel := context.WithCancel(context.Background())
@@ -46,7 +46,7 @@ func makeFunc(clientName string, pathPrefix string, additionalPathPrefixes []str
 			ClientName:   clientName,
 			PathPrefix:   pathPrefix,
 			PathPrefixes: additionalPathPrefixes,
-			SinceNs:      lastTsNs,
+			SinceNs:      *lastTsNs,
 			Signature:    selfSignature,
 		})
 		if err != nil {
@@ -69,7 +69,7 @@ func makeFunc(clientName string, pathPrefix string, additionalPathPrefixes []str
 					glog.Errorf("process %v: %v", resp, err)
 				}
 			}
-			lastTsNs = resp.TsNs
+			*lastTsNs = resp.TsNs
 		}
 	}
 }

weed/pb/server_address.go

@@ -13,7 +13,7 @@ type ServerAddress string
 type ServerAddresses string
 
 func NewServerAddress(host string, port int, grpcPort int) ServerAddress {
-	if grpcPort == port+10000 {
+	if grpcPort == 0 || grpcPort == port+10000 {
 		return ServerAddress(util.JoinHostPort(host, port))
 	}
 	return ServerAddress(util.JoinHostPort(host, port) + "." + strconv.Itoa(grpcPort))

weed/pb/volume_server.proto

@@ -58,6 +58,8 @@ service VolumeServer {
     }
     rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
     }
+    rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
+    }
 
     rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
     }
@@ -284,6 +286,16 @@ message WriteNeedleBlobRequest {
 message WriteNeedleBlobResponse {
 }
 
+message ReadAllNeedlesRequest {
+    repeated uint32 volume_ids = 1;
+}
+message ReadAllNeedlesResponse {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    uint32 cookie = 3;
+    bytes needle_blob = 5;
+}
+
 message VolumeTailSenderRequest {
     uint32 volume_id = 1;
     uint64 since_ns = 2;

weed/remote_storage/azure/azure_storage_client.go

@@ -3,17 +3,17 @@ package azure
 import (
 	"context"
 	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"reflect"
+
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
 	"github.com/chrislusf/seaweedfs/weed/remote_storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"io"
-	"io/ioutil"
-	"net/url"
-	"os"
-	"reflect"
 )
 
 func init() {
@@ -115,7 +115,7 @@ func (az *azureRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocatio
 	bodyStream := downloadResponse.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20})
 	defer bodyStream.Close()
 
-	data, err = ioutil.ReadAll(bodyStream)
+	data, err = io.ReadAll(bodyStream)
 
 	if err != nil {
 		return nil, fmt.Errorf("failed to download file %s%s: %v", loc.Bucket, loc.Path, err)