2.96

avoid connect to the old filler address

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+      interval: "daily"
+- package-ecosystem: gomod
+  directory: "/"
+  schedule:
+    interval: daily

.github/workflows/binaries_dev.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
 
       - name: Delete old release assets
-        uses: mknejp/delete-release-assets@v1
+        uses: mknejp/delete-release-assets@a8aaab13272b1eaac16cc46dddd3f725b97ee05a # v1
         with:
           token: ${{ github.token }}
           tag: dev
@@ -20,24 +20,24 @@ jobs:
           assets: |
             weed-*
 
-  build_dev_linux:
+  build_dev_linux_windows:
     needs: cleanup
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        goos: [linux]
+        goos: [linux, windows]
         goarch: [amd64]
 
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -53,7 +53,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -68,7 +68,7 @@ jobs:
           asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
   build_dev_darwin:
-    needs: build_dev_linux
+    needs: build_dev_linux_windows
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -78,13 +78,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -100,7 +100,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release0.yml

@@ -23,9 +23,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -39,7 +39,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release1.yml

@@ -23,9 +23,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -39,7 +39,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release2.yml

@@ -23,9 +23,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -39,7 +39,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release3.yml

@@ -23,9 +23,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -39,7 +39,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/container_dev.yml

@@ -5,6 +5,9 @@ on:
     branches: [ master ]
   workflow_dispatch: []
 
+permissions:
+  contents: read
+
 jobs:
 
   build-dev-containers:
@@ -13,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -30,30 +33,30 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -6,6 +6,9 @@ on:
       - '*'
   workflow_dispatch: []
 
+permissions:
+  contents: read
+
 jobs:
 
   build-latest-container:
@@ -14,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -31,30 +34,30 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -6,6 +6,9 @@ on:
       - '*'
   workflow_dispatch: []
 
+permissions:
+  contents: read
+
 jobs:
   build-default-release-container:
     runs-on: [ubuntu-latest]
@@ -13,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -31,20 +34,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -6,6 +6,9 @@ on:
       - '*'
   workflow_dispatch: []
 
+permissions:
+  contents: read
+
 jobs:
 
   build-large-release-container:
@@ -14,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -32,20 +35,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -6,6 +6,9 @@ on:
       - '*'
   workflow_dispatch: []
 
+permissions:
+  contents: read
+
 jobs:
 
   build-large-release-container_rocksdb:
@@ -14,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -32,20 +35,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/go.yml

@@ -10,6 +10,9 @@ concurrency:
   group: ${{ github.head_ref }}/go
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
 
   build:
@@ -18,13 +21,13 @@ jobs:
     steps:
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@f6164bd8c8acb4a71fb2791a8b6c4024ff038dab # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
 
     - name: Get dependencies
       run: |

README.md

@@ -33,7 +33,7 @@ Your support will be really appreciated by me and other supporters!
 
 
 ### Gold Sponsors
-![shuguang](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/shuguang.png)
+- [![nodion](https://www.nodion.com/img/logo.svg)](https://www.nodion.com)
 
 ---
 
@@ -41,6 +41,7 @@ Your support will be really appreciated by me and other supporters!
 - [Download Binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest)
 - [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
 - [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
+- [SeaweedFS on Telegram](https://t.me/Seaweedfs) 
 - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
 - [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
@@ -51,6 +52,8 @@ Table of Contents
 =================
 
 * [Quick Start](#quick-start)
+    * [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
+    * [Quick Start with Single Binary](#quick-start-with-single-binary)
 * [Introduction](#introduction)
 * [Features](#features)
     * [Additional Features](#additional-features)
@@ -73,7 +76,7 @@ Table of Contents
 
 `docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
 
-## Quick Start with single binary ##
+## Quick Start with Single Binary ##
 * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
 * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
 

docker/Dockerfile.rocksdb_large

@@ -9,7 +9,7 @@ ENV ROCKSDB_VERSION v6.22.1
 RUN cd /tmp && \
     git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
     cd rocksdb && \
-    make static_lib && \
+    PORTABLE=1 make static_lib && \
     make install-static
 
 ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
@@ -29,7 +29,7 @@ FROM alpine AS final
 LABEL author="Chris Lu"
 COPY --from=builder /go/bin/weed /usr/bin/
 RUN mkdir -p /etc/seaweedfs
-COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
 COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
 RUN apk add fuse snappy gflags
 
@@ -50,7 +50,7 @@ EXPOSE 8333
 # webdav server http port
 EXPOSE 7333
 
-RUN mkdir -p /data/filerldb2
+RUN mkdir -p /data/filer_rocksdb
 
 VOLUME /data
 

docker/compose/local-cluster-compose.yml

@@ -36,7 +36,7 @@ services:
     ports:
       - 8080:8080
       - 18080:18080
-    command: 'volume -dataCenter dc1 -rack v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
+    command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -46,7 +46,7 @@ services:
     ports:
       - 8082:8082
       - 18082:18082
-    command: 'volume -dataCenter dc2 -rack v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
+    command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -56,7 +56,7 @@ services:
     ports:
       - 8083:8083
       - 18083:18083
-    command: 'volume -dataCenter dc3 -rack v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
+    command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -67,7 +67,7 @@ services:
       - 8888:8888
       - 18888:18888
       - 8111:8111
-    command: 'filer -defaultReplicaPlacement 100 -iam -master="master0:9333,master1:9334,master2:9335"'
+    command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"'
     depends_on:
       - master0
       - master1
@@ -85,4 +85,4 @@ services:
       - master2
       - volume1
       - volume2
-      - filer
+      - filer

docker/entrypoint.sh

@@ -24,7 +24,7 @@ case "$1" in
   'master')
   	ARGS="-mdir=/data -volumePreallocate -volumeSizeLimitMB=1024"
   	shift
-  	exec /usr/bin/weed master $ARGS $@
+  	exec /usr/bin/weed -logtostderr=true master $ARGS $@
 	;;
 
   'volume')
@@ -33,7 +33,7 @@ case "$1" in
   	  ARGS="-dir=/data"
   	fi
   	shift
-  	exec /usr/bin/weed volume $ARGS $@
+  	exec /usr/bin/weed -logtostderr=true volume $ARGS $@
 	;;
 
   'server')
@@ -42,19 +42,19 @@ case "$1" in
   	  ARGS="-dir=/data -master.volumePreallocate -master.volumeSizeLimitMB=1024"
   	fi
  	shift
-  	exec /usr/bin/weed server $ARGS $@
+  	exec /usr/bin/weed -logtostderr=true server $ARGS $@
   	;;
 
   'filer')
   	ARGS=""
   	shift
-  	exec /usr/bin/weed filer $ARGS $@
+  	exec /usr/bin/weed -logtostderr=true filer $ARGS $@
 	;;
 
   's3')
   	ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE"
   	shift
-  	exec /usr/bin/weed s3 $ARGS $@
+  	exec /usr/bin/weed -logtostderr=true s3 $ARGS $@
 	;;
 
   *)

docker/filer_rocksdb.toml

@@ -0,0 +1,3 @@
+[rocksdb]
+enabled = true
+dir = "/data/filer_rocksdb"

go.mod

@@ -1,24 +1,23 @@
 module github.com/chrislusf/seaweedfs
 
-go 1.17
+go 1.18
 
 require (
-	cloud.google.com/go v0.94.1 // indirect
-	cloud.google.com/go/pubsub v1.3.1
-	cloud.google.com/go/storage v1.16.1
+	cloud.google.com/go v0.100.2 // indirect
+	cloud.google.com/go/pubsub v1.19.0
+	cloud.google.com/go/storage v1.21.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.14.0
-	github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
 	github.com/OneOfOne/xxhash v1.2.2
 	github.com/Shopify/sarama v1.23.1
-	github.com/aws/aws-sdk-go v1.35.3
+	github.com/aws/aws-sdk-go v1.43.25
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash v1.1.0
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
-	github.com/chrislusf/raft v1.0.7
-	github.com/colinmarc/hdfs/v2 v2.2.0
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/chrislusf/raft v1.0.8
+	github.com/colinmarc/hdfs/v2 v2.3.0
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -33,26 +32,25 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
-	github.com/fclairamb/ftpserverlib v0.8.0
+	github.com/fclairamb/ftpserverlib v0.17.0
 	github.com/frankban/quicktest v1.7.2 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
-	github.com/go-redis/redis/v8 v8.4.4
+	github.com/go-redis/redis/v8 v8.11.5
 	github.com/go-redsync/redsync/v4 v4.4.1
-	github.com/go-sql-driver/mysql v1.5.0
+	github.com/go-sql-driver/mysql v1.6.0
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/go-zookeeper/zk v1.0.2 // indirect
 	github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
-	github.com/golang-jwt/jwt v3.2.1+incompatible
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
+	github.com/golang-jwt/jwt v3.2.2+incompatible
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/btree v1.0.0
-	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/btree v1.0.1
+	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/uuid v1.3.0
-	github.com/google/wire v0.4.0 // indirect
-	github.com/googleapis/gax-go v2.0.2+incompatible // indirect
-	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
+	github.com/google/wire v0.5.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
 	github.com/gorilla/mux v1.7.4
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
@@ -61,34 +59,29 @@ require (
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
-	github.com/jcmturner/gokrb5/v8 v8.4.1
-	github.com/jinzhu/copier v0.2.8
+	github.com/jcmturner/gokrb5/v8 v8.4.2
+	github.com/jinzhu/copier v0.3.5
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/json-iterator/go v1.1.11
-	github.com/karlseguin/ccache/v2 v2.0.7
+	github.com/json-iterator/go v1.1.12
+	github.com/karlseguin/ccache/v2 v2.0.8
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/klauspost/cpuid v1.2.1 // indirect
 	github.com/klauspost/reedsolomon v1.9.2
-	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
 	github.com/kurin/blazer v0.5.3
-	github.com/lib/pq v1.10.0
+	github.com/lib/pq v1.10.4
 	github.com/linxGnu/grocksdb v1.6.38
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/mailru/easyjson v0.7.1 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/nats-io/jwt v1.0.1 // indirect
-	github.com/nats-io/nats.go v1.10.0 // indirect
-	github.com/nats-io/nkeys v0.2.0 // indirect
-	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/olivere/elastic/v7 v7.0.19
 	github.com/pelletier/go-toml v1.7.0 // indirect
-	github.com/peterh/liner v1.1.0
+	github.com/peterh/liner v1.2.2
 	github.com/pierrec/lz4 v2.2.7+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -100,19 +93,18 @@ require (
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
-	github.com/seaweedfs/fuse v1.2.1
-	github.com/seaweedfs/goexif v1.0.2
-	github.com/sirupsen/logrus v1.6.0 // indirect
+	github.com/seaweedfs/goexif v2.0.0+incompatible
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/afero v1.7.0 // indirect
 	github.com/spf13/cast v1.3.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.4.0
-	github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
-	github.com/stretchr/testify v1.7.0
+	github.com/streadway/amqp v1.0.0
+	github.com/stretchr/testify v1.7.1
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tidwall/gjson v1.10.2
+	github.com/tidwall/gjson v1.14.0
 	github.com/tidwall/match v1.1.1
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
@@ -126,25 +118,24 @@ require (
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
 	go.etcd.io/etcd/client/v3 v3.5.0
-	go.mongodb.org/mongo-driver v1.8.0
+	go.mongodb.org/mongo-driver v1.8.4
 	go.opencensus.io v0.23.0 // indirect
-	go.opentelemetry.io/otel v0.15.0 // indirect
-	gocloud.dev v0.20.0
+	gocloud.dev v0.24.0
 	gocloud.dev/pubsub/natspubsub v0.20.0
-	gocloud.dev/pubsub/rabbitpubsub v0.20.0
-	golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
+	gocloud.dev/pubsub/rabbitpubsub v0.24.0
+	golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
-	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
+	golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect
+	golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/api v0.57.0
+	google.golang.org/api v0.73.0
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 // indirect
-	google.golang.org/grpc v1.40.0
-	google.golang.org/protobuf v1.27.1
+	google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6 // indirect
+	google.golang.org/grpc v1.45.0
+	google.golang.org/protobuf v1.28.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
 	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
@@ -152,44 +143,62 @@ require (
 	gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
 	gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
 	modernc.org/b v1.0.0 // indirect
-	modernc.org/cc/v3 v3.33.5 // indirect
-	modernc.org/ccgo/v3 v3.9.4 // indirect
-	modernc.org/libc v1.9.5 // indirect
-	modernc.org/mathutil v1.2.2 // indirect
-	modernc.org/memory v1.0.4 // indirect
+	modernc.org/cc/v3 v3.35.24 // indirect
+	modernc.org/ccgo/v3 v3.15.17 // indirect
+	modernc.org/libc v1.14.12 // indirect
+	modernc.org/mathutil v1.4.1 // indirect
+	modernc.org/memory v1.0.7 // indirect
 	modernc.org/opt v0.1.1 // indirect
-	modernc.org/sqlite v1.10.7
-	modernc.org/strutil v1.1.0 // indirect
+	modernc.org/sqlite v1.15.3
+	modernc.org/strutil v1.1.1 // indirect
 	modernc.org/token v1.0.0 // indirect
 )
 
 require (
-	cloud.google.com/go/kms v1.0.0 // indirect
-	github.com/apache/thrift v0.15.0 // indirect
+	github.com/fluent/fluent-logger-golang v1.8.0
+	github.com/hanwen/go-fuse/v2 v2.1.0
+)
+
+require (
+	cloud.google.com/go/compute v1.5.0 // indirect
+	cloud.google.com/go/iam v0.1.1 // indirect
+	github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.9.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect
+	github.com/aws/smithy-go v1.8.0 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
-	github.com/fluent/fluent-logger-golang v1.8.0 // indirect
+	github.com/fclairamb/go-log v0.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
-	github.com/jcmturner/rpc/v2 v2.0.2 // indirect
+	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/mattn/go-runewidth v0.0.7 // indirect
 	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
-	github.com/miekg/dns v1.1.44 // indirect
+	github.com/nats-io/nats.go v1.11.0 // indirect
+	github.com/nats-io/nkeys v0.3.0 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/philhofer/fwd v1.1.1 // indirect
+	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tinylib/msgp v1.1.6 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.0 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
-	go.uber.org/zap v1.17.0 // indirect
-	golang.org/x/mod v0.4.2 // indirect
+	go.uber.org/zap v1.19.0 // indirect
+	golang.org/x/mod v0.5.0 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	lukechampine.com/uint128 v1.1.1 // indirect
 )
 
-// replace github.com/seaweedfs/fuse => /Users/chrislu/go/src/github.com/seaweedfs/fuse
 // replace github.com/chrislusf/raft => /Users/chrislu/go/src/github.com/chrislusf/raft

k8s/helm_charts2/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.85"
-version: "2.85"
+appVersion: "2.96"
+version: "2.96"

k8s/helm_charts2/templates/s3-deployment.yaml

@@ -130,7 +130,7 @@ spec:
               name: swfs-s3
           readinessProbe:
             httpGet:
-              path: /
+              path: /status
               port: {{ .Values.s3.port }}
               scheme: HTTP
             initialDelaySeconds: 15
@@ -140,7 +140,7 @@ spec:
             timeoutSeconds: 10
           livenessProbe:
             httpGet:
-              path: /
+              path: /status
               port: {{ .Values.s3.port }}
               scheme: HTTP
             initialDelaySeconds: 20

k8s/helm_charts2/templates/volume-statefulset.yaml

@@ -40,7 +40,7 @@ spec:
       imagePullSecrets:
         - name: {{ .Values.global.imagePullSecrets }}
       {{- end }}
-      terminationGracePeriodSeconds: 10
+      terminationGracePeriodSeconds: 150
       {{- if .Values.volume.priorityClassName }}
       priorityClassName: {{ .Values.volume.priorityClassName | quote }}
       {{- end }}

other/java/client/src/main/proto/filer.proto

@@ -114,6 +114,7 @@ message Entry {
     bytes content = 9; // if not empty, the file content
 
     RemoteEntry remote_entry = 10;
+    int64 quota = 11; // for bucket only. Positive/Negative means enabled/disabled.
 }
 
 message FullEntry {
@@ -170,6 +171,8 @@ message FuseAttributes {
     string symlink_target = 13;
     bytes md5 = 14;
     string disk_type = 15;
+    uint32 rdev = 16;
+    uint64 inode = 17;
 }
 
 message CreateEntryRequest {
@@ -178,6 +181,7 @@ message CreateEntryRequest {
     bool o_excl = 3;
     bool is_from_other_cluster = 4;
     repeated int32 signatures = 5;
+    bool skip_check_parent_directory = 6;
 }
 
 message CreateEntryResponse {

other/java/examples/pom.xml

@@ -23,7 +23,7 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
-            <version>2.9.2</version>
+            <version>2.10.1</version>
             <scope>compile</scope>
         </dependency>
     </dependencies>

other/java/hdfs-over-ftp/pom.xml

@@ -36,7 +36,7 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
-            <version>3.2.1</version>
+            <version>3.2.2</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>

other/java/hdfs2/pom.xml

@@ -6,7 +6,7 @@
 
     <properties>
         <seaweedfs.client.version>2.85</seaweedfs.client.version>
-        <hadoop.version>2.9.2</hadoop.version>
+        <hadoop.version>2.10.1</hadoop.version>
     </properties>
 
     <groupId>com.github.chrislusf</groupId>

other/java/hdfs3/pom.xml

@@ -6,7 +6,7 @@
 
     <properties>
         <seaweedfs.client.version>2.85</seaweedfs.client.version>
-        <hadoop.version>3.1.1</hadoop.version>
+        <hadoop.version>3.1.4</hadoop.version>
     </properties>
 
     <groupId>com.github.chrislusf</groupId>

weed/Makefile

@@ -19,15 +19,15 @@ debug_shell:
 
 debug_mount:
 	go build -gcflags="all=-N -l"
-	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/ -umask=000
 
 debug_server:
 	go build -gcflags="all=-N -l"
-	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=/Volumes/mobile_disk/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1
 
 debug_volume:
 	go build -gcflags="all=-N -l"
-	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- volume -dir=/Volumes/mobile_disk/100 -port 8564 -max=30 -preStopSeconds=2
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- volume -dir=~/tmp/100 -port 8564 -max=30 -preStopSeconds=2
 
 debug_webdav:
 	go build -gcflags="all=-N -l"
@@ -52,4 +52,3 @@ debug_filer_remote_sync_buckets:
 debug_master_follower:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 master.follower
-

weed/command/benchmark.go

@@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
 		defer pprof.StopCPUProfile()
 	}
 
-	b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddresses())
+	b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
 	go b.masterClient.KeepConnectedToMaster()
 	b.masterClient.WaitUntilConnected()
 

weed/command/filer.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"fmt"
+	"net"
 	"net/http"
 	"os"
 	"time"
@@ -12,7 +13,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
 	stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
@@ -28,7 +29,7 @@ var (
 )
 
 type FilerOptions struct {
-	masters                 []pb.ServerAddress
+	masters                 map[string]pb.ServerAddress
 	mastersString           *string
 	ip                      *string
 	bindIp                  *string
@@ -51,6 +52,7 @@ type FilerOptions struct {
 	concurrentUploadLimitMB *int
 	debug                   *bool
 	debugPort               *int
+	localSocket             *string
 }
 
 func init() {
@@ -58,7 +60,7 @@ func init() {
 	f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
 	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
 	f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
-	f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to")
+	f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
 	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
 	f.portGrpc = cmdFiler.Flag.Int("port.grpc", 0, "filer server grpc listen port")
 	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
@@ -76,6 +78,7 @@ func init() {
 	f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
 	f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
 	f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging")
+	f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
 
 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -96,10 +99,11 @@ func init() {
 	filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file")
 	filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
 	filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
-	filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB")
+	filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
 
 	// start iam on filer
 	filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
+	filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
 	filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
 }
 
@@ -139,11 +143,14 @@ func runFiler(cmd *Command, args []string) bool {
 	if *filerStartS3 {
 		filerS3Options.filer = &filerAddress
 		filerS3Options.bindIp = f.bindIp
+		filerS3Options.localFilerSocket = f.localSocket
 		go func() {
 			time.Sleep(startDelay * time.Second)
 			filerS3Options.startS3Server()
 		}()
 		startDelay++
+	} else {
+		*f.localSocket = ""
 	}
 
 	if *filerStartWebDav {
@@ -164,7 +171,7 @@ func runFiler(cmd *Command, args []string) bool {
 		}()
 	}
 
-	f.masters = pb.ServerAddresses(*f.mastersString).ToAddresses()
+	f.masters = pb.ServerAddresses(*f.mastersString).ToAddressMap()
 
 	f.startFiler()
 
@@ -182,6 +189,9 @@ func (fo *FilerOptions) startFiler() {
 	if *fo.portGrpc == 0 {
 		*fo.portGrpc = 10000 + *fo.port
 	}
+	if *fo.bindIp == "" {
+		*fo.bindIp = *fo.ip
+	}
 
 	defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")
 
@@ -210,7 +220,7 @@ func (fo *FilerOptions) startFiler() {
 	if *fo.publicPort != 0 {
 		publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
 		glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
-		publicListener, e := util.NewListener(publicListeningAddress, 0)
+		publicListener, localPublicListner, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
 		if e != nil {
 			glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
 		}
@@ -219,29 +229,61 @@ func (fo *FilerOptions) startFiler() {
 				glog.Fatalf("Volume server fail to serve public: %v", e)
 			}
 		}()
+		if localPublicListner != nil {
+			go func() {
+				if e := http.Serve(localPublicListner, publicVolumeMux); e != nil {
+					glog.Errorf("Volume server fail to serve public: %v", e)
+				}
+			}()
+		}
 	}
 
 	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
-	filerListener, e := util.NewListener(
-		util.JoinHostPort(*fo.bindIp, *fo.port),
+	filerListener, filerLocalListener, e := util.NewIpAndLocalListeners(
+		*fo.bindIp, *fo.port,
 		time.Duration(10)*time.Second,
 	)
 	if e != nil {
 		glog.Fatalf("Filer listener error: %v", e)
 	}
 
+	// start on local unix socket
+	if *fo.localSocket == "" {
+		*fo.localSocket = fmt.Sprintf("/tmp/seaweefs-filer-%d.sock", *fo.port)
+		if err := os.Remove(*fo.localSocket); err != nil && !os.IsNotExist(err) {
+			glog.Fatalf("Failed to remove %s, error: %s", *fo.localSocket, err.Error())
+		}
+	}
+	filerSocketListener, err := net.Listen("unix", *fo.localSocket)
+	if err != nil {
+		glog.Fatalf("Failed to listen on %s: %v", *fo.localSocket, err)
+	}
+
 	// starting grpc server
 	grpcPort := *fo.portGrpc
-	grpcL, err := util.NewListener(util.JoinHostPort(*fo.bindIp, grpcPort), 0)
+	grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0)
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
 	}
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer"))
 	filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
 	reflection.Register(grpcS)
+	if grpcLocalL != nil {
+		go grpcS.Serve(grpcLocalL)
+	}
 	go grpcS.Serve(grpcL)
 
 	httpS := &http.Server{Handler: defaultMux}
+	go func() {
+		httpS.Serve(filerSocketListener)
+	}()
+	if filerLocalListener != nil {
+		go func() {
+			if err := httpS.Serve(filerLocalListener); err != nil {
+				glog.Errorf("Filer Fail to serve: %v", e)
+			}
+		}()
+	}
 	if err := httpS.Serve(filerListener); err != nil {
 		glog.Fatalf("Filer Fail to serve: %v", e)
 	}

weed/command/filer_cat.go

@@ -8,7 +8,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 	"google.golang.org/grpc"
-	"math"
 	"net/url"
 	"os"
 	"strings"
@@ -115,7 +114,7 @@ func runFilerCat(cmd *Command, args []string) bool {
 
 		filerCat.filerClient = client
 
-		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, int64(filer.FileSize(respLookupEntry.Entry)))
 
 	})
 

weed/command/filer_copy.go

@@ -212,7 +212,7 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
 	if mode.IsDir() {
 		files, _ := os.ReadDir(fileOrDir)
 		for _, subFileOrDir := range files {
-			cleanedDestDirectory := filepath.Clean(destPath + fi.Name())
+			cleanedDestDirectory := destPath + fi.Name()
 			if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), cleanedDestDirectory+"/", fileCopyTaskChan); err != nil {
 				return err
 			}
@@ -482,7 +482,8 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 				})
 			})
 			if err != nil {
-				fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
+				uploadError = fmt.Errorf("Failed to assign from %v: %v\n", worker.options.masters, err)
+				return
 			}
 
 			targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId

weed/command/filer_meta_backup.go

@@ -162,24 +162,21 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
 		ctx := context.Background()
 		message := resp.EventNotification
 
-		if message.OldEntry == nil && message.NewEntry == nil {
+		if filer_pb.IsEmpty(resp) {
 			return nil
-		}
-		if message.OldEntry == nil && message.NewEntry != nil {
+		} else if filer_pb.IsCreate(resp) {
 			println("+", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
 			entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry)
 			return store.InsertEntry(ctx, entry)
-		}
-		if message.OldEntry != nil && message.NewEntry == nil {
+		} else if filer_pb.IsDelete(resp) {
 			println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name))
 			return store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name))
-		}
-		if message.OldEntry != nil && message.NewEntry != nil {
-			if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
-				println("~", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
-				entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry)
-				return store.UpdateEntry(ctx, entry)
-			}
+		} else if filer_pb.IsUpdate(resp) {
+			println("~", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
+			entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry)
+			return store.UpdateEntry(ctx, entry)
+		} else {
+			// renaming
 			println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name))
 			if err := store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name)); err != nil {
 				return err

weed/command/filer_meta_tail.go

@@ -74,7 +74,7 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
 	}
 
 	shouldPrint := func(resp *filer_pb.SubscribeMetadataResponse) bool {
-		if resp.EventNotification.OldEntry == nil && resp.EventNotification.NewEntry == nil {
+		if filer_pb.IsEmpty(resp) {
 			return false
 		}
 		if filterFunc == nil {

weed/command/filer_remote_gateway_buckets.go

@@ -174,10 +174,10 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 			return handleEtcRemoteChanges(resp)
 		}
 
-		if message.OldEntry == nil && message.NewEntry == nil {
+		if filer_pb.IsEmpty(resp) {
 			return nil
 		}
-		if message.OldEntry == nil && message.NewEntry != nil {
+		if filer_pb.IsCreate(resp) {
 			if message.NewParentPath == option.bucketsDir {
 				return handleCreateBucket(message.NewEntry)
 			}
@@ -212,7 +212,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 			}
 			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
 		}
-		if message.OldEntry != nil && message.NewEntry == nil {
+		if filer_pb.IsDelete(resp) {
 			if resp.Directory == option.bucketsDir {
 				return handleDeleteBucket(message.OldEntry)
 			}
@@ -264,7 +264,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 						// update directory property
 						return nil
 					}
-					if filer.IsSameData(message.OldEntry, message.NewEntry) {
+					if message.OldEntry.RemoteEntry != nil && filer.IsSameData(message.OldEntry, message.NewEntry) {
 						glog.V(2).Infof("update meta: %+v", resp)
 						oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation)
 						return client.UpdateFileMetadata(oldDest, message.OldEntry, message.NewEntry)

weed/command/filer_remote_sync_dir.go

@@ -91,10 +91,10 @@ func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string,
 			return handleEtcRemoteChanges(resp)
 		}
 
-		if message.OldEntry == nil && message.NewEntry == nil {
+		if filer_pb.IsEmpty(resp) {
 			return nil
 		}
-		if message.OldEntry == nil && message.NewEntry != nil {
+		if filer_pb.IsCreate(resp) {
 			if !filer.HasData(message.NewEntry) {
 				return nil
 			}
@@ -115,7 +115,7 @@ func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string,
 			}
 			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
 		}
-		if message.OldEntry != nil && message.NewEntry == nil {
+		if filer_pb.IsDelete(resp) {
 			glog.V(2).Infof("delete: %+v", resp)
 			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
 			if message.OldEntry.IsDirectory {

weed/command/filer_sync.go

@@ -169,8 +169,11 @@ func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption,
 		return persistEventFn(resp)
 	}
 
+	var lastLogTsNs = time.Now().Nanosecond()
 	processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error {
-		glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, lastTsNs), float64(counter)/float64(3))
+		now := time.Now().Nanosecond()
+		glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, lastTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
+		lastLogTsNs = now
 		return setOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature, lastTsNs)
 	})
 
@@ -259,7 +262,7 @@ func genProcessFunction(sourcePath string, targetPath string, dataSink sink.Repl
 		}
 
 		// handle deletions
-		if message.OldEntry != nil && message.NewEntry == nil {
+		if filer_pb.IsDelete(resp) {
 			if !strings.HasPrefix(string(sourceOldKey), sourcePath) {
 				return nil
 			}
@@ -268,7 +271,7 @@ func genProcessFunction(sourcePath string, targetPath string, dataSink sink.Repl
 		}
 
 		// handle new entries
-		if message.OldEntry == nil && message.NewEntry != nil {
+		if filer_pb.IsCreate(resp) {
 			if !strings.HasPrefix(string(sourceNewKey), sourcePath) {
 				return nil
 			}
@@ -277,7 +280,7 @@ func genProcessFunction(sourcePath string, targetPath string, dataSink sink.Repl
 		}
 
 		// this is something special?
-		if message.OldEntry == nil && message.NewEntry == nil {
+		if filer_pb.IsEmpty(resp) {
 			return nil
 		}
 

weed/command/fix.go

@@ -2,7 +2,7 @@ package command
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io/fs"
 	"os"
 	"path"
 	"strconv"
@@ -22,17 +22,15 @@ func init() {
 }
 
 var cmdFix = &Command{
-	UsageLine: "fix -dir=/tmp -volumeId=234",
-	Short:     "run weed tool fix on index file if corrupted",
-	Long: `Fix runs the SeaweedFS fix command to re-create the index .idx file.
-
+	UsageLine: "fix [-volumeId=234] [-collection=bigData] /tmp",
+	Short:     "run weed tool fix on files or whole folders to recreate index file(s) if corrupted",
+	Long: `Fix runs the SeaweedFS fix command on dat files or whole folders to re-create the index .idx file.
   `,
 }
 
 var (
-	fixVolumePath       = cmdFix.Flag.String("dir", ".", "data directory to store files")
-	fixVolumeCollection = cmdFix.Flag.String("collection", "", "the volume collection name")
-	fixVolumeId         = cmdFix.Flag.Int("volumeId", 0, "an optional volume id.")
+	fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
+	fixVolumeId         = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
 )
 
 type VolumeFileScanner4Fix struct {
@@ -62,63 +60,68 @@ func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64
 }
 
 func runFix(cmd *Command, args []string) bool {
+	for _, arg := range args {
+		basePath, f := path.Split(util.ResolvePath(arg))
 
-	dir := util.ResolvePath(*fixVolumePath)
-	if *fixVolumeId != 0 {
-		doFixOneVolume(dir, *fixVolumeCollection, needle.VolumeId(*fixVolumeId))
-		return true
-	}
-
-	files, err := ioutil.ReadDir(dir)
-	if err != nil {
-		fmt.Println(err)
-		return false
-	}
-
-	for _, file := range files {
-		if !strings.HasSuffix(file.Name(), ".dat") {
-			continue
+		files := []fs.DirEntry{}
+		if f == "" {
+			fileInfo, err := os.ReadDir(basePath)
+			if err != nil {
+				fmt.Println(err)
+				return false
+			}
+			files = fileInfo
+		} else {
+			fileInfo, err := os.Stat(basePath + f)
+			if err != nil {
+				fmt.Println(err)
+				return false
+			}
+			files = []fs.DirEntry{fs.FileInfoToDirEntry(fileInfo)}
 		}
-		if *fixVolumeCollection != "" {
-			if !strings.HasPrefix(file.Name(), *fixVolumeCollection+"_") {
+
+		for _, file := range files {
+			if !strings.HasSuffix(file.Name(), ".dat") {
 				continue
 			}
+			if *fixVolumeCollection != "" {
+				if !strings.HasPrefix(file.Name(), *fixVolumeCollection+"_") {
+					continue
+				}
+			}
+			baseFileName := file.Name()[:len(file.Name())-4]
+			collection, volumeIdStr := "", baseFileName
+			if sepIndex := strings.LastIndex(baseFileName, "_"); sepIndex > 0 {
+				collection = baseFileName[:sepIndex]
+				volumeIdStr = baseFileName[sepIndex+1:]
+			}
+			volumeId, parseErr := strconv.ParseInt(volumeIdStr, 10, 64)
+			if parseErr != nil {
+				fmt.Printf("Failed to parse volume id from %s: %v\n", baseFileName, parseErr)
+				return false
+			}
+			if *fixVolumeId != 0 && *fixVolumeId != volumeId {
+				continue
+			}
+			doFixOneVolume(basePath, baseFileName, collection, volumeId)
 		}
-		baseFileName := file.Name()[:len(file.Name())-4]
-		collection, volumeIdStr := "", baseFileName
-		if sepIndex := strings.LastIndex(baseFileName, "_"); sepIndex > 0 {
-			collection = baseFileName[:sepIndex]
-			volumeIdStr = baseFileName[sepIndex+1:]
-		}
-		volumeId, parseErr := strconv.ParseInt(volumeIdStr, 10, 64)
-		if parseErr != nil {
-			fmt.Printf("Failed to parse volume id from %s: %v\n", baseFileName, parseErr)
-			return false
-		}
-		doFixOneVolume(dir, collection, needle.VolumeId(volumeId))
 	}
-
 	return true
 }
 
-func doFixOneVolume(dir, collection string, volumeId needle.VolumeId) {
+func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64) {
 
-	baseFileName := strconv.Itoa(int(volumeId))
-	if collection != "" {
-		baseFileName = collection + "_" + baseFileName
-	}
-
-	indexFileName := path.Join(dir, baseFileName+".idx")
+	indexFileName := path.Join(basepath, baseFileName+".idx")
 
 	nm := needle_map.NewMemDb()
 	defer nm.Close()
 
-	vid := needle.VolumeId(*fixVolumeId)
+	vid := needle.VolumeId(volumeId)
 	scanner := &VolumeFileScanner4Fix{
 		nm: nm,
 	}
 
-	if err := storage.ScanVolumeFile(dir, collection, vid, storage.NeedleMapInMemory, scanner); err != nil {
+	if err := storage.ScanVolumeFile(basepath, collection, vid, storage.NeedleMapInMemory, scanner); err != nil {
 		glog.Fatalf("scan .dat File: %v", err)
 		os.Remove(indexFileName)
 	}

weed/command/iam.go

@@ -22,6 +22,7 @@ var (
 type IamOptions struct {
 	filer   *string
 	masters *string
+	ip      *string
 	port    *int
 }
 
@@ -29,6 +30,7 @@ func init() {
 	cmdIam.Run = runIam // break init cycle
 	iamStandaloneOptions.filer = cmdIam.Flag.String("filer", "localhost:8888", "filer server address")
 	iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers")
+	iamStandaloneOptions.ip = cmdIam.Flag.String("ip", util.DetectedHostAddress(), "iam server http listen ip address")
 	iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port")
 }
 
@@ -65,7 +67,7 @@ func (iamopt *IamOptions) startIamServer() bool {
 		}
 	}
 
-	masters := pb.ServerAddresses(*iamopt.masters).ToAddresses()
+	masters := pb.ServerAddresses(*iamopt.masters).ToAddressMap()
 	router := mux.NewRouter().SkipClean(true)
 	_, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{
 		Masters:        masters,
@@ -81,12 +83,19 @@ func (iamopt *IamOptions) startIamServer() bool {
 	httpS := &http.Server{Handler: router}
 
 	listenAddress := fmt.Sprintf(":%d", *iamopt.port)
-	iamApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
+	iamApiListener, iamApiLocalListener, err := util.NewIpAndLocalListeners(*iamopt.ip, *iamopt.port, time.Duration(10)*time.Second)
 	if err != nil {
 		glog.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err)
 	}
 
 	glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", util.Version(), *iamopt.port)
+	if iamApiLocalListener != nil {
+		go func() {
+			if err = httpS.Serve(iamApiLocalListener); err != nil {
+				glog.Errorf("IAM API Server Fail to serve: %v", err)
+			}
+		}()
+	}
 	if err = httpS.Serve(iamApiListener); err != nil {
 		glog.Fatalf("IAM API Server Fail to serve: %v", err)
 	}

weed/command/master.go

@@ -1,22 +1,25 @@
 package command
 
 import (
-	"github.com/chrislusf/raft/protobuf"
-	"github.com/gorilla/mux"
-	"google.golang.org/grpc/reflection"
 	"net/http"
 	"os"
 	"sort"
 	"strings"
 	"time"
 
+	"github.com/chrislusf/raft/protobuf"
+	stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
+	"github.com/gorilla/mux"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc/reflection"
+
 	"github.com/chrislusf/seaweedfs/weed/util/grace"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
@@ -42,6 +45,9 @@ type MasterOptions struct {
 	metricsAddress     *string
 	metricsIntervalSec *int
 	raftResumeState    *bool
+	metricsHttpPort    *int
+	heartbeatInterval  *time.Duration
+	electionTimeout    *time.Duration
 }
 
 func init() {
@@ -49,7 +55,7 @@ func init() {
 	m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
 	m.portGrpc = cmdMaster.Flag.Int("port.grpc", 0, "grpc listen port")
 	m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address, also used as identifier")
-	m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to")
+	m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
 	m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
 	m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
 	m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
@@ -61,7 +67,10 @@ func init() {
 	m.disableHttp = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
 	m.metricsAddress = cmdMaster.Flag.String("metrics.address", "", "Prometheus gateway address <host>:<port>")
 	m.metricsIntervalSec = cmdMaster.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
+	m.metricsHttpPort = cmdMaster.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	m.raftResumeState = cmdMaster.Flag.Bool("resumeState", false, "resume previous state on start master server")
+	m.heartbeatInterval = cmdMaster.Flag.Duration("heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
+	m.electionTimeout = cmdMaster.Flag.Duration("electionTimeout", 10*time.Second, "election timeout of master servers")
 }
 
 var cmdMaster = &Command{
@@ -104,6 +113,7 @@ func runMaster(cmd *Command, args []string) bool {
 		glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
 	}
 
+	go stats_collect.StartMetricsServer(*m.metricsHttpPort)
 	startMaster(m, masterWhiteList)
 
 	return true
@@ -116,20 +126,38 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	if *masterOption.portGrpc == 0 {
 		*masterOption.portGrpc = 10000 + *masterOption.port
 	}
+	if *masterOption.ipBind == "" {
+		*masterOption.ipBind = *masterOption.ip
+	}
 
 	myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.portGrpc, *masterOption.peers)
 
+	masterPeers := make(map[string]pb.ServerAddress)
+	for _, peer := range peers {
+		masterPeers[peer.String()] = peer
+	}
+
 	r := mux.NewRouter()
-	ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), peers)
+	ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers)
 	listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
 	glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
-	masterListener, e := util.NewListener(listeningAddress, 0)
+	masterListener, masterLocalListner, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
 	}
+
 	// start raftServer
-	raftServer, err := weed_server.NewRaftServer(security.LoadClientTLS(util.GetViper(), "grpc.master"),
-		peers, myMasterAddress, util.ResolvePath(*masterOption.metaFolder), ms.Topo, *masterOption.raftResumeState)
+	raftServerOption := &weed_server.RaftServerOption{
+		GrpcDialOption:    security.LoadClientTLS(util.GetViper(), "grpc.master"),
+		Peers:             masterPeers,
+		ServerAddr:        myMasterAddress,
+		DataDir:           util.ResolvePath(*masterOption.metaFolder),
+		Topo:              ms.Topo,
+		RaftResumeState:   *masterOption.raftResumeState,
+		HeartbeatInterval: *masterOption.heartbeatInterval,
+		ElectionTimeout:   *masterOption.electionTimeout,
+	}
+	raftServer, err := weed_server.NewRaftServer(raftServerOption)
 	if raftServer == nil {
 		glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
 	}
@@ -137,7 +165,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
 	// starting grpc server
 	grpcPort := *masterOption.portGrpc
-	grpcL, err := util.NewListener(util.JoinHostPort(*masterOption.ipBind, grpcPort), 0)
+	grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOption.ipBind, grpcPort, 0)
 	if err != nil {
 		glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
 	}
@@ -146,6 +174,9 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	protobuf.RegisterRaftServer(grpcS, raftServer)
 	reflection.Register(grpcS)
 	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOption.ipBind, grpcPort)
+	if grpcLocalL != nil {
+		go grpcS.Serve(grpcLocalL)
+	}
 	go grpcS.Serve(grpcL)
 
 	go func() {
@@ -160,8 +191,39 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	go ms.MasterClient.KeepConnectedToMaster()
 
 	// start http server
+	var (
+		clientCertFile,
+		certFile,
+		keyFile string
+	)
+	useTLS := false
+	useMTLS := false
+
+	if viper.GetString("https.master.key") != "" {
+		useTLS = true
+		certFile = viper.GetString("https.master.cert")
+		keyFile = viper.GetString("https.master.key")
+	}
+
+	if viper.GetString("https.master.ca") != "" {
+		useMTLS = true
+		clientCertFile = viper.GetString("https.master.ca")
+	}
+
 	httpS := &http.Server{Handler: r}
-	go httpS.Serve(masterListener)
+	if masterLocalListner != nil {
+		go httpS.Serve(masterLocalListner)
+	}
+
+	if useMTLS {
+		httpS.TLSConfig = security.LoadClientTLSHTTP(clientCertFile)
+	}
+
+	if useTLS {
+		go httpS.ServeTLS(masterListener, certFile, keyFile)
+	} else {
+		go httpS.Serve(masterListener)
+	}
 
 	select {}
 }

weed/command/master_follower.go

@@ -3,17 +3,18 @@ package command
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"time"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
 	"google.golang.org/grpc/reflection"
-	"net/http"
-	"time"
 )
 
 var (
@@ -24,7 +25,7 @@ func init() {
 	cmdMasterFollower.Run = runMasterFollower // break init cycle
 	mf.port = cmdMasterFollower.Flag.Int("port", 9334, "http listen port")
 	mf.portGrpc = cmdMasterFollower.Flag.Int("port.grpc", 0, "grpc listen port")
-	mf.ipBind = cmdMasterFollower.Flag.String("ip.bind", "", "ip address to bind to")
+	mf.ipBind = cmdMasterFollower.Flag.String("ip.bind", "", "ip address to bind to. Default to localhost.")
 	mf.peers = cmdMasterFollower.Flag.String("masters", "localhost:9333", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
 
 	mf.ip = aws.String(util.DetectedHostAddress())
@@ -45,13 +46,13 @@ var cmdMasterFollower = &Command{
 	Short:     "start a master follower",
 	Long: `start a master follower to provide volume=>location mapping service
 
-	The master follower does not participate in master election. 
+	The master follower does not participate in master election.
 	It just follow the existing masters, and listen for any volume location changes.
 
 	In most cases, the master follower is not needed. In big data centers with thousands of volume
 	servers. In theory, the master may have trouble to keep up with the write requests and read requests.
 
-	The master follower can relieve the master from from read requests, which only needs to 
+	The master follower can relieve the master from from read requests, which only needs to
 	lookup a fileId or volumeId.
 
 	The master follower currently can handle fileId lookup requests:
@@ -82,7 +83,7 @@ func runMasterFollower(cmd *Command, args []string) bool {
 func startMasterFollower(masterOptions MasterOptions) {
 
 	// collect settings from main masters
-	masters := pb.ServerAddresses(*mf.peers).ToAddresses()
+	masters := pb.ServerAddresses(*mf.peers).ToAddressMap()
 
 	var err error
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.master")
@@ -111,18 +112,22 @@ func startMasterFollower(masterOptions MasterOptions) {
 	option := masterOptions.toMasterOption(nil)
 	option.IsFollower = true
 
+	if *masterOptions.ipBind == "" {
+		*masterOptions.ipBind = *masterOptions.ip
+	}
+
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, option, masters)
 	listeningAddress := util.JoinHostPort(*masterOptions.ipBind, *masterOptions.port)
 	glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
-	masterListener, e := util.NewListener(listeningAddress, 0)
+	masterListener, masterLocalListner, e := util.NewIpAndLocalListeners(*masterOptions.ipBind, *masterOptions.port, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
 	}
 
 	// starting grpc server
 	grpcPort := *masterOptions.portGrpc
-	grpcL, err := util.NewListener(util.JoinHostPort(*masterOptions.ipBind, grpcPort), 0)
+	grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOptions.ipBind, grpcPort, 0)
 	if err != nil {
 		glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
 	}
@@ -130,12 +135,18 @@ func startMasterFollower(masterOptions MasterOptions) {
 	master_pb.RegisterSeaweedServer(grpcS, ms)
 	reflection.Register(grpcS)
 	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOptions.ip, grpcPort)
+	if grpcLocalL != nil {
+		go grpcS.Serve(grpcLocalL)
+	}
 	go grpcS.Serve(grpcL)
 
 	go ms.MasterClient.KeepConnectedToMaster()
 
 	// start http server
 	httpS := &http.Server{Handler: r}
+	if masterLocalListner != nil {
+		go httpS.Serve(masterLocalListner)
+	}
 	go httpS.Serve(masterListener)
 
 	select {}

weed/command/mount.go

@@ -11,6 +11,7 @@ type MountOptions struct {
 	dir                *string
 	dirAutoCreate      *bool
 	collection         *string
+	collectionQuota    *int
 	replication        *string
 	diskType           *string
 	ttlSec             *int
@@ -26,6 +27,8 @@ type MountOptions struct {
 	uidMap             *string
 	gidMap             *string
 	readOnly           *bool
+	debug              *bool
+	debugPort          *int
 }
 
 var (
@@ -42,13 +45,14 @@ func init() {
 	mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
 	mountOptions.dirAutoCreate = cmdMount.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to")
 	mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files")
+	mountOptions.collectionQuota = cmdMount.Flag.Int("collectionQuotaMB", 0, "quota for the collection")
 	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
 	mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
 	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0")
 	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
-	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)")
+	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 0, "local file chunk cache capacity in MB")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
 	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
 	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")
@@ -57,6 +61,8 @@ func init() {
 	mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
 	mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
 	mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only")
+	mountOptions.debug = cmdMount.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
+	mountOptions.debugPort = cmdMount.Flag.Int("debug.port", 6061, "http port for debugging")
 
 	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
 	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")

weed/command/mount_darwin.go

@@ -1,13 +1,5 @@
 package command
 
-import (
-	"github.com/seaweedfs/fuse"
-)
-
-func osSpecificMountOptions() []fuse.MountOption {
-	return []fuse.MountOption{}
-}
-
 func checkMountPointAvailable(dir string) bool {
 	return true
 }

weed/command/mount_freebsd.go

@@ -1,13 +0,0 @@
-package command
-
-import (
-	"github.com/seaweedfs/fuse"
-)
-
-func osSpecificMountOptions() []fuse.MountOption {
-	return []fuse.MountOption{}
-}
-
-func checkMountPointAvailable(dir string) bool {
-	return true
-}

weed/command/mount_linux.go

@@ -6,8 +6,6 @@ import (
 	"io"
 	"os"
 	"strings"
-
-	"github.com/seaweedfs/fuse"
 )
 
 const (
@@ -137,10 +135,6 @@ func parseInfoFile(r io.Reader) ([]*Info, error) {
 	return out, nil
 }
 
-func osSpecificMountOptions() []fuse.MountOption {
-	return []fuse.MountOption{}
-}
-
 func checkMountPointAvailable(dir string) bool {
 	mountPoint := dir
 	if mountPoint != "/" && strings.HasSuffix(mountPoint, "/") {

weed/command/mount_notsupported.go

@@ -1,5 +1,5 @@
-//go:build !linux && !darwin && !freebsd
-// +build !linux,!darwin,!freebsd
+//go:build !linux && !darwin
+// +build !linux,!darwin
 
 package command
 

weed/command/mount_std.go

@@ -1,39 +1,38 @@
-//go:build linux || darwin || freebsd
-// +build linux darwin freebsd
+//go:build linux || darwin
+// +build linux darwin
 
 package command
 
 import (
 	"context"
 	"fmt"
-	"os"
-	"os/user"
-	"path"
-	"path/filepath"
-	"runtime"
-	"strconv"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/chrislusf/seaweedfs/weed/storage/types"
-
-	"github.com/chrislusf/seaweedfs/weed/filesys/meta_cache"
-
-	"github.com/seaweedfs/fuse"
-	"github.com/seaweedfs/fuse/fs"
-
-	"github.com/chrislusf/seaweedfs/weed/filesys"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/mount"
+	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
+	"github.com/chrislusf/seaweedfs/weed/mount/unmount"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"net/http"
+	"os"
+	"os/user"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/chrislusf/seaweedfs/weed/util/grace"
 )
 
 func runMount(cmd *Command, args []string) bool {
 
+	if *mountOptions.debug {
+		go http.ListenAndServe(fmt.Sprintf(":%d", *mountOptions.debugPort), nil)
+	}
+
 	grace.SetupProfiling(*mountCpuProfile, *mountMemProfile)
 	if *mountReadRetryTime < time.Second {
 		*mountReadRetryTime = time.Second
@@ -53,27 +52,18 @@ func runMount(cmd *Command, args []string) bool {
 	return RunMount(&mountOptions, os.FileMode(umask))
 }
 
-func getParentInode(mountDir string) (uint64, error) {
-	parentDir := filepath.Clean(filepath.Join(mountDir, ".."))
-	fi, err := os.Stat(parentDir)
-	if err != nil {
-		return 0, err
-	}
-
-	stat, ok := fi.Sys().(*syscall.Stat_t)
-	if !ok {
-		return 0, nil
-	}
-
-	return stat.Ino, nil
-}
-
 func RunMount(option *MountOptions, umask os.FileMode) bool {
 
-	filerAddresses := pb.ServerAddresses(*option.filer).ToAddresses()
+	// basic checks
+	chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB
+	if chunkSizeLimitMB <= 0 {
+		fmt.Printf("Please specify a reasonable buffer size.")
+		return false
+	}
 
+	// try to connect to filer
+	filerAddresses := pb.ServerAddresses(*option.filer).ToAddresses()
 	util.LoadConfiguration("security", false)
-	// try to connect to filer, filerBucketsPath may be useful later
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 	var cipher bool
 	var err error
@@ -98,26 +88,15 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	}
 
 	filerMountRootPath := *option.filerMountRootPath
-	dir := util.ResolvePath(*option.dir)
-	parentInode, err := getParentInode(dir)
-	if err != nil {
-		glog.Errorf("failed to retrieve inode for parent directory of %s: %v", dir, err)
-		return true
-	}
 
-	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)
+	// clean up mount point
+	dir := util.ResolvePath(*option.dir)
 	if dir == "" {
 		fmt.Printf("Please specify the mount directory via \"-dir\"")
 		return false
 	}
 
-	chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB
-	if chunkSizeLimitMB <= 0 {
-		fmt.Printf("Please specify a reasonable buffer size.")
-		return false
-	}
-
-	fuse.Unmount(dir)
+	unmount.Unmount(dir)
 
 	// detect mount folder mode
 	if *option.dirAutoCreate {
@@ -125,6 +104,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	}
 	fileInfo, err := os.Stat(dir)
 
+	// collect uid, gid
 	uid, gid := uint32(0), uint32(0)
 	mountMode := os.ModeDir | 0777
 	if err == nil {
@@ -136,6 +116,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		return false
 	}
 
+	// detect uid, gid
 	if uid == 0 {
 		if u, err := user.Current(); err == nil {
 			if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
@@ -161,34 +142,51 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		return true
 	}
 
-	mountName := path.Base(dir)
+	serverFriendlyName := strings.ReplaceAll(*option.filer, ",", "+")
 
-	options := []fuse.MountOption{
-		fuse.VolumeName(mountName),
-		fuse.FSName(*option.filer + ":" + filerMountRootPath),
-		fuse.Subtype("seaweedfs"),
-		// fuse.NoAppleDouble(), // include .DS_Store, otherwise can not delete non-empty folders
-		fuse.NoAppleXattr(),
-		fuse.ExclCreate(),
-		fuse.DaemonTimeout("3600"),
-		fuse.AllowSUID(),
-		fuse.DefaultPermissions(),
-		fuse.MaxReadahead(1024 * 512),
-		fuse.AsyncRead(),
-		// fuse.WritebackCache(),
-		// fuse.MaxBackground(1024),
-		// fuse.CongestionThreshold(1024),
-	}
-
-	options = append(options, osSpecificMountOptions()...)
-	if *option.allowOthers {
-		options = append(options, fuse.AllowOther())
+	// mount fuse
+	fuseMountOptions := &fuse.MountOptions{
+		AllowOther:               *option.allowOthers,
+		Options:                  nil,
+		MaxBackground:            128,
+		MaxWrite:                 1024 * 1024 * 2,
+		MaxReadAhead:             1024 * 1024 * 2,
+		IgnoreSecurityLabels:     false,
+		RememberInodes:           false,
+		FsName:                   serverFriendlyName + ":" + filerMountRootPath,
+		Name:                     "seaweedfs",
+		SingleThreaded:           false,
+		DisableXAttrs:            false,
+		Debug:                    *option.debug,
+		EnableLocks:              false,
+		ExplicitDataCacheControl: false,
+		DirectMount:              true,
+		DirectMountFlags:         0,
+		//SyncRead:                 false, // set to false to enable the FUSE_CAP_ASYNC_READ capability
+		//EnableAcl:                true,
 	}
 	if *option.nonempty {
-		options = append(options, fuse.AllowNonEmptyMount())
+		fuseMountOptions.Options = append(fuseMountOptions.Options, "nonempty")
 	}
 	if *option.readOnly {
-		options = append(options, fuse.ReadOnly())
+		if runtime.GOOS == "darwin" {
+			fuseMountOptions.Options = append(fuseMountOptions.Options, "rdonly")
+		} else {
+			fuseMountOptions.Options = append(fuseMountOptions.Options, "ro")
+		}
+	}
+	if runtime.GOOS == "darwin" {
+		// https://github-wiki-see.page/m/macfuse/macfuse/wiki/Mount-Options
+		ioSizeMB := 1
+		for ioSizeMB*2 <= *option.chunkSizeLimitMB && ioSizeMB*2 <= 32 {
+			ioSizeMB *= 2
+		}
+		fuseMountOptions.Options = append(fuseMountOptions.Options, "daemon_timeout=600")
+		fuseMountOptions.Options = append(fuseMountOptions.Options, "noapplexattr")
+		// fuseMountOptions.Options = append(fuseMountOptions.Options, "novncache") // need to test effectiveness
+		fuseMountOptions.Options = append(fuseMountOptions.Options, "slow_statfs")
+		fuseMountOptions.Options = append(fuseMountOptions.Options, "volname="+serverFriendlyName)
+		fuseMountOptions.Options = append(fuseMountOptions.Options, fmt.Sprintf("iosize=%d", ioSizeMB*1024*1024))
 	}
 
 	// find mount point
@@ -197,9 +195,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		mountRoot = mountRoot[0 : len(mountRoot)-1]
 	}
 
-	diskType := types.ToDiskType(*option.diskType)
-
-	seaweedFileSystem := filesys.NewSeaweedFileSystem(&filesys.Option{
+	seaweedFileSystem := mount.NewSeaweedFileSystem(&mount.Option{
 		MountDirectory:     dir,
 		FilerAddresses:     filerAddresses,
 		GrpcDialOption:     grpcDialOption,
@@ -207,49 +203,37 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		Collection:         *option.collection,
 		Replication:        *option.replication,
 		TtlSec:             int32(*option.ttlSec),
-		DiskType:           diskType,
+		DiskType:           types.ToDiskType(*option.diskType),
 		ChunkSizeLimit:     int64(chunkSizeLimitMB) * 1024 * 1024,
 		ConcurrentWriters:  *option.concurrentWriters,
 		CacheDir:           *option.cacheDir,
 		CacheSizeMB:        *option.cacheSizeMB,
 		DataCenter:         *option.dataCenter,
+		Quota:              int64(*option.collectionQuota) * 1024 * 1024,
 		MountUid:           uid,
 		MountGid:           gid,
 		MountMode:          mountMode,
 		MountCtime:         fileInfo.ModTime(),
 		MountMtime:         time.Now(),
-		MountParentInode:   parentInode,
 		Umask:              umask,
 		VolumeServerAccess: *mountOptions.volumeServerAccess,
 		Cipher:             cipher,
 		UidGidMapper:       uidGidMapper,
 	})
 
-	// mount
-	c, err := fuse.Mount(dir, options...)
+	server, err := fuse.NewServer(seaweedFileSystem, dir, fuseMountOptions)
 	if err != nil {
-		glog.V(0).Infof("mount: %v", err)
-		return true
+		glog.Fatalf("Mount fail: %v", err)
 	}
-	defer fuse.Unmount(dir)
-
 	grace.OnInterrupt(func() {
-		fuse.Unmount(dir)
-		c.Close()
+		unmount.Unmount(dir)
 	})
 
-	glog.V(0).Infof("mounted %s%s to %v", *option.filer, mountRoot, dir)
-	server := fs.New(c, nil)
-	seaweedFileSystem.Server = server
 	seaweedFileSystem.StartBackgroundTasks()
-	err = server.Serve(seaweedFileSystem)
 
-	// check if the mount process has an error to report
-	<-c.Ready
-	if err := c.MountError; err != nil {
-		glog.V(0).Infof("mount process: %v", err)
-		return true
-	}
+	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)
+
+	server.Serve()
 
 	return true
 }

weed/command/msg_broker.go

@@ -95,7 +95,7 @@ func (msgBrokerOpt *MessageBrokerOptions) startQueueServer() bool {
 	}, grpcDialOption)
 
 	// start grpc listener
-	grpcL, err := util.NewListener(util.JoinHostPort("", *msgBrokerOpt.port), 0)
+	grpcL, _, err := util.NewIpAndLocalListeners("", *msgBrokerOpt.port, 0)
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", *msgBrokerOpt.port, err)
 	}

weed/command/s3.go

@@ -34,12 +34,13 @@ type S3Options struct {
 	metricsHttpPort  *int
 	allowEmptyFolder *bool
 	auditLogConfig   *string
+	localFilerSocket *string
 }
 
 func init() {
 	cmdS3.Run = runS3 // break init cycle
 	s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
-	s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to")
+	s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to. Default to localhost.")
 	s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
 	s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
 	s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file")
@@ -184,6 +185,7 @@ func (s3opt *S3Options) startS3Server() bool {
 		BucketsPath:      filerBucketsPath,
 		GrpcDialOption:   grpcDialOption,
 		AllowEmptyFolder: *s3opt.allowEmptyFolder,
+		LocalFilerSocket: s3opt.localFilerSocket,
 	})
 	if s3ApiServer_err != nil {
 		glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
@@ -191,8 +193,12 @@ func (s3opt *S3Options) startS3Server() bool {
 
 	httpS := &http.Server{Handler: router}
 
+	if *s3opt.bindIp == "" {
+		*s3opt.bindIp = "localhost"
+	}
+
 	listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port)
-	s3ApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
+	s3ApiListener, s3ApiLocalListner, err := util.NewIpAndLocalListeners(*s3opt.bindIp, *s3opt.port, time.Duration(10)*time.Second)
 	if err != nil {
 		glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
 	}
@@ -206,11 +212,25 @@ func (s3opt *S3Options) startS3Server() bool {
 
 	if *s3opt.tlsPrivateKey != "" {
 		glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
+		if s3ApiLocalListner != nil {
+			go func() {
+				if err = httpS.ServeTLS(s3ApiLocalListner, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
+					glog.Fatalf("S3 API Server Fail to serve: %v", err)
+				}
+			}()
+		}
 		if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
 			glog.Fatalf("S3 API Server Fail to serve: %v", err)
 		}
 	} else {
 		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.Version(), *s3opt.port)
+		if s3ApiLocalListner != nil {
+			go func() {
+				if err = httpS.Serve(s3ApiLocalListner); err != nil {
+					glog.Fatalf("S3 API Server Fail to serve: %v", err)
+				}
+			}()
+		}
 		if err = httpS.Serve(s3ApiListener); err != nil {
 			glog.Fatalf("S3 API Server Fail to serve: %v", err)
 		}

weed/command/scaffold.go

@@ -2,7 +2,7 @@ package command
 
 import (
 	"fmt"
-	"os"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"path/filepath"
 
 	"github.com/chrislusf/seaweedfs/weed/command/scaffold"
@@ -56,7 +56,7 @@ func runScaffold(cmd *Command, args []string) bool {
 	}
 
 	if *outputPath != "" {
-		os.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
+		util.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
 	} else {
 		fmt.Println(content)
 	}

weed/command/scaffold/filer.toml

@@ -153,6 +153,8 @@ password = ""
 superLargeDirectories = []
 # Name of the datacenter local to this filer, used as host selection fallback.
 localDC = ""
+# Gocql connection timeout, default: 600ms
+connection_timeout_millisecond = 600
 
 [hbase]
 enabled = false
@@ -193,6 +195,40 @@ routeByLatency = false
 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
 superLargeDirectories = []
 
+[redis_lua]
+enabled = false
+address = "localhost:6379"
+password = ""
+database = 0
+# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
+superLargeDirectories = []
+
+[redis_lua_sentinel]
+enabled = false
+addresses = ["172.22.12.7:26379","172.22.12.8:26379","172.22.12.9:26379"]
+masterName = "master"
+username = ""
+password = ""
+database = 0
+
+[redis_lua_cluster]
+enabled = false
+addresses = [
+    "localhost:30001",
+    "localhost:30002",
+    "localhost:30003",
+    "localhost:30004",
+    "localhost:30005",
+    "localhost:30006",
+]
+password = ""
+# allows reads from slave servers or the master, but all writes still go to the master
+readOnly = false
+# automatically use the closest Redis server for reads
+routeByLatency = false
+# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
+superLargeDirectories = []
+
 [redis3] # beta
 enabled = false
 address = "localhost:6379"

weed/command/scaffold/master.toml

@@ -19,9 +19,6 @@ scripts = """
 """
 sleep_minutes = 17          # sleep minutes between each script execution
 
-[master.filer]
-default = "localhost:8888"    # used by maintenance scripts if the scripts needs to use fs related commands
-
 
 [master.sequencer]
 type = "raft"     # Choose [raft|snowflake] type for storing the file id sequence

weed/command/scaffold/security.toml

@@ -83,7 +83,13 @@ key = ""
 #     this does not work with other clients, e.g., "weed filer|mount" etc, yet.
 [https.client]
 enabled = true
+
 [https.volume]
 cert = ""
 key = ""
+ca = ""
 
+[https.master]
+cert = ""
+key = ""
+ca = ""

weed/command/server.go

@@ -28,6 +28,7 @@ var (
 	masterOptions    MasterOptions
 	filerOptions     FilerOptions
 	s3Options        S3Options
+	iamOptions       IamOptions
 	webdavOptions    WebDavOption
 	msgBrokerOptions MessageBrokerOptions
 )
@@ -54,14 +55,14 @@ var cmdServer = &Command{
 
 var (
 	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
-	serverBindIp              = cmdServer.Flag.String("ip.bind", "", "ip address to bind to")
+	serverBindIp              = cmdServer.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
 	serverTimeout             = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
 	serverDataCenter          = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
 	serverRack                = cmdServer.Flag.String("rack", "", "current volume server's rack name")
 	serverWhiteListOption     = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
 	serverDisableHttp         = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
 	volumeDataFolders         = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
-	volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
+	volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured as free disk space divided by volume size.")
 	volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).")
 	volumeMinFreeSpace        = cmdServer.Flag.String("volume.minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
 	serverMetricsHttpPort     = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
@@ -71,6 +72,7 @@ var (
 	isStartingVolumeServer = cmdServer.Flag.Bool("volume", true, "whether to start volume server")
 	isStartingFiler        = cmdServer.Flag.Bool("filer", false, "whether to start filer")
 	isStartingS3           = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
+	isStartingIam          = cmdServer.Flag.Bool("iam", false, "whether to start IAM service")
 	isStartingWebDav       = cmdServer.Flag.Bool("webdav", false, "whether to start WebDAV gateway")
 	isStartingMsgBroker    = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker")
 
@@ -96,6 +98,8 @@ func init() {
 	masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
 	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
 	masterOptions.raftResumeState = cmdServer.Flag.Bool("resumeState", false, "resume previous state on start master server")
+	masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
+	masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
 
 	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
@@ -108,6 +112,7 @@ func init() {
 	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
 	filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.")
 	filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
+	filerOptions.localSocket = cmdServer.Flag.String("filer.localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
@@ -134,6 +139,8 @@ func init() {
 	s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
 	s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
 
+	iamOptions.port = cmdServer.Flag.Int("iam.port", 8111, "iam server http listen port")
+
 	webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port")
 	webdavOptions.collection = cmdServer.Flag.String("webdav.collection", "", "collection to create the files")
 	webdavOptions.replication = cmdServer.Flag.String("webdav.replication", "", "replication to create the files")
@@ -141,7 +148,7 @@ func init() {
 	webdavOptions.tlsPrivateKey = cmdServer.Flag.String("webdav.key.file", "", "path to the TLS private key file")
 	webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
 	webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
-	webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB")
+	webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
 
 	msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port")
 
@@ -161,6 +168,9 @@ func runServer(cmd *Command, args []string) bool {
 	if *isStartingS3 {
 		*isStartingFiler = true
 	}
+	if *isStartingIam {
+		*isStartingFiler = true
+	}
 	if *isStartingWebDav {
 		*isStartingFiler = true
 	}
@@ -174,13 +184,19 @@ func runServer(cmd *Command, args []string) bool {
 		masterOptions.peers = &peers
 	}
 
+	if *serverBindIp == "" {
+		serverBindIp = serverIp
+	}
+
 	// ip address
 	masterOptions.ip = serverIp
 	masterOptions.ipBind = serverBindIp
-	filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
+	filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddressMap()
 	filerOptions.ip = serverIp
 	filerOptions.bindIp = serverBindIp
 	s3Options.bindIp = serverBindIp
+	iamOptions.ip = serverBindIp
+	iamOptions.masters = masterOptions.peers
 	serverOptions.v.ip = serverIp
 	serverOptions.v.bindIp = serverBindIp
 	serverOptions.v.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
@@ -201,6 +217,7 @@ func runServer(cmd *Command, args []string) bool {
 
 	filerAddress := string(pb.NewServerAddress(*serverIp, *filerOptions.port, *filerOptions.portGrpc))
 	s3Options.filer = &filerAddress
+	iamOptions.filer = &filerAddress
 	webdavOptions.filer = &filerAddress
 	msgBrokerOptions.filer = &filerAddress
 
@@ -227,25 +244,28 @@ func runServer(cmd *Command, args []string) bool {
 	if *isStartingFiler {
 		go func() {
 			time.Sleep(1 * time.Second)
-
 			filerOptions.startFiler()
-
 		}()
 	}
 
 	if *isStartingS3 {
 		go func() {
 			time.Sleep(2 * time.Second)
-
+			s3Options.localFilerSocket = filerOptions.localSocket
 			s3Options.startS3Server()
+		}()
+	}
 
+	if *isStartingIam {
+		go func() {
+			time.Sleep(2 * time.Second)
+			iamOptions.startIamServer()
 		}()
 	}
 
 	if *isStartingWebDav {
 		go func() {
 			time.Sleep(2 * time.Second)
-
 			webdavOptions.startWebDav()
 
 		}()

weed/command/volume.go

@@ -2,7 +2,6 @@ package command
 
 import (
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"net/http"
 	httppprof "net/http/pprof"
 	"os"
@@ -11,6 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
+
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
 
@@ -24,7 +25,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/chrislusf/seaweedfs/weed/server"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
 	stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -74,7 +75,7 @@ func init() {
 	v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
 	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
 	v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
-	v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to")
+	v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
 	v.mastersString = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
 	v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
 	// v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
@@ -94,7 +95,7 @@ func init() {
 	v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
 	v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
-	v.enableTcp = cmdVolume.Flag.Bool("tcp", false, "<exprimental> enable tcp port")
+	v.enableTcp = cmdVolume.Flag.Bool("tcp", false, "<experimental> enable tcp port")
 }
 
 var cmdVolume = &Command{
@@ -107,7 +108,7 @@ var cmdVolume = &Command{
 
 var (
 	volumeFolders         = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
-	maxVolumeCounts       = cmdVolume.Flag.String("max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
+	maxVolumeCounts       = cmdVolume.Flag.String("max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured as free disk space divided by volume size.")
 	volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
 	minFreeSpacePercent   = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).")
 	minFreeSpace          = cmdVolume.Flag.String("minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
@@ -193,6 +194,9 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		*v.ip = util.DetectedHostAddress()
 		glog.V(0).Infof("detected volume server ip address: %v", *v.ip)
 	}
+	if *v.bindIp == "" {
+		*v.bindIp = *v.ip
+	}
 
 	if *v.publicPort == 0 {
 		*v.publicPort = *v.port
@@ -364,11 +368,18 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpd
 	}
 
 	httpDown := httpdown.HTTP{
-		KillTimeout: 5 * time.Minute,
-		StopTimeout: 5 * time.Minute,
+		KillTimeout: time.Minute,
+		StopTimeout: 30 * time.Second,
 		CertFile:    certFile,
 		KeyFile:     keyFile}
-	clusterHttpServer := httpDown.Serve(&http.Server{Handler: handler}, listener)
+	httpS := &http.Server{Handler: handler}
+
+	if viper.GetString("https.volume.ca") != "" {
+		clientCertFile := viper.GetString("https.volume.ca")
+		httpS.TLSConfig = security.LoadClientTLSHTTP(clientCertFile)
+	}
+
+	clusterHttpServer := httpDown.Serve(httpS, listener)
 	go func() {
 		if e := clusterHttpServer.Wait(); e != nil {
 			glog.Fatalf("Volume server fail to serve: %v", e)

weed/command/webdav.go

@@ -43,7 +43,7 @@ func init() {
 	webDavStandaloneOptions.tlsPrivateKey = cmdWebDav.Flag.String("key.file", "", "path to the TLS private key file")
 	webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file")
 	webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
-	webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 1000, "local cache capacity in MB")
+	webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 0, "local cache capacity in MB")
 }
 
 var cmdWebDav = &Command{

weed/filer/cassandra/cassandra_store.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/gocql/gocql"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -33,6 +34,7 @@ func (store *CassandraStore) Initialize(configuration util.Configuration, prefix
 		configuration.GetString(prefix+"password"),
 		configuration.GetStringSlice(prefix+"superLargeDirectories"),
 		configuration.GetString(prefix+"localDC"),
+		configuration.GetInt(prefix+"connection_timeout_millisecond"),
 	)
 }
 
@@ -41,12 +43,14 @@ func (store *CassandraStore) isSuperLargeDirectory(dir string) (dirHash string,
 	return
 }
 
-func (store *CassandraStore) initialize(keyspace string, hosts []string, username string, password string, superLargeDirectories []string, localDC string) (err error) {
+func (store *CassandraStore) initialize(keyspace string, hosts []string, username string, password string, superLargeDirectories []string, localDC string, timeout int) (err error) {
 	store.cluster = gocql.NewCluster(hosts...)
 	if username != "" && password != "" {
 		store.cluster.Authenticator = gocql.PasswordAuthenticator{Username: username, Password: password}
 	}
 	store.cluster.Keyspace = keyspace
+	store.cluster.Timeout = time.Duration(timeout) * time.Millisecond
+	glog.V(0).Infof("timeout = %d", timeout)
 	fallback := gocql.RoundRobinHostPolicy()
 	if localDC != "" {
 		fallback = gocql.DCAwareRoundRobinPolicy(localDC)

weed/filer/entry.go

@@ -24,6 +24,8 @@ type Attr struct {
 	SymlinkTarget string
 	Md5           []byte
 	FileSize      uint64
+	Rdev          uint32
+	Inode         uint64
 }
 
 func (attr Attr) IsDirectory() bool {
@@ -43,6 +45,7 @@ type Entry struct {
 	HardLinkCounter int32
 	Content         []byte
 	Remote          *filer_pb.RemoteEntry
+	Quota           int64
 }
 
 func (entry *Entry) Size() uint64 {
@@ -70,6 +73,7 @@ func (entry *Entry) ShallowClone() *Entry {
 	newEntry.HardLinkCounter = entry.HardLinkCounter
 	newEntry.Content = entry.Content
 	newEntry.Remote = entry.Remote
+	newEntry.Quota = entry.Quota
 
 	return newEntry
 }
@@ -96,6 +100,7 @@ func (entry *Entry) ToExistingProtoEntry(message *filer_pb.Entry) {
 	message.HardLinkCounter = entry.HardLinkCounter
 	message.Content = entry.Content
 	message.RemoteEntry = entry.Remote
+	message.Quota = entry.Quota
 }
 
 func FromPbEntryToExistingEntry(message *filer_pb.Entry, fsEntry *Entry) {
@@ -106,6 +111,7 @@ func FromPbEntryToExistingEntry(message *filer_pb.Entry, fsEntry *Entry) {
 	fsEntry.HardLinkCounter = message.HardLinkCounter
 	fsEntry.Content = message.Content
 	fsEntry.Remote = message.RemoteEntry
+	fsEntry.Quota = message.Quota
 }
 
 func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry {

weed/filer/entry_codec.go

@@ -48,6 +48,8 @@ func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes {
 		SymlinkTarget: entry.Attr.SymlinkTarget,
 		Md5:           entry.Attr.Md5,
 		FileSize:      entry.Attr.FileSize,
+		Rdev:          entry.Attr.Rdev,
+		Inode:         entry.Attr.Inode,
 	}
 }
 
@@ -74,6 +76,8 @@ func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr {
 	t.SymlinkTarget = attr.SymlinkTarget
 	t.Md5 = attr.Md5
 	t.FileSize = attr.FileSize
+	t.Rdev = attr.Rdev
+	t.Inode = attr.Inode
 
 	return t
 }
@@ -118,6 +122,9 @@ func EqualEntry(a, b *Entry) bool {
 	if !proto.Equal(a.Remote, b.Remote) {
 		return false
 	}
+	if a.Quota != b.Quota {
+		return false
+	}
 	return true
 }
 

weed/filer/filechunk_manifest.go

@@ -8,6 +8,7 @@ import (
 	"math"
 	"net/url"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/golang/protobuf/proto"
@@ -21,6 +22,12 @@ const (
 	ManifestBatch = 10000
 )
 
+var bytesBufferPool = sync.Pool{
+	New: func() interface{} {
+		return new(bytes.Buffer)
+	},
+}
+
 func HasChunkManifest(chunks []*filer_pb.FileChunk) bool {
 	for _, chunk := range chunks {
 		if chunk.IsChunkManifest {
@@ -61,12 +68,12 @@ func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chun
 
 		manifestChunks = append(manifestChunks, chunk)
 		// recursive
-		dchunks, mchunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset)
+		dataChunks, manifestChunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset)
 		if subErr != nil {
 			return chunks, nil, subErr
 		}
-		dataChunks = append(dataChunks, dchunks...)
-		manifestChunks = append(manifestChunks, mchunks...)
+		dataChunks = append(dataChunks, dataChunks...)
+		manifestChunks = append(manifestChunks, manifestChunks...)
 	}
 	return
 }
@@ -77,12 +84,15 @@ func ResolveOneChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, c
 	}
 
 	// IsChunkManifest
-	data, err := fetchChunk(lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed)
+	bytesBuffer := bytesBufferPool.Get().(*bytes.Buffer)
+	bytesBuffer.Reset()
+	defer bytesBufferPool.Put(bytesBuffer)
+	err := fetchWholeChunk(bytesBuffer, lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed)
 	if err != nil {
 		return nil, fmt.Errorf("fail to read manifest %s: %v", chunk.GetFileIdString(), err)
 	}
 	m := &filer_pb.FileChunkManifest{}
-	if err := proto.Unmarshal(data, m); err != nil {
+	if err := proto.Unmarshal(bytesBuffer.Bytes(), m); err != nil {
 		return nil, fmt.Errorf("fail to unmarshal manifest %s: %v", chunk.GetFileIdString(), err)
 	}
 
@@ -92,38 +102,43 @@ func ResolveOneChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, c
 }
 
 // TODO fetch from cache for weed mount?
-func fetchChunk(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool) ([]byte, error) {
+func fetchWholeChunk(bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool) error {
 	urlStrings, err := lookupFileIdFn(fileId)
 	if err != nil {
 		glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err)
-		return nil, err
+		return err
 	}
-	return retriedFetchChunkData(urlStrings, cipherKey, isGzipped, true, 0, 0)
+	err = retriedStreamFetchChunkData(bytesBuffer, urlStrings, cipherKey, isGzipped, true, 0, 0)
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
-func fetchChunkRange(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool, offset int64, size int) ([]byte, error) {
+func fetchChunkRange(buffer []byte, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool, offset int64) (int, error) {
 	urlStrings, err := lookupFileIdFn(fileId)
 	if err != nil {
 		glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err)
-		return nil, err
+		return 0, err
 	}
-	return retriedFetchChunkData(urlStrings, cipherKey, isGzipped, false, offset, size)
+	return retriedFetchChunkData(buffer, urlStrings, cipherKey, isGzipped, false, offset)
 }
 
-func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) {
+func retriedFetchChunkData(buffer []byte, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64) (n int, err error) {
 
-	var err error
 	var shouldRetry bool
-	receivedData := make([]byte, 0, size)
 
 	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
 		for _, urlString := range urlStrings {
-			receivedData = receivedData[:0]
+			n = 0
 			if strings.Contains(urlString, "%") {
 				urlString = url.PathEscape(urlString)
 			}
-			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
-				receivedData = append(receivedData, data...)
+			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, len(buffer), func(data []byte) {
+				if n < len(buffer) {
+					x := copy(buffer[n:], data)
+					n += x
+				}
 			})
 			if !shouldRetry {
 				break
@@ -142,7 +157,7 @@ func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool
 		}
 	}
 
-	return receivedData, err
+	return n, err
 
 }
 

weed/filer/filechunks.go

@@ -23,7 +23,13 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
 }
 
 func FileSize(entry *filer_pb.Entry) (size uint64) {
-	return maxUint64(TotalSize(entry.Chunks), entry.Attributes.FileSize)
+	fileSize := entry.Attributes.FileSize
+	if entry.RemoteEntry != nil {
+		if entry.RemoteEntry.RemoteMtime > entry.Attributes.Mtime {
+			fileSize = maxUint64(fileSize, uint64(entry.RemoteEntry.RemoteSize))
+		}
+	}
+	return maxUint64(TotalSize(entry.Chunks), fileSize)
 }
 
 func ETag(entry *filer_pb.Entry) (etag string) {
@@ -101,6 +107,21 @@ func DoMinusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {
 	return
 }
 
+func DoMinusChunksBySourceFileId(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {
+
+	fileIds := make(map[string]bool)
+	for _, interval := range bs {
+		fileIds[interval.GetFileIdString()] = true
+	}
+	for _, chunk := range as {
+		if _, found := fileIds[chunk.GetSourceFileId()]; !found {
+			delta = append(delta, chunk)
+		}
+	}
+
+	return
+}
+
 type ChunkView struct {
 	FileId      string
 	Offset      int64

weed/filer/filer.go

@@ -49,7 +49,7 @@ type Filer struct {
 	UniqueFileId        uint32
 }
 
-func NewFiler(masters []pb.ServerAddress, grpcDialOption grpc.DialOption,
+func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption,
 	filerHost pb.ServerAddress, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
 	f := &Filer{
 		MasterClient:        wdclient.NewMasterClient(grpcDialOption, cluster.FilerType, filerHost, dataCenter, masters),
@@ -151,7 +151,7 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error {
 	return f.Store.RollbackTransaction(ctx)
 }
 
-func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32) error {
+func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool) error {
 
 	if string(entry.FullPath) == "/" {
 		return nil
@@ -169,9 +169,11 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr
 
 	if oldEntry == nil {
 
-		dirParts := strings.Split(string(entry.FullPath), "/")
-		if err := f.ensureParentDirecotryEntry(ctx, entry, dirParts, len(dirParts)-1, isFromOtherCluster); err != nil {
-			return err
+		if !skipCreateParentDir {
+			dirParts := strings.Split(string(entry.FullPath), "/")
+			if err := f.ensureParentDirecotryEntry(ctx, entry, dirParts, len(dirParts)-1, isFromOtherCluster); err != nil {
+				return err
+			}
 		}
 
 		glog.V(4).Infof("InsertEntry %s: new entry: %v", entry.FullPath, entry.Name())

weed/filer/filer_conf.go

@@ -75,12 +75,12 @@ func (fc *FilerConf) loadFromFiler(filer *Filer) (err error) {
 		return fc.LoadFromBytes(entry.Content)
 	}
 
-	return fc.loadFromChunks(filer, entry.Content, entry.Chunks)
+	return fc.loadFromChunks(filer, entry.Content, entry.Chunks, entry.Size())
 }
 
-func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk) (err error) {
+func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk, size uint64) (err error) {
 	if len(content) == 0 {
-		content, err = filer.readEntry(chunks)
+		content, err = filer.readEntry(chunks, size)
 		if err != nil {
 			glog.Errorf("read filer conf content: %v", err)
 			return

weed/filer/filer_delete_entry.go

@@ -9,8 +9,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
-type HardLinkId []byte
-
 const (
 	MsgFailDelNonEmptyFolder = "fail to delete non-empty folder"
 )
@@ -127,7 +125,11 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
 
 	glog.V(3).Infof("deleting entry %v, delete chunks: %v", entry.FullPath, shouldDeleteChunks)
 
-	if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil {
+	if !entry.IsDirectory() && !shouldDeleteChunks {
+		if storeDeletionErr := f.Store.DeleteOneEntrySkipHardlink(ctx, entry.FullPath); storeDeletionErr != nil {
+			return fmt.Errorf("filer store delete skip hardlink: %v", storeDeletionErr)
+		}
+	} else if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil {
 		return fmt.Errorf("filer store delete: %v", storeDeletionErr)
 	}
 	if !entry.IsDirectory() {

weed/filer/filer_deletion.go

@@ -1,6 +1,7 @@
 package filer
 
 import (
+	"math"
 	"strings"
 	"time"
 
@@ -129,6 +130,12 @@ func (f *Filer) DeleteChunks(chunks []*filer_pb.FileChunk) {
 	}
 }
 
+func (f *Filer) DeleteChunksNotRecursive(chunks []*filer_pb.FileChunk) {
+	for _, chunk := range chunks {
+		f.fileIdDeletionQueue.EnQueue(chunk.GetFileIdString())
+	}
+}
+
 func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
 
 	if oldEntry == nil {
@@ -141,14 +148,36 @@ func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
 
 	var toDelete []*filer_pb.FileChunk
 	newChunkIds := make(map[string]bool)
-	for _, newChunk := range newEntry.Chunks {
+	newDataChunks, newManifestChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(),
+		newEntry.Chunks, 0, math.MaxInt64)
+	if err != nil {
+		glog.Errorf("Failed to resolve new entry chunks when delete old entry chunks. new: %s, old: %s",
+			newEntry.Chunks, oldEntry.Chunks)
+		return
+	}
+	for _, newChunk := range newDataChunks {
+		newChunkIds[newChunk.GetFileIdString()] = true
+	}
+	for _, newChunk := range newManifestChunks {
 		newChunkIds[newChunk.GetFileIdString()] = true
 	}
 
-	for _, oldChunk := range oldEntry.Chunks {
+	oldDataChunks, oldManifestChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(),
+		oldEntry.Chunks, 0, math.MaxInt64)
+	if err != nil {
+		glog.Errorf("Failed to resolve old entry chunks when delete old entry chunks. new: %s, old: %s",
+			newEntry.Chunks, oldEntry.Chunks)
+		return
+	}
+	for _, oldChunk := range oldDataChunks {
 		if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found {
 			toDelete = append(toDelete, oldChunk)
 		}
 	}
-	f.DeleteChunks(toDelete)
+	for _, oldChunk := range oldManifestChunks {
+		if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found {
+			toDelete = append(toDelete, oldChunk)
+		}
+	}
+	f.DeleteChunksNotRecursive(toDelete)
 }

weed/filer/filer_hardlink.go

@@ -0,0 +1,16 @@
+package filer
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+const (
+	HARD_LINK_MARKER = '\x01'
+)
+
+type HardLinkId []byte // 16 bytes + 1 marker byte
+
+func NewHardLinkId() HardLinkId {
+	bytes := append(util.RandomBytes(16), HARD_LINK_MARKER)
+	return bytes
+}

weed/filer/filer_notify_append.go

@@ -43,7 +43,7 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error {
 	entry.Chunks = append(entry.Chunks, uploadResult.ToPbFileChunk(assignResult.Fid, offset))
 
 	// update the entry
-	err = f.CreateEntry(context.Background(), entry, false, false, nil)
+	err = f.CreateEntry(context.Background(), entry, false, false, nil, false)
 
 	return err
 }

weed/filer/filer_on_meta_event.go

@@ -2,8 +2,6 @@ package filer
 
 import (
 	"bytes"
-	"math"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -24,12 +22,12 @@ func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
 		}
 	}
 	if f.DirBucketsPath == event.Directory {
-		if message.OldEntry == nil && message.NewEntry != nil {
+		if filer_pb.IsCreate(event) {
 			if message.NewEntry.IsDirectory {
 				f.Store.OnBucketCreation(message.NewEntry.Name)
 			}
 		}
-		if message.OldEntry != nil && message.NewEntry == nil {
+		if filer_pb.IsDelete(event) {
 			if message.OldEntry.IsDirectory {
 				f.Store.OnBucketDeletion(message.OldEntry.Name)
 			}
@@ -55,9 +53,9 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR
 	}
 }
 
-func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
+func (f *Filer) readEntry(chunks []*filer_pb.FileChunk, size uint64) ([]byte, error) {
 	var buf bytes.Buffer
-	err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64)
+	err := StreamContent(f.MasterClient, &buf, chunks, 0, int64(size))
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +64,7 @@ func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
 
 func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) {
 	fc := NewFilerConf()
-	err := fc.loadFromChunks(f, entry.Content, entry.Chunks)
+	err := fc.loadFromChunks(f, entry.Content, entry.Chunks, FileSize(entry))
 	if err != nil {
 		glog.Errorf("read filer conf chunks: %v", err)
 		return

weed/filer/filerstore.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"io"
 )
 
 var (
@@ -45,3 +46,7 @@ type BucketAware interface {
 	OnBucketDeletion(bucket string)
 	CanDropWholeBucket() bool
 }
+
+type Debuggable interface {
+	Debug(writer io.Writer)
+}

weed/filer/filerstore_hardlink.go

@@ -9,16 +9,20 @@ import (
 )
 
 func (fsw *FilerStoreWrapper) handleUpdateToHardLinks(ctx context.Context, entry *Entry) error {
-	if len(entry.HardLinkId) == 0 {
+
+	if entry.IsDirectory() {
 		return nil
 	}
-	// handle hard links
-	if err := fsw.setHardLink(ctx, entry); err != nil {
-		return fmt.Errorf("setHardLink %d: %v", entry.HardLinkId, err)
+
+	if len(entry.HardLinkId) > 0 {
+		// handle hard links
+		if err := fsw.setHardLink(ctx, entry); err != nil {
+			return fmt.Errorf("setHardLink %d: %v", entry.HardLinkId, err)
+		}
 	}
 
 	// check what is existing entry
-	glog.V(4).Infof("handleUpdateToHardLinks FindEntry %s", entry.FullPath)
+	// glog.V(4).Infof("handleUpdateToHardLinks FindEntry %s", entry.FullPath)
 	actualStore := fsw.getActualStore(entry.FullPath)
 	existingEntry, err := actualStore.FindEntry(ctx, entry.FullPath)
 	if err != nil && err != filer_pb.ErrNotFound {
@@ -46,6 +50,8 @@ func (fsw *FilerStoreWrapper) setHardLink(ctx context.Context, entry *Entry) err
 		return encodeErr
 	}
 
+	glog.V(4).Infof("setHardLink %v nlink:%d", entry.FullPath, entry.HardLinkCounter)
+
 	return fsw.KvPut(ctx, key, newBlob)
 }
 
@@ -55,7 +61,6 @@ func (fsw *FilerStoreWrapper) maybeReadHardLink(ctx context.Context, entry *Entr
 	}
 	key := entry.HardLinkId
 
-	glog.V(4).Infof("maybeReadHardLink KvGet %v", key)
 	value, err := fsw.KvGet(ctx, key)
 	if err != nil {
 		glog.Errorf("read %s hardlink %d: %v", entry.FullPath, entry.HardLinkId, err)
@@ -67,6 +72,8 @@ func (fsw *FilerStoreWrapper) maybeReadHardLink(ctx context.Context, entry *Entr
 		return err
 	}
 
+	glog.V(4).Infof("maybeReadHardLink %v nlink:%d", entry.FullPath, entry.HardLinkCounter)
+
 	return nil
 }
 

weed/filer/filerstore_wrapper.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/viant/ptrie"
+	"io"
 	"math"
 	"strings"
 	"time"
@@ -15,12 +16,14 @@ import (
 
 var (
 	_ = VirtualFilerStore(&FilerStoreWrapper{})
+	_ = Debuggable(&FilerStoreWrapper{})
 )
 
 type VirtualFilerStore interface {
 	FilerStore
 	DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error
 	DeleteOneEntry(ctx context.Context, entry *Entry) error
+	DeleteOneEntrySkipHardlink(ctx context.Context, fullpath util.FullPath) error
 	AddPathSpecificStore(path string, storeId string, store FilerStore)
 	OnBucketCreation(bucket string)
 	OnBucketDeletion(bucket string)
@@ -125,7 +128,7 @@ func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) err
 		return err
 	}
 
-	glog.V(4).Infof("InsertEntry %s", entry.FullPath)
+	// glog.V(4).Infof("InsertEntry %s", entry.FullPath)
 	return actualStore.InsertEntry(ctx, entry)
 }
 
@@ -146,7 +149,7 @@ func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) err
 		return err
 	}
 
-	glog.V(4).Infof("UpdateEntry %s", entry.FullPath)
+	// glog.V(4).Infof("UpdateEntry %s", entry.FullPath)
 	return actualStore.UpdateEntry(ctx, entry)
 }
 
@@ -190,7 +193,7 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath)
 		}
 	}
 
-	glog.V(4).Infof("DeleteEntry %s", fp)
+	// glog.V(4).Infof("DeleteEntry %s", fp)
 	return actualStore.DeleteEntry(ctx, fp)
 }
 
@@ -210,10 +213,22 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
 		}
 	}
 
-	glog.V(4).Infof("DeleteOneEntry %s", existingEntry.FullPath)
+	// glog.V(4).Infof("DeleteOneEntry %s", existingEntry.FullPath)
 	return actualStore.DeleteEntry(ctx, existingEntry.FullPath)
 }
 
+func (fsw *FilerStoreWrapper) DeleteOneEntrySkipHardlink(ctx context.Context, fullpath util.FullPath) (err error) {
+	actualStore := fsw.getActualStore(fullpath)
+	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "delete").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "delete").Observe(time.Since(start).Seconds())
+	}()
+
+	glog.V(4).Infof("DeleteOneEntrySkipHardlink %s", fullpath)
+	return actualStore.DeleteEntry(ctx, fullpath)
+}
+
 func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
 	actualStore := fsw.getActualStore(fp + "/")
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Inc()
@@ -222,7 +237,7 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.
 		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Observe(time.Since(start).Seconds())
 	}()
 
-	glog.V(4).Infof("DeleteFolderChildren %s", fp)
+	// glog.V(4).Infof("DeleteFolderChildren %s", fp)
 	return actualStore.DeleteFolderChildren(ctx, fp)
 }
 
@@ -234,7 +249,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath
 		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "list").Observe(time.Since(start).Seconds())
 	}()
 
-	glog.V(4).Infof("ListDirectoryEntries %s from %s limit %d", dirPath, startFileName, limit)
+	// glog.V(4).Infof("ListDirectoryEntries %s from %s limit %d", dirPath, startFileName, limit)
 	return actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *Entry) bool {
 		fsw.maybeReadHardLink(ctx, entry)
 		filer_pb.AfterEntryDeserialization(entry.Chunks)
@@ -252,7 +267,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context,
 	if limit > math.MaxInt32-1 {
 		limit = math.MaxInt32 - 1
 	}
-	glog.V(4).Infof("ListDirectoryPrefixedEntries %s from %s prefix %s limit %d", dirPath, startFileName, prefix, limit)
+	// glog.V(4).Infof("ListDirectoryPrefixedEntries %s from %s prefix %s limit %d", dirPath, startFileName, prefix, limit)
 	adjustedEntryFunc := func(entry *Entry) bool {
 		fsw.maybeReadHardLink(ctx, entry)
 		filer_pb.AfterEntryDeserialization(entry.Chunks)
@@ -283,8 +298,10 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u
 
 	count := int64(0)
 	for count < limit && len(notPrefixed) > 0 {
+		var isLastItemHasPrefix bool
 		for _, entry := range notPrefixed {
 			if strings.HasPrefix(entry.Name(), prefix) {
+				isLastItemHasPrefix = true
 				count++
 				if !eachEntryFunc(entry) {
 					return
@@ -292,9 +309,11 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u
 				if count >= limit {
 					break
 				}
+			} else {
+				isLastItemHasPrefix = false
 			}
 		}
-		if count < limit {
+		if count < limit && isLastItemHasPrefix && len(notPrefixed) == int(limit) {
 			notPrefixed = notPrefixed[:0]
 			lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool {
 				notPrefixed = append(notPrefixed, entry)
@@ -303,6 +322,8 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u
 			if err != nil {
 				return
 			}
+		} else {
+			break
 		}
 	}
 	return
@@ -333,3 +354,9 @@ func (fsw *FilerStoreWrapper) KvGet(ctx context.Context, key []byte) (value []by
 func (fsw *FilerStoreWrapper) KvDelete(ctx context.Context, key []byte) (err error) {
 	return fsw.getDefaultStore().KvDelete(ctx, key)
 }
+
+func (fsw *FilerStoreWrapper) Debug(writer io.Writer) {
+	if debuggable, ok := fsw.getDefaultStore().(Debuggable); ok {
+		debuggable.Debug(writer)
+	}
+}

weed/filer/leveldb/leveldb_store.go

@@ -9,6 +9,7 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/filter"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
+	"io"
 	"os"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
@@ -21,6 +22,10 @@ const (
 	DIR_FILE_SEPARATOR = byte(0x00)
 )
 
+var (
+	_ = filer.Debuggable(&LevelDBStore{})
+)
+
 func init() {
 	filer.Stores = append(filer.Stores, &LevelDBStore{})
 }
@@ -242,3 +247,13 @@ func getNameFromKey(key []byte) string {
 func (store *LevelDBStore) Shutdown() {
 	store.db.Close()
 }
+
+func (store *LevelDBStore) Debug(writer io.Writer) {
+	iter := store.db.NewIterator(&leveldb_util.Range{}, nil)
+	for iter.Next() {
+		key := iter.Key()
+		fullName := bytes.Replace(key, []byte{DIR_FILE_SEPARATOR}, []byte{' '}, 1)
+		fmt.Fprintf(writer, "%v\n", string(fullName))
+	}
+	iter.Release()
+}

weed/filer/leveldb/leveldb_store_test.go

@@ -13,8 +13,7 @@ import (
 
 func TestCreateAndFind(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test")
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	store := &LevelDBStore{}
 	store.initialize(dir)
 	testFiler.SetStore(store)
@@ -32,7 +31,7 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil {
+	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
@@ -67,8 +66,7 @@ func TestCreateAndFind(t *testing.T) {
 
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2")
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	store := &LevelDBStore{}
 	store.initialize(dir)
 	testFiler.SetStore(store)
@@ -90,8 +88,7 @@ func TestEmptyRoot(t *testing.T) {
 
 func BenchmarkInsertEntry(b *testing.B) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := os.MkdirTemp("", "seaweedfs_filer_bench")
-	defer os.RemoveAll(dir)
+	dir := b.TempDir()
 	store := &LevelDBStore{}
 	store.initialize(dir)
 	testFiler.SetStore(store)

weed/filer/leveldb2/leveldb2_store_test.go

@@ -2,7 +2,6 @@ package leveldb
 
 import (
 	"context"
-	"os"
 	"testing"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
@@ -11,8 +10,7 @@ import (
 
 func TestCreateAndFind(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test")
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	store := &LevelDB2Store{}
 	store.initialize(dir, 2)
 	testFiler.SetStore(store)
@@ -30,7 +28,7 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil {
+	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
@@ -65,8 +63,7 @@ func TestCreateAndFind(t *testing.T) {
 
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2")
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	store := &LevelDB2Store{}
 	store.initialize(dir, 2)
 	testFiler.SetStore(store)

weed/filer/leveldb3/leveldb3_store.go

@@ -72,8 +72,8 @@ func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) {
 	}
 	if name != DEFAULT {
 		opts = &opt.Options{
-			BlockCacheCapacity: 4 * 1024 * 1024, // default value is 8MiB
-			WriteBuffer:        2 * 1024 * 1024, // default value is 4MiB
+			BlockCacheCapacity: 16 * 1024 * 1024, // default value is 8MiB
+			WriteBuffer:        8 * 1024 * 1024,  // default value is 4MiB
 			Filter:             bloom,
 		}
 	}

weed/filer/leveldb3/leveldb3_store_test.go

@@ -2,7 +2,6 @@ package leveldb
 
 import (
 	"context"
-	"os"
 	"testing"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
@@ -11,8 +10,7 @@ import (
 
 func TestCreateAndFind(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test")
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	store := &LevelDB3Store{}
 	store.initialize(dir)
 	testFiler.SetStore(store)
@@ -30,7 +28,7 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil {
+	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
@@ -65,8 +63,7 @@ func TestCreateAndFind(t *testing.T) {
 
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2")
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	store := &LevelDB3Store{}
 	store.initialize(dir)
 	testFiler.SetStore(store)

weed/filer/meta_aggregator.go

@@ -76,9 +76,6 @@ func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (no
 		}
 	} else {
 		if _, found := ma.peerStatues[address]; found {
-			ma.peerStatues[address] -= 1
-		}
-		if ma.peerStatues[address] <= 0 {
 			delete(ma.peerStatues, address)
 		}
 	}
@@ -165,13 +162,14 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self pb.ServerAddress, p
 
 	for {
 		glog.V(4).Infof("subscribing remote %s meta change: %v", peer, time.Unix(0, lastTsNs))
-		err := pb.WithFilerClient(false, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithFilerClient(true, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 			stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
 				ClientName: "filer:" + string(self),
 				PathPrefix: "/",
 				SinceNs:    lastTsNs,
+				ClientId:   int32(ma.filer.UniqueFileId),
 			})
 			if err != nil {
 				return fmt.Errorf("subscribe: %v", err)

weed/filer/mongodb/mongodb_store.go

@@ -159,7 +159,7 @@ func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPa
 	dir, name := fullpath.DirAndName()
 
 	where := bson.M{"directory": dir, "name": name}
-	_, err := store.connect.Database(store.database).Collection(store.collectionName).DeleteOne(ctx, where)
+	_, err := store.connect.Database(store.database).Collection(store.collectionName).DeleteMany(ctx, where)
 	if err != nil {
 		return fmt.Errorf("delete %s : %v", fullpath, err)
 	}
@@ -199,9 +199,9 @@ func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath uti
 
 	for cur.Next(ctx) {
 		var data Model
-		err := cur.Decode(&data)
-		if err != nil && err != mongo.ErrNoDocuments {
-			return lastFileName, err
+		err = cur.Decode(&data)
+		if err != nil {
+			break
 		}
 
 		entry := &filer.Entry{

weed/filer/mysql2/mysql2_store.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
@@ -82,7 +83,7 @@ func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpse
 		return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
 	}
 
-	if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {
+	if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil && !strings.Contains(err.Error(), "table already exist") {
 		return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
 	}
 

weed/filer/permission.go

@@ -1,22 +0,0 @@
-package filer
-
-func hasWritePermission(dir *Entry, entry *Entry) bool {
-
-	if dir == nil {
-		return false
-	}
-
-	if dir.Uid == entry.Uid && dir.Mode&0200 > 0 {
-		return true
-	}
-
-	if dir.Gid == entry.Gid && dir.Mode&0020 > 0 {
-		return true
-	}
-
-	if dir.Mode&0002 > 0 {
-		return true
-	}
-
-	return false
-}

weed/filer/read_write.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
-	"math"
 	"time"
 )
 
@@ -23,7 +22,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed
 		return err
 	}
 
-	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, int64(FileSize(respLookupEntry.Entry)))
 
 }
 
@@ -63,6 +62,7 @@ func SaveInsideFiler(client filer_pb.SeaweedFilerClient, dir, name string, conte
 				},
 				Content: content,
 			},
+			SkipCheckParentDirectory: true,
 		})
 	} else if err == nil {
 		entry := resp.Entry

weed/filer/reader_at.go

@@ -12,21 +12,16 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
-	"github.com/golang/groupcache/singleflight"
 )
 
 type ChunkReadAt struct {
-	masterClient *wdclient.MasterClient
-	chunkViews   []*ChunkView
-	lookupFileId wdclient.LookupFileIdFunctionType
-	readerLock   sync.Mutex
-	fileSize     int64
-
-	fetchGroup      singleflight.Group
-	chunkCache      chunk_cache.ChunkCache
-	lastChunkFileId string
-	lastChunkData   []byte
-	readerPattern   *ReaderPattern
+	masterClient  *wdclient.MasterClient
+	chunkViews    []*ChunkView
+	readerLock    sync.Mutex
+	fileSize      int64
+	readerCache   *ReaderCache
+	readerPattern *ReaderPattern
+	lastChunkFid  string
 }
 
 var _ = io.ReaderAt(&ChunkReadAt{})
@@ -90,16 +85,14 @@ func NewChunkReaderAtFromClient(lookupFn wdclient.LookupFileIdFunctionType, chun
 
 	return &ChunkReadAt{
 		chunkViews:    chunkViews,
-		lookupFileId:  lookupFn,
-		chunkCache:    chunkCache,
 		fileSize:      fileSize,
+		readerCache:   newReaderCache(32, chunkCache, lookupFn),
 		readerPattern: NewReaderPattern(),
 	}
 }
 
 func (c *ChunkReadAt) Close() error {
-	c.lastChunkData = nil
-	c.lastChunkFileId = ""
+	c.readerCache.destroy()
 	return nil
 }
 
@@ -117,19 +110,17 @@ func (c *ChunkReadAt) ReadAt(p []byte, offset int64) (n int, err error) {
 func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) {
 
 	startOffset, remaining := offset, int64(len(p))
-	var nextChunk *ChunkView
+	var nextChunks []*ChunkView
 	for i, chunk := range c.chunkViews {
 		if remaining <= 0 {
 			break
 		}
 		if i+1 < len(c.chunkViews) {
-			nextChunk = c.chunkViews[i+1]
-		} else {
-			nextChunk = nil
+			nextChunks = c.chunkViews[i+1:]
 		}
 		if startOffset < chunk.LogicOffset {
 			gap := int(chunk.LogicOffset - startOffset)
-			glog.V(4).Infof("zero [%d,%d)", startOffset, startOffset+int64(gap))
+			glog.V(4).Infof("zero [%d,%d)", startOffset, chunk.LogicOffset)
 			n += int(min(int64(gap), remaining))
 			startOffset, remaining = chunk.LogicOffset, remaining-int64(gap)
 			if remaining <= 0 {
@@ -142,16 +133,13 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) {
 			continue
 		}
 		// glog.V(4).Infof("read [%d,%d), %d/%d chunk %s [%d,%d)", chunkStart, chunkStop, i, len(c.chunkViews), chunk.FileId, chunk.LogicOffset-chunk.Offset, chunk.LogicOffset-chunk.Offset+int64(chunk.Size))
-		var buffer []byte
 		bufferOffset := chunkStart - chunk.LogicOffset + chunk.Offset
-		bufferLength := chunkStop - chunkStart
-		buffer, err = c.readChunkSlice(chunk, nextChunk, uint64(bufferOffset), uint64(bufferLength))
+		copied, err := c.readChunkSliceAt(p[startOffset-offset:chunkStop-chunkStart+startOffset-offset], chunk, nextChunks, uint64(bufferOffset))
 		if err != nil {
 			glog.Errorf("fetching chunk %+v: %v\n", chunk, err)
-			return
+			return copied, err
 		}
 
-		copied := copy(p[startOffset-offset:chunkStop-chunkStart+startOffset-offset], buffer)
 		n += copied
 		startOffset, remaining = startOffset+int64(copied), remaining-int64(copied)
 	}
@@ -173,104 +161,25 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) {
 
 }
 
-func (c *ChunkReadAt) readChunkSlice(chunkView *ChunkView, nextChunkViews *ChunkView, offset, length uint64) ([]byte, error) {
+func (c *ChunkReadAt) readChunkSliceAt(buffer []byte, chunkView *ChunkView, nextChunkViews []*ChunkView, offset uint64) (n int, err error) {
 
-	var chunkSlice []byte
-	if chunkView.LogicOffset == 0 {
-		chunkSlice = c.chunkCache.GetChunkSlice(chunkView.FileId, offset, length)
-	}
-	if len(chunkSlice) > 0 {
-		return chunkSlice, nil
-	}
-	if c.lookupFileId == nil {
-		return nil, nil
-	}
 	if c.readerPattern.IsRandomMode() {
-		return c.doFetchRangeChunkData(chunkView, offset, length)
-	}
-	chunkData, err := c.readFromWholeChunkData(chunkView, nextChunkViews)
-	if err != nil {
-		return nil, err
-	}
-	wanted := min(int64(length), int64(len(chunkData))-int64(offset))
-	return chunkData[offset : int64(offset)+wanted], nil
-}
-
-func (c *ChunkReadAt) readFromWholeChunkData(chunkView *ChunkView, nextChunkViews ...*ChunkView) (chunkData []byte, err error) {
-
-	if c.lastChunkFileId == chunkView.FileId {
-		return c.lastChunkData, nil
+		return fetchChunkRange(buffer, c.readerCache.lookupFileIdFn, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset))
 	}
 
-	v, doErr := c.readOneWholeChunk(chunkView)
-
-	if doErr != nil {
-		return nil, doErr
-	}
-
-	chunkData = v.([]byte)
-
-	c.lastChunkData = chunkData
-	c.lastChunkFileId = chunkView.FileId
-
-	for _, nextChunkView := range nextChunkViews {
-		if c.chunkCache != nil && nextChunkView != nil {
-			go c.readOneWholeChunk(nextChunkView)
+	n, err = c.readerCache.ReadChunkAt(buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), chunkView.LogicOffset == 0)
+	if c.lastChunkFid != chunkView.FileId {
+		if chunkView.Offset == 0 { // start of a new chunk
+			if c.lastChunkFid != "" {
+				c.readerCache.UnCache(c.lastChunkFid)
+				c.readerCache.MaybeCache(nextChunkViews)
+			} else {
+				if len(nextChunkViews) >= 1 {
+					c.readerCache.MaybeCache(nextChunkViews[:1]) // just read the next chunk if at the very beginning
+				}
+			}
 		}
 	}
-
+	c.lastChunkFid = chunkView.FileId
 	return
 }
-
-func (c *ChunkReadAt) readOneWholeChunk(chunkView *ChunkView) (interface{}, error) {
-
-	var err error
-
-	return c.fetchGroup.Do(chunkView.FileId, func() (interface{}, error) {
-
-		glog.V(4).Infof("readFromWholeChunkData %s offset %d [%d,%d) size at least %d", chunkView.FileId, chunkView.Offset, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size), chunkView.ChunkSize)
-
-		var data []byte
-		if chunkView.LogicOffset == 0 {
-			data = c.chunkCache.GetChunk(chunkView.FileId, chunkView.ChunkSize)
-		}
-		if data != nil {
-			glog.V(4).Infof("cache hit %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset-chunkView.Offset, chunkView.LogicOffset-chunkView.Offset+int64(len(data)))
-		} else {
-			var err error
-			data, err = c.doFetchFullChunkData(chunkView)
-			if err != nil {
-				return data, err
-			}
-			if chunkView.LogicOffset == 0 {
-				// only cache the first chunk
-				c.chunkCache.SetChunk(chunkView.FileId, data)
-			}
-		}
-		return data, err
-	})
-}
-
-func (c *ChunkReadAt) doFetchFullChunkData(chunkView *ChunkView) ([]byte, error) {
-
-	glog.V(4).Infof("+ doFetchFullChunkData %s", chunkView.FileId)
-
-	data, err := fetchChunk(c.lookupFileId, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped)
-
-	glog.V(4).Infof("- doFetchFullChunkData %s", chunkView.FileId)
-
-	return data, err
-
-}
-
-func (c *ChunkReadAt) doFetchRangeChunkData(chunkView *ChunkView, offset, length uint64) ([]byte, error) {
-
-	glog.V(4).Infof("+ doFetchFullChunkData %s", chunkView.FileId)
-
-	data, err := fetchChunkRange(c.lookupFileId, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(length))
-
-	glog.V(4).Infof("- doFetchFullChunkData %s", chunkView.FileId)
-
-	return data, err
-
-}

weed/filer/reader_at_test.go

@@ -21,8 +21,12 @@ func (m *mockChunkCache) GetChunk(fileId string, minSize uint64) (data []byte) {
 	return data
 }
 
-func (m *mockChunkCache) GetChunkSlice(fileId string, offset, length uint64) []byte {
-	return m.GetChunk(fileId, length)
+func (m *mockChunkCache) ReadChunkAt(data []byte, fileId string, offset uint64) (n int, err error) {
+	x, _ := strconv.Atoi(fileId)
+	for i := 0; i < len(data); i++ {
+		data[i] = byte(x)
+	}
+	return len(data), nil
 }
 
 func (m *mockChunkCache) SetChunk(fileId string, data []byte) {
@@ -65,10 +69,9 @@ func TestReaderAt(t *testing.T) {
 
 	readerAt := &ChunkReadAt{
 		chunkViews:    ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
-		lookupFileId:  nil,
 		readerLock:    sync.Mutex{},
 		fileSize:      10,
-		chunkCache:    &mockChunkCache{},
+		readerCache:   newReaderCache(3, &mockChunkCache{}, nil),
 		readerPattern: NewReaderPattern(),
 	}
 
@@ -81,7 +84,7 @@ func TestReaderAt(t *testing.T) {
 
 func testReadAt(t *testing.T, readerAt *ChunkReadAt, offset int64, size int, expected int, expectedErr error) {
 	data := make([]byte, size)
-	n, err := readerAt.ReadAt(data, offset)
+	n, err := readerAt.doReadAt(data, offset)
 
 	for _, d := range data {
 		fmt.Printf("%x", d)
@@ -116,10 +119,9 @@ func TestReaderAt0(t *testing.T) {
 
 	readerAt := &ChunkReadAt{
 		chunkViews:    ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
-		lookupFileId:  nil,
 		readerLock:    sync.Mutex{},
 		fileSize:      10,
-		chunkCache:    &mockChunkCache{},
+		readerCache:   newReaderCache(3, &mockChunkCache{}, nil),
 		readerPattern: NewReaderPattern(),
 	}
 
@@ -145,10 +147,9 @@ func TestReaderAt1(t *testing.T) {
 
 	readerAt := &ChunkReadAt{
 		chunkViews:    ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
-		lookupFileId:  nil,
 		readerLock:    sync.Mutex{},
 		fileSize:      20,
-		chunkCache:    &mockChunkCache{},
+		readerCache:   newReaderCache(3, &mockChunkCache{}, nil),
 		readerPattern: NewReaderPattern(),
 	}
 

weed/filer/reader_cache.go

@@ -0,0 +1,192 @@
+package filer
+
+import (
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
+	"github.com/chrislusf/seaweedfs/weed/util/mem"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
+	"sync"
+	"time"
+)
+
+type ReaderCache struct {
+	chunkCache     chunk_cache.ChunkCache
+	lookupFileIdFn wdclient.LookupFileIdFunctionType
+	sync.Mutex
+	downloaders map[string]*SingleChunkCacher
+	limit       int
+}
+
+type SingleChunkCacher struct {
+	sync.RWMutex
+	parent        *ReaderCache
+	chunkFileId   string
+	data          []byte
+	err           error
+	cipherKey     []byte
+	isGzipped     bool
+	chunkSize     int
+	shouldCache   bool
+	wg            sync.WaitGroup
+	completedTime time.Time
+}
+
+func newReaderCache(limit int, chunkCache chunk_cache.ChunkCache, lookupFileIdFn wdclient.LookupFileIdFunctionType) *ReaderCache {
+	return &ReaderCache{
+		limit:          limit,
+		chunkCache:     chunkCache,
+		lookupFileIdFn: lookupFileIdFn,
+		downloaders:    make(map[string]*SingleChunkCacher),
+	}
+}
+
+func (rc *ReaderCache) MaybeCache(chunkViews []*ChunkView) {
+	if rc.lookupFileIdFn == nil {
+		return
+	}
+
+	rc.Lock()
+	defer rc.Unlock()
+
+	for _, chunkView := range chunkViews {
+		if _, found := rc.downloaders[chunkView.FileId]; found {
+			continue
+		}
+
+		if len(rc.downloaders) >= rc.limit {
+			// if still no slots, return
+			return
+		}
+
+		// glog.V(4).Infof("prefetch %s offset %d", chunkView.FileId, chunkView.LogicOffset)
+		// cache this chunk if not yet
+		cacher := newSingleChunkCacher(rc, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int(chunkView.ChunkSize), false)
+		cacher.wg.Add(1)
+		go cacher.startCaching()
+		cacher.wg.Wait()
+		rc.downloaders[chunkView.FileId] = cacher
+
+	}
+
+	return
+}
+
+func (rc *ReaderCache) ReadChunkAt(buffer []byte, fileId string, cipherKey []byte, isGzipped bool, offset int64, chunkSize int, shouldCache bool) (int, error) {
+	rc.Lock()
+	defer rc.Unlock()
+	if cacher, found := rc.downloaders[fileId]; found {
+		return cacher.readChunkAt(buffer, offset)
+	}
+	if shouldCache || rc.lookupFileIdFn == nil {
+		n, err := rc.chunkCache.ReadChunkAt(buffer, fileId, uint64(offset))
+		if n > 0 {
+			return n, err
+		}
+	}
+
+	if len(rc.downloaders) >= rc.limit {
+		oldestFid, oldestTime := "", time.Now()
+		for fid, downloader := range rc.downloaders {
+			if !downloader.completedTime.IsZero() {
+				if downloader.completedTime.Before(oldestTime) {
+					oldestFid, oldestTime = fid, downloader.completedTime
+				}
+			}
+		}
+		if oldestFid != "" {
+			oldDownloader := rc.downloaders[oldestFid]
+			delete(rc.downloaders, oldestFid)
+			oldDownloader.destroy()
+		}
+	}
+
+	// glog.V(4).Infof("cache1 %s", fileId)
+
+	cacher := newSingleChunkCacher(rc, fileId, cipherKey, isGzipped, chunkSize, shouldCache)
+	cacher.wg.Add(1)
+	go cacher.startCaching()
+	cacher.wg.Wait()
+	rc.downloaders[fileId] = cacher
+
+	return cacher.readChunkAt(buffer, offset)
+}
+
+func (rc *ReaderCache) UnCache(fileId string) {
+	rc.Lock()
+	defer rc.Unlock()
+	// glog.V(4).Infof("uncache %s", fileId)
+	if downloader, found := rc.downloaders[fileId]; found {
+		downloader.destroy()
+		delete(rc.downloaders, fileId)
+	}
+}
+
+func (rc *ReaderCache) destroy() {
+	rc.Lock()
+	defer rc.Unlock()
+
+	for _, downloader := range rc.downloaders {
+		downloader.destroy()
+	}
+
+}
+
+func newSingleChunkCacher(parent *ReaderCache, fileId string, cipherKey []byte, isGzipped bool, chunkSize int, shouldCache bool) *SingleChunkCacher {
+	t := &SingleChunkCacher{
+		parent:      parent,
+		chunkFileId: fileId,
+		cipherKey:   cipherKey,
+		isGzipped:   isGzipped,
+		chunkSize:   chunkSize,
+		shouldCache: shouldCache,
+	}
+	return t
+}
+
+func (s *SingleChunkCacher) startCaching() {
+	s.Lock()
+	defer s.Unlock()
+
+	s.wg.Done() // means this has been started
+
+	urlStrings, err := s.parent.lookupFileIdFn(s.chunkFileId)
+	if err != nil {
+		s.err = fmt.Errorf("operation LookupFileId %s failed, err: %v", s.chunkFileId, err)
+		return
+	}
+
+	s.data = mem.Allocate(s.chunkSize)
+
+	_, s.err = retriedFetchChunkData(s.data, urlStrings, s.cipherKey, s.isGzipped, true, 0)
+	if s.err != nil {
+		mem.Free(s.data)
+		s.data = nil
+		return
+	}
+
+	s.completedTime = time.Now()
+	if s.shouldCache {
+		s.parent.chunkCache.SetChunk(s.chunkFileId, s.data)
+	}
+
+	return
+}
+
+func (s *SingleChunkCacher) destroy() {
+	if s.data != nil {
+		mem.Free(s.data)
+		s.data = nil
+	}
+}
+
+func (s *SingleChunkCacher) readChunkAt(buf []byte, offset int64) (int, error) {
+	s.RLock()
+	defer s.RUnlock()
+
+	if s.err != nil {
+		return 0, s.err
+	}
+
+	return copy(buf, s.data[offset:]), nil
+
+}

weed/filer/redis_lua/redis_cluster_store.go

@@ -0,0 +1,44 @@
+package redis_lua
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/go-redis/redis/v8"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &RedisLuaClusterStore{})
+}
+
+type RedisLuaClusterStore struct {
+	UniversalRedisLuaStore
+}
+
+func (store *RedisLuaClusterStore) GetName() string {
+	return "redis_lua_cluster"
+}
+
+func (store *RedisLuaClusterStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+
+	configuration.SetDefault(prefix+"useReadOnly", false)
+	configuration.SetDefault(prefix+"routeByLatency", false)
+
+	return store.initialize(
+		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetBool(prefix+"useReadOnly"),
+		configuration.GetBool(prefix+"routeByLatency"),
+		configuration.GetStringSlice(prefix+"superLargeDirectories"),
+	)
+}
+
+func (store *RedisLuaClusterStore) initialize(addresses []string, password string, readOnly, routeByLatency bool, superLargeDirectories []string) (err error) {
+	store.Client = redis.NewClusterClient(&redis.ClusterOptions{
+		Addrs:          addresses,
+		Password:       password,
+		ReadOnly:       readOnly,
+		RouteByLatency: routeByLatency,
+	})
+	store.loadSuperLargeDirectories(superLargeDirectories)
+	return
+}

weed/filer/redis_lua/redis_sentinel_store.go

@@ -0,0 +1,45 @@
+package redis_lua
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/go-redis/redis/v8"
+	"time"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &RedisLuaSentinelStore{})
+}
+
+type RedisLuaSentinelStore struct {
+	UniversalRedisLuaStore
+}
+
+func (store *RedisLuaSentinelStore) GetName() string {
+	return "redis_lua_sentinel"
+}
+
+func (store *RedisLuaSentinelStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"masterName"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetInt(prefix+"database"),
+	)
+}
+
+func (store *RedisLuaSentinelStore) initialize(addresses []string, masterName string, username string, password string, database int) (err error) {
+	store.Client = redis.NewFailoverClient(&redis.FailoverOptions{
+		MasterName:      masterName,
+		SentinelAddrs:   addresses,
+		Username:        username,
+		Password:        password,
+		DB:              database,
+		MinRetryBackoff: time.Millisecond * 100,
+		MaxRetryBackoff: time.Minute * 1,
+		ReadTimeout:     time.Second * 30,
+		WriteTimeout:    time.Second * 5,
+	})
+	return
+}

weed/filer/redis_lua/redis_store.go

@@ -0,0 +1,38 @@
+package redis_lua
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/go-redis/redis/v8"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &RedisLuaStore{})
+}
+
+type RedisLuaStore struct {
+	UniversalRedisLuaStore
+}
+
+func (store *RedisLuaStore) GetName() string {
+	return "redis_lua"
+}
+
+func (store *RedisLuaStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetString(prefix+"address"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetInt(prefix+"database"),
+		configuration.GetStringSlice(prefix+"superLargeDirectories"),
+	)
+}
+
+func (store *RedisLuaStore) initialize(hostPort string, password string, database int, superLargeDirectories []string) (err error) {
+	store.Client = redis.NewClient(&redis.Options{
+		Addr:     hostPort,
+		Password: password,
+		DB:       database,
+	})
+	store.loadSuperLargeDirectories(superLargeDirectories)
+	return
+}

weed/filer/redis_lua/stored_procedure/delete_entry.lua

@@ -0,0 +1,19 @@
+-- KEYS[1]: full path of entry
+local fullpath = KEYS[1]
+-- KEYS[2]: full path of entry
+local fullpath_list_key = KEYS[2]
+-- KEYS[3]: dir of the entry
+local dir_list_key = KEYS[3]
+
+-- ARGV[1]: isSuperLargeDirectory
+local isSuperLargeDirectory = ARGV[1] == "1"
+-- ARGV[2]: name of the entry
+local name = ARGV[2]
+
+redis.call("DEL", fullpath, fullpath_list_key)
+
+if not isSuperLargeDirectory and name ~= "" then
+    redis.call("ZREM", dir_list_key, name)
+end
+
+return 0

weed/filer/redis_lua/stored_procedure/delete_folder_children.lua

@@ -0,0 +1,15 @@
+-- KEYS[1]: full path of entry
+local fullpath = KEYS[1]
+
+if fullpath ~= "" and string.sub(fullpath, -1) == "/" then
+    fullpath = string.sub(fullpath, 0, -2)
+end
+
+local files = redis.call("ZRANGE", fullpath .. "\0", "0", "-1")
+
+for _, name in ipairs(files) do
+    local file_path = fullpath .. "/" .. name
+    redis.call("DEL", file_path, file_path .. "\0")
+end
+
+return 0

weed/filer/redis_lua/stored_procedure/init.go

@@ -0,0 +1,24 @@
+package stored_procedure
+
+import (
+	_ "embed"
+	"github.com/go-redis/redis/v8"
+)
+
+func init() {
+	InsertEntryScript = redis.NewScript(insertEntry)
+	DeleteEntryScript = redis.NewScript(deleteEntry)
+	DeleteFolderChildrenScript = redis.NewScript(deleteFolderChildren)
+}
+
+//go:embed insert_entry.lua
+var insertEntry string
+var InsertEntryScript *redis.Script
+
+//go:embed delete_entry.lua
+var deleteEntry string
+var DeleteEntryScript *redis.Script
+
+//go:embed delete_folder_children.lua
+var deleteFolderChildren string
+var DeleteFolderChildrenScript *redis.Script

weed/filer/redis_lua/stored_procedure/insert_entry.lua

@@ -0,0 +1,27 @@
+-- KEYS[1]: full path of entry
+local full_path = KEYS[1]
+-- KEYS[2]: dir of the entry
+local dir_list_key = KEYS[2]
+
+-- ARGV[1]: content of the entry
+local entry = ARGV[1]
+-- ARGV[2]: TTL of the entry
+local ttlSec = tonumber(ARGV[2])
+-- ARGV[3]: isSuperLargeDirectory
+local isSuperLargeDirectory = ARGV[3] == "1"
+-- ARGV[4]: zscore of the entry in zset
+local zscore = tonumber(ARGV[4])
+-- ARGV[5]: name of the entry
+local name = ARGV[5]
+
+if ttlSec > 0 then
+    redis.call("SET", full_path, entry, "EX", ttlSec)
+else
+    redis.call("SET", full_path, entry)
+end
+
+if not isSuperLargeDirectory and name ~= "" then
+    redis.call("ZADD", dir_list_key, "NX", zscore, name)
+end
+
+return 0

weed/filer/redis_lua/universal_redis_store.go

@@ -0,0 +1,191 @@
+package redis_lua
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/go-redis/redis/v8"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/filer/redis_lua/stored_procedure"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+const (
+	DIR_LIST_MARKER = "\x00"
+)
+
+type UniversalRedisLuaStore struct {
+	Client                  redis.UniversalClient
+	superLargeDirectoryHash map[string]bool
+}
+
+func (store *UniversalRedisLuaStore) isSuperLargeDirectory(dir string) (isSuperLargeDirectory bool) {
+	_, isSuperLargeDirectory = store.superLargeDirectoryHash[dir]
+	return
+}
+
+func (store *UniversalRedisLuaStore) loadSuperLargeDirectories(superLargeDirectories []string) {
+	// set directory hash
+	store.superLargeDirectoryHash = make(map[string]bool)
+	for _, dir := range superLargeDirectories {
+		store.superLargeDirectoryHash[dir] = true
+	}
+}
+
+func (store *UniversalRedisLuaStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *UniversalRedisLuaStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *UniversalRedisLuaStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *UniversalRedisLuaStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+
+	if len(entry.Chunks) > 50 {
+		value = util.MaybeGzipData(value)
+	}
+
+	dir, name := entry.FullPath.DirAndName()
+
+	err = stored_procedure.InsertEntryScript.Run(ctx, store.Client,
+		[]string{string(entry.FullPath), genDirectoryListKey(dir)},
+		value, entry.TtlSec,
+		store.isSuperLargeDirectory(dir), 0, name).Err()
+
+	if err != nil {
+		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
+	}
+
+	return nil
+}
+
+func (store *UniversalRedisLuaStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *UniversalRedisLuaStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+
+	data, err := store.Client.Get(ctx, string(fullpath)).Result()
+	if err == redis.Nil {
+		return nil, filer_pb.ErrNotFound
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("get %s : %v", fullpath, err)
+	}
+
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data)))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *UniversalRedisLuaStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
+
+	dir, name := fullpath.DirAndName()
+
+	err = stored_procedure.DeleteEntryScript.Run(ctx, store.Client,
+		[]string{string(fullpath), genDirectoryListKey(string(fullpath)), genDirectoryListKey(dir)},
+		store.isSuperLargeDirectory(dir), name).Err()
+
+	if err != nil {
+		return fmt.Errorf("DeleteEntry %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *UniversalRedisLuaStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
+
+	if store.isSuperLargeDirectory(string(fullpath)) {
+		return nil
+	}
+
+	err = stored_procedure.DeleteFolderChildrenScript.Run(ctx, store.Client,
+		[]string{string(fullpath)}).Err()
+
+	if err != nil {
+		return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *UniversalRedisLuaStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
+}
+
+func (store *UniversalRedisLuaStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+
+	dirListKey := genDirectoryListKey(string(dirPath))
+
+	min := "-"
+	if startFileName != "" {
+		if includeStartFile {
+			min = "[" + startFileName
+		} else {
+			min = "(" + startFileName
+		}
+	}
+
+	members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{
+		Min:    min,
+		Max:    "+",
+		Offset: 0,
+		Count:  limit,
+	}).Result()
+	if err != nil {
+		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
+	}
+
+	// fetch entry meta
+	for _, fileName := range members {
+		path := util.NewFullPath(string(dirPath), fileName)
+		entry, err := store.FindEntry(ctx, path)
+		lastFileName = fileName
+		if err != nil {
+			glog.V(0).Infof("list %s : %v", path, err)
+			if err == filer_pb.ErrNotFound {
+				continue
+			}
+		} else {
+			if entry.TtlSec > 0 {
+				if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+					store.DeleteEntry(ctx, path)
+					continue
+				}
+			}
+			if !eachEntryFunc(entry) {
+				break
+			}
+		}
+	}
+
+	return lastFileName, err
+}
+
+func genDirectoryListKey(dir string) (dirList string) {
+	return dir + DIR_LIST_MARKER
+}
+
+func (store *UniversalRedisLuaStore) Shutdown() {
+	store.Client.Close()
+}

weed/filer/redis_lua/universal_redis_store_kv.go

@@ -0,0 +1,42 @@
+package redis_lua
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/go-redis/redis/v8"
+)
+
+func (store *UniversalRedisLuaStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+
+	_, err = store.Client.Set(ctx, string(key), value, 0).Result()
+
+	if err != nil {
+		return fmt.Errorf("kv put: %v", err)
+	}
+
+	return nil
+}
+
+func (store *UniversalRedisLuaStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+
+	data, err := store.Client.Get(ctx, string(key)).Result()
+
+	if err == redis.Nil {
+		return nil, filer.ErrKvNotFound
+	}
+
+	return []byte(data), err
+}
+
+func (store *UniversalRedisLuaStore) KvDelete(ctx context.Context, key []byte) (err error) {
+
+	_, err = store.Client.Del(ctx, string(key)).Result()
+
+	if err != nil {
+		return fmt.Errorf("kv delete: %v", err)
+	}
+
+	return nil
+}

weed/filer/rocksdb/rocksdb_store_test.go

@@ -16,8 +16,7 @@ import (
 
 func TestCreateAndFind(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
-	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test")
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	store := &RocksDBStore{}
 	store.initialize(dir)
 	testFiler.SetStore(store)
@@ -35,7 +34,7 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil {
+	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
@@ -70,8 +69,7 @@ func TestCreateAndFind(t *testing.T) {
 
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
-	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2")
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	store := &RocksDBStore{}
 	store.initialize(dir)
 	testFiler.SetStore(store)
@@ -93,8 +91,7 @@ func TestEmptyRoot(t *testing.T) {
 
 func BenchmarkInsertEntry(b *testing.B) {
 	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
-	dir, _ := os.MkdirTemp("", "seaweedfs_filer_bench")
-	defer os.RemoveAll(dir)
+	dir := b.TempDir()
 	store := &RocksDBStore{}
 	store.initialize(dir)
 	testFiler.SetStore(store)

weed/filer/stream.go

@@ -62,7 +62,7 @@ func NewFileReader(filerClient filer_pb.FilerClient, entry *filer_pb.Entry) io.R
 
 func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
 
-	glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks)
+	glog.V(4).Infof("start to stream content for chunks: %+v", chunks)
 	chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
 
 	fileId2Url := make(map[string][]string)
@@ -80,11 +80,23 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writ
 		fileId2Url[chunkView.FileId] = urlStrings
 	}
 
+	remaining := size
 	for _, chunkView := range chunkViews {
-
+		if offset < chunkView.LogicOffset {
+			gap := chunkView.LogicOffset - offset
+			remaining -= gap
+			glog.V(4).Infof("zero [%d,%d)", offset, chunkView.LogicOffset)
+			err := writeZero(writer, gap)
+			if err != nil {
+				return fmt.Errorf("write zero [%d,%d)", offset, chunkView.LogicOffset)
+			}
+			offset = chunkView.LogicOffset
+		}
 		urlStrings := fileId2Url[chunkView.FileId]
 		start := time.Now()
 		err := retriedStreamFetchChunkData(writer, urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
+		offset += int64(chunkView.Size)
+		remaining -= int64(chunkView.Size)
 		stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
 		if err != nil {
 			stats.FilerRequestCounter.WithLabelValues("chunkDownloadError").Inc()
@@ -92,6 +104,13 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writ
 		}
 		stats.FilerRequestCounter.WithLabelValues("chunkDownload").Inc()
 	}
+	if remaining > 0 {
+		glog.V(4).Infof("zero [%d,%d)", offset, offset+remaining)
+		err := writeZero(writer, remaining)
+		if err != nil {
+			return fmt.Errorf("write zero [%d,%d)", offset, offset+remaining)
+		}
+	}
 
 	return nil
 
@@ -99,30 +118,47 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writ
 
 // ----------------  ReadAllReader ----------------------------------
 
-func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) {
+func writeZero(w io.Writer, size int64) (err error) {
+	zeroPadding := make([]byte, 1024)
+	var written int
+	for size > 0 {
+		if size > 1024 {
+			written, err = w.Write(zeroPadding)
+		} else {
+			written, err = w.Write(zeroPadding[:size])
+		}
+		size -= int64(written)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
 
-	buffer := bytes.Buffer{}
+func ReadAll(buffer []byte, masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) error {
 
 	lookupFileIdFn := func(fileId string) (targetUrls []string, err error) {
 		return masterClient.LookupFileId(fileId)
 	}
 
-	chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
+	chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, int64(len(buffer)))
+
+	idx := 0
 
 	for _, chunkView := range chunkViews {
 		urlStrings, err := lookupFileIdFn(chunkView.FileId)
 		if err != nil {
 			glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
-			return nil, err
+			return err
 		}
 
-		data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
+		n, err := retriedFetchChunkData(buffer[idx:idx+int(chunkView.Size)], urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset)
 		if err != nil {
-			return nil, err
+			return err
 		}
-		buffer.Write(data)
+		idx += n
 	}
-	return buffer.Bytes(), nil
+	return nil
 }
 
 // ----------------  ChunkStreamReader ----------------------------------

weed/filesys/dir.go

@@ -1,618 +0,0 @@
-package filesys
-
-import (
-	"bytes"
-	"context"
-	"math"
-	"os"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/seaweedfs/fuse"
-	"github.com/seaweedfs/fuse/fs"
-
-	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/chrislusf/seaweedfs/weed/filesys/meta_cache"
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/util"
-)
-
-type Dir struct {
-	name   string
-	wfs    *WFS
-	entry  *filer_pb.Entry
-	parent *Dir
-	id     uint64
-}
-
-var _ = fs.Node(&Dir{})
-
-var _ = fs.NodeIdentifier(&Dir{})
-var _ = fs.NodeCreater(&Dir{})
-var _ = fs.NodeMknoder(&Dir{})
-var _ = fs.NodeMkdirer(&Dir{})
-var _ = fs.NodeFsyncer(&Dir{})
-var _ = fs.NodeRequestLookuper(&Dir{})
-var _ = fs.HandleReadDirAller(&Dir{})
-var _ = fs.NodeRemover(&Dir{})
-var _ = fs.NodeRenamer(&Dir{})
-var _ = fs.NodeSetattrer(&Dir{})
-var _ = fs.NodeGetxattrer(&Dir{})
-var _ = fs.NodeSetxattrer(&Dir{})
-var _ = fs.NodeRemovexattrer(&Dir{})
-var _ = fs.NodeListxattrer(&Dir{})
-var _ = fs.NodeForgetter(&Dir{})
-
-func (dir *Dir) Id() uint64 {
-	if dir.parent == nil {
-		return 1
-	}
-	return dir.id
-}
-
-func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
-
-	entry, err := dir.maybeLoadEntry()
-	if err != nil {
-		glog.V(3).Infof("dir Attr %s, err: %+v", dir.FullPath(), err)
-		return err
-	}
-
-	// https://github.com/bazil/fuse/issues/196
-	attr.Valid = time.Second
-	attr.Inode = dir.Id()
-	attr.Mode = os.FileMode(entry.Attributes.FileMode) | os.ModeDir
-	attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
-	attr.Crtime = time.Unix(entry.Attributes.Crtime, 0)
-	attr.Ctime = time.Unix(entry.Attributes.Crtime, 0)
-	attr.Atime = time.Unix(entry.Attributes.Mtime, 0)
-	attr.Gid = entry.Attributes.Gid
-	attr.Uid = entry.Attributes.Uid
-
-	if dir.FullPath() == dir.wfs.option.FilerMountRootPath {
-		attr.BlockSize = blockSize
-	}
-
-	glog.V(4).Infof("dir Attr %s, attr: %+v", dir.FullPath(), attr)
-
-	return nil
-}
-
-func (dir *Dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
-
-	glog.V(4).Infof("dir Getxattr %s", dir.FullPath())
-
-	entry, err := dir.maybeLoadEntry()
-	if err != nil {
-		return err
-	}
-
-	return getxattr(entry, req, resp)
-}
-
-func (dir *Dir) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
-	// fsync works at OS level
-	// write the file chunks to the filerGrpcAddress
-	glog.V(3).Infof("dir %s fsync %+v", dir.FullPath(), req)
-
-	return nil
-}
-
-func (dir *Dir) newFile(name string) fs.Node {
-
-	fileFullPath := util.NewFullPath(dir.FullPath(), name)
-	fileId := fileFullPath.AsInode()
-	dir.wfs.handlesLock.Lock()
-	existingHandle, found := dir.wfs.handles[fileId]
-	dir.wfs.handlesLock.Unlock()
-
-	if found {
-		glog.V(4).Infof("newFile found opened file handle: %+v", fileFullPath)
-		return existingHandle.f
-	}
-	return &File{
-		Name: name,
-		dir:  dir,
-		wfs:  dir.wfs,
-		id:   fileId,
-	}
-}
-
-func (dir *Dir) newDirectory(fullpath util.FullPath) fs.Node {
-
-	return &Dir{name: fullpath.Name(), wfs: dir.wfs, parent: dir, id: fullpath.AsInode()}
-
-}
-
-func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
-	resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
-
-	exclusive := req.Flags&fuse.OpenExclusive != 0
-	isDirectory := req.Mode&os.ModeDir > 0
-
-	if exclusive || isDirectory {
-		_, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, exclusive)
-		if err != nil {
-			return nil, nil, err
-		}
-	}
-	var node fs.Node
-	if isDirectory {
-		node = dir.newDirectory(util.NewFullPath(dir.FullPath(), req.Name))
-		return node, node, nil
-	}
-
-	node = dir.newFile(req.Name)
-	file := node.(*File)
-	file.entry = &filer_pb.Entry{
-		Name:        req.Name,
-		IsDirectory: req.Mode&os.ModeDir > 0,
-		Attributes: &filer_pb.FuseAttributes{
-			Mtime:       time.Now().Unix(),
-			Crtime:      time.Now().Unix(),
-			FileMode:    uint32(req.Mode &^ dir.wfs.option.Umask),
-			Uid:         req.Uid,
-			Gid:         req.Gid,
-			Collection:  dir.wfs.option.Collection,
-			Replication: dir.wfs.option.Replication,
-			TtlSec:      dir.wfs.option.TtlSec,
-		},
-	}
-	file.dirtyMetadata = true
-	fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid)
-	return file, fh, nil
-
-}
-
-func (dir *Dir) Mknod(ctx context.Context, req *fuse.MknodRequest) (fs.Node, error) {
-
-	_, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, false)
-
-	if err != nil {
-		return nil, err
-	}
-	var node fs.Node
-	node = dir.newFile(req.Name)
-	return node, nil
-}
-
-func (dir *Dir) doCreateEntry(name string, mode os.FileMode, uid, gid uint32, exclusive bool) (*filer_pb.CreateEntryRequest, error) {
-	dirFullPath := dir.FullPath()
-	request := &filer_pb.CreateEntryRequest{
-		Directory: dirFullPath,
-		Entry: &filer_pb.Entry{
-			Name:        name,
-			IsDirectory: mode&os.ModeDir > 0,
-			Attributes: &filer_pb.FuseAttributes{
-				Mtime:       time.Now().Unix(),
-				Crtime:      time.Now().Unix(),
-				FileMode:    uint32(mode &^ dir.wfs.option.Umask),
-				Uid:         uid,
-				Gid:         gid,
-				Collection:  dir.wfs.option.Collection,
-				Replication: dir.wfs.option.Replication,
-				TtlSec:      dir.wfs.option.TtlSec,
-			},
-		},
-		OExcl:      exclusive,
-		Signatures: []int32{dir.wfs.signature},
-	}
-	glog.V(1).Infof("create %s/%s", dirFullPath, name)
-
-	err := dir.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-
-		dir.wfs.mapPbIdFromLocalToFiler(request.Entry)
-		defer dir.wfs.mapPbIdFromFilerToLocal(request.Entry)
-
-		if err := filer_pb.CreateEntry(client, request); err != nil {
-			if strings.Contains(err.Error(), "EEXIST") {
-				return fuse.EEXIST
-			}
-			glog.V(0).Infof("create %s/%s: %v", dirFullPath, name, err)
-			return fuse.EIO
-		}
-
-		if err := dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
-			glog.Errorf("local InsertEntry dir %s/%s: %v", dirFullPath, name, err)
-			return fuse.EIO
-		}
-
-		return nil
-	})
-	return request, err
-}
-
-func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
-
-	glog.V(4).Infof("mkdir %s: %s", dir.FullPath(), req.Name)
-
-	newEntry := &filer_pb.Entry{
-		Name:        req.Name,
-		IsDirectory: true,
-		Attributes: &filer_pb.FuseAttributes{
-			Mtime:    time.Now().Unix(),
-			Crtime:   time.Now().Unix(),
-			FileMode: uint32(req.Mode &^ dir.wfs.option.Umask),
-			Uid:      req.Uid,
-			Gid:      req.Gid,
-		},
-	}
-
-	dirFullPath := dir.FullPath()
-
-	err := dir.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-
-		dir.wfs.mapPbIdFromLocalToFiler(newEntry)
-		defer dir.wfs.mapPbIdFromFilerToLocal(newEntry)
-
-		request := &filer_pb.CreateEntryRequest{
-			Directory:  dirFullPath,
-			Entry:      newEntry,
-			Signatures: []int32{dir.wfs.signature},
-		}
-
-		glog.V(1).Infof("mkdir: %v", request)
-		if err := filer_pb.CreateEntry(client, request); err != nil {
-			glog.V(0).Infof("mkdir %s/%s: %v", dirFullPath, req.Name, err)
-			return err
-		}
-
-		if err := dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
-			glog.Errorf("local mkdir dir %s/%s: %v", dirFullPath, req.Name, err)
-			return fuse.EIO
-		}
-
-		return nil
-	})
-
-	if err == nil {
-		node := dir.newDirectory(util.NewFullPath(dirFullPath, req.Name))
-
-		return node, nil
-	}
-
-	glog.V(0).Infof("mkdir %s/%s: %v", dirFullPath, req.Name, err)
-
-	return nil, fuse.EIO
-}
-
-func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) {
-
-	dirPath := util.FullPath(dir.FullPath())
-	// glog.V(4).Infof("dir Lookup %s: %s by %s", dirPath, req.Name, req.Header.String())
-
-	fullFilePath := dirPath.Child(req.Name)
-	visitErr := meta_cache.EnsureVisited(dir.wfs.metaCache, dir.wfs, dirPath)
-	if visitErr != nil {
-		glog.Errorf("dir Lookup %s: %v", dirPath, visitErr)
-		return nil, fuse.EIO
-	}
-	localEntry, cacheErr := dir.wfs.metaCache.FindEntry(context.Background(), fullFilePath)
-	if cacheErr == filer_pb.ErrNotFound {
-		return nil, fuse.ENOENT
-	}
-
-	if localEntry == nil {
-		// glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath)
-		entry, err := filer_pb.GetEntry(dir.wfs, fullFilePath)
-		if err != nil {
-			glog.V(1).Infof("dir GetEntry %s: %v", fullFilePath, err)
-			return nil, fuse.ENOENT
-		}
-		localEntry = filer.FromPbEntry(string(dirPath), entry)
-	} else {
-		glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath)
-	}
-
-	if localEntry != nil {
-		if localEntry.IsDirectory() {
-			node = dir.newDirectory(fullFilePath)
-		} else {
-			node = dir.newFile(req.Name)
-		}
-
-		// resp.EntryValid = time.Second
-		resp.Attr.Inode = fullFilePath.AsInode()
-		resp.Attr.Valid = time.Second
-		resp.Attr.Size = localEntry.FileSize
-		resp.Attr.Mtime = localEntry.Attr.Mtime
-		resp.Attr.Crtime = localEntry.Attr.Crtime
-		resp.Attr.Mode = localEntry.Attr.Mode
-		resp.Attr.Gid = localEntry.Attr.Gid
-		resp.Attr.Uid = localEntry.Attr.Uid
-		if localEntry.HardLinkCounter > 0 {
-			resp.Attr.Nlink = uint32(localEntry.HardLinkCounter)
-		}
-
-		return node, nil
-	}
-
-	glog.V(4).Infof("not found dir GetEntry %s: %v", fullFilePath, err)
-	return nil, fuse.ENOENT
-}
-
-func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
-
-	dirPath := util.FullPath(dir.FullPath())
-	glog.V(4).Infof("dir ReadDirAll %s", dirPath)
-
-	processEachEntryFn := func(entry *filer.Entry, isLast bool) {
-		if entry.IsDirectory() {
-			dirent := fuse.Dirent{Name: entry.Name(), Type: fuse.DT_Dir, Inode: dirPath.Child(entry.Name()).AsInode()}
-			ret = append(ret, dirent)
-		} else {
-			dirent := fuse.Dirent{Name: entry.Name(), Type: findFileType(uint16(entry.Attr.Mode)), Inode: dirPath.Child(entry.Name()).AsInode()}
-			ret = append(ret, dirent)
-		}
-	}
-
-	if err = meta_cache.EnsureVisited(dir.wfs.metaCache, dir.wfs, dirPath); err != nil {
-		glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
-		return nil, fuse.EIO
-	}
-	listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, "", false, int64(math.MaxInt32), func(entry *filer.Entry) bool {
-		processEachEntryFn(entry, false)
-		return true
-	})
-	if listErr != nil {
-		glog.Errorf("list meta cache: %v", listErr)
-		return nil, fuse.EIO
-	}
-
-	// create proper . and .. directories
-	ret = append(ret, fuse.Dirent{
-		Inode: dirPath.AsInode(),
-		Name:  ".",
-		Type:  fuse.DT_Dir,
-	})
-
-	// return the correct parent inode for the mount root
-	var inode uint64
-	if string(dirPath) == dir.wfs.option.FilerMountRootPath {
-		inode = dir.wfs.option.MountParentInode
-	} else {
-		inode = util.FullPath(dir.parent.FullPath()).AsInode()
-	}
-
-	ret = append(ret, fuse.Dirent{
-		Inode: inode,
-		Name:  "..",
-		Type:  fuse.DT_Dir,
-	})
-
-	return
-}
-
-func findFileType(mode uint16) fuse.DirentType {
-	switch mode & (syscall.S_IFMT & 0xffff) {
-	case syscall.S_IFSOCK:
-		return fuse.DT_Socket
-	case syscall.S_IFLNK:
-		return fuse.DT_Link
-	case syscall.S_IFREG:
-		return fuse.DT_File
-	case syscall.S_IFBLK:
-		return fuse.DT_Block
-	case syscall.S_IFDIR:
-		return fuse.DT_Dir
-	case syscall.S_IFCHR:
-		return fuse.DT_Char
-	case syscall.S_IFIFO:
-		return fuse.DT_FIFO
-	}
-	return fuse.DT_File
-}
-
-func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
-
-	if !req.Dir {
-		return dir.removeOneFile(req)
-	}
-
-	return dir.removeFolder(req)
-
-}
-
-func (dir *Dir) removeOneFile(req *fuse.RemoveRequest) error {
-
-	dirFullPath := dir.FullPath()
-	filePath := util.NewFullPath(dirFullPath, req.Name)
-	entry, err := filer_pb.GetEntry(dir.wfs, filePath)
-	if err != nil {
-		return err
-	}
-
-	// first, ensure the filer store can correctly delete
-	glog.V(3).Infof("remove file: %v", req)
-	isDeleteData := entry != nil && entry.HardLinkCounter <= 1
-	err = filer_pb.Remove(dir.wfs, dirFullPath, req.Name, isDeleteData, false, false, false, []int32{dir.wfs.signature})
-	if err != nil {
-		glog.V(3).Infof("not found remove file %s: %v", filePath, err)
-		return fuse.ENOENT
-	}
-
-	// then, delete meta cache and fsNode cache
-	if err = dir.wfs.metaCache.DeleteEntry(context.Background(), filePath); err != nil {
-		glog.V(3).Infof("local DeleteEntry %s: %v", filePath, err)
-		return fuse.ESTALE
-	}
-
-	// remove current file handle if any
-	dir.wfs.handlesLock.Lock()
-	defer dir.wfs.handlesLock.Unlock()
-	inodeId := filePath.AsInode()
-	if fh, ok := dir.wfs.handles[inodeId]; ok {
-		delete(dir.wfs.handles, inodeId)
-		fh.isDeleted = true
-	}
-
-	return nil
-
-}
-
-func (dir *Dir) removeFolder(req *fuse.RemoveRequest) error {
-
-	dirFullPath := dir.FullPath()
-	glog.V(3).Infof("remove directory entry: %v", req)
-	ignoreRecursiveErr := true // ignore recursion error since the OS should manage it
-	err := filer_pb.Remove(dir.wfs, dirFullPath, req.Name, true, true, ignoreRecursiveErr, false, []int32{dir.wfs.signature})
-	if err != nil {
-		glog.V(0).Infof("remove %s/%s: %v", dirFullPath, req.Name, err)
-		if strings.Contains(err.Error(), "non-empty") {
-			return fuse.EEXIST
-		}
-		return fuse.ENOENT
-	}
-
-	t := util.NewFullPath(dirFullPath, req.Name)
-	dir.wfs.metaCache.DeleteEntry(context.Background(), t)
-
-	return nil
-
-}
-
-func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
-
-	glog.V(4).Infof("%v dir setattr %+v", dir.FullPath(), req)
-
-	entry, err := dir.maybeLoadEntry()
-	if err != nil {
-		return err
-	}
-
-	if req.Valid.Mode() {
-		entry.Attributes.FileMode = uint32(req.Mode)
-	}
-
-	if req.Valid.Uid() {
-		entry.Attributes.Uid = req.Uid
-	}
-
-	if req.Valid.Gid() {
-		entry.Attributes.Gid = req.Gid
-	}
-
-	if req.Valid.Mtime() {
-		entry.Attributes.Mtime = req.Mtime.Unix()
-	}
-
-	return dir.saveEntry(entry)
-
-}
-
-func (dir *Dir) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {
-
-	glog.V(4).Infof("dir Setxattr %s: %s", dir.FullPath(), req.Name)
-
-	entry, err := dir.maybeLoadEntry()
-	if err != nil {
-		return err
-	}
-
-	if err := setxattr(entry, req); err != nil {
-		return err
-	}
-
-	return dir.saveEntry(entry)
-
-}
-
-func (dir *Dir) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {
-
-	glog.V(4).Infof("dir Removexattr %s: %s", dir.FullPath(), req.Name)
-
-	entry, err := dir.maybeLoadEntry()
-	if err != nil {
-		return err
-	}
-
-	if err := removexattr(entry, req); err != nil {
-		return err
-	}
-
-	return dir.saveEntry(entry)
-
-}
-
-func (dir *Dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
-
-	glog.V(4).Infof("dir Listxattr %s", dir.FullPath())
-
-	entry, err := dir.maybeLoadEntry()
-	if err != nil {
-		return err
-	}
-
-	if err := listxattr(entry, req, resp); err != nil {
-		return err
-	}
-
-	return nil
-
-}
-
-func (dir *Dir) Forget() {
-	glog.V(4).Infof("Forget dir %s", dir.FullPath())
-}
-
-func (dir *Dir) maybeLoadEntry() (*filer_pb.Entry, error) {
-	parentDirPath, name := util.FullPath(dir.FullPath()).DirAndName()
-	return dir.wfs.maybeLoadEntry(parentDirPath, name)
-}
-
-func (dir *Dir) saveEntry(entry *filer_pb.Entry) error {
-
-	parentDir, name := util.FullPath(dir.FullPath()).DirAndName()
-
-	return dir.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-
-		dir.wfs.mapPbIdFromLocalToFiler(entry)
-		defer dir.wfs.mapPbIdFromFilerToLocal(entry)
-
-		request := &filer_pb.UpdateEntryRequest{
-			Directory:  parentDir,
-			Entry:      entry,
-			Signatures: []int32{dir.wfs.signature},
-		}
-
-		glog.V(1).Infof("save dir entry: %v", request)
-		_, err := client.UpdateEntry(context.Background(), request)
-		if err != nil {
-			glog.Errorf("UpdateEntry dir %s/%s: %v", parentDir, name, err)
-			return fuse.EIO
-		}
-
-		if err := dir.wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
-			glog.Errorf("UpdateEntry dir %s/%s: %v", parentDir, name, err)
-			return fuse.ESTALE
-		}
-
-		return nil
-	})
-}
-
-func (dir *Dir) FullPath() string {
-	var parts []string
-	for p := dir; p != nil; p = p.parent {
-		if strings.HasPrefix(p.name, "/") {
-			if len(p.name) > 1 {
-				parts = append(parts, p.name[1:])
-			}
-		} else {
-			parts = append(parts, p.name)
-		}
-	}
-
-	if len(parts) == 0 {
-		return "/"
-	}
-
-	var buf bytes.Buffer
-	for i := len(parts) - 1; i >= 0; i-- {
-		buf.WriteString("/")
-		buf.WriteString(parts[i])
-	}
-	return buf.String()
-}

weed/filesys/dir_link.go

@@ -1,160 +0,0 @@
-package filesys
-
-import (
-	"context"
-	"github.com/chrislusf/seaweedfs/weed/util"
-	"os"
-	"syscall"
-	"time"
-
-	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/fuse"
-	"github.com/seaweedfs/fuse/fs"
-)
-
-var _ = fs.NodeLinker(&Dir{})
-var _ = fs.NodeSymlinker(&Dir{})
-var _ = fs.NodeReadlinker(&File{})
-
-const (
-	HARD_LINK_MARKER = '\x01'
-)
-
-func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (fs.Node, error) {
-
-	oldFile, ok := old.(*File)
-	if !ok {
-		glog.Errorf("old node is not a file: %+v", old)
-	}
-
-	glog.V(4).Infof("Link: %v/%v -> %v/%v", oldFile.dir.FullPath(), oldFile.Name, dir.FullPath(), req.NewName)
-
-	oldEntry, err := oldFile.maybeLoadEntry(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	if oldEntry == nil {
-		return nil, fuse.EIO
-	}
-
-	// update old file to hardlink mode
-	if len(oldEntry.HardLinkId) == 0 {
-		oldEntry.HardLinkId = append(util.RandomBytes(16), HARD_LINK_MARKER)
-		oldEntry.HardLinkCounter = 1
-	}
-	oldEntry.HardLinkCounter++
-	updateOldEntryRequest := &filer_pb.UpdateEntryRequest{
-		Directory:  oldFile.dir.FullPath(),
-		Entry:      oldEntry,
-		Signatures: []int32{dir.wfs.signature},
-	}
-
-	// CreateLink 1.2 : update new file to hardlink mode
-	request := &filer_pb.CreateEntryRequest{
-		Directory: dir.FullPath(),
-		Entry: &filer_pb.Entry{
-			Name:            req.NewName,
-			IsDirectory:     false,
-			Attributes:      oldEntry.Attributes,
-			Chunks:          oldEntry.Chunks,
-			Extended:        oldEntry.Extended,
-			HardLinkId:      oldEntry.HardLinkId,
-			HardLinkCounter: oldEntry.HardLinkCounter,
-		},
-		Signatures: []int32{dir.wfs.signature},
-	}
-
-	// apply changes to the filer, and also apply to local metaCache
-	err = dir.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-
-		dir.wfs.mapPbIdFromLocalToFiler(request.Entry)
-		defer dir.wfs.mapPbIdFromFilerToLocal(request.Entry)
-
-		if err := filer_pb.UpdateEntry(client, updateOldEntryRequest); err != nil {
-			glog.V(0).Infof("Link %v/%v -> %s/%s: %v", oldFile.dir.FullPath(), oldFile.Name, dir.FullPath(), req.NewName, err)
-			return fuse.EIO
-		}
-		dir.wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(updateOldEntryRequest.Directory, updateOldEntryRequest.Entry))
-
-		if err := filer_pb.CreateEntry(client, request); err != nil {
-			glog.V(0).Infof("Link %v/%v -> %s/%s: %v", oldFile.dir.FullPath(), oldFile.Name, dir.FullPath(), req.NewName, err)
-			return fuse.EIO
-		}
-		dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry))
-
-		return nil
-	})
-
-	if err != nil {
-		return nil, fuse.EIO
-	}
-
-	// create new file node
-	newNode := dir.newFile(req.NewName)
-	newFile := newNode.(*File)
-
-	return newFile, err
-
-}
-
-func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node, error) {
-
-	glog.V(4).Infof("Symlink: %v/%v to %v", dir.FullPath(), req.NewName, req.Target)
-
-	request := &filer_pb.CreateEntryRequest{
-		Directory: dir.FullPath(),
-		Entry: &filer_pb.Entry{
-			Name:        req.NewName,
-			IsDirectory: false,
-			Attributes: &filer_pb.FuseAttributes{
-				Mtime:         time.Now().Unix(),
-				Crtime:        time.Now().Unix(),
-				FileMode:      uint32((os.FileMode(0777) | os.ModeSymlink) &^ dir.wfs.option.Umask),
-				Uid:           req.Uid,
-				Gid:           req.Gid,
-				SymlinkTarget: req.Target,
-			},
-		},
-		Signatures: []int32{dir.wfs.signature},
-	}
-
-	err := dir.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-
-		dir.wfs.mapPbIdFromLocalToFiler(request.Entry)
-		defer dir.wfs.mapPbIdFromFilerToLocal(request.Entry)
-
-		if err := filer_pb.CreateEntry(client, request); err != nil {
-			glog.V(0).Infof("symlink %s/%s: %v", dir.FullPath(), req.NewName, err)
-			return fuse.EIO
-		}
-
-		dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry))
-
-		return nil
-	})
-
-	symlink := dir.newFile(req.NewName)
-
-	return symlink, err
-
-}
-
-func (file *File) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) {
-
-	entry, err := file.maybeLoadEntry(ctx)
-	if err != nil {
-		return "", err
-	}
-
-	if os.FileMode(entry.Attributes.FileMode)&os.ModeSymlink == 0 {
-		return "", fuse.Errno(syscall.EINVAL)
-	}
-
-	glog.V(4).Infof("Readlink: %v/%v => %v", file.dir.FullPath(), file.Name, entry.Attributes.SymlinkTarget)
-
-	return entry.Attributes.SymlinkTarget, nil
-
-}

weed/filesys/dir_rename.go

@@ -1,131 +0,0 @@
-package filesys
-
-import (
-	"context"
-	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/seaweedfs/fuse"
-	"github.com/seaweedfs/fuse/fs"
-	"io"
-
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/util"
-)
-
-func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {
-
-	newDir := newDirectory.(*Dir)
-
-	newPath := util.NewFullPath(newDir.FullPath(), req.NewName)
-	oldPath := util.NewFullPath(dir.FullPath(), req.OldName)
-
-	glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath)
-
-	// update remote filer
-	err := dir.wfs.WithFilerClient(true, func(client filer_pb.SeaweedFilerClient) error {
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel()
-
-		request := &filer_pb.StreamRenameEntryRequest{
-			OldDirectory: dir.FullPath(),
-			OldName:      req.OldName,
-			NewDirectory: newDir.FullPath(),
-			NewName:      req.NewName,
-			Signatures:   []int32{dir.wfs.signature},
-		}
-
-		stream, err := client.StreamRenameEntry(ctx, request)
-		if err != nil {
-			glog.Errorf("dir AtomicRenameEntry %s => %s : %v", oldPath, newPath, err)
-			return fuse.EXDEV
-		}
-
-		for {
-			resp, recvErr := stream.Recv()
-			if recvErr != nil {
-				if recvErr == io.EOF {
-					break
-				} else {
-					return recvErr
-				}
-			}
-
-			if err = dir.handleRenameResponse(ctx, resp); err != nil {
-				return err
-			}
-
-		}
-
-		return nil
-
-	})
-	if err != nil {
-		glog.V(0).Infof("dir Rename %s => %s : %v", oldPath, newPath, err)
-		return fuse.EIO
-	}
-
-	return nil
-}
-
-func (dir *Dir) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamRenameEntryResponse) error {
-	// comes from filer StreamRenameEntry, can only be create or delete entry
-
-	if resp.EventNotification.NewEntry != nil {
-		// with new entry, the old entry name also exists. This is the first step to create new entry
-		newEntry := filer.FromPbEntry(resp.EventNotification.NewParentPath, resp.EventNotification.NewEntry)
-		if err := dir.wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, "", newEntry); err != nil {
-			return err
-		}
-
-		oldParent, newParent := util.FullPath(resp.Directory), util.FullPath(resp.EventNotification.NewParentPath)
-		oldName, newName := resp.EventNotification.OldEntry.Name, resp.EventNotification.NewEntry.Name
-
-		oldPath := oldParent.Child(oldName)
-		newPath := newParent.Child(newName)
-		oldFsNode := NodeWithId(oldPath.AsInode())
-		newFsNode := NodeWithId(newPath.AsInode())
-		newDirNode, found := dir.wfs.Server.FindInternalNode(NodeWithId(newParent.AsInode()))
-		var newDir *Dir
-		if found {
-			newDir = newDirNode.(*Dir)
-		}
-		dir.wfs.Server.InvalidateInternalNode(oldFsNode, newFsNode, func(internalNode fs.Node) {
-			if file, ok := internalNode.(*File); ok {
-				glog.V(4).Infof("internal file node %s", oldParent.Child(oldName))
-				file.Name = newName
-				file.id = uint64(newFsNode)
-				if found {
-					file.dir = newDir
-				}
-			}
-			if dir, ok := internalNode.(*Dir); ok {
-				glog.V(4).Infof("internal dir node %s", oldParent.Child(oldName))
-				dir.name = newName
-				dir.id = uint64(newFsNode)
-				if found {
-					dir.parent = newDir
-				}
-			}
-		})
-
-		// change file handle
-		inodeId := oldPath.AsInode()
-		dir.wfs.handlesLock.Lock()
-		if existingHandle, found := dir.wfs.handles[inodeId]; found && existingHandle != nil {
-			glog.V(4).Infof("opened file handle %s => %s", oldPath, newPath)
-			delete(dir.wfs.handles, inodeId)
-			existingHandle.f.entry.Name = newName
-			dir.wfs.handles[newPath.AsInode()] = existingHandle
-		}
-		dir.wfs.handlesLock.Unlock()
-
-	} else if resp.EventNotification.OldEntry != nil {
-		// without new entry, only old entry name exists. This is the second step to delete old entry
-		if err := dir.wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath(resp.Directory, resp.EventNotification.OldEntry.Name), nil); err != nil {
-			return err
-		}
-	}
-
-	return nil
-
-}

weed/filesys/dirty_pages_continuous.go

@@ -1,138 +0,0 @@
-package filesys
-
-import (
-	"bytes"
-	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filesys/page_writer"
-	"io"
-	"sync"
-	"time"
-
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-)
-
-type ContinuousDirtyPages struct {
-	intervals      *page_writer.ContinuousIntervals
-	f              *File
-	writeWaitGroup sync.WaitGroup
-	chunkAddLock   sync.Mutex
-	lastErr        error
-	collection     string
-	replication    string
-}
-
-func newContinuousDirtyPages(file *File) *ContinuousDirtyPages {
-	dirtyPages := &ContinuousDirtyPages{
-		intervals: &page_writer.ContinuousIntervals{},
-		f:         file,
-	}
-	return dirtyPages
-}
-
-func (pages *ContinuousDirtyPages) AddPage(offset int64, data []byte) {
-
-	glog.V(4).Infof("%s AddPage [%d,%d)", pages.f.fullpath(), offset, offset+int64(len(data)))
-
-	if len(data) > int(pages.f.wfs.option.ChunkSizeLimit) {
-		// this is more than what buffer can hold.
-		pages.flushAndSave(offset, data)
-	}
-
-	pages.intervals.AddInterval(data, offset)
-
-	if pages.intervals.TotalSize() >= pages.f.wfs.option.ChunkSizeLimit {
-		pages.saveExistingLargestPageToStorage()
-	}
-
-	return
-}
-
-func (pages *ContinuousDirtyPages) flushAndSave(offset int64, data []byte) {
-
-	// flush existing
-	pages.saveExistingPagesToStorage()
-
-	// flush the new page
-	pages.saveToStorage(bytes.NewReader(data), offset, int64(len(data)))
-
-	return
-}
-
-func (pages *ContinuousDirtyPages) FlushData() error {
-
-	pages.saveExistingPagesToStorage()
-	pages.writeWaitGroup.Wait()
-	if pages.lastErr != nil {
-		return fmt.Errorf("flush data: %v", pages.lastErr)
-	}
-	return nil
-}
-
-func (pages *ContinuousDirtyPages) saveExistingPagesToStorage() {
-	for pages.saveExistingLargestPageToStorage() {
-	}
-}
-
-func (pages *ContinuousDirtyPages) saveExistingLargestPageToStorage() (hasSavedData bool) {
-
-	maxList := pages.intervals.RemoveLargestIntervalLinkedList()
-	if maxList == nil {
-		return false
-	}
-
-	entry := pages.f.getEntry()
-	if entry == nil {
-		return false
-	}
-
-	fileSize := int64(entry.Attributes.FileSize)
-
-	chunkSize := min(maxList.Size(), fileSize-maxList.Offset())
-	if chunkSize == 0 {
-		return false
-	}
-
-	pages.saveToStorage(maxList.ToReader(), maxList.Offset(), chunkSize)
-
-	return true
-}
-
-func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, size int64) {
-
-	mtime := time.Now().UnixNano()
-	pages.writeWaitGroup.Add(1)
-	writer := func() {
-		defer pages.writeWaitGroup.Done()
-
-		reader = io.LimitReader(reader, size)
-		chunk, collection, replication, err := pages.f.wfs.saveDataAsChunk(pages.f.fullpath())(reader, pages.f.Name, offset)
-		if err != nil {
-			glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), offset, offset+size, err)
-			pages.lastErr = err
-			return
-		}
-		chunk.Mtime = mtime
-		pages.collection, pages.replication = collection, replication
-		pages.chunkAddLock.Lock()
-		defer pages.chunkAddLock.Unlock()
-		pages.f.addChunks([]*filer_pb.FileChunk{chunk})
-		glog.V(3).Infof("%s saveToStorage [%d,%d)", pages.f.fullpath(), offset, offset+size)
-	}
-
-	if pages.f.wfs.concurrentWriters != nil {
-		pages.f.wfs.concurrentWriters.Execute(writer)
-	} else {
-		go writer()
-	}
-}
-
-func (pages *ContinuousDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) {
-	return pages.intervals.ReadDataAt(data, startOffset)
-}
-
-func (pages *ContinuousDirtyPages) GetStorageOptions() (collection, replication string) {
-	return pages.collection, pages.replication
-}
-func (pages ContinuousDirtyPages) Destroy() {
-}

weed/filesys/dirty_pages_stream.go

@@ -1,106 +0,0 @@
-package filesys
-
-import (
-	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filesys/page_writer"
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"io"
-	"sync"
-	"time"
-)
-
-type StreamDirtyPages struct {
-	f              *File
-	writeWaitGroup sync.WaitGroup
-	pageAddLock    sync.Mutex
-	chunkAddLock   sync.Mutex
-	lastErr        error
-	collection     string
-	replication    string
-	chunkedStream  *page_writer.ChunkedStreamWriter
-}
-
-func newStreamDirtyPages(file *File, chunkSize int64) *StreamDirtyPages {
-
-	dirtyPages := &StreamDirtyPages{
-		f:             file,
-		chunkedStream: page_writer.NewChunkedStreamWriter(chunkSize),
-	}
-
-	dirtyPages.chunkedStream.SetSaveToStorageFunction(dirtyPages.saveChunkedFileIntevalToStorage)
-
-	return dirtyPages
-}
-
-func (pages *StreamDirtyPages) AddPage(offset int64, data []byte) {
-
-	pages.pageAddLock.Lock()
-	defer pages.pageAddLock.Unlock()
-
-	glog.V(4).Infof("%v stream AddPage [%d, %d)", pages.f.fullpath(), offset, offset+int64(len(data)))
-	if _, err := pages.chunkedStream.WriteAt(data, offset); err != nil {
-		pages.lastErr = err
-	}
-
-	return
-}
-
-func (pages *StreamDirtyPages) FlushData() error {
-	pages.saveChunkedFileToStorage()
-	pages.writeWaitGroup.Wait()
-	if pages.lastErr != nil {
-		return fmt.Errorf("flush data: %v", pages.lastErr)
-	}
-	pages.chunkedStream.Reset()
-	return nil
-}
-
-func (pages *StreamDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) {
-	return pages.chunkedStream.ReadDataAt(data, startOffset)
-}
-
-func (pages *StreamDirtyPages) GetStorageOptions() (collection, replication string) {
-	return pages.collection, pages.replication
-}
-
-func (pages *StreamDirtyPages) saveChunkedFileToStorage() {
-
-	pages.chunkedStream.FlushAll()
-
-}
-
-func (pages *StreamDirtyPages) saveChunkedFileIntevalToStorage(reader io.Reader, offset int64, size int64, cleanupFn func()) {
-
-	mtime := time.Now().UnixNano()
-	pages.writeWaitGroup.Add(1)
-	writer := func() {
-		defer pages.writeWaitGroup.Done()
-		defer cleanupFn()
-
-		chunk, collection, replication, err := pages.f.wfs.saveDataAsChunk(pages.f.fullpath())(reader, pages.f.Name, offset)
-		if err != nil {
-			glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), offset, offset+size, err)
-			pages.lastErr = err
-			return
-		}
-		chunk.Mtime = mtime
-		pages.collection, pages.replication = collection, replication
-		pages.chunkAddLock.Lock()
-		pages.f.addChunks([]*filer_pb.FileChunk{chunk})
-		glog.V(3).Infof("%s saveToStorage %s [%d,%d)", pages.f.fullpath(), chunk.FileId, offset, offset+size)
-		pages.chunkAddLock.Unlock()
-
-	}
-
-	if pages.f.wfs.concurrentWriters != nil {
-		pages.f.wfs.concurrentWriters.Execute(writer)
-	} else {
-		go writer()
-	}
-
-}
-
-func (pages StreamDirtyPages) Destroy() {
-	pages.chunkedStream.Reset()
-}