temp disable locking

.github/workflows/binaries_dev.yml

@@ -38,13 +38,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -60,7 +60,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -87,13 +87,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -109,7 +109,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release0.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release1.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release2.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release3.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release4.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -45,7 +45,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/codeql.yml

@@ -18,7 +18,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

.github/workflows/container_dev.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -36,7 +36,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
         with:
           buildkitd-flags: "--debug"
       -
@@ -56,7 +56,7 @@ jobs:
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
         with:
           buildkitd-flags: "--debug"
       -
@@ -57,7 +57,7 @@ jobs:
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
@@ -47,7 +47,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -38,7 +38,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
@@ -48,7 +48,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -38,7 +38,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
@@ -48,7 +48,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release4.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
@@ -47,7 +47,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release5.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
@@ -47,7 +47,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/depsreview.yml

@@ -9,6 +9,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@f46c48ed6d4f1227fb2d9ea62bf6bcbed315589e
+        uses: actions/dependency-review-action@11310527b429536e263dc6cc47873e608189ba21

.github/workflows/e2e.yml

@@ -24,13 +24,13 @@ jobs:
     timeout-minutes: 15
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v2
+      uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
 
     - name: Install dependencies
       run: |
@@ -42,43 +42,28 @@ jobs:
       run: make build_e2e && docker compose -f ./compose/e2e-mount.yml up --wait
 
     - name: Run FIO
-      timeout-minutes: 15
+      timeout-minutes: 5
       run: |
         echo "Starting FIO at: $(date)"
         # Concurrent r/w
         echo 'Run randrw with size=16M bs=4k'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
-
-        echo "Verify FIO at: $(date)"
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
+        
+        echo 'Run randrw with size=16M bs=128k'
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
+        
+        echo 'Run randrw with size=16M bs=1m'
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
+        
         # Verified write
         echo 'Run randwrite with size=16M bs=4k'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
-
-    - name: Run FIO
-      timeout-minutes: 15
-      run: |
-        echo "Starting FIO at: $(date)"
-        # Concurrent r/w
-        echo 'Run randrw with size=16M bs=128k'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
-
-        echo "Verify FIO at: $(date)"
-        # Verified write
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
+        
         echo 'Run randwrite with size=16M bs=128k'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
-
-    - name: Run FIO
-      timeout-minutes: 15
-      run: |
-        echo "Starting FIO at: $(date)"
-        # Concurrent r/w
-        echo 'Run randrw with size=16M bs=1m'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
         
-        echo "Verify FIO at: $(date)"
-        # Verified write
         echo 'Run randwrite with size=16M bs=1m'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
 
     - name: Save logs
       if: always()

.github/workflows/go.yml

@@ -21,13 +21,13 @@ jobs:
     steps:
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v2
+      uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
 
     - name: Get dependencies
       run: |

.github/workflows/helm_chart_release.yml

@@ -1,22 +0,0 @@
-name: "helm: publish charts"
-on:
-  push:
-    tags: 
-      - '*'
-
-permissions:
-  contents: write
-  pages: write
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Publish Helm charts
-        uses: stefanprodan/helm-gh-pages@master
-        with:          
-          token: ${{ secrets.GITHUB_TOKEN }}
-          charts_dir: k8s/charts
-          target_dir: helm
-          branch: gh-pages

.github/workflows/helm_ci.yml

@@ -1,51 +0,0 @@
-name: "helm: lint and test charts"
-
-on:
-  push:
-    branches: [ master ]
-    paths: ['k8s/**']
-  pull_request:
-    branches: [ master ]
-    paths: ['k8s/**']
-
-permissions:
-  contents: read
-
-jobs:
-  lint-test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Set up Helm
-        uses: azure/setup-helm@v3
-        with:
-          version: v3.10.0
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.9'
-          check-latest: true
-
-      - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.4.0
-
-      - name: Run chart-testing (list-changed)
-        id: list-changed
-        run: |
-          changed=$(ct list-changed --chart-dirs k8s/charts)
-          if [[ -n "$changed" ]]; then
-            echo "::set-output name=changed::true"
-          fi
-
-      - name: Run chart-testing (lint)
-        run: ct lint --all --validate-maintainers=false --chart-dirs k8s/charts
-
-      - name: Create kind cluster
-        uses: helm/kind-action@v1.7.0
-
-      - name: Run chart-testing (install)
-        run: ct install --all --chart-dirs k8s/charts

CODE_OF_CONDUCT.md

@@ -1,74 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to make participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, gender identity and expression, level of experience,
-nationality, personal appearance, race, religion, or sexual identity and
-orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-- Using welcoming and inclusive language
-- Being respectful of differing viewpoints and experiences
-- Gracefully accepting constructive criticism
-- Focusing on what is best for the community
-- Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-- The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
-- Trolling, insulting/derogatory comments, and personal or political attacks
-- Public or private harassment
-- Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-- Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at <enteremailhere>. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/

README.md

@@ -3,12 +3,12 @@
 
 [![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
 [![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs)
-[![Build Status](https://img.shields.io/github/actions/workflow/status/seaweedfs/seaweedfs/go.yml)](https://github.com/seaweedfs/seaweedfs/actions/workflows/go.yml)
+[![Build Status](https://img.shields.io/github/workflow/status/chrislusf/seaweedfs/Go)](https://github.com/seaweedfs/seaweedfs/actions/workflows/go.yml)
 [![GoDoc](https://godoc.org/github.com/seaweedfs/seaweedfs/weed?status.svg)](https://godoc.org/github.com/seaweedfs/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/seaweedfs/seaweedfs/wiki)
 [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
 [![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf)
-[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/seaweedfs)](https://artifacthub.io/packages/search?repo=seaweedfs)
+
 
 ![SeaweedFS Logo](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/seaweedfs.png)
 
@@ -32,7 +32,7 @@ Your support will be really appreciated by me and other supporters!
 -->
 
 ### Gold Sponsors
-[![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com)
+[![nodion](https://www.nodion.com/img/logo.svg)](https://www.nodion.com)
 [![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com)
 
 ---
@@ -66,7 +66,6 @@ Table of Contents
     * [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
     * [Compared to GlusterFS](#compared-to-glusterfs)
     * [Compared to Ceph](#compared-to-ceph)
-    * [Compared to Minio](#compared-to-minio)
 * [Dev Plan](#dev-plan)
 * [Installation Guide](#installation-guide)
 * [Disk Related Topics](#disk-related-topics)
@@ -80,7 +79,7 @@ Table of Contents
 `docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
 
 ## Quick Start with Single Binary ##
-* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`. Or run `go install github.com/seaweedfs/seaweedfs/weed@latest`.
+* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
 * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
 
 Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!

backers.md

@@ -7,7 +7,6 @@
 
 - [Evercam Camera Management Software](https://evercam.io/)
 - [Spherical Elephant GmbH](https://www.sphericalelephant.com)
-- [WizardTales GmbH](https://www.wizardtales.com)
 
 - <h2 align="center">Backers</h2>
 

docker/Dockerfile.go_build

@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine as builder
+FROM golang:1.19-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/seaweedfs/
 RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs

docker/Dockerfile.rocksdb_dev_env

@@ -1,9 +1,9 @@
-FROM golang:1.20-buster as builder
+FROM golang:1.19-buster as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v8.1.1
+ENV ROCKSDB_VERSION v7.5.3
 
 # build RocksDB
 RUN cd /tmp && \

docker/Dockerfile.rocksdb_large

@@ -1,9 +1,9 @@
-FROM golang:1.20-buster as builder
+FROM golang:1.19-buster as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v8.1.1
+ENV ROCKSDB_VERSION v7.5.3
 
 # build RocksDB
 RUN cd /tmp && \

docker/Makefile

@@ -29,7 +29,7 @@ go_build_large_disk:
 build_rocksdb_dev_env:
 	docker build --no-cache -t chrislusf/rocksdb_dev_env -f Dockerfile.rocksdb_dev_env .
 
-build_rocksdb_local: build_rocksdb_dev_env
+build_rocksdb_local:
 	cd .. ; docker build --no-cache -t chrislusf/seaweedfs:rocksdb_local -f docker/Dockerfile.rocksdb_large_local .
 
 build_rocksdb:
@@ -62,8 +62,7 @@ k8s: build
 dev_registry: build
 	docker compose -f compose/local-registry-compose.yml -p seaweedfs up
 
-dev_replicate:
-	docker build --build-arg TAGS=gocdk --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.go_build .
+dev_replicate: build
 	docker compose -f compose/local-replicate-compose.yml -p seaweedfs up
 
 dev_auditlog: build

docker/seaweedfs.sql

@@ -3,10 +3,10 @@ CREATE USER IF NOT EXISTS 'seaweedfs'@'%' IDENTIFIED BY 'secret';
 GRANT ALL PRIVILEGES ON seaweedfs.* TO 'seaweedfs'@'%';
 FLUSH PRIVILEGES;
 USE seaweedfs;
-CREATE TABLE IF NOT EXISTS `filemeta` (
-    `dirhash`   BIGINT NOT NULL       COMMENT 'first 64 bits of MD5 hash value of directory field',
-    `name`      VARCHAR(766) NOT NULL COMMENT 'directory or file name',
-    `directory` TEXT NOT NULL         COMMENT 'full path to parent directory',
-    `meta`      LONGBLOB,
-    PRIMARY KEY (`dirhash`, `name`)
-) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+CREATE TABLE IF NOT EXISTS filemeta (
+    dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
+    name        VARCHAR(1000)  COMMENT 'directory or file name',
+    directory   TEXT           COMMENT 'full path to parent directory',
+    meta        LONGBLOB,
+    PRIMARY KEY (dirhash, name)
+) DEFAULT CHARSET=utf8;

go.mod

@@ -1,50 +1,49 @@
 module github.com/seaweedfs/seaweedfs
 
-go 1.20
+go 1.19
 
 require (
-	cloud.google.com/go v0.110.2 // indirect
-	cloud.google.com/go/pubsub v1.31.0
-	cloud.google.com/go/storage v1.30.1
+	cloud.google.com/go v0.105.0 // indirect
+	cloud.google.com/go/pubsub v1.27.1
+	cloud.google.com/go/storage v1.28.1
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
-	github.com/Shopify/sarama v1.38.1
-	github.com/aws/aws-sdk-go v1.44.271
+	github.com/Shopify/sarama v1.37.2
+	github.com/aws/aws-sdk-go v1.44.146
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
-	github.com/cenkalti/backoff/v4 v4.2.1
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/coreos/go-semver v0.3.1 // indirect
-	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/disintegration/imaging v1.6.2
-	github.com/dustin/go-humanize v1.0.1
+	github.com/dustin/go-humanize v1.0.0
 	github.com/eapache/go-resiliency v1.3.0 // indirect
-	github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
+	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a
 	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
-	github.com/fclairamb/ftpserverlib v0.21.0
+	github.com/fclairamb/ftpserverlib v0.20.0
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/go-redsync/redsync/v4 v4.8.1
-	github.com/go-sql-driver/mysql v1.7.1
-	github.com/go-zookeeper/zk v1.0.3 // indirect
-	github.com/gocql/gocql v1.4.0
+	github.com/go-redsync/redsync/v4 v4.6.0
+	github.com/go-sql-driver/mysql v1.7.0
+	github.com/go-zookeeper/zk v1.0.2 // indirect
+	github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3
+	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/uuid v1.3.0
 	github.com/google/wire v0.5.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.9.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
 	github.com/gorilla/mux v1.8.0
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -52,45 +51,46 @@ require (
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jcmturner/gofork v1.7.6 // indirect
-	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
+	github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect
 	github.com/jinzhu/copier v0.3.5
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.16.0 // indirect
-	github.com/klauspost/reedsolomon v1.11.7
+	github.com/klauspost/compress v1.15.11 // indirect
+	github.com/klauspost/reedsolomon v1.11.3
 	github.com/kurin/blazer v0.5.3
-	github.com/lib/pq v1.10.9
-	github.com/linxGnu/grocksdb v1.8.0
-	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/lib/pq v1.10.7
+	github.com/linxGnu/grocksdb v1.7.10
+	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mattn/go-ieproxy v0.0.11 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mattn/go-ieproxy v0.0.9 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/olivere/elastic/v7 v7.0.32
+	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/peterh/liner v1.2.2
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/posener/complete v1.2.3
-	github.com/pquerna/cachecontrol v0.2.0
-	github.com/prometheus/client_golang v1.15.0
+	github.com/pquerna/cachecontrol v0.1.0
+	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.10.1
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	github.com/seaweedfs/goexif v1.0.3
-	github.com/seaweedfs/raft v1.1.3
+	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
+	github.com/seaweedfs/goexif v2.0.0+incompatible
+	github.com/seaweedfs/raft v1.1.0
 	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/spf13/afero v1.9.3 // indirect
+	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.15.0
-	github.com/stretchr/testify v1.8.3
+	github.com/spf13/viper v1.14.0
+	github.com/stretchr/testify v1.8.1
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
 	github.com/tidwall/gjson v1.14.4
@@ -103,203 +103,136 @@ require (
 	github.com/viant/ptrie v0.3.0
 	github.com/viant/toolbox v0.33.2 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.1.2 // indirect
-	github.com/xdg-go/stringprep v1.0.4 // indirect
-	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	go.etcd.io/etcd/client/v3 v3.5.9
-	go.mongodb.org/mongo-driver v1.11.6
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.etcd.io/etcd/client/v3 v3.5.6
+	go.mongodb.org/mongo-driver v1.11.0
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.29.0
-	gocloud.dev/pubsub/natspubsub v0.29.0
-	gocloud.dev/pubsub/rabbitpubsub v0.29.0
-	golang.org/x/crypto v0.7.0 // indirect
-	golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874
-	golang.org/x/image v0.7.0
-	golang.org/x/net v0.10.0
-	golang.org/x/oauth2 v0.8.0 // indirect
-	golang.org/x/sys v0.8.0
-	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/tools v0.9.1
+	gocloud.dev v0.27.0
+	gocloud.dev/pubsub/natspubsub v0.27.0
+	gocloud.dev/pubsub/rabbitpubsub v0.27.0
+	golang.org/x/crypto v0.1.0 // indirect
+	golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
+	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
+	golang.org/x/net v0.2.0
+	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
+	golang.org/x/sys v0.3.0
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/tools v0.3.0
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.124.0
+	google.golang.org/api v0.103.0
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
-	google.golang.org/grpc v1.55.0
-	google.golang.org/protobuf v1.30.0
+	google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c // indirect
+	google.golang.org/grpc v1.51.0
+	google.golang.org/protobuf v1.28.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
 	modernc.org/cc/v3 v3.40.0 // indirect
 	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.22.5 // indirect
+	modernc.org/libc v1.21.5 // indirect
 	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.5.0 // indirect
+	modernc.org/memory v1.4.0 // indirect
 	modernc.org/opt v0.1.3 // indirect
-	modernc.org/sqlite v1.22.1
+	modernc.org/sqlite v1.20.0
 	modernc.org/strutil v1.1.3
 	modernc.org/token v1.0.1 // indirect
 )
 
 require (
-	github.com/Jille/raft-grpc-transport v1.4.0
-	github.com/arangodb/go-driver v1.5.2
+	github.com/Jille/raft-grpc-transport v1.3.0
+	github.com/arangodb/go-driver v1.4.0
 	github.com/armon/go-metrics v0.4.1
 	github.com/fluent/fluent-logger-golang v1.9.0
-	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
-	github.com/hanwen/go-fuse/v2 v2.3.0
-	github.com/hashicorp/raft v1.5.0
+	github.com/google/flatbuffers v22.11.23+incompatible
+	github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
+	github.com/hashicorp/raft v1.3.11
 	github.com/hashicorp/raft-boltdb/v2 v2.2.2
-	github.com/rabbitmq/amqp091-go v1.8.1
-	github.com/rclone/rclone v1.62.2
-	github.com/schollz/progressbar/v3 v3.13.1
-	github.com/tikv/client-go/v2 v2.0.7
-	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.3
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.47.1
+	github.com/rabbitmq/amqp091-go v1.5.0
+	github.com/tikv/client-go/v2 v2.0.2
+	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1
+	golang.org/x/sync v0.1.0
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
 )
 
 require (
-	cloud.google.com/go/compute v1.19.0 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.0.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
-	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect
-	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
-	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/Unknwon/goconfig v1.0.0 // indirect
-	github.com/abbot/go-http-auth v0.4.0 // indirect
+	cloud.google.com/go/compute v1.12.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.1 // indirect
+	cloud.google.com/go/iam v0.7.0 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2 v1.17.4 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.18.12 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.20.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.20.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect
-	github.com/aws/smithy-go v1.13.5 // indirect
-	github.com/benbjohnson/clock v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.16.8 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.15.15 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.12.10 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.17.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.19.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.11.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.16.10 // indirect
+	github.com/aws/smithy-go v1.12.0 // indirect
+	github.com/benbjohnson/clock v1.1.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
-	github.com/buengese/sgzip v0.1.1 // indirect
-	github.com/calebcase/tmpfile v1.0.3 // indirect
-	github.com/colinmarc/hdfs/v2 v2.3.0 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
-	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
-	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/fclairamb/go-log v0.4.1 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
-	github.com/geoffgarside/ber v1.1.0 // indirect
-	github.com/go-chi/chi/v5 v5.0.8 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
-	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/s2a-go v0.1.4 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
-	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-hclog v1.2.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-msgpack v1.1.5 // indirect
-	github.com/hashicorp/golang-lru v0.6.0 // indirect
-	github.com/hirochachacha/go-smb2 v1.1.0 // indirect
-	github.com/iguanesolutions/go-systemd/v5 v5.1.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
-	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
-	github.com/jlaffaye/ftp v0.1.1-0.20230214004652-d84bf4be2b6e // indirect
-	github.com/jonboulle/clockwork v0.3.0 // indirect
+	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
-	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
-	github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
-	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
-	github.com/kr/fs v0.1.0 // indirect
-	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/klauspost/cpuid/v2 v2.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.7 // indirect
 	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
-	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/montanaflynn/stats v0.7.0 // indirect
-	github.com/nats-io/nats.go v1.23.0 // indirect
+	github.com/montanaflynn/stats v0.6.6 // indirect
+	github.com/nats-io/nats.go v1.16.0 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
-	github.com/ncw/swift/v2 v2.0.1 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.32.0 // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
-	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
 	github.com/philhofer/fwd v1.1.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.17 // indirect
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
-	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
-	github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
-	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
-	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
-	github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6 // indirect
-	github.com/pkg/xattr v0.4.9 // indirect
-	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
-	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
-	github.com/rfjakob/eme v1.1.2 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/shirou/gopsutil/v3 v3.23.2 // indirect
-	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
-	github.com/sony/gobreaker v0.5.0 // indirect
-	github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect
+	github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
+	github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 // indirect
+	github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect
+	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.4.2 // indirect
-	github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca // indirect
-	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
-	github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
+	github.com/stathat/consistent v1.0.0 // indirect
+	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 // indirect
 	github.com/tinylib/msgp v1.1.6 // indirect
-	github.com/tklauser/go-sysconf v0.3.11 // indirect
-	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
-	github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
-	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f // indirect
-	github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba // indirect
+	github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
-	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
-	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	github.com/zeebo/blake3 v0.2.3 // indirect
-	github.com/zeebo/errs v1.3.0 // indirect
-	go.etcd.io/bbolt v1.3.7 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/multierr v1.9.0 // indirect
-	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/mod v0.10.0 // indirect
-	golang.org/x/sync v0.2.0 // indirect
-	golang.org/x/term v0.8.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
+	go.etcd.io/bbolt v1.3.6 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.6 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.6 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	go.uber.org/zap v1.22.0 // indirect
+	golang.org/x/mod v0.7.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/uint128 v1.2.0 // indirect
-	storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
-	storj.io/drpc v0.0.32 // indirect
-	storj.io/uplink v1.10.0 // indirect
 )
 
 // replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft

k8s/charts/artifacthub-repo.yml

@@ -1,16 +0,0 @@
-# Artifact Hub repository metadata file
-#
-# Some settings like the verified publisher flag or the ignored packages won't
-# be applied until the next time the repository is processed. Please keep in
-# mind that the repository won't be processed if it has not changed since the
-# last time it was processed. Depending on the repository kind, this is checked
-# in a different way. For Helm http based repositories, we consider it has
-# changed if the `index.yaml` file changes. For git based repositories, it does
-# when the hash of the last commit in the branch you set up changes. This does
-# NOT apply to ownership claim operations, which are processed immediately.
-#
-
-repositoryID: 5b2f1fe2-20e5-486e-9746-183484642aa2
-# owners: # (optional, used to claim repository ownership)
-#   - name: username
-#     email: email

k8s/charts/seaweedfs/README.md

@@ -1,81 +0,0 @@
-# SEAWEEDFS - helm chart (2.x+)
-
-## Getting Started
-
-### Add the helm repo
-
-`helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm`
-
-### Install the helm chart
-
-`helm install seaweedfs seaweedfs/seaweedfs`
-
-### (Recommended) Provide `values.yaml`
-
-`helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs`
-
-## Info:
-* master/filer/volume are stateful sets with anti-affinity on the hostname,
-so your deployment will be spread/HA.
-* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances) and backup/HA memsql can provide.
-* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer with ENV.
-* cert config exists and can be enabled, but not been tested, requires cert-manager to be installed.
-
-## Prerequisites
-### Database
-
-leveldb is the default database this only supports one filer replica.
-
-To have multiple filers a external datastore is recommened.
-
-Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`. 
-This database should be pre-configured and initialized by running:
-```sql
-CREATE TABLE IF NOT EXISTS `filemeta` (
-  `dirhash`   BIGINT NOT NULL       COMMENT 'first 64 bits of MD5 hash value of directory field',
-  `name`      VARCHAR(766) NOT NULL COMMENT 'directory or file name',
-  `directory` TEXT NOT NULL         COMMENT 'full path to parent directory',
-  `meta`      LONGBLOB,
-  PRIMARY KEY (`dirhash`, `name`)
-) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
-```
-
-Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
-
-### Node Labels
-Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
-
-Here is an example:
-* s3/filer/master needs the label **sw-backend=true**
-* volume need the label **sw-volume=true**
-
-to label a node to be able to run all pod types in k8s:
-```
-kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
-```
-
-on production k8s deployment you will want each pod to have a different host,
-especially the volume server and the masters, all pods (master/volume/filer)
-should have anti-affinity rules to disallow running multiple component pods  on the same host.
-
-If you still want to run multiple pods of the same component (master/volume/filer) on the same host please set/update the corresponding affinity rule in values.yaml to an empty one:
-
-```affinity: ""```
-
-## PVC - storage class ###
-
-On the volume stateful set added support for k8s PVC, currently example
-with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
-https://github.com/rancher/local-path-provisioner
-
-you can use ANY storage class you like, just update the correct storage-class
-for your deployment.
-
-## current instances config (AIO):
-
-1 instance for each type (master/filer+s3/volume)
-
-You can update the replicas count for each node type in values.yaml,
-need to add more nodes with the corresponding labels if applicable.
-
-Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing).

k8s/helm_charts2/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.52"
-version: "3.52"
+appVersion: "3.36"
+version: "3.36"

k8s/helm_charts2/README.md

@@ -0,0 +1,62 @@
+# SEAWEEDFS - helm chart (2.x)
+
+## Info:
+* master/filer/volume are stateful sets with anti-affinity on the hostname,
+so your deployment will be spread/HA.
+* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
+and backup/HA memsql can provide.
+* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer
+with ENV.
+* cert config exists and can be enabled, but not been tested.
+
+## Prerequisites
+### Database
+A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`. 
+This database should be pre-configured and initialized by running:
+```sql
+CREATE TABLE IF NOT EXISTS filemeta (
+  dirhash     BIGINT               COMMENT 'first 64 bits of MD5 hash value of directory field',
+  name        VARCHAR(1000) BINARY COMMENT 'directory or file name',
+  directory   TEXT BINARY          COMMENT 'full path to parent directory',
+  meta        LONGBLOB,
+  PRIMARY KEY (dirhash, name)
+) DEFAULT CHARSET=utf8;
+```
+
+Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`.
+
+### Node Labels
+Kubernetes node have labels which help to define which node(Host) will run which pod:
+* s3/filer/master needs the label **sw-backend=true**
+* volume need the label **sw-volume=true**
+
+to label a node to be able to run all pod types in k8s:
+```
+kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
+```
+
+on production k8s deployment you will want each pod to have a different host,
+especially the volume server & the masters, currently all pods (master/volume/filer)
+have anti-affinity rule to disallow running multiple pod type on the same host.
+if you still want to run multiple pods of the same type (master/volume/filer) on the same host
+please set/update the corresponding affinity rule in values.yaml to an empty one:
+
+```affinity: ""```
+
+## PVC - storage class ###
+
+on the volume stateful set added support for K8S PVC, currently example
+with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
+https://github.com/rancher/local-path-provisioner
+
+you can use ANY storage class you like, just update the correct storage-class
+for your deployment.
+
+## current instances config (AIO):
+1 instance for each type (master/filer+s3/volume)
+
+you can update the replicas count for each node type in values.yaml,
+need to add more nodes with the corresponding labels.
+
+most of the configuration are available through values.yaml
+

k8s/helm_charts2/templates/filer-service.yaml

@@ -20,12 +20,6 @@ spec:
     port: {{ .Values.filer.grpcPort }}
     targetPort: {{ .Values.filer.grpcPort }}
     protocol: TCP
-  {{- if .Values.filer.s3.enabled }}
-  - name: "swfs-s3"
-    port: {{ .Values.filer.s3.port }}
-    targetPort: {{ .Values.filer.s3.port }}
-    protocol: TCP
-  {{- end }}
   {{- if .Values.filer.metricsPort }}
   - name: "metrics"
     port: {{ .Values.filer.metricsPort }}

k8s/helm_charts2/templates/filer-statefulset.yaml

@@ -97,22 +97,11 @@ spec:
               value: {{ $value | quote }}
             {{- end }}
             {{- end }}
-            {{- if .Values.filer.secretExtraEnvironmentVars }}
-            {{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
-            - name: {{ $key }}
-              valueFrom: {{ toYaml $value | nindent 16 }}
-            {{- end }}
-            {{- end }}             
           command:
             - "/bin/sh"
             - "-ec"
             - | 
-              exec /usr/bin/weed \
-              {{- if eq .Values.filer.logs.type "hostPath" }}
-              -logdir=/logs \
-              {{- else }}
-              -logtostderr=true \
-              {{- end }}
+              exec /usr/bin/weed -logdir=/logs \
               {{- if .Values.filer.loggingOverrideLevel }}
               -v={{ .Values.filer.loggingOverrideLevel }} \
               {{- else }}
@@ -148,9 +137,6 @@ spec:
               -encryptVolumeData \
               {{- end }}
               -ip=${POD_IP} \
-              {{- if .Values.filer.filerGroup}}
-              -filerGroup={{ .Values.filer.filerGroup}} \
-              {{- end }}
               {{- if .Values.filer.s3.enabled }}
               -s3 \
               -s3.port={{ .Values.filer.s3.port }} \
@@ -171,14 +157,14 @@ spec:
               -s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
               {{- end }}
               {{- end }}
-              -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
+              -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
           volumeMounts:
             - name: seaweedfs-filer-log-volume
               mountPath: "/logs/"
             - mountPath: /etc/sw
               name: config-users
               readOnly: true
-            {{- if (or .Values.filer.enablePVC (or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.data.type "persistentVolumeClaim"))) }}
+            {{- if .Values.filer.enablePVC }}
             - name: data-filer
               mountPath: /data
             {{- end }}
@@ -239,13 +225,13 @@ spec:
         {{- if eq .Values.filer.logs.type "hostPath" }}
         - name: seaweedfs-filer-log-volume
           hostPath:
-            path: {{ .Values.filer.logs.hostPathPrefix }}/logs/seaweedfs/filer
+            path: /storage/logs/seaweedfs/filer
             type: DirectoryOrCreate
         {{- end }}
         {{- if eq .Values.filer.data.type "hostPath" }}
         - name: data-filer
           hostPath:
-            path: {{ .Values.filer.data.hostPathPrefix }}/filer_store
+            path: /storage/filer_store
             type: DirectoryOrCreate
         {{- end }}
         - name: db-schema-config-volume

k8s/helm_charts2/templates/master-statefulset.yaml

@@ -90,12 +90,7 @@ spec:
             - "/bin/sh"
             - "-ec"
             - | 
-              exec /usr/bin/weed \
-              {{- if eq .Values.master.logs.type "hostPath" }}
-              -logdir=/logs \
-              {{- else }}
-              -logtostderr=true \
-              {{- end }}
+              exec /usr/bin/weed -logdir=/logs \
               {{- if .Values.master.loggingOverrideLevel }}
               -v={{ .Values.master.loggingOverrideLevel }} \
               {{- else }}
@@ -133,8 +128,8 @@ spec:
               {{- if .Values.master.garbageThreshold }}
               -garbageThreshold={{ .Values.master.garbageThreshold }} \
               {{- end }}
-              -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master.{{ .Release.Namespace }} \
-              -peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
+              -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \
+              -peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
           volumeMounts:
             - name : data-{{ .Release.Namespace }}
               mountPath: /data
@@ -201,13 +196,13 @@ spec:
         {{- if eq .Values.master.logs.type "hostPath" }}
         - name: seaweedfs-master-log-volume
           hostPath:
-            path: {{ .Values.master.logs.hostPathPrefix }}/logs/seaweedfs/master
+            path: /storage/logs/seaweedfs/master
             type: DirectoryOrCreate
         {{- end }}
         {{- if eq .Values.master.data.type "hostPath" }}
         - name: data-{{ .Release.Namespace }}
           hostPath:
-            path: {{ .Values.master.data.hostPathPrefix }}/seaweed-master/
+            path: /ssd/seaweed-master/
             type: DirectoryOrCreate
         {{- end }}
         {{- if .Values.global.enableSecurity }}

k8s/helm_charts2/templates/s3-deployment.yaml

@@ -72,19 +72,13 @@ spec:
             - "/bin/sh"
             - "-ec"
             - | 
-              exec /usr/bin/weed \
-               {{- if eq .Values.s3.logs.type "hostPath" }}
-              -logdir=/logs \
-              {{- else }}
-              -logtostderr=true \
-              {{- end }}
+              exec /usr/bin/weed -logdir=/logs \
               {{- if .Values.s3.loggingOverrideLevel }}
               -v={{ .Values.s3.loggingOverrideLevel }} \
               {{- else }}
               -v={{ .Values.global.loggingLevel }} \
               {{- end }}
               s3 \
-              -ip.bind={{ .Values.s3.bindAddress }} \
               -port={{ .Values.s3.port }} \
               {{- if .Values.s3.metricsPort }}
               -metricsPort {{ .Values.s3.metricsPort }} \
@@ -105,7 +99,7 @@ spec:
               {{- if .Values.s3.auditLogConfig }}
               -auditLogConfig=/etc/sw/s3_auditLogConfig.json \
               {{- end }}
-              -filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
+              -filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }}
           volumeMounts:
             - name: logs
               mountPath: "/logs/"
@@ -173,7 +167,7 @@ spec:
         {{- if eq .Values.s3.logs.type "hostPath" }}
         - name: logs
           hostPath:
-            path: {{ .Values.s3.logs.hostPathPrefix }}/logs/seaweedfs/s3
+            path: /storage/logs/seaweedfs/s3
             type: DirectoryOrCreate
         {{- end }}
         {{- if .Values.global.enableSecurity }}

k8s/helm_charts2/templates/volume-statefulset.yaml

@@ -93,12 +93,7 @@ spec:
             - "/bin/sh"
             - "-ec"
             - |
-              exec /usr/bin/weed \
-                {{- if eq .Values.volume.logs.type "hostPath" }}
-                -logdir=/logs \
-                {{- else }}
-                -logtostderr=true \
-                {{- end }}
+              exec /usr/bin/weed -logdir=/logs \
                 {{- if .Values.volume.loggingOverrideLevel }}
                 -v={{ .Values.volume.loggingOverrideLevel }} \
                 {{- else }}
@@ -138,9 +133,9 @@ spec:
                 -fileSizeLimitMB={{ .Values.volume.fileSizeLimitMB }} \
                 {{- end }}
                 -minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \
-                -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume.{{ .Release.Namespace }} \
+                -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume \
                 -compactionMBps={{ .Values.volume.compactionMBps }} \
-                -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
+                -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
           volumeMounts:
             - name: data
               mountPath: "{{ .Values.volume.dir }}/"
@@ -187,7 +182,7 @@ spec:
               port: {{ .Values.volume.port }}
               scheme: HTTP
             initialDelaySeconds: 15
-            periodSeconds: 15
+            periodSeconds: 90
             successThreshold: 1
             failureThreshold: 100
             timeoutSeconds: 30
@@ -211,19 +206,19 @@ spec:
         {{- if eq .Values.volume.data.type "hostPath" }}
         - name: data
           hostPath:
-            path: {{ .Values.volume.data.hostPathPrefix }}/object_store/
+            path: /storage/object_store/
             type: DirectoryOrCreate
         {{- end }}
         {{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
         - name: idx
           hostPath:
-            path: {{ .Values.volume.idx.hostPathPrefix }}/seaweedfs-volume-idx/
+            path: /ssd/seaweedfs-volume-idx/
             type: DirectoryOrCreate
         {{- end }}
         {{- if eq .Values.volume.logs.type "hostPath" }}
         - name: logs
           hostPath:
-            path: {{ .Values.volume.logs.hostPathPrefix }}/logs/seaweedfs/volume
+            path: /storage/logs/seaweedfs/volume
             type: DirectoryOrCreate
         {{- end }}
         {{- if .Values.global.enableSecurity }}

k8s/helm_charts2/values.yaml

@@ -5,7 +5,7 @@ global:
   repository: ""
   imageName: chrislusf/seaweedfs
   imagePullPolicy: IfNotPresent
-  imagePullSecrets: ""
+  imagePullSecrets: imagepullsecret
   restartPolicy: Always
   loggingLevel: 1
   enableSecurity: false
@@ -22,8 +22,8 @@ global:
   replicationPlacment: "001"
   extraEnvironmentVars:
     WEED_CLUSTER_DEFAULT: "sw"
-    WEED_CLUSTER_SW_MASTER: "seaweedfs-master.seaweedfs:9333"
-    WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client.seaweedfs:8888"
+    WEED_CLUSTER_SW_MASTER: "seaweedfs-master:9333"
+    WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client:8888"
 
 image:
   registry: ""
@@ -44,11 +44,11 @@ master:
   volumePreallocate: false
   volumeSizeLimitMB: 1000
   loggingOverrideLevel: null
-  # number of seconds between heartbeats, default 5
+  #number of seconds between heartbeats, default 5
   pulseSeconds: null
-  # threshold to vacuum and reclaim spaces, default 0.3 (30%)
+  #threshold to vacuum and reclaim spaces, default 0.3 (30%)
   garbageThreshold: null
-  # Prometheus push interval in seconds, default 15
+  #Prometheus push interval in seconds, default 15
   metricsIntervalSec: 15
   #  replication type is XYZ:
   # X number of replica in other data centers
@@ -68,13 +68,11 @@ master:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /ssd
 
   logs:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /storage
 
   initContainers: ""
 
@@ -112,10 +110,10 @@ master:
   # nodeSelector labels for master pod assignment, formatted as a muli-line string.
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
-  nodeSelector: |
-    beta.kubernetes.io/arch: amd64
   # nodeSelector: |
-  #   sw-backend: "true"
+  #   beta.kubernetes.io/arch: amd64
+  nodeSelector: |
+    sw-backend: "true"
 
   # used to assign priority to master pods
   # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
@@ -178,19 +176,15 @@ volume:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /storage
-
   idx:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /ssd
 
   logs:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /storage
 
   # limit background compaction or copying speed in mega bytes per second
   compactionMBps: "50"
@@ -241,7 +235,7 @@ volume:
   # should map directly to the value of the resources field for a PodSpec,
   # formatted as a multi-line string. By default no direct resource request
   # is made.
-  resources: {}
+  resources: null
 
   # Toleration Settings for server pods
   # This should be a multi-line string matching the Toleration array
@@ -251,10 +245,10 @@ volume:
   # nodeSelector labels for server pod assignment, formatted as a muli-line string.
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
-  nodeSelector: |
-   beta.kubernetes.io/arch: amd64
   # nodeSelector: |
-  #   sw-volume: "true"
+  #   beta.kubernetes.io/arch: amd64
+  nodeSelector: |
+    sw-volume: "true"
 
   # used to assign priority to server pods
   # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
@@ -273,7 +267,6 @@ filer:
   grpcPort: 18888
   metricsPort: 9327
   loggingOverrideLevel: null
-  filerGroup: ""
   #  replication type is XYZ:
   # X number of replica in other data centers
   # Y number of replica in other racks in the same data center
@@ -315,13 +308,11 @@ filer:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /storage
 
   logs:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /storage
 
   initContainers: ""
 
@@ -359,10 +350,10 @@ filer:
   # nodeSelector labels for server pod assignment, formatted as a muli-line string.
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
-  nodeSelector: |
-    beta.kubernetes.io/arch: amd64
   # nodeSelector: |
-  #   sw-backend: "true"
+  #   beta.kubernetes.io/arch: amd64
+  nodeSelector: |
+    sw-backend: "true"
 
   # used to assign priority to server pods
   # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
@@ -390,7 +381,7 @@ filer:
 
   # extraEnvVars is a list of extra enviroment variables to set with the stateful set.
   extraEnvironmentVars:
-    WEED_MYSQL_ENABLED: "false"
+    WEED_MYSQL_ENABLED: "true"
     WEED_MYSQL_HOSTNAME: "mysql-db-host"
     WEED_MYSQL_PORT: "3306"
     WEED_MYSQL_DATABASE: "sw_database"
@@ -401,29 +392,17 @@ filer:
     # enable usage of memsql as filer backend
     WEED_MYSQL_INTERPOLATEPARAMS: "true"
     # if you want to use leveldb2, then should enable "enablePVC". or you may lose your data.
-    WEED_LEVELDB2_ENABLED: "true"
+    WEED_LEVELDB2_ENABLED: "false"
     # with http DELETE, by default the filer would check whether a folder is empty.
     # recursive_delete will delete all sub folders and files, similar to "rm -Rf"
     WEED_FILER_OPTIONS_RECURSIVE_DELETE: "false"
     # directories under this folder will be automatically creating a separate bucket
     WEED_FILER_BUCKETS_FOLDER: "/buckets"
 
-
-  # secret env variables
-  secretExtraEnvironmentVars: []
-      # WEED_POSTGRES_USERNAME:
-      #   secretKeyRef:
-      #     name: postgres-credentials
-      #     key: username
-      # WEED_POSTGRES_PASSWORD:
-      #   secretKeyRef:
-      #     name: postgres-credentials
-      #     key: password
-
   s3:
     enabled: true
     port: 8333
-    # allow empty folders
+    #allow empty folders
     allowEmptyFolder: false
     # Suffix of the host name, {bucket}.{domainName}
     domainName: ""
@@ -439,11 +418,10 @@ s3:
   imageTag: null
   restartPolicy: null
   replicas: 1
-  bindAddress: 0.0.0.0
   port: 8333
   metricsPort: 9327
   loggingOverrideLevel: null
-  # allow empty folders
+  #allow empty folders
   allowEmptyFolder: true
   # enable user & permission to s3 (need to inject to all services)
   enableAuth: false
@@ -462,7 +440,7 @@ s3:
   # should map directly to the value of the resources field for a PodSpec,
   # formatted as a multi-line string. By default no direct resource request
   # is made.
-  resources: {}
+  resources: null
 
   # Toleration Settings for server pods
   # This should be a multi-line string matching the Toleration array
@@ -472,10 +450,10 @@ s3:
   # nodeSelector labels for server pod assignment, formatted as a muli-line string.
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
-  nodeSelector: |
-    beta.kubernetes.io/arch: amd64
   # nodeSelector: |
-  #   sw-backend: "true"
+  #   beta.kubernetes.io/arch: amd64
+  nodeSelector: |
+    sw-backend: "true"
 
   # used to assign priority to server pods
   # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
@@ -485,12 +463,12 @@ s3:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /storage
+
 
 certificates:
   commonName: "SeaweedFS CA"
   ipAddresses: []
   keyAlgorithm: rsa
   keySize: 2048
-  duration: 2160h  # 90d
-  renewBefore: 360h  # 15d
+  duration: 2160h # 90d
+  renewBefore: 360h # 15d

other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java

@@ -21,7 +21,7 @@ public class FilerSslContext {
         String home = System.getProperty("user.home");
         File f1 = new File("./"+securityFileName);
         File f2 = new File(home + "/.seaweedfs/"+securityFileName);
-        File f3 = new File("/etc/seaweedfs/"+securityFileName);
+        File f3 = new File(home + "/etc/seaweedfs/"+securityFileName);
 
         File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null;
 

other/java/s3copier/pom.xml

@@ -5,10 +5,6 @@
   <artifactId>copier</artifactId>
   <packaging>jar</packaging>
   <version>1.0-SNAPSHOT</version>
-  <properties>
-    <maven.compiler.source>18</maven.compiler.source>
-    <maven.compiler.target>18</maven.compiler.target>
-  </properties>
   <name>copier</name>
   <url>http://maven.apache.org</url>
 

other/java/s3copier/src/main/java/com/seaweedfs/s3/HighLevelMultipartUpload.java

@@ -10,8 +10,6 @@ import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.regions.Regions;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.model.CreateBucketRequest;
-import com.amazonaws.services.s3.model.GetBucketLocationRequest;
 import com.amazonaws.services.s3.transfer.TransferManager;
 import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
 import com.amazonaws.services.s3.transfer.Upload;
@@ -44,19 +42,6 @@ public class HighLevelMultipartUpload {
                 .withS3Client(s3Client)
                 .build();
 
-
-            if (!s3Client.doesBucketExistV2(bucketName)) {
-                // Because the CreateBucketRequest object doesn't specify a region, the
-                // bucket is created in the region specified in the client.
-                s3Client.createBucket(new CreateBucketRequest(bucketName));
-
-                // Verify that the bucket was created by retrieving it and checking its location.
-                String bucketLocation = s3Client.getBucketLocation(new GetBucketLocationRequest(bucketName));
-                System.out.println("Bucket location: " + bucketLocation);
-            } else {
-                System.out.println("Bucket already exists");
-            }
-
             // TransferManager processes all transfers asynchronously,
             // so this call returns immediately.
             Upload upload = tm.upload(bucketName, keyName, file);

unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go

@@ -6,7 +6,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"strconv"
@@ -53,7 +52,7 @@ func main() {
 }
 
 func startGenerateMetadata() {
-	pb.WithFilerClient(false, util.RandomInt32(), pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), func(client filer_pb.SeaweedFilerClient) error {
+	pb.WithFilerClient(false, pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), func(client filer_pb.SeaweedFilerClient) error {
 
 		for i := 0; i < *n; i++ {
 			name := fmt.Sprintf("file%d", i)
@@ -79,19 +78,7 @@ func startGenerateMetadata() {
 
 func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
 
-	metadataFollowOption := &pb.MetadataFollowOption{
-		ClientName:             "tail",
-		ClientId:               0,
-		ClientEpoch:            0,
-		SelfSignature:          0,
-		PathPrefix:             *dir,
-		AdditionalPathPrefixes: nil,
-		DirectoriesToWatch:     nil,
-		StartTsNs:              0,
-		StopTsNs:               0,
-		EventErrorType:         pb.TrivialOnError,
-	}
-	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), metadataFollowOption, eachEntryFunc)
+	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), "tail", 0, 0, *dir, nil, 0, 0, 0, eachEntryFunc, pb.TrivialOnError)
 
 	if tailErr != nil {
 		fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)

weed/command/filer.go

@@ -60,7 +60,6 @@ type FilerOptions struct {
 	localSocket             *string
 	showUIDirectoryDelete   *bool
 	downloadMaxMBps         *int
-	diskType                *string
 }
 
 func init() {
@@ -90,7 +89,6 @@ func init() {
 	f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
 	f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
 	f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
-	f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 
 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -115,7 +113,6 @@ func init() {
 	filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
 	filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
 	filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
-	filerWebDavOptions.filerRootPath = cmdFiler.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
 
 	// start iam on filer
 	filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
@@ -185,11 +182,6 @@ func runFiler(cmd *Command, args []string) bool {
 
 	if *filerStartWebDav {
 		filerWebDavOptions.filer = &filerAddress
-
-		if *filerWebDavOptions.disk == "" {
-			filerWebDavOptions.disk = f.diskType
-		}
-
 		go func(delay time.Duration) {
 			time.Sleep(delay * time.Second)
 			filerWebDavOptions.startWebDav()
@@ -250,7 +242,6 @@ func (fo *FilerOptions) startFiler() {
 		ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
 		ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
 		DownloadMaxBytesPs:    int64(*fo.downloadMaxMBps) * 1024 * 1024,
-		DiskType:              *fo.diskType,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)

weed/command/filer_backup.go

@@ -138,19 +138,6 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 		}()
 	}
 
-	metadataFollowOption := &pb.MetadataFollowOption{
-		ClientName:             "backup_" + dataSink.GetName(),
-		ClientId:               clientId,
-		ClientEpoch:            clientEpoch,
-		SelfSignature:          0,
-		PathPrefix:             sourcePath,
-		AdditionalPathPrefixes: nil,
-		DirectoriesToWatch:     nil,
-		StartTsNs:              startFrom.UnixNano(),
-		StopTsNs:               0,
-		EventErrorType:         pb.TrivialOnError,
-	}
-
-	return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
+	return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), clientId, clientEpoch, sourcePath, nil, startFrom.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
 
 }

weed/command/filer_cat.go

@@ -96,7 +96,7 @@ func runFilerCat(cmd *Command, args []string) bool {
 		writer = f
 	}
 
-	pb.WithFilerClient(false, util.RandomInt32(), filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	pb.WithFilerClient(false, filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Name:      name,

weed/command/filer_copy.go

@@ -159,7 +159,6 @@ func runCopy(cmd *Command, args []string) bool {
 			worker := FileCopyWorker{
 				options:      &copy,
 				filerAddress: filerAddress,
-				signature:    util.RandomInt32(),
 			}
 			if err := worker.copyFiles(fileCopyTaskChan); err != nil {
 				fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
@@ -173,7 +172,7 @@ func runCopy(cmd *Command, args []string) bool {
 }
 
 func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress pb.ServerAddress) (masters []string, collection, replication string, dirBuckets string, maxMB uint32, cipher bool, err error) {
-	err = pb.WithGrpcFilerClient(false, 0, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithGrpcFilerClient(false, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 		if err != nil {
 			return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
@@ -226,7 +225,6 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
 type FileCopyWorker struct {
 	options      *CopyOptions
 	filerAddress pb.ServerAddress
-	signature    int32
 }
 
 func (worker *FileCopyWorker) copyFiles(fileCopyTaskChan chan FileCopyTask) error {
@@ -304,7 +302,7 @@ func (worker *FileCopyWorker) checkExistingFileFirst(task FileCopyTask, f *os.Fi
 		return
 	}
 
-	err = pb.WithGrpcFilerClient(false, worker.signature, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: task.destinationUrlPath,
@@ -367,10 +365,10 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 		if flushErr != nil {
 			return flushErr
 		}
-		chunks = append(chunks, uploadResult.ToPbFileChunk(finalFileId, 0, time.Now().UnixNano()))
+		chunks = append(chunks, uploadResult.ToPbFileChunk(finalFileId, 0))
 	}
 
-	if err := pb.WithGrpcFilerClient(false, worker.signature, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
@@ -452,7 +450,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 				uploadError = fmt.Errorf("upload %v result: %v\n", fileName, uploadResult.Error)
 				return
 			}
-			chunksChan <- uploadResult.ToPbFileChunk(fileId, i*chunkSize, time.Now().UnixNano())
+			chunksChan <- uploadResult.ToPbFileChunk(fileId, i*chunkSize)
 
 			fmt.Printf("uploaded %s-%d [%d,%d)\n", fileName, i+1, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
 		}(i)
@@ -481,7 +479,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 		return fmt.Errorf("create manifest: %v", manifestErr)
 	}
 
-	if err := pb.WithGrpcFilerClient(false, worker.signature, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
@@ -532,7 +530,7 @@ func detectMimeType(f *os.File) string {
 	return mimeType
 }
 
-func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64, tsNs int64) (chunk *filer_pb.FileChunk, err error) {
+func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, err error) {
 
 	finalFileId, uploadResult, flushErr, _ := operation.UploadWithRetry(
 		worker,
@@ -563,7 +561,7 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off
 	if uploadResult.Error != "" {
 		return nil, fmt.Errorf("upload result: %v", uploadResult.Error)
 	}
-	return uploadResult.ToPbFileChunk(finalFileId, offset, tsNs), nil
+	return uploadResult.ToPbFileChunk(finalFileId, offset), nil
 }
 
 var _ = filer_pb.FilerClient(&FileCopyWorker{})
@@ -571,7 +569,7 @@ var _ = filer_pb.FilerClient(&FileCopyWorker{})
 func (worker *FileCopyWorker) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) {
 
 	filerGrpcAddress := worker.filerAddress.ToGrpcAddress()
-	err = pb.WithGrpcClient(streamingMode, worker.signature, func(grpcConnection *grpc.ClientConn) error {
+	err = pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
 		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 		return fn(client)
 	}, filerGrpcAddress, false, worker.options.grpcDialOption)

weed/command/filer_meta_backup.go

@@ -196,21 +196,8 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
 	})
 
 	metaBackup.clientEpoch++
-
-	metadataFollowOption := &pb.MetadataFollowOption{
-		ClientName:             "meta_backup",
-		ClientId:               metaBackup.clientId,
-		ClientEpoch:            metaBackup.clientEpoch,
-		SelfSignature:          0,
-		PathPrefix:             *metaBackup.filerDirectory,
-		AdditionalPathPrefixes: nil,
-		DirectoriesToWatch:     nil,
-		StartTsNs:              startTime.UnixNano(),
-		StopTsNs:               0,
-		EventErrorType:         pb.TrivialOnError,
-	}
-
-	return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
+	return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, "meta_backup", metaBackup.clientId, metaBackup.clientEpoch,
+		*metaBackup.filerDirectory, nil, startTime.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
 
 }
 
@@ -238,7 +225,7 @@ var _ = filer_pb.FilerClient(&FilerMetaBackupOptions{})
 
 func (metaBackup *FilerMetaBackupOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	return pb.WithFilerClient(streamingMode, metaBackup.clientId, pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(streamingMode, pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		return fn(client)
 	})
 

weed/command/filer_meta_tail.go

@@ -107,28 +107,16 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
 		untilTsNs = time.Now().Add(-*tailStop).UnixNano()
 	}
 
-	metadataFollowOption := &pb.MetadataFollowOption{
-		ClientName:             "tail",
-		ClientId:               clientId,
-		ClientEpoch:            0,
-		SelfSignature:          0,
-		PathPrefix:             *tailTarget,
-		AdditionalPathPrefixes: nil,
-		DirectoriesToWatch:     nil,
-		StartTsNs:              time.Now().Add(-*tailStart).UnixNano(),
-		StopTsNs:               untilTsNs,
-		EventErrorType:         pb.TrivialOnError,
-	}
-
-	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, metadataFollowOption, func(resp *filer_pb.SubscribeMetadataResponse) error {
-		if !shouldPrint(resp) {
+	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, "tail", clientId, 0, *tailTarget, nil,
+		time.Now().Add(-*tailStart).UnixNano(), untilTsNs, 0, func(resp *filer_pb.SubscribeMetadataResponse) error {
+			if !shouldPrint(resp) {
+				return nil
+			}
+			if err := eachEntryFunc(resp); err != nil {
+				return err
+			}
 			return nil
-		}
-		if err := eachEntryFunc(resp); err != nil {
-			return err
-		}
-		return nil
-	})
+		}, pb.TrivialOnError)
 
 	if tailErr != nil {
 		fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)

weed/command/filer_remote_gateway.go

@@ -35,7 +35,7 @@ type RemoteGatewayOptions struct {
 var _ = filer_pb.FilerClient(&RemoteGatewayOptions{})
 
 func (option *RemoteGatewayOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
-	return pb.WithFilerClient(streamingMode, option.clientId, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(streamingMode, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		return fn(client)
 	})
 }

weed/command/filer_remote_gateway_buckets.go

@@ -39,21 +39,8 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo
 	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo)
 
 	option.clientEpoch++
-
-	metadataFollowOption := &pb.MetadataFollowOption{
-		ClientName:             "filer.remote.sync",
-		ClientId:               option.clientId,
-		ClientEpoch:            option.clientEpoch,
-		SelfSignature:          0,
-		PathPrefix:             option.bucketsDir,
-		AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
-		DirectoriesToWatch:     nil,
-		StartTsNs:              lastOffsetTs.UnixNano(),
-		StopTsNs:               0,
-		EventErrorType:         pb.TrivialOnError,
-	}
-
-	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
+	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId, option.clientEpoch,
+		option.bucketsDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
 }
 
 func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
@@ -385,9 +372,6 @@ func extractBucketPath(bucketsDir, dir string) (util.FullPath, bool) {
 func (option *RemoteGatewayOptions) collectRemoteStorageConf() (err error) {
 
 	if mappings, err := filer.ReadMountMappings(option.grpcDialOption, pb.ServerAddress(*option.filerAddress)); err != nil {
-		if err == filer_pb.ErrNotFound {
-			return fmt.Errorf("remote storage is not configured in filer server")
-		}
 		return err
 	} else {
 		option.mappings = mappings

weed/command/filer_remote_sync.go

@@ -26,7 +26,7 @@ type RemoteSyncOptions struct {
 var _ = filer_pb.FilerClient(&RemoteSyncOptions{})
 
 func (option *RemoteSyncOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
-	return pb.WithFilerClient(streamingMode, option.clientId, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(streamingMode, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		return fn(client)
 	})
 }

weed/command/filer_remote_sync_dir.go

@@ -63,21 +63,8 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
 	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)
 
 	option.clientEpoch++
-
-	metadataFollowOption := &pb.MetadataFollowOption{
-		ClientName:             "filer.remote.sync",
-		ClientId:               option.clientId,
-		ClientEpoch:            option.clientEpoch,
-		SelfSignature:          0,
-		PathPrefix:             mountedDir,
-		AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
-		DirectoriesToWatch:     nil,
-		StartTsNs:              lastOffsetTs.UnixNano(),
-		StopTsNs:               0,
-		EventErrorType:         pb.TrivialOnError,
-	}
-
-	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
+	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId, option.clientEpoch,
+		mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
 }
 
 func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {

weed/command/filer_sync.go

@@ -287,20 +287,8 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 		return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, processor.processedTsWatermark)
 	})
 
-	metadataFollowOption := &pb.MetadataFollowOption{
-		ClientName:             clientName,
-		ClientId:               clientId,
-		ClientEpoch:            clientEpoch,
-		SelfSignature:          targetFilerSignature,
-		PathPrefix:             sourcePath,
-		AdditionalPathPrefixes: nil,
-		DirectoriesToWatch:     nil,
-		StartTsNs:              sourceFilerOffsetTsNs,
-		StopTsNs:               0,
-		EventErrorType:         pb.RetryForeverOnError,
-	}
-
-	return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
+	return pb.FollowMetadata(sourceFiler, grpcDialOption, clientName, clientId, clientEpoch,
+		sourcePath, nil, sourceFilerOffsetTsNs, 0, targetFilerSignature, processEventFnWithOffset, pb.RetryForeverOnError)
 
 }
 
@@ -316,7 +304,7 @@ func getSignaturePrefixByPath(path string) string {
 
 func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) {
 
-	readErr = pb.WithFilerClient(false, signature, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	readErr = pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		syncKey := []byte(signaturePrefix + "____")
 		util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature))
 
@@ -342,7 +330,7 @@ func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signature
 }
 
 func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, offsetTsNs int64) error {
-	return pb.WithFilerClient(false, signature, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 		syncKey := []byte(signaturePrefix + "____")
 		util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature))
@@ -411,11 +399,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 				return nil
 			}
 			key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
-			if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
-				return fmt.Errorf("create entry1 : %v", err)
-			} else {
-				return nil
-			}
+			return dataSink.CreateEntry(key, message.NewEntry, message.Signatures)
 		}
 
 		// this is something special?
@@ -443,11 +427,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 				}
 				// create the new entry
 				newKey := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
-				if err := dataSink.CreateEntry(newKey, message.NewEntry, message.Signatures); err != nil {
-					return fmt.Errorf("create entry2 : %v", err)
-				} else {
-					return nil
-				}
+				return dataSink.CreateEntry(newKey, message.NewEntry, message.Signatures)
 
 			} else {
 				// new key is outside of the watched directory
@@ -461,11 +441,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			if strings.HasPrefix(string(sourceNewKey), sourcePath) {
 				// new key is in the watched directory
 				key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
-				if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
-					return fmt.Errorf("create entry3 : %v", err)
-				} else {
-					return nil
-				}
+				return dataSink.CreateEntry(key, message.NewEntry, message.Signatures)
 			} else {
 				// new key is also outside of the watched directory
 				// skip

weed/command/iam.go

@@ -50,7 +50,7 @@ func (iamopt *IamOptions) startIamServer() bool {
 	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 	for {
-		err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)

weed/command/master.go

@@ -2,14 +2,13 @@ package command
 
 import (
 	"fmt"
+	hashicorpRaft "github.com/hashicorp/raft"
 	"net/http"
 	"os"
 	"path"
 	"strings"
 	"time"
 
-	hashicorpRaft "github.com/hashicorp/raft"
-
 	"golang.org/x/exp/slices"
 
 	"github.com/gorilla/mux"
@@ -164,11 +163,11 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		RaftResumeState:   *masterOption.raftResumeState,
 		HeartbeatInterval: *masterOption.heartbeatInterval,
 		ElectionTimeout:   *masterOption.electionTimeout,
-		RaftBootstrap:     *masterOption.raftBootstrap,
+		RaftBootstrap:     *m.raftBootstrap,
 	}
 	var raftServer *weed_server.RaftServer
 	var err error
-	if *masterOption.raftHashicorp {
+	if *m.raftHashicorp {
 		if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
 			glog.Fatalf("NewHashicorpRaftServer: %s", err)
 		}
@@ -181,7 +180,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	ms.SetRaftServer(raftServer)
 	r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
 	r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD")
-	if *masterOption.raftHashicorp {
+	if *m.raftHashicorp {
 		r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
 	}
 	// starting grpc server
@@ -192,7 +191,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	}
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
 	master_pb.RegisterSeaweedServer(grpcS, ms)
-	if *masterOption.raftHashicorp {
+	if *m.raftHashicorp {
 		raftServer.TransportManager.Register(grpcS)
 	} else {
 		protobuf.RegisterRaftServer(grpcS, raftServer)
@@ -205,7 +204,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	go grpcS.Serve(grpcL)
 
 	timeSleep := 1500 * time.Millisecond
-	if !*masterOption.raftHashicorp {
+	if !*m.raftHashicorp {
 		go func() {
 			time.Sleep(timeSleep)
 

weed/command/mount.go

@@ -53,9 +53,9 @@ func init() {
 	mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
-	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers")
+	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0")
 	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
-	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 0, "file chunk read cache capacity in MB")
+	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 0, "local file chunk cache capacity in MB")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
 	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
 	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")

weed/command/mount_std.go

@@ -182,7 +182,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		DirectMount:              true,
 		DirectMountFlags:         0,
 		//SyncRead:                 false, // set to false to enable the FUSE_CAP_ASYNC_READ capability
-		EnableAcl: true,
+		//EnableAcl:                true,
 	}
 	if *option.nonempty {
 		fuseMountOptions.Options = append(fuseMountOptions.Options, "nonempty")
@@ -256,8 +256,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 
 	seaweedFileSystem.StartBackgroundTasks()
 
-	glog.V(0).Infof("mounted %s%s to %v", *option.filer, mountRoot, dir)
-	glog.V(0).Infof("This is SeaweedFS version %s %s %s", util.Version(), runtime.GOOS, runtime.GOARCH)
+	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)
 
 	server.Serve()
 

weed/command/s3.go

@@ -2,11 +2,8 @@ package command
 
 import (
 	"context"
-	"crypto/tls"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
-	"google.golang.org/grpc/credentials/tls/certprovider"
-	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
 	"google.golang.org/grpc/reflection"
 	"net/http"
 	"time"
@@ -43,7 +40,6 @@ type S3Options struct {
 	auditLogConfig            *string
 	localFilerSocket          *string
 	dataCenter                *string
-	certProvider              certprovider.Provider
 }
 
 func init() {
@@ -154,18 +150,11 @@ func runS3(cmd *Command, args []string) bool {
 
 }
 
-// GetCertificateWithUpdate Auto refreshing TSL certificate
-func (S3opt *S3Options) GetCertificateWithUpdate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
-	certs, err := S3opt.certProvider.KeyMaterial(context.Background())
-	return &certs.Certs[0], err
-}
-
 func (s3opt *S3Options) startS3Server() bool {
 
 	filerAddress := pb.ServerAddress(*s3opt.filer)
 
 	filerBucketsPath := "/buckets"
-	filerGroup := ""
 
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
@@ -174,13 +163,12 @@ func (s3opt *S3Options) startS3Server() bool {
 	var metricsIntervalSec int
 
 	for {
-		err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
 			}
 			filerBucketsPath = resp.DirBuckets
-			filerGroup = resp.FilerGroup
 			metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec)
 			glog.V(0).Infof("S3 read filer buckets dir: %s", filerBucketsPath)
 			return nil
@@ -212,7 +200,6 @@ func (s3opt *S3Options) startS3Server() bool {
 		AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty,
 		LocalFilerSocket:          localFilerSocket,
 		DataCenter:                *s3opt.dataCenter,
-		FilerGroup:                filerGroup,
 	})
 	if s3ApiServer_err != nil {
 		glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
@@ -255,24 +242,15 @@ func (s3opt *S3Options) startS3Server() bool {
 	go grpcS.Serve(grpcL)
 
 	if *s3opt.tlsPrivateKey != "" {
-		pemfileOptions := pemfile.Options{
-			CertFile:        *s3opt.tlsCertificate,
-			KeyFile:         *s3opt.tlsPrivateKey,
-			RefreshDuration: security.CredRefreshingInterval,
-		}
-		if s3opt.certProvider, err = pemfile.NewProvider(pemfileOptions); err != nil {
-			glog.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err)
-		}
-		httpS.TLSConfig = &tls.Config{GetCertificate: s3opt.GetCertificateWithUpdate}
 		glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
 		if s3ApiLocalListener != nil {
 			go func() {
-				if err = httpS.ServeTLS(s3ApiLocalListener, "", ""); err != nil {
+				if err = httpS.ServeTLS(s3ApiLocalListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
 					glog.Fatalf("S3 API Server Fail to serve: %v", err)
 				}
 			}()
 		}
-		if err = httpS.ServeTLS(s3ApiListener, "", ""); err != nil {
+		if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
 			glog.Fatalf("S3 API Server Fail to serve: %v", err)
 		}
 	} else {

weed/command/scaffold/filer.toml

@@ -41,13 +41,13 @@ enabled = false
 dbFile = "./filer.db"                # sqlite db file
 
 [mysql]  # or memsql, tidb
-# CREATE TABLE IF NOT EXISTS `filemeta` (
-#   `dirhash`   BIGINT NOT NULL       COMMENT 'first 64 bits of MD5 hash value of directory field',
-#   `name`      VARCHAR(766) NOT NULL COMMENT 'directory or file name',
-#   `directory` TEXT NOT NULL         COMMENT 'full path to parent directory',
-#   `meta`      LONGBLOB,
-#   PRIMARY KEY (`dirhash`, `name`)
-# ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+# CREATE TABLE IF NOT EXISTS filemeta (
+#   dirhash     BIGINT               COMMENT 'first 64 bits of MD5 hash value of directory field',
+#   name        VARCHAR(1000) BINARY COMMENT 'directory or file name',
+#   directory   TEXT BINARY          COMMENT 'full path to parent directory',
+#   meta        LONGBLOB,
+#   PRIMARY KEY (dirhash, name)
+# ) DEFAULT CHARSET=utf8;
 
 enabled = false
 hostname = "localhost"
@@ -61,18 +61,18 @@ connection_max_lifetime_seconds = 0
 interpolateParams = false
 # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
 enableUpsert = true
-upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`"""
+upsertQuery = """INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
 
 [mysql2]  # or memsql, tidb
 enabled = false
 createTable = """
   CREATE TABLE IF NOT EXISTS `%s` (
-    `dirhash`   BIGINT NOT NULL,
-    `name`      VARCHAR(766) NOT NULL,
-    `directory` TEXT NOT NULL,
-    `meta`      LONGBLOB,
-    PRIMARY KEY (`dirhash`, `name`)
-  ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+    dirhash BIGINT,
+    name VARCHAR(1000) BINARY,
+    directory TEXT BINARY,
+    meta LONGBLOB,
+    PRIMARY KEY (dirhash, name)
+  ) DEFAULT CHARSET=utf8;
 """
 hostname = "localhost"
 port = 3306
@@ -85,7 +85,7 @@ connection_max_lifetime_seconds = 0
 interpolateParams = false
 # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
 enableUpsert = true
-upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`"""
+upsertQuery = """INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
 
 [postgres] # or cockroachdb, YugabyteDB
 # CREATE TABLE IF NOT EXISTS filemeta (
@@ -108,7 +108,7 @@ connection_max_open = 100
 connection_max_lifetime_seconds = 0
 # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
 enableUpsert = true
-upsertQuery = """UPSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)"""
+upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
 
 [postgres2]
 enabled = false
@@ -133,7 +133,7 @@ connection_max_open = 100
 connection_max_lifetime_seconds = 0
 # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
 enableUpsert = true
-upsertQuery = """UPSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)"""
+upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
 
 [cassandra]
 # CREATE TABLE filemeta (
@@ -262,9 +262,6 @@ routeByLatency = false
 [etcd]
 enabled = false
 servers = "localhost:2379"
-username = ""
-password = ""
-key_prefix = "seaweedfs."
 timeout = "3s"
 
 [mongodb]
@@ -347,4 +344,4 @@ cert_path=""
 # Set the private key path
 key_path=""
 # The name list used to verify the cn name
-verify_cn=""
+verify_cn=""

weed/command/scaffold/replication.toml

@@ -68,7 +68,6 @@ is_incremental = false
 enabled = false
 b2_account_id = ""
 b2_master_application_key = ""
-b2_region = ""
 bucket = "mybucket"            # an existing bucket
 directory = "/"                # destination directory
 is_incremental = false

weed/command/server.go

@@ -97,7 +97,6 @@ func init() {
 	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
 	masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
 	masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
-	masterOptions.raftBootstrap = cmdMaster.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster")
 	masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
 	masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
 
@@ -116,7 +115,6 @@ func init() {
 	filerOptions.localSocket = cmdServer.Flag.String("filer.localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
 	filerOptions.showUIDirectoryDelete = cmdServer.Flag.Bool("filer.ui.deleteDir", true, "enable filer UI show delete directory button")
 	filerOptions.downloadMaxMBps = cmdServer.Flag.Int("filer.downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
-	filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
@@ -158,7 +156,6 @@ func init() {
 	webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
 	webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
 	webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
-	webdavOptions.filerRootPath = cmdServer.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
 
 	mqBrokerOptions.port = cmdServer.Flag.Int("mq.broker.port", 17777, "message queue broker gRPC listen port")
 
@@ -255,9 +252,6 @@ func runServer(cmd *Command, args []string) bool {
 	serverWhiteList := util.StringSplit(*serverWhiteListOption, ",")
 
 	if *isStartingFiler {
-		if *filerOptions.diskType == "" && *serverOptions.v.diskType != "" {
-			filerOptions.diskType = serverOptions.v.diskType
-		}
 		go func() {
 			time.Sleep(1 * time.Second)
 			filerOptions.startFiler()

weed/command/update.go

@@ -11,6 +11,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -214,7 +215,7 @@ func GitHubLatestRelease(ctx context.Context, ver string, owner, repo string) (R
 		return Release{}, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
 	}
 
-	buf, err := io.ReadAll(res.Body)
+	buf, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		return Release{}, err
 	}
@@ -264,7 +265,7 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) {
 		return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
 	}
 
-	buf, err := io.ReadAll(res.Body)
+	buf, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		return nil, err
 	}
@@ -336,7 +337,7 @@ func extractToFile(buf []byte, filename, target string) error {
 
 	// Write everything to a temp file
 	dir := filepath.Dir(target)
-	new, err := os.CreateTemp(dir, "weed")
+	new, err := ioutil.TempFile(dir, "weed")
 	if err != nil {
 		return err
 	}

weed/command/webdav.go

@@ -13,7 +13,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
-	weed_server "github.com/seaweedfs/seaweedfs/weed/server"
+	"github.com/seaweedfs/seaweedfs/weed/server"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
@@ -23,7 +23,6 @@ var (
 
 type WebDavOption struct {
 	filer          *string
-	filerRootPath  *string
 	port           *int
 	collection     *string
 	replication    *string
@@ -45,7 +44,6 @@ func init() {
 	webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file")
 	webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
 	webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 0, "local cache capacity in MB")
-	webDavStandaloneOptions.filerRootPath = cmdWebDav.Flag.String("filer.path", "/", "use this remote path from filer server")
 }
 
 var cmdWebDav = &Command{
@@ -87,7 +85,7 @@ func (wo *WebDavOption) startWebDav() bool {
 	var cipher bool
 	// connect to filer
 	for {
-		err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
@@ -106,7 +104,6 @@ func (wo *WebDavOption) startWebDav() bool {
 
 	ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{
 		Filer:          filerAddress,
-		FilerRootPath:  *wo.filerRootPath,
 		GrpcDialOption: grpcDialOption,
 		Collection:     *wo.collection,
 		Replication:    *wo.replication,

weed/filer/configuration.go

@@ -33,7 +33,7 @@ func (f *Filer) LoadConfiguration(config *util.ViperProxy) (isFresh bool) {
 
 	if !hasDefaultStoreConfigured {
 		println()
-		println("Supported filer stores are the following. If not found, check the full version.")
+		println("Supported filer stores are:")
 		for _, store := range Stores {
 			println("    " + store.GetName())
 		}

weed/filer/etcd/etcd_store.go

@@ -25,7 +25,6 @@ func init() {
 
 type EtcdStore struct {
 	client *clientv3.Client
-	etcdKeyPrefix string
 }
 
 func (store *EtcdStore) GetName() string {
@@ -38,19 +37,15 @@ func (store *EtcdStore) Initialize(configuration weed_util.Configuration, prefix
 		servers = "localhost:2379"
 	}
 
-        username := configuration.GetString(prefix + "username")
-        password := configuration.GetString(prefix + "password")
-        store.etcdKeyPrefix = configuration.GetString(prefix + "key_prefix")
-
 	timeout := configuration.GetString(prefix + "timeout")
 	if timeout == "" {
 		timeout = "3s"
 	}
 
-	return store.initialize(servers, username, password, timeout)
+	return store.initialize(servers, timeout)
 }
 
-func (store *EtcdStore) initialize(servers string, username string, password string, timeout string) (err error) {
+func (store *EtcdStore) initialize(servers string, timeout string) (err error) {
 	glog.Infof("filer store etcd: %s", servers)
 
 	to, err := time.ParseDuration(timeout)
@@ -60,8 +55,6 @@ func (store *EtcdStore) initialize(servers string, username string, password str
 
 	store.client, err = clientv3.New(clientv3.Config{
 		Endpoints:   strings.Split(servers, ","),
-		Username:    username,
-		Password:    password,
 		DialTimeout: to,
 	})
 	if err != nil {
@@ -93,7 +86,7 @@ func (store *EtcdStore) InsertEntry(ctx context.Context, entry *filer.Entry) (er
 		meta = weed_util.MaybeGzipData(meta)
 	}
 
-	if _, err := store.client.Put(ctx, store.etcdKeyPrefix + string(key), string(meta)); err != nil {
+	if _, err := store.client.Put(ctx, string(key), string(meta)); err != nil {
 		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
 	}
 
@@ -107,7 +100,7 @@ func (store *EtcdStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (er
 func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {
 	key := genKey(fullpath.DirAndName())
 
-	resp, err := store.client.Get(ctx, store.etcdKeyPrefix + string(key))
+	resp, err := store.client.Get(ctx, string(key))
 	if err != nil {
 		return nil, fmt.Errorf("get %s : %v", fullpath, err)
 	}
@@ -130,7 +123,7 @@ func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPa
 func (store *EtcdStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
 	key := genKey(fullpath.DirAndName())
 
-	if _, err := store.client.Delete(ctx, store.etcdKeyPrefix + string(key)); err != nil {
+	if _, err := store.client.Delete(ctx, string(key)); err != nil {
 		return fmt.Errorf("delete %s : %v", fullpath, err)
 	}
 
@@ -140,7 +133,7 @@ func (store *EtcdStore) DeleteEntry(ctx context.Context, fullpath weed_util.Full
 func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
 	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
 
-	if _, err := store.client.Delete(ctx, store.etcdKeyPrefix + string(directoryPrefix), clientv3.WithPrefix()); err != nil {
+	if _, err := store.client.Delete(ctx, string(directoryPrefix), clientv3.WithPrefix()); err != nil {
 		return fmt.Errorf("deleteFolderChildren %s : %v", fullpath, err)
 	}
 
@@ -158,7 +151,7 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u
 		lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
 	}
 
-	resp, err := store.client.Get(ctx, store.etcdKeyPrefix + string(lastFileStart),
+	resp, err := store.client.Get(ctx, string(lastFileStart),
 		clientv3.WithFromKey(), clientv3.WithLimit(limit+1))
 	if err != nil {
 		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)

weed/filer/etcd/etcd_store_kv.go

@@ -8,7 +8,7 @@ import (
 
 func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
 
-	_, err = store.client.Put(ctx, store.etcdKeyPrefix + string(key), string(value))
+	_, err = store.client.Put(ctx, string(key), string(value))
 
 	if err != nil {
 		return fmt.Errorf("kv put: %v", err)
@@ -19,7 +19,7 @@ func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (er
 
 func (store *EtcdStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
 
-	resp, err := store.client.Get(ctx, store.etcdKeyPrefix + string(key))
+	resp, err := store.client.Get(ctx, string(key))
 
 	if err != nil {
 		return nil, fmt.Errorf("kv get: %v", err)
@@ -34,7 +34,7 @@ func (store *EtcdStore) KvGet(ctx context.Context, key []byte) (value []byte, er
 
 func (store *EtcdStore) KvDelete(ctx context.Context, key []byte) (err error) {
 
-	_, err = store.client.Delete(ctx, store.etcdKeyPrefix + string(key))
+	_, err = store.client.Delete(ctx, string(key))
 
 	if err != nil {
 		return fmt.Errorf("kv delete: %v", err)

weed/filer/etcd/etcd_store_test.go

@@ -10,7 +10,7 @@ func TestStore(t *testing.T) {
 	// to set up local env
 	if false {
 		store := &EtcdStore{}
-		store.initialize("localhost:2379", "", "", "3s")
+		store.initialize("localhost:2379", "3s")
 		store_test.TestFilerStore(t, store)
 	}
 }

weed/filer/filechunk_group.go

@@ -1,155 +0,0 @@
-package filer
-
-import (
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
-	"github.com/seaweedfs/seaweedfs/weed/wdclient"
-	"sync"
-)
-
-type ChunkGroup struct {
-	lookupFn     wdclient.LookupFileIdFunctionType
-	chunkCache   chunk_cache.ChunkCache
-	sections     map[SectionIndex]*FileChunkSection
-	sectionsLock sync.RWMutex
-	readerCache  *ReaderCache
-}
-
-func NewChunkGroup(lookupFn wdclient.LookupFileIdFunctionType, chunkCache chunk_cache.ChunkCache, chunks []*filer_pb.FileChunk) (*ChunkGroup, error) {
-	group := &ChunkGroup{
-		lookupFn:    lookupFn,
-		chunkCache:  chunkCache,
-		sections:    make(map[SectionIndex]*FileChunkSection),
-		readerCache: NewReaderCache(32, chunkCache, lookupFn),
-	}
-
-	err := group.SetChunks(chunks)
-	return group, err
-}
-
-func (group *ChunkGroup) AddChunk(chunk *filer_pb.FileChunk) error {
-
-	group.sectionsLock.Lock()
-	defer group.sectionsLock.Unlock()
-
-	sectionIndexStart, sectionIndexStop := SectionIndex(chunk.Offset/SectionSize), SectionIndex((chunk.Offset+int64(chunk.Size))/SectionSize)
-	for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
-		section, found := group.sections[si]
-		if !found {
-			section = NewFileChunkSection(si)
-			group.sections[si] = section
-		}
-		section.addChunk(chunk)
-	}
-	return nil
-}
-
-func (group *ChunkGroup) ReadDataAt(fileSize int64, buff []byte, offset int64) (n int, tsNs int64, err error) {
-
-	group.sectionsLock.RLock()
-	defer group.sectionsLock.RUnlock()
-
-	sectionIndexStart, sectionIndexStop := SectionIndex(offset/SectionSize), SectionIndex((offset+int64(len(buff)))/SectionSize)
-	for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
-		section, found := group.sections[si]
-		rangeStart, rangeStop := max(offset, int64(si*SectionSize)), min(offset+int64(len(buff)), int64((si+1)*SectionSize))
-		if !found {
-			for i := rangeStart; i < rangeStop; i++ {
-				buff[i-offset] = 0
-			}
-			continue
-		}
-		xn, xTsNs, xErr := section.readDataAt(group, fileSize, buff[rangeStart-offset:rangeStop-offset], rangeStart)
-		if xErr != nil {
-			err = xErr
-		}
-		n += xn
-		tsNs = max(tsNs, xTsNs)
-	}
-	return
-}
-
-func (group *ChunkGroup) SetChunks(chunks []*filer_pb.FileChunk) error {
-	group.sectionsLock.RLock()
-	defer group.sectionsLock.RUnlock()
-
-	var dataChunks []*filer_pb.FileChunk
-	for _, chunk := range chunks {
-
-		if !chunk.IsChunkManifest {
-			dataChunks = append(dataChunks, chunk)
-			continue
-		}
-
-		resolvedChunks, err := ResolveOneChunkManifest(group.lookupFn, chunk)
-		if err != nil {
-			return err
-		}
-
-		dataChunks = append(dataChunks, resolvedChunks...)
-	}
-
-	sections := make(map[SectionIndex]*FileChunkSection)
-
-	for _, chunk := range dataChunks {
-		sectionIndexStart, sectionIndexStop := SectionIndex(chunk.Offset/SectionSize), SectionIndex((chunk.Offset+int64(chunk.Size))/SectionSize)
-		for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
-			section, found := sections[si]
-			if !found {
-				section = NewFileChunkSection(si)
-				sections[si] = section
-			}
-			section.chunks = append(section.chunks, chunk)
-		}
-	}
-
-	group.sections = sections
-	return nil
-}
-
-const (
-	// see weedfs_file_lseek.go
-	SEEK_DATA uint32 = 3 // seek to next data after the offset
-	// SEEK_HOLE uint32 = 4 // seek to next hole after the offset
-)
-
-// FIXME: needa tests
-func (group *ChunkGroup) SearchChunks(offset, fileSize int64, whence uint32) (found bool, out int64) {
-	group.sectionsLock.RLock()
-	defer group.sectionsLock.RUnlock()
-
-	return group.doSearchChunks(offset, fileSize, whence)
-}
-
-func (group *ChunkGroup) doSearchChunks(offset, fileSize int64, whence uint32) (found bool, out int64) {
-
-	sectionIndex, maxSectionIndex := SectionIndex(offset/SectionSize), SectionIndex(fileSize/SectionSize)
-	if whence == SEEK_DATA {
-		for si := sectionIndex; si < maxSectionIndex+1; si++ {
-			section, foundSection := group.sections[si]
-			if !foundSection {
-				continue
-			}
-			sectionStart := section.DataStartOffset(group, offset, fileSize)
-			if sectionStart == -1 {
-				continue
-			}
-			return true, sectionStart
-		}
-		return false, 0
-	} else {
-		// whence == SEEK_HOLE
-		for si := sectionIndex; si < maxSectionIndex; si++ {
-			section, foundSection := group.sections[si]
-			if !foundSection {
-				return true, offset
-			}
-			holeStart := section.NextStopOffset(group, offset, fileSize)
-			if holeStart%SectionSize == 0 {
-				continue
-			}
-			return true, holeStart
-		}
-		return true, fileSize
-	}
-}

weed/filer/filechunk_group_test.go

@@ -1,36 +0,0 @@
-package filer
-
-import (
-	"github.com/stretchr/testify/assert"
-	"testing"
-)
-
-func TestChunkGroup_doSearchChunks(t *testing.T) {
-	type fields struct {
-		sections map[SectionIndex]*FileChunkSection
-	}
-	type args struct {
-		offset   int64
-		fileSize int64
-		whence   uint32
-	}
-	tests := []struct {
-		name      string
-		fields    fields
-		args      args
-		wantFound bool
-		wantOut   int64
-	}{
-		// TODO: Add test cases.
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			group := &ChunkGroup{
-				sections: tt.fields.sections,
-			}
-			gotFound, gotOut := group.doSearchChunks(tt.args.offset, tt.args.fileSize, tt.args.whence)
-			assert.Equalf(t, tt.wantFound, gotFound, "doSearchChunks(%v, %v, %v)", tt.args.offset, tt.args.fileSize, tt.args.whence)
-			assert.Equalf(t, tt.wantOut, gotOut, "doSearchChunks(%v, %v, %v)", tt.args.offset, tt.args.fileSize, tt.args.whence)
-		})
-	}
-}

weed/filer/filechunk_manifest.go

@@ -264,7 +264,7 @@ func mergeIntoManifest(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer
 		}
 	}
 
-	manifestChunk, err = saveFunc(bytes.NewReader(data), "", 0, 0)
+	manifestChunk, err = saveFunc(bytes.NewReader(data), "", 0)
 	if err != nil {
 		return nil, err
 	}
@@ -275,4 +275,4 @@ func mergeIntoManifest(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer
 	return
 }
 
-type SaveDataAsChunkFunctionType func(reader io.Reader, name string, offset int64, tsNs int64) (chunk *filer_pb.FileChunk, err error)
+type SaveDataAsChunkFunctionType func(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, err error)

weed/filer/filechunk_section.go

@@ -1,150 +0,0 @@
-package filer
-
-import (
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"sync"
-)
-
-const SectionSize = 2 * 1024 * 1024 * 32 // 64MiB
-type SectionIndex int64
-type FileChunkSection struct {
-	sectionIndex     SectionIndex
-	chunks           []*filer_pb.FileChunk
-	visibleIntervals *IntervalList[*VisibleInterval]
-	chunkViews       *IntervalList[*ChunkView]
-	reader           *ChunkReadAt
-	lock             sync.RWMutex
-	isPrepared       bool
-}
-
-func NewFileChunkSection(si SectionIndex) *FileChunkSection {
-	return &FileChunkSection{
-		sectionIndex: si,
-	}
-}
-
-func (section *FileChunkSection) addChunk(chunk *filer_pb.FileChunk) error {
-	section.lock.Lock()
-	defer section.lock.Unlock()
-
-	start, stop := max(int64(section.sectionIndex)*SectionSize, chunk.Offset), min(((int64(section.sectionIndex)+1)*SectionSize), chunk.Offset+int64(chunk.Size))
-
-	section.chunks = append(section.chunks, chunk)
-
-	if section.visibleIntervals == nil {
-		section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
-	} else {
-		MergeIntoVisibles(section.visibleIntervals, start, stop, chunk)
-		garbageFileIds := FindGarbageChunks(section.visibleIntervals, start, stop)
-		removeGarbageChunks(section, garbageFileIds)
-	}
-
-	if section.chunkViews != nil {
-		MergeIntoChunkViews(section.chunkViews, start, stop, chunk)
-	}
-
-	return nil
-}
-
-func removeGarbageChunks(section *FileChunkSection, garbageFileIds map[string]struct{}) {
-	for i := 0; i < len(section.chunks); {
-		t := section.chunks[i]
-		length := len(section.chunks)
-		if _, found := garbageFileIds[t.FileId]; found {
-			if i < length-1 {
-				section.chunks[i] = section.chunks[length-1]
-			}
-			section.chunks = section.chunks[:length-1]
-		} else {
-			i++
-		}
-	}
-}
-
-func (section *FileChunkSection) setupForRead(group *ChunkGroup, fileSize int64) {
-	if section.isPrepared {
-		section.reader.fileSize = fileSize
-		return
-	}
-
-	section.lock.Lock()
-	defer section.lock.Unlock()
-
-	if section.isPrepared {
-		section.reader.fileSize = fileSize
-		return
-	}
-
-	if section.visibleIntervals == nil {
-		section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
-		section.chunks, _ = SeparateGarbageChunks(section.visibleIntervals, section.chunks)
-		if section.reader != nil {
-			_ = section.reader.Close()
-			section.reader = nil
-		}
-	}
-	if section.chunkViews == nil {
-		section.chunkViews = ViewFromVisibleIntervals(section.visibleIntervals, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
-	}
-
-	if section.reader == nil {
-		section.reader = NewChunkReaderAtFromClient(group.readerCache, section.chunkViews, min(int64(section.sectionIndex+1)*SectionSize, fileSize))
-	}
-
-	section.isPrepared = true
-	section.reader.fileSize = fileSize
-}
-
-func (section *FileChunkSection) readDataAt(group *ChunkGroup, fileSize int64, buff []byte, offset int64) (n int, tsNs int64, err error) {
-
-	section.setupForRead(group, fileSize)
-	section.lock.RLock()
-	defer section.lock.RUnlock()
-
-	return section.reader.ReadAtWithTime(buff, offset)
-}
-
-func (section *FileChunkSection) DataStartOffset(group *ChunkGroup, offset int64, fileSize int64) int64 {
-
-	section.setupForRead(group, fileSize)
-	section.lock.RLock()
-	defer section.lock.RUnlock()
-
-	for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
-		visible := x.Value
-		if visible.stop <= offset {
-			continue
-		}
-		if offset < visible.start {
-			return offset
-		}
-		return offset
-	}
-	return -1
-}
-
-func (section *FileChunkSection) NextStopOffset(group *ChunkGroup, offset int64, fileSize int64) int64 {
-
-	section.setupForRead(group, fileSize)
-	section.lock.RLock()
-	defer section.lock.RUnlock()
-
-	isAfterOffset := false
-	for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
-		visible := x.Value
-		if !isAfterOffset {
-			if visible.stop <= offset {
-				continue
-			}
-			isAfterOffset = true
-		}
-		if offset < visible.start {
-			return offset
-		}
-		// now visible.start <= offset
-		if offset < visible.stop {
-			offset = visible.stop
-		}
-	}
-	return offset
-}

weed/filer/filechunk_section_test.go

@@ -1,48 +0,0 @@
-package filer
-
-import (
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"testing"
-)
-
-func Test_removeGarbageChunks(t *testing.T) {
-	section := NewFileChunkSection(0)
-	section.chunks = append(section.chunks, &filer_pb.FileChunk{
-		FileId:       "0",
-		Offset:       0,
-		Size:         1,
-		ModifiedTsNs: 0,
-	})
-	section.chunks = append(section.chunks, &filer_pb.FileChunk{
-		FileId:       "1",
-		Offset:       1,
-		Size:         1,
-		ModifiedTsNs: 1,
-	})
-	section.chunks = append(section.chunks, &filer_pb.FileChunk{
-		FileId:       "2",
-		Offset:       2,
-		Size:         1,
-		ModifiedTsNs: 2,
-	})
-	section.chunks = append(section.chunks, &filer_pb.FileChunk{
-		FileId:       "3",
-		Offset:       3,
-		Size:         1,
-		ModifiedTsNs: 3,
-	})
-	section.chunks = append(section.chunks, &filer_pb.FileChunk{
-		FileId:       "4",
-		Offset:       4,
-		Size:         1,
-		ModifiedTsNs: 4,
-	})
-	garbageFileIds := make(map[string]struct{})
-	garbageFileIds["0"] = struct{}{}
-	garbageFileIds["2"] = struct{}{}
-	garbageFileIds["4"] = struct{}{}
-	removeGarbageChunks(section, garbageFileIds)
-	if len(section.chunks) != 2 {
-		t.Errorf("remove chunk 2 failed")
-	}
-}

weed/filer/filechunks.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"golang.org/x/exp/slices"
 	"math"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -41,7 +42,7 @@ func ETag(entry *filer_pb.Entry) (etag string) {
 }
 
 func ETagEntry(entry *Entry) (etag string) {
-	if entry.IsInRemoteOnly() {
+        if entry.IsInRemoteOnly() {
 		return entry.Remote.RemoteETag
 	}
 	if entry.Attr.Md5 == nil {
@@ -65,15 +66,8 @@ func CompactFileChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks
 
 	visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks, 0, math.MaxInt64)
 
-	compacted, garbage = SeparateGarbageChunks(visibles, chunks)
-
-	return
-}
-
-func SeparateGarbageChunks(visibles *IntervalList[*VisibleInterval], chunks []*filer_pb.FileChunk) (compacted []*filer_pb.FileChunk, garbage []*filer_pb.FileChunk) {
 	fileIds := make(map[string]bool)
-	for x := visibles.Front(); x != nil; x = x.Next {
-		interval := x.Value
+	for _, interval := range visibles {
 		fileIds[interval.fileId] = true
 	}
 	for _, chunk := range chunks {
@@ -83,18 +77,7 @@ func SeparateGarbageChunks(visibles *IntervalList[*VisibleInterval], chunks []*f
 			garbage = append(garbage, chunk)
 		}
 	}
-	return compacted, garbage
-}
 
-func FindGarbageChunks(visibles *IntervalList[*VisibleInterval], start int64, stop int64) (garbageFileIds map[string]struct{}) {
-	garbageFileIds = make(map[string]struct{})
-	for x := visibles.Front(); x != nil; x = x.Next {
-		interval := x.Value
-		offset := interval.start - interval.offsetInChunk
-		if start <= offset && offset+int64(interval.chunkSize) <= stop {
-			garbageFileIds[interval.fileId] = struct{}{}
-		}
-	}
 	return
 }
 
@@ -148,39 +131,20 @@ func DoMinusChunksBySourceFileId(as, bs []*filer_pb.FileChunk) (delta []*filer_p
 }
 
 type ChunkView struct {
-	FileId        string
-	OffsetInChunk int64 // offset within the chunk
-	ViewSize      uint64
-	ViewOffset    int64 // actual offset in the file, for the data specified via [offset, offset+size) in current chunk
-	ChunkSize     uint64
-	CipherKey     []byte
-	IsGzipped     bool
-	ModifiedTsNs  int64
-}
-
-func (cv *ChunkView) SetStartStop(start, stop int64) {
-	cv.OffsetInChunk += start - cv.ViewOffset
-	cv.ViewOffset = start
-	cv.ViewSize = uint64(stop - start)
-}
-func (cv *ChunkView) Clone() IntervalValue {
-	return &ChunkView{
-		FileId:        cv.FileId,
-		OffsetInChunk: cv.OffsetInChunk,
-		ViewSize:      cv.ViewSize,
-		ViewOffset:    cv.ViewOffset,
-		ChunkSize:     cv.ChunkSize,
-		CipherKey:     cv.CipherKey,
-		IsGzipped:     cv.IsGzipped,
-		ModifiedTsNs:  cv.ModifiedTsNs,
-	}
+	FileId      string
+	Offset      int64
+	Size        uint64
+	LogicOffset int64 // actual offset in the file, for the data specified via [offset, offset+size) in current chunk
+	ChunkSize   uint64
+	CipherKey   []byte
+	IsGzipped   bool
 }
 
 func (cv *ChunkView) IsFullChunk() bool {
-	return cv.ViewSize == cv.ChunkSize
+	return cv.Size == cv.ChunkSize
 }
 
-func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (chunkViews *IntervalList[*ChunkView]) {
+func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (views []*ChunkView) {
 
 	visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks, offset, offset+size)
 
@@ -188,7 +152,7 @@ func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*
 
 }
 
-func ViewFromVisibleIntervals(visibles *IntervalList[*VisibleInterval], offset int64, size int64) (chunkViews *IntervalList[*ChunkView]) {
+func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int64) (views []*ChunkView) {
 
 	stop := offset + size
 	if size == math.MaxInt64 {
@@ -198,112 +162,164 @@ func ViewFromVisibleIntervals(visibles *IntervalList[*VisibleInterval], offset i
 		stop = math.MaxInt64
 	}
 
-	chunkViews = NewIntervalList[*ChunkView]()
-	for x := visibles.Front(); x != nil; x = x.Next {
-		chunk := x.Value
+	for _, chunk := range visibles {
 
 		chunkStart, chunkStop := max(offset, chunk.start), min(stop, chunk.stop)
 
 		if chunkStart < chunkStop {
-			chunkView := &ChunkView{
-				FileId:        chunk.fileId,
-				OffsetInChunk: chunkStart - chunk.start + chunk.offsetInChunk,
-				ViewSize:      uint64(chunkStop - chunkStart),
-				ViewOffset:    chunkStart,
-				ChunkSize:     chunk.chunkSize,
-				CipherKey:     chunk.cipherKey,
-				IsGzipped:     chunk.isGzipped,
-				ModifiedTsNs:  chunk.modifiedTsNs,
-			}
-			chunkViews.AppendInterval(&Interval[*ChunkView]{
-				StartOffset: chunkStart,
-				StopOffset:  chunkStop,
-				TsNs:        chunk.modifiedTsNs,
-				Value:       chunkView,
-				Prev:        nil,
-				Next:        nil,
+			views = append(views, &ChunkView{
+				FileId:      chunk.fileId,
+				Offset:      chunkStart - chunk.start + chunk.chunkOffset,
+				Size:        uint64(chunkStop - chunkStart),
+				LogicOffset: chunkStart,
+				ChunkSize:   chunk.chunkSize,
+				CipherKey:   chunk.cipherKey,
+				IsGzipped:   chunk.isGzipped,
 			})
 		}
 	}
 
-	return chunkViews
+	return views
 
 }
 
-func MergeIntoVisibles(visibles *IntervalList[*VisibleInterval], start int64, stop int64, chunk *filer_pb.FileChunk) {
+func logPrintf(name string, visibles []VisibleInterval) {
 
-	newV := &VisibleInterval{
-		start:         start,
-		stop:          stop,
-		fileId:        chunk.GetFileIdString(),
-		modifiedTsNs:  chunk.ModifiedTsNs,
-		offsetInChunk: start - chunk.Offset, // the starting position in the chunk
-		chunkSize:     chunk.Size,           // size of the chunk
-		cipherKey:     chunk.CipherKey,
-		isGzipped:     chunk.IsCompressed,
-	}
-
-	visibles.InsertInterval(start, stop, chunk.ModifiedTsNs, newV)
+	/*
+		glog.V(0).Infof("%s len %d", name, len(visibles))
+		for _, v := range visibles {
+			glog.V(0).Infof("%s:  [%d,%d) %s %d", name, v.start, v.stop, v.fileId, v.chunkOffset)
+		}
+	*/
 }
 
-func MergeIntoChunkViews(chunkViews *IntervalList[*ChunkView], start int64, stop int64, chunk *filer_pb.FileChunk) {
+func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (newVisibles []VisibleInterval) {
 
-	chunkView := &ChunkView{
-		FileId:        chunk.GetFileIdString(),
-		OffsetInChunk: start - chunk.Offset,
-		ViewSize:      uint64(stop - start),
-		ViewOffset:    start,
-		ChunkSize:     chunk.Size,
-		CipherKey:     chunk.CipherKey,
-		IsGzipped:     chunk.IsCompressed,
-		ModifiedTsNs:  chunk.ModifiedTsNs,
+	newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.ModifiedTsNs, 0, chunk.Size, chunk.CipherKey, chunk.IsCompressed)
+
+	length := len(visibles)
+	if length == 0 {
+		return append(visibles, newV)
+	}
+	last := visibles[length-1]
+	if last.stop <= chunk.Offset {
+		return append(visibles, newV)
 	}
 
-	chunkViews.InsertInterval(start, stop, chunk.ModifiedTsNs, chunkView)
+	logPrintf("  before", visibles)
+	// glog.V(0).Infof("newVisibles %d adding chunk [%d,%d) %s size:%d", len(newVisibles), chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Size)
+	chunkStop := chunk.Offset + int64(chunk.Size)
+	for _, v := range visibles {
+		if v.start < chunk.Offset && chunk.Offset < v.stop {
+			t := newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTsNs, v.chunkOffset, v.chunkSize, v.cipherKey, v.isGzipped)
+			newVisibles = append(newVisibles, t)
+			// glog.V(0).Infof("visible %d [%d,%d) =1> [%d,%d)", i, v.start, v.stop, t.start, t.stop)
+		}
+		if v.start < chunkStop && chunkStop < v.stop {
+			t := newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTsNs, v.chunkOffset+(chunkStop-v.start), v.chunkSize, v.cipherKey, v.isGzipped)
+			newVisibles = append(newVisibles, t)
+			// glog.V(0).Infof("visible %d [%d,%d) =2> [%d,%d)", i, v.start, v.stop, t.start, t.stop)
+		}
+		if chunkStop <= v.start || v.stop <= chunk.Offset {
+			newVisibles = append(newVisibles, v)
+			// glog.V(0).Infof("visible %d [%d,%d) =3> [%d,%d)", i, v.start, v.stop, v.start, v.stop)
+		}
+	}
+	newVisibles = append(newVisibles, newV)
+
+	logPrintf("  append", newVisibles)
+
+	for i := len(newVisibles) - 1; i >= 0; i-- {
+		if i > 0 && newV.start < newVisibles[i-1].start {
+			newVisibles[i] = newVisibles[i-1]
+		} else {
+			newVisibles[i] = newV
+			break
+		}
+	}
+	logPrintf("  sorted", newVisibles)
+
+	return newVisibles
 }
 
 // NonOverlappingVisibleIntervals translates the file chunk into VisibleInterval in memory
 // If the file chunk content is a chunk manifest
-func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles *IntervalList[*VisibleInterval], err error) {
+func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles []VisibleInterval, err error) {
 
 	chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks, startOffset, stopOffset)
 	if err != nil {
 		return
 	}
 
-	visibles2 := readResolvedChunks(chunks, 0, math.MaxInt64)
+	visibles2 := readResolvedChunks(chunks)
 
-	return visibles2, err
+	if true {
+		return visibles2, err
+	}
+	slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
+		if a.ModifiedTsNs == b.ModifiedTsNs {
+			filer_pb.EnsureFid(a)
+			filer_pb.EnsureFid(b)
+			if a.Fid == nil || b.Fid == nil {
+				return true
+			}
+			return a.Fid.FileKey < b.Fid.FileKey
+		}
+		return a.ModifiedTsNs < b.ModifiedTsNs
+	})
+	for _, chunk := range chunks {
+
+		// glog.V(0).Infof("merge [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))
+		visibles = MergeIntoVisibles(visibles, chunk)
+
+		logPrintf("add", visibles)
+
+	}
+
+	if len(visibles) != len(visibles2) {
+		fmt.Printf("different visibles size %d : %d\n", len(visibles), len(visibles2))
+	} else {
+		for i := 0; i < len(visibles); i++ {
+			checkDifference(visibles[i], visibles2[i])
+		}
+	}
+
+	return
+}
+
+func checkDifference(x, y VisibleInterval) {
+	if x.start != y.start ||
+		x.stop != y.stop ||
+		x.fileId != y.fileId ||
+		x.modifiedTsNs != y.modifiedTsNs {
+		fmt.Printf("different visible %+v : %+v\n", x, y)
+	}
 }
 
 // find non-overlapping visible intervals
 // visible interval map to one file chunk
 
 type VisibleInterval struct {
-	start         int64
-	stop          int64
-	modifiedTsNs  int64
-	fileId        string
-	offsetInChunk int64
-	chunkSize     uint64
-	cipherKey     []byte
-	isGzipped     bool
+	start        int64
+	stop         int64
+	modifiedTsNs int64
+	fileId       string
+	chunkOffset  int64
+	chunkSize    uint64
+	cipherKey    []byte
+	isGzipped    bool
 }
 
-func (v *VisibleInterval) SetStartStop(start, stop int64) {
-	v.offsetInChunk += start - v.start
-	v.start, v.stop = start, stop
-}
-func (v *VisibleInterval) Clone() IntervalValue {
-	return &VisibleInterval{
-		start:         v.start,
-		stop:          v.stop,
-		modifiedTsNs:  v.modifiedTsNs,
-		fileId:        v.fileId,
-		offsetInChunk: v.offsetInChunk,
-		chunkSize:     v.chunkSize,
-		cipherKey:     v.cipherKey,
-		isGzipped:     v.isGzipped,
+func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, chunkOffset int64, chunkSize uint64, cipherKey []byte, isGzipped bool) VisibleInterval {
+	return VisibleInterval{
+		start:        start,
+		stop:         stop,
+		fileId:       fileId,
+		modifiedTsNs: modifiedTime,
+		chunkOffset:  chunkOffset, // the starting position in the chunk
+		chunkSize:    chunkSize,
+		cipherKey:    cipherKey,
+		isGzipped:    isGzipped,
 	}
 }
 

weed/filer/filechunks_read.go

@@ -1,22 +1,14 @@
 package filer
 
 import (
-	"container/list"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"golang.org/x/exp/slices"
 )
 
-func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles *IntervalList[*VisibleInterval]) {
+func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) {
 
 	var points []*Point
 	for _, chunk := range chunks {
-		if chunk.IsChunkManifest {
-			println("This should not happen! A manifest chunk found:", chunk.GetFileIdString())
-		}
-		start, stop := max(chunk.Offset, startOffset), min(chunk.Offset+int64(chunk.Size), stopOffset)
-		if start >= stop {
-			continue
-		}
 		points = append(points, &Point{
 			x:       chunk.Offset,
 			ts:      chunk.ModifiedTsNs,
@@ -41,45 +33,40 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOff
 	})
 
 	var prevX int64
-	queue := list.New() // points with higher ts are at the tail
-	visibles = NewIntervalList[*VisibleInterval]()
-	var prevPoint *Point
+	var queue []*Point
 	for _, point := range points {
-		if queue.Len() > 0 {
-			prevPoint = queue.Back().Value.(*Point)
-		} else {
-			prevPoint = nil
-		}
 		if point.isStart {
-			if prevPoint != nil {
-				if point.x != prevX && prevPoint.ts < point.ts {
-					addToVisibles(visibles, prevX, prevPoint, point)
+			if len(queue) > 0 {
+				lastIndex := len(queue) - 1
+				lastPoint := queue[lastIndex]
+				if point.x != prevX && lastPoint.ts < point.ts {
+					visibles = addToVisibles(visibles, prevX, lastPoint, point)
 					prevX = point.x
 				}
 			}
 			// insert into queue
-			if prevPoint == nil || prevPoint.ts < point.ts {
-				queue.PushBack(point)
-				prevX = point.x
-			} else {
-				for e := queue.Front(); e != nil; e = e.Next() {
-					if e.Value.(*Point).ts > point.ts {
-						queue.InsertBefore(point, e)
-						break
+			for i := len(queue); i >= 0; i-- {
+				if i == 0 || queue[i-1].ts <= point.ts {
+					if i == len(queue) {
+						prevX = point.x
 					}
+					queue = addToQueue(queue, i, point)
+					break
 				}
 			}
 		} else {
-			isLast := true
-			for e := queue.Back(); e != nil; e = e.Prev() {
-				if e.Value.(*Point).ts == point.ts {
-					queue.Remove(e)
+			lastIndex := len(queue) - 1
+			index := lastIndex
+			var startPoint *Point
+			for ; index >= 0; index-- {
+				startPoint = queue[index]
+				if startPoint.ts == point.ts {
+					queue = removeFromQueue(queue, index)
 					break
 				}
-				isLast = false
 			}
-			if isLast && prevPoint != nil {
-				addToVisibles(visibles, prevX, prevPoint, point)
+			if index == lastIndex && startPoint != nil {
+				visibles = addToVisibles(visibles, prevX, startPoint, point)
 				prevX = point.x
 			}
 		}
@@ -88,30 +75,37 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOff
 	return
 }
 
-func addToVisibles(visibles *IntervalList[*VisibleInterval], prevX int64, startPoint *Point, point *Point) {
-	if prevX < point.x {
-		chunk := startPoint.chunk
-		visible := &VisibleInterval{
-			start:         prevX,
-			stop:          point.x,
-			fileId:        chunk.GetFileIdString(),
-			modifiedTsNs:  chunk.ModifiedTsNs,
-			offsetInChunk: prevX - chunk.Offset,
-			chunkSize:     chunk.Size,
-			cipherKey:     chunk.CipherKey,
-			isGzipped:     chunk.IsCompressed,
-		}
-		appendVisibleInterfal(visibles, visible)
+func removeFromQueue(queue []*Point, index int) []*Point {
+	for i := index; i < len(queue)-1; i++ {
+		queue[i] = queue[i+1]
 	}
+	queue = queue[:len(queue)-1]
+	return queue
 }
 
-func appendVisibleInterfal(visibles *IntervalList[*VisibleInterval], visible *VisibleInterval) {
-	visibles.AppendInterval(&Interval[*VisibleInterval]{
-		StartOffset: visible.start,
-		StopOffset:  visible.stop,
-		TsNs:        visible.modifiedTsNs,
-		Value:       visible,
-	})
+func addToQueue(queue []*Point, index int, point *Point) []*Point {
+	queue = append(queue, point)
+	for i := len(queue) - 1; i > index; i-- {
+		queue[i], queue[i-1] = queue[i-1], queue[i]
+	}
+	return queue
+}
+
+func addToVisibles(visibles []VisibleInterval, prevX int64, startPoint *Point, point *Point) []VisibleInterval {
+	if prevX < point.x {
+		chunk := startPoint.chunk
+		visibles = append(visibles, VisibleInterval{
+			start:        prevX,
+			stop:         point.x,
+			fileId:       chunk.GetFileIdString(),
+			modifiedTsNs: chunk.ModifiedTsNs,
+			chunkOffset:  prevX - chunk.Offset,
+			chunkSize:    chunk.Size,
+			cipherKey:    chunk.CipherKey,
+			isGzipped:    chunk.IsCompressed,
+		})
+	}
+	return visibles
 }
 
 type Point struct {

weed/filer/filechunks_read_test.go

@@ -3,7 +3,6 @@ package filer
 import (
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"math"
 	"math/rand"
 	"testing"
 )
@@ -43,38 +42,9 @@ func TestReadResolvedChunks(t *testing.T) {
 		},
 	}
 
-	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+	visibles := readResolvedChunks(chunks)
 
-	fmt.Printf("resolved to %d visible intervales\n", visibles.Len())
-	for x := visibles.Front(); x != nil; x = x.Next {
-		visible := x.Value
-		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
-	}
-
-}
-
-func TestReadResolvedChunks2(t *testing.T) {
-
-	chunks := []*filer_pb.FileChunk{
-		{
-			FileId:       "c",
-			Offset:       200,
-			Size:         50,
-			ModifiedTsNs: 3,
-		},
-		{
-			FileId:       "e",
-			Offset:       200,
-			Size:         25,
-			ModifiedTsNs: 5,
-		},
-	}
-
-	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
-
-	fmt.Printf("resolved to %d visible intervales\n", visibles.Len())
-	for x := visibles.Front(); x != nil; x = x.Next {
-		visible := x.Value
+	for _, visible := range visibles {
 		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
 	}
 
@@ -102,10 +72,9 @@ func TestRandomizedReadResolvedChunks(t *testing.T) {
 		chunks = append(chunks, randomWrite(array, start, size, ts))
 	}
 
-	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+	visibles := readResolvedChunks(chunks)
 
-	for x := visibles.Front(); x != nil; x = x.Next {
-		visible := x.Value
+	for _, visible := range visibles {
 		for i := visible.start; i < visible.stop; i++ {
 			if array[i] != visible.modifiedTsNs {
 				t.Errorf("position %d expected ts %d actual ts %d", i, array[i], visible.modifiedTsNs)
@@ -143,9 +112,9 @@ func TestSequentialReadResolvedChunks(t *testing.T) {
 		})
 	}
 
-	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+	visibles := readResolvedChunks(chunks)
 
-	fmt.Printf("visibles %d", visibles.Len())
+	fmt.Printf("visibles %d", len(visibles))
 
 }
 
@@ -232,48 +201,9 @@ func TestActualReadResolvedChunks(t *testing.T) {
 		},
 	}
 
-	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+	visibles := readResolvedChunks(chunks)
 
-	for x := visibles.Front(); x != nil; x = x.Next {
-		visible := x.Value
-		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
-	}
-
-}
-
-func TestActualReadResolvedChunks2(t *testing.T) {
-
-	chunks := []*filer_pb.FileChunk{
-		{
-			FileId:       "1,e7b96fef48",
-			Offset:       0,
-			Size:         184320,
-			ModifiedTsNs: 1,
-		},
-		{
-			FileId:       "2,22562640b9",
-			Offset:       184320,
-			Size:         4096,
-			ModifiedTsNs: 2,
-		},
-		{
-			FileId:       "2,33562640b9",
-			Offset:       184320,
-			Size:         4096,
-			ModifiedTsNs: 4,
-		},
-		{
-			FileId:       "4,df033e0fe4",
-			Offset:       188416,
-			Size:         2097152,
-			ModifiedTsNs: 3,
-		},
-	}
-
-	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
-
-	for x := visibles.Front(); x != nil; x = x.Next {
-		visible := x.Value
+	for _, visible := range visibles {
 		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
 	}